EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

docs: fix module docstrings and copyright headers (#2077)

Browse Source
This commit is contained in:
Xuehai Pan
2024-11-22 22:16:53 +08:00
committed by GitHub
parent 2797bc3422
commit 62a4a339eb
83 changed files with 1102 additions and 1210 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
LICENSE
View File

@@ -186,7 +186,7 @@
same "printed page" as the copyright notice for easier same "printed page" as the copyright notice for easier
identification within third-party archives. identification within third-party archives.
Copyright [yyyy] [name of copyright owner] Copyright 2023-2024 SGLang Team
Licensed under the Apache License, Version 2.0 (the "License"); Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. you may not use this file except in compliance with the License.

27
benchmark/lora/lora_bench.py
View File

@@ -1,17 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
import argparse import argparse
import asyncio import asyncio

27
python/sglang/srt/configs/model_config.py
View File

@@ -1,17 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
import json import json
import logging import logging

27
python/sglang/srt/constrained/__init__.py
View File

@@ -1,17 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# TODO(lmzheng): make this an optional dependency # TODO(lmzheng): make this an optional dependency
from sglang.srt.constrained.outlines_backend import build_regex_from_object from sglang.srt.constrained.outlines_backend import build_regex_from_object

28
python/sglang/srt/constrained/base_grammar_backend.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""The baseclass of a backend for grammar-guided constrained decoding.""" """The baseclass of a backend for grammar-guided constrained decoding."""
from concurrent.futures import Future, ThreadPoolExecutor from concurrent.futures import Future, ThreadPoolExecutor

28
python/sglang/srt/constrained/outlines_backend.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""Constrained decoding with outlines backend.""" """Constrained decoding with outlines backend."""
import json import json

28
python/sglang/srt/constrained/outlines_jump_forward.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
""" """
Faster constrained decoding with jump forward decoding / compressed finite state machine. Faster constrained decoding with jump forward decoding / compressed finite state machine.
Reference: https://lmsys.org/blog/2024-02-05-compressed-fsm/ Reference: https://lmsys.org/blog/2024-02-05-compressed-fsm/

28
python/sglang/srt/constrained/xgrammar_backend.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""Constrained decoding with xgrammar backend.""" """Constrained decoding with xgrammar backend."""
import logging import logging

28
python/sglang/srt/conversation.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""Conversation chat templates.""" """Conversation chat templates."""
# Adapted from # Adapted from

28
python/sglang/srt/hf_transformers_utils.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""Utilities for Huggingface Transformers.""" """Utilities for Huggingface Transformers."""
import contextlib import contextlib

26
python/sglang/srt/layers/activation.py
View File

@@ -1,16 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
http://www.apache.org/licenses/LICENSE-2.0 # http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software #
distributed under the License is distributed on an "AS IS" BASIS, # Unless required by applicable law or agreed to in writing, software
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # distributed under the License is distributed on an "AS IS" BASIS,
See the License for the specific language governing permissions and # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
limitations under the License. # See the License for the specific language governing permissions and
""" # limitations under the License.
# ==============================================================================
"""Fused operators for activation layers.""" """Fused operators for activation layers."""
import logging import logging

28
python/sglang/srt/layers/attention/triton_ops/decode_attention.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
""" """
Memory-efficient attention for decoding. Memory-efficient attention for decoding.
It supports page size = 1. It supports page size = 1.

28
python/sglang/srt/layers/attention/triton_ops/extend_attention.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
""" """
Memory-efficient attention for prefill. Memory-efficient attention for prefill.
It supports page size = 1 and prefill with KV cache (i.e. extend). It supports page size = 1 and prefill with KV cache (i.e. extend).

28
python/sglang/srt/layers/attention/triton_ops/prefill_attention.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
""" """
Memory-efficient attention for prefill. Memory-efficient attention for prefill.
It supporst page size = 1. It supporst page size = 1.

27
python/sglang/srt/layers/custom_op_util.py
View File

@@ -1,17 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
from vllm.model_executor.custom_op import CustomOp from vllm.model_executor.custom_op import CustomOp

28
python/sglang/srt/layers/layernorm.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""Fused operators for normalization layers.""" """Fused operators for normalization layers."""
import logging import logging

28
python/sglang/srt/layers/logits_processor.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""Logits processing.""" """Logits processing."""
import dataclasses import dataclasses

28
python/sglang/srt/layers/radix_attention.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""Radix attention.""" """Radix attention."""
from torch import nn from torch import nn

26
python/sglang/srt/layers/rotary_embedding.py
View File

@@ -1,16 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
http://www.apache.org/licenses/LICENSE-2.0 # http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software #
distributed under the License is distributed on an "AS IS" BASIS, # Unless required by applicable law or agreed to in writing, software
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # distributed under the License is distributed on an "AS IS" BASIS,
See the License for the specific language governing permissions and # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
limitations under the License. # See the License for the specific language governing permissions and
""" # limitations under the License.
# ==============================================================================
"""MRotaryEmbedding""" """MRotaryEmbedding"""
from typing import Any, Dict, List, Optional, Tuple, Union from typing import Any, Dict, List, Optional, Tuple, Union

27
python/sglang/srt/lora/lora.py
View File

@@ -1,17 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# Integrates "S-LoRA: Serving Thousands of Concurrent LoRA Adapters" # Integrates "S-LoRA: Serving Thousands of Concurrent LoRA Adapters"
# and "Punica: Multi-Tenant LoRA Serving" # and "Punica: Multi-Tenant LoRA Serving"

27
python/sglang/srt/lora/lora_config.py
View File

@@ -1,17 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
import json import json
import os import os

46
python/sglang/srt/lora/lora_manager.py
View File

@@ -1,22 +1,20 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# Integrates "S-LoRA: Serving Thousands of Concurrent LoRA Adapters" # Integrates "S-LoRA: Serving Thousands of Concurrent LoRA Adapters"
# and "Punica: Multi-Tenant LoRA Serving" # and "Punica: Multi-Tenant LoRA Serving"
import logging import logging
import re import re
@@ -146,9 +144,9 @@ class LoRAManager:
} }
else: else:
logger.warning( logger.warning(
f"WARNING: get_module_name() is not defined, " "WARNING: get_module_name() is not defined, "
f"which is used to map config module name to model implementation module name." "which is used to map config module name to model implementation module name."
f"Use the default one, but please check if it is correct for your model." "Use the default one, but please check if it is correct for your model."
) )
self.target_modules = { self.target_modules = {
get_module_name(module) for module in self.origin_target_modules get_module_name(module) for module in self.origin_target_modules
@@ -194,9 +192,9 @@ class LoRAManager:
hidden_dim_A, _ = self.base_model.get_hidden_dim(module_A) hidden_dim_A, _ = self.base_model.get_hidden_dim(module_A)
else: else:
logger.warning( logger.warning(
f"WARNING: get_hidden_dim() is not defined, " "WARNING: get_hidden_dim() is not defined, "
f"which is used to get the hidden dim for different lora modules" "which is used to get the hidden dim for different lora modules"
f"Use the default one, but please check if it is correct for your model." "Use the default one, but please check if it is correct for your model."
) )
hidden_dim_A, _ = get_hidden_dim(module_A, self.base_hf_config) hidden_dim_A, _ = get_hidden_dim(module_A, self.base_hf_config)
c = self.loras[-1].get_stacked_multiply(module_A) c = self.loras[-1].get_stacked_multiply(module_A)
@@ -218,9 +216,9 @@ class LoRAManager:
_, hidden_dim_B = self.base_model.get_hidden_dim(module_B) _, hidden_dim_B = self.base_model.get_hidden_dim(module_B)
else: else:
logger.warning( logger.warning(
f"WARNING: get_hidden_dim() is not defined, " "WARNING: get_hidden_dim() is not defined, "
f"which is used to get the hidden dim for different lora modules" "which is used to get the hidden dim for different lora modules"
f"Use the default one, but please check if it is correct for your model." "Use the default one, but please check if it is correct for your model."
) )
_, hidden_dim_B = get_hidden_dim(module_B, self.base_hf_config) _, hidden_dim_B = get_hidden_dim(module_B, self.base_hf_config)
c = self.loras[-1].get_stacked_multiply(module_B) c = self.loras[-1].get_stacked_multiply(module_B)

28
python/sglang/srt/managers/data_parallel_controller.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""A controller that dispatches requests to multiple data parallel workers.""" """A controller that dispatches requests to multiple data parallel workers."""
import logging import logging

28
python/sglang/srt/managers/detokenizer_manager.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""DetokenizerManager is a process that detokenizes the token ids.""" """DetokenizerManager is a process that detokenizes the token ids."""
import dataclasses import dataclasses

28
python/sglang/srt/managers/io_struct.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
""" """
The definition of objects transfered between different The definition of objects transfered between different
processes (TokenizerManager, DetokenizerManager, Controller). processes (TokenizerManager, DetokenizerManager, Controller).

28
python/sglang/srt/managers/schedule_batch.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
""" """
Store information about requests and batches. Store information about requests and batches.

28
python/sglang/srt/managers/schedule_policy.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""Request scheduler policy""" """Request scheduler policy"""
import os import os

28
python/sglang/srt/managers/scheduler.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""A scheduler that manages a tensor parallel GPU worker.""" """A scheduler that manages a tensor parallel GPU worker."""
import dataclasses import dataclasses

23
python/sglang/srt/managers/session_controller.py
View File

@@ -1,15 +1,14 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
import copy import copy
import uuid import uuid

28
python/sglang/srt/managers/tokenizer_manager.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""TokenizerManager is a process that tokenizes the text.""" """TokenizerManager is a process that tokenizes the text."""
import asyncio import asyncio

28
python/sglang/srt/managers/tp_worker.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""A tensor parallel worker.""" """A tensor parallel worker."""
import logging import logging

28
python/sglang/srt/managers/tp_worker_overlap_thread.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""A tensor parallel worker.""" """A tensor parallel worker."""
import dataclasses import dataclasses

28
python/sglang/srt/metrics/collector.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""Utilities for Prometheus Metrics Collection.""" """Utilities for Prometheus Metrics Collection."""
from dataclasses import dataclass from dataclasses import dataclass

28
python/sglang/srt/metrics/func_timer.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
""" """
Records the latency of some functions Records the latency of some functions
""" """

27
python/sglang/srt/mm_utils.py
View File

@@ -1,17 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# Source: https://github.com/LLaVA-VL/LLaVA-NeXT/blob/main/llava/mm_utils.py # Source: https://github.com/LLaVA-VL/LLaVA-NeXT/blob/main/llava/mm_utils.py
""" """

32
python/sglang/srt/model_executor/cuda_graph_runner.py
View File

@@ -1,22 +1,20 @@
from __future__ import annotations # Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
""" # you may not use this file except in compliance with the License.
Copyright 2023-2024 SGLang Team # You may obtain a copy of the License at
Licensed under the Apache License, Version 2.0 (the "License"); #
you may not use this file except in compliance with the License. # http://www.apache.org/licenses/LICENSE-2.0
You may obtain a copy of the License at #
# Unless required by applicable law or agreed to in writing, software
http://www.apache.org/licenses/LICENSE-2.0 # distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Unless required by applicable law or agreed to in writing, software # See the License for the specific language governing permissions and
distributed under the License is distributed on an "AS IS" BASIS, # limitations under the License.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # ==============================================================================
See the License for the specific language governing permissions and
limitations under the License.
"""
"""Run the model with cuda graph and torch.compile.""" """Run the model with cuda graph and torch.compile."""
from __future__ import annotations
import bisect import bisect
from contextlib import contextmanager from contextlib import contextmanager
from typing import TYPE_CHECKING, Callable from typing import TYPE_CHECKING, Callable

32
python/sglang/srt/model_executor/forward_batch_info.py
View File

@@ -1,20 +1,16 @@
from __future__ import annotations # Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
""" # you may not use this file except in compliance with the License.
Copyright 2023-2024 SGLang Team # You may obtain a copy of the License at
Licensed under the Apache License, Version 2.0 (the "License"); #
you may not use this file except in compliance with the License. # http://www.apache.org/licenses/LICENSE-2.0
You may obtain a copy of the License at #
# Unless required by applicable law or agreed to in writing, software
http://www.apache.org/licenses/LICENSE-2.0 # distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Unless required by applicable law or agreed to in writing, software # See the License for the specific language governing permissions and
distributed under the License is distributed on an "AS IS" BASIS, # limitations under the License.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # ==============================================================================
See the License for the specific language governing permissions and
limitations under the License.
"""
""" """
Store information about a forward batch. Store information about a forward batch.
@@ -31,6 +27,8 @@ ScheduleBatch -> ModelWorkerBatch -> ForwardBatch
It contains low-level tensor data. Most of the data consists of GPU tensors. It contains low-level tensor data. Most of the data consists of GPU tensors.
""" """
from __future__ import annotations
from dataclasses import dataclass from dataclasses import dataclass
from enum import IntEnum, auto from enum import IntEnum, auto
from typing import TYPE_CHECKING, List, Optional from typing import TYPE_CHECKING, List, Optional

28
python/sglang/srt/model_executor/model_runner.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""ModelRunner runs the forward passes of the models.""" """ModelRunner runs the forward passes of the models."""
import gc import gc

29
python/sglang/srt/models/chatglm.py
View File

@@ -1,22 +1,21 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
# coding=utf-8
# Adapted from # Adapted from
# https://github.com/THUDM/ChatGLM2-6B # https://github.com/THUDM/ChatGLM2-6B
"""Inference-only ChatGLM model compatible with THUDM weights.""" """Inference-only ChatGLM model compatible with THUDM weights."""
from typing import Iterable, Optional, Tuple from typing import Iterable, Optional, Tuple
import torch import torch

31
python/sglang/srt/models/commandr.py
View File

@@ -1,19 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# coding=utf-8
# Copyright 2024 Cohere and the HuggingFace Inc. team. All rights reserved. # Copyright 2024 Cohere and the HuggingFace Inc. team. All rights reserved.
# #
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
@@ -32,12 +29,14 @@ limitations under the License.
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# ==============================================================================
# Adapted from # Adapted from
# https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/commandr.py#L1 # https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/commandr.py#L1
# This file is based on the LLama model definition file in transformers # This file is based on the LLama model definition file in transformers
"""PyTorch Cohere model.""" """PyTorch Cohere model."""
from typing import Iterable, Optional, Tuple from typing import Iterable, Optional, Tuple
import torch import torch

29
python/sglang/srt/models/dbrx.py
View File

@@ -1,21 +1,20 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# Adapted from: # Adapted from:
# https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/dbrx.py#L1 # https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/dbrx.py#L1
# coding=utf-8
from typing import Iterable, Optional, Tuple from typing import Iterable, Optional, Tuple
import torch import torch

28
python/sglang/srt/models/deepseek.py
View File

@@ -1,21 +1,21 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# Adapted from: # Adapted from:
# https://github.com/vllm-project/vllm/blob/14f91fe67c2342f2fe859dc6a5c40810df0e1c61/vllm/model_executor/models/deepseek.py # https://github.com/vllm-project/vllm/blob/14f91fe67c2342f2fe859dc6a5c40810df0e1c61/vllm/model_executor/models/deepseek.py
"""Inference-only Deepseek model.""" """Inference-only Deepseek model."""
from typing import Any, Dict, Iterable, Optional, Tuple from typing import Any, Dict, Iterable, Optional, Tuple
import torch import torch

28
python/sglang/srt/models/deepseek_v2.py
View File

@@ -1,21 +1,21 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# Adapted from: # Adapted from:
# https://github.com/vllm-project/vllm/blob/fb6af8bc086328ca6659e72d11ffd4309ce4de22/vllm/model_executor/models/deepseek_v2.py # https://github.com/vllm-project/vllm/blob/fb6af8bc086328ca6659e72d11ffd4309ce4de22/vllm/model_executor/models/deepseek_v2.py
"""Inference-only DeepseekV2 model.""" """Inference-only DeepseekV2 model."""
from typing import Any, Dict, Iterable, Optional, Tuple from typing import Any, Dict, Iterable, Optional, Tuple
import torch import torch

29
python/sglang/srt/models/exaone.py
View File

@@ -1,18 +1,17 @@
""" # Copyright 2024 The LGcns AI Engineering Team
Copyright 2024 The LGcns AI Engineering Team # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# Adapted from llama2.py # Adapted from llama2.py
"""Inference-only Exaone model compatible with HuggingFace weights.""" """Inference-only Exaone model compatible with HuggingFace weights."""

28
python/sglang/srt/models/gemma.py
View File

@@ -1,21 +1,21 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# Adapted from: # Adapted from:
# https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/gemma.py#L1 # https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/gemma.py#L1
"""Inference-only Gemma model compatible with HuggingFace weights.""" """Inference-only Gemma model compatible with HuggingFace weights."""
from typing import Iterable, Optional, Tuple from typing import Iterable, Optional, Tuple
import torch import torch

28
python/sglang/srt/models/gemma2.py
View File

@@ -1,20 +1,20 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# Adapted from: # Adapted from:
# https://github.com/vllm-project/vllm/blob/56b325e977435af744f8b3dca7af0ca209663558/vllm/model_executor/models/gemma2.py # https://github.com/vllm-project/vllm/blob/56b325e977435af744f8b3dca7af0ca209663558/vllm/model_executor/models/gemma2.py
from typing import Iterable, Optional, Set, Tuple, Union from typing import Iterable, Optional, Set, Tuple, Union
import torch import torch

27
python/sglang/srt/models/gemma2_reward.py
View File

@@ -1,17 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
from typing import Iterable, Optional, Tuple from typing import Iterable, Optional, Tuple

28
python/sglang/srt/models/gpt_bigcode.py
View File

@@ -1,21 +1,21 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# Adapted from: # Adapted from:
# https://github.com/vllm-project/vllm/blob/07eb6f19f3b0ee9f7adf6eb689607028aa40bfd5/vllm/model_executor/models/gpt_bigcode.py # https://github.com/vllm-project/vllm/blob/07eb6f19f3b0ee9f7adf6eb689607028aa40bfd5/vllm/model_executor/models/gpt_bigcode.py
"""Inference-only GPTBigCode model compatible with HuggingFace weights.""" """Inference-only GPTBigCode model compatible with HuggingFace weights."""
from typing import Iterable, Optional, Tuple from typing import Iterable, Optional, Tuple
import torch import torch

28
python/sglang/srt/models/grok.py
View File

@@ -1,21 +1,21 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# Adapted from # Adapted from
# https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/mixtral.py#L1 # https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/mixtral.py#L1
"""Inference-only Grok1 model.""" """Inference-only Grok1 model."""
import warnings import warnings
from typing import Iterable, List, Optional, Tuple from typing import Iterable, List, Optional, Tuple

28
python/sglang/srt/models/internlm2.py
View File

@@ -1,19 +1,17 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
# -*- coding: utf-8 -*-
# Adapted from https://raw.githubusercontent.com/vllm-project/vllm/7f62077af5159c625fe3ad1c812e6c1a2b93ba3b/vllm/model_executor/models/internlm2.py # Adapted from https://raw.githubusercontent.com/vllm-project/vllm/7f62077af5159c625fe3ad1c812e6c1a2b93ba3b/vllm/model_executor/models/internlm2.py
from typing import Any, Dict, Iterable, Optional, Tuple from typing import Any, Dict, Iterable, Optional, Tuple

27
python/sglang/srt/models/internlm2_reward.py
View File

@@ -1,17 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
from typing import Iterable, Optional, Tuple from typing import Iterable, Optional, Tuple

27
python/sglang/srt/models/llama.py
View File

@@ -1,17 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# Adapted from # Adapted from
# https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/llama.py#L1 # https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/llama.py#L1

27
python/sglang/srt/models/llama_classification.py
View File

@@ -1,17 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
from typing import Iterable, Optional, Tuple from typing import Iterable, Optional, Tuple

27
python/sglang/srt/models/llama_reward.py
View File

@@ -1,17 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
from typing import Iterable, Optional, Tuple from typing import Iterable, Optional, Tuple

28
python/sglang/srt/models/llava.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""Inference-only LLaVa model compatible with HuggingFace weights.""" """Inference-only LLaVa model compatible with HuggingFace weights."""
import math import math

28
python/sglang/srt/models/llavavid.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""Inference-only LLaVa video model compatible with HuggingFace weights.""" """Inference-only LLaVa video model compatible with HuggingFace weights."""
from typing import Iterable, List, Optional, Tuple from typing import Iterable, List, Optional, Tuple

28
python/sglang/srt/models/minicpm.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""Inference-only MiniCPM model compatible with HuggingFace weights.""" """Inference-only MiniCPM model compatible with HuggingFace weights."""
import math import math

28
python/sglang/srt/models/minicpm3.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""Inference-only MiniCPM3 model compatible with HuggingFace weights.""" """Inference-only MiniCPM3 model compatible with HuggingFace weights."""
import math import math

28
python/sglang/srt/models/mistral.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""Inference-only Mistral model.""" """Inference-only Mistral model."""
from sglang.srt.models.llama import LlamaForCausalLM from sglang.srt.models.llama import LlamaForCausalLM

28
python/sglang/srt/models/mixtral.py
View File

@@ -1,21 +1,21 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# Adapted from # Adapted from
# https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/mixtral.py#L1 # https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/mixtral.py#L1
"""Inference-only Mixtral model.""" """Inference-only Mixtral model."""
from typing import Iterable, Optional, Tuple from typing import Iterable, Optional, Tuple
import torch import torch

28
python/sglang/srt/models/mixtral_quant.py
View File

@@ -1,21 +1,21 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# Adapted from # Adapted from
# https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/mixtral_quant.py#L1 # https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/mixtral_quant.py#L1
"""Inference-only Mixtral model.""" """Inference-only Mixtral model."""
from typing import Iterable, Optional, Tuple from typing import Iterable, Optional, Tuple
import numpy as np import numpy as np

27
python/sglang/srt/models/olmo.py
View File

@@ -1,17 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# Adapted from # Adapted from
# https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/olmo.py#L1 # https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/olmo.py#L1

28
python/sglang/srt/models/olmoe.py
View File

@@ -1,22 +1,22 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# Adapted from: # Adapted from:
# https://github.com/vllm-project/vllm/pull/7922 # https://github.com/vllm-project/vllm/pull/7922
"""Inference-only OLMoE model compatible with HuggingFace weights.""" """Inference-only OLMoE model compatible with HuggingFace weights."""
from typing import Any, Dict, Iterable, List, Optional, Tuple from typing import Any, Dict, Iterable, List, Optional, Tuple
import torch import torch

28
python/sglang/srt/models/qwen.py
View File

@@ -1,20 +1,20 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# Adapted from # Adapted from
# https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/qwen.py#L1 # https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/qwen.py#L1
from typing import Any, Dict, Iterable, Optional, Tuple from typing import Any, Dict, Iterable, Optional, Tuple
import torch import torch

28
python/sglang/srt/models/qwen2.py
View File

@@ -1,21 +1,21 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# Adapted from llama2.py # Adapted from llama2.py
# Modify details for the adaptation of Qwen2 model. # Modify details for the adaptation of Qwen2 model.
"""Inference-only Qwen2 model compatible with HuggingFace weights.""" """Inference-only Qwen2 model compatible with HuggingFace weights."""
from typing import Any, Dict, Iterable, Optional, Tuple from typing import Any, Dict, Iterable, Optional, Tuple
import torch import torch

31
python/sglang/srt/models/qwen2_moe.py
View File

@@ -1,23 +1,22 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
# coding=utf-8
# Adapted from # Adapted from
# https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/qwen2_moe.py # https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/qwen2_moe.py
"""Inference-only Qwen2MoE model compatible with HuggingFace weights.""" """Inference-only Qwen2MoE model compatible with HuggingFace weights."""
from typing import Any, Dict, Iterable, List, Optional, Tuple
from typing import Any, Dict, Iterable, Optional, Tuple
import torch import torch
import torch.nn.functional as F import torch.nn.functional as F

34
python/sglang/srt/models/stablelm.py
View File

@@ -1,22 +1,24 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# Adapted from: # Adapted from:
# https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/stablelm.py#L1 # https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/stablelm.py#L1
"""Inference-only StableLM-2 (https://huggingface.co/stabilityai/stablelm-2-1_6b) """
model compatible with HuggingFace weights.""" Inference-only StableLM-2 (https://huggingface.co/stabilityai/stablelm-2-1_6b)
model compatible with HuggingFace weights.
"""
from typing import Iterable, Optional, Tuple from typing import Iterable, Optional, Tuple
import torch import torch

32
python/sglang/srt/models/torch_native_llama.py
View File

@@ -1,24 +1,22 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# Adapted from # Adapted from
# https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/llama.py#L1 # https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/llama.py#L1
"""Inference-only LLaMA model compatible with HuggingFace weights."""
# PyTorch Tensor Parallel Available for This Model
""" """
Inference-only LLaMA model compatible with HuggingFace weights.
This model supports tensor parallelism (TP) using the PyTorch tensor parallel package. This model supports tensor parallelism (TP) using the PyTorch tensor parallel package.
Reference: https://pytorch.org/docs/stable/distributed.tensor.parallel.html Reference: https://pytorch.org/docs/stable/distributed.tensor.parallel.html

27
python/sglang/srt/models/xverse.py
View File

@@ -1,17 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# Adapted from # Adapted from
# https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/xverse.py#L1 # https://github.com/vllm-project/vllm/blob/c7f2cf2b7f67bce5842fedfdba508440fe257375/vllm/model_executor/models/xverse.py#L1

29
python/sglang/srt/models/xverse_moe.py
View File

@@ -1,19 +1,18 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""Inference-only XVERSE MoE model.""" """Inference-only XVERSE MoE model."""
from typing import Any, Dict, Iterable, Optional, Tuple from typing import Any, Dict, Iterable, Optional, Tuple
import torch import torch

28
python/sglang/srt/models/yivl.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""Inference-only Yi-VL model.""" """Inference-only Yi-VL model."""
from typing import Iterable, Optional, Tuple from typing import Iterable, Optional, Tuple

28
python/sglang/srt/openai_api/adapter.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""Conversion between OpenAI APIs and native SRT APIs""" """Conversion between OpenAI APIs and native SRT APIs"""
import asyncio import asyncio

28
python/sglang/srt/openai_api/protocol.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""Pydantic models for OpenAI API protocol""" """Pydantic models for OpenAI API protocol"""
import time import time

28
python/sglang/srt/sampling/sampling_params.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""Sampling parameters for text generation.""" """Sampling parameters for text generation."""
from typing import List, Optional, Union from typing import List, Optional, Union

28
python/sglang/srt/server.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
""" """
The entry point of inference server. The entry point of inference server.
SRT = SGLang Runtime. SRT = SGLang Runtime.

28
python/sglang/srt/server_args.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""The arguments of the server.""" """The arguments of the server."""
import argparse import argparse

28
python/sglang/srt/utils.py
View File

@@ -1,18 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
"""Common utilities.""" """Common utilities."""
import base64 import base64

27
python/sglang/test/runners.py
View File

@@ -1,17 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
import json import json
import multiprocessing as mp import multiprocessing as mp

23
scripts/playground/test_session_id.py
View File

@@ -1,15 +1,14 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
# FIXME: Make it a CI test # FIXME: Make it a CI test

27
test/srt/models/test_embedding_models.py
View File

@@ -1,17 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
import multiprocessing as mp import multiprocessing as mp
import unittest import unittest

28
test/srt/models/test_generation_models.py
View File

@@ -1,3 +1,16 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
""" """
Usage: Usage:
@@ -6,21 +19,6 @@ To test a specific model:
2. Run `ONLY_RUN=Qwen/Qwen2-1.5B python3 -m unittest test_generation_models.TestGenerationModels.test_others` 2. Run `ONLY_RUN=Qwen/Qwen2-1.5B python3 -m unittest test_generation_models.TestGenerationModels.test_others`
""" """
"""
Copyright 2023-2024 SGLang Team
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import dataclasses import dataclasses
import multiprocessing as mp import multiprocessing as mp
import os import os

27
test/srt/models/test_lora.py
View File

@@ -1,17 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
import multiprocessing as mp import multiprocessing as mp
import unittest import unittest

27
test/srt/models/test_reward_models.py
View File

@@ -1,17 +1,16 @@
""" # Copyright 2023-2024 SGLang Team
Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License");
Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.
you may not use this file except in compliance with the License. # You may obtain a copy of the License at
You may obtain a copy of the License at #
# http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0 #
# Unless required by applicable law or agreed to in writing, software
Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS,
distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and
See the License for the specific language governing permissions and # limitations under the License.
limitations under the License. # ==============================================================================
"""
import multiprocessing as mp import multiprocessing as mp
import unittest import unittest