297 lines
8.1 KiB
Python
297 lines
8.1 KiB
Python
"""
|
|
Copyright 2023-2024 SGLang Team
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
"""
|
|
|
|
"""Pydantic models for OpenAI API protocol"""
|
|
|
|
import time
|
|
from typing import Dict, List, Optional, Union
|
|
|
|
from pydantic import BaseModel, Field
|
|
from typing_extensions import Literal
|
|
|
|
|
|
class ModelCard(BaseModel):
|
|
"""Model cards."""
|
|
|
|
id: str
|
|
object: str = "model"
|
|
created: int = Field(default_factory=lambda: int(time.time()))
|
|
owned_by: str = "sglang"
|
|
root: Optional[str] = None
|
|
|
|
|
|
class ModelList(BaseModel):
|
|
"""Model list consists of model cards."""
|
|
|
|
object: str = "list"
|
|
data: List[ModelCard] = []
|
|
|
|
|
|
class ErrorResponse(BaseModel):
|
|
object: str = "error"
|
|
message: str
|
|
type: str
|
|
param: Optional[str] = None
|
|
code: int
|
|
|
|
|
|
class LogProbs(BaseModel):
|
|
text_offset: List[int] = Field(default_factory=list)
|
|
token_logprobs: List[Optional[float]] = Field(default_factory=list)
|
|
tokens: List[str] = Field(default_factory=list)
|
|
top_logprobs: List[Optional[Dict[str, float]]] = Field(default_factory=list)
|
|
|
|
|
|
class TopLogprob(BaseModel):
|
|
token: str
|
|
bytes: List[int]
|
|
logprob: float
|
|
|
|
|
|
class ChatCompletionTokenLogprob(BaseModel):
|
|
token: str
|
|
bytes: List[int]
|
|
logprob: float
|
|
top_logprobs: List[TopLogprob]
|
|
|
|
|
|
class ChoiceLogprobs(BaseModel):
|
|
# build for v1/chat/completions response
|
|
content: List[ChatCompletionTokenLogprob]
|
|
|
|
|
|
class UsageInfo(BaseModel):
|
|
prompt_tokens: int = 0
|
|
total_tokens: int = 0
|
|
completion_tokens: Optional[int] = 0
|
|
|
|
|
|
class FileRequest(BaseModel):
|
|
# https://platform.openai.com/docs/api-reference/files/create
|
|
file: bytes # The File object (not file name) to be uploaded
|
|
purpose: str = (
|
|
"batch" # The intended purpose of the uploaded file, default is "batch"
|
|
)
|
|
|
|
|
|
class FileResponse(BaseModel):
|
|
id: str
|
|
object: str = "file"
|
|
bytes: int
|
|
created_at: int
|
|
filename: str
|
|
purpose: str
|
|
|
|
|
|
class FileDeleteResponse(BaseModel):
|
|
id: str
|
|
object: str = "file"
|
|
deleted: bool
|
|
|
|
|
|
class BatchRequest(BaseModel):
|
|
input_file_id: (
|
|
str # The ID of an uploaded file that contains requests for the new batch
|
|
)
|
|
endpoint: str # The endpoint to be used for all requests in the batch
|
|
completion_window: str # The time frame within which the batch should be processed
|
|
metadata: Optional[dict] = None # Optional custom metadata for the batch
|
|
|
|
|
|
class BatchResponse(BaseModel):
|
|
id: str
|
|
object: str = "batch"
|
|
endpoint: str
|
|
errors: Optional[dict] = None
|
|
input_file_id: str
|
|
completion_window: str
|
|
status: str = "validating"
|
|
output_file_id: Optional[str] = None
|
|
error_file_id: Optional[str] = None
|
|
created_at: int
|
|
in_progress_at: Optional[int] = None
|
|
expires_at: Optional[int] = None
|
|
finalizing_at: Optional[int] = None
|
|
completed_at: Optional[int] = None
|
|
failed_at: Optional[int] = None
|
|
expired_at: Optional[int] = None
|
|
cancelling_at: Optional[int] = None
|
|
cancelled_at: Optional[int] = None
|
|
request_counts: dict = {"total": 0, "completed": 0, "failed": 0}
|
|
metadata: Optional[dict] = None
|
|
|
|
|
|
class CompletionRequest(BaseModel):
|
|
# Ordered by official OpenAI API documentation
|
|
# https://platform.openai.com/docs/api-reference/completions/create
|
|
model: str
|
|
prompt: Union[List[int], List[List[int]], str, List[str]]
|
|
best_of: Optional[int] = None
|
|
echo: Optional[bool] = False
|
|
frequency_penalty: Optional[float] = 0.0
|
|
logit_bias: Optional[Dict[str, float]] = None
|
|
logprobs: Optional[int] = None
|
|
max_tokens: Optional[int] = 16
|
|
n: int = 1
|
|
presence_penalty: Optional[float] = 0.0
|
|
seed: Optional[int] = None
|
|
stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
|
|
stream: Optional[bool] = False
|
|
suffix: Optional[str] = None
|
|
temperature: Optional[float] = 1.0
|
|
top_p: Optional[float] = 1.0
|
|
user: Optional[str] = None
|
|
|
|
# Extra parameters for SRT backend only and will be ignored by OpenAI models.
|
|
regex: Optional[str] = None
|
|
ignore_eos: Optional[bool] = False
|
|
|
|
|
|
class CompletionResponseChoice(BaseModel):
|
|
index: int
|
|
text: str
|
|
logprobs: Optional[LogProbs] = None
|
|
finish_reason: Optional[str] = None
|
|
|
|
|
|
class CompletionResponse(BaseModel):
|
|
id: str
|
|
object: str = "text_completion"
|
|
created: int = Field(default_factory=lambda: int(time.time()))
|
|
model: str
|
|
choices: List[CompletionResponseChoice]
|
|
usage: UsageInfo
|
|
|
|
|
|
class CompletionResponseStreamChoice(BaseModel):
|
|
index: int
|
|
text: str
|
|
logprobs: Optional[LogProbs] = None
|
|
finish_reason: Optional[str] = None
|
|
|
|
|
|
class CompletionStreamResponse(BaseModel):
|
|
id: str
|
|
object: str = "text_completion"
|
|
created: int = Field(default_factory=lambda: int(time.time()))
|
|
model: str
|
|
choices: List[CompletionResponseStreamChoice]
|
|
usage: UsageInfo
|
|
|
|
|
|
class ChatCompletionMessageGenericParam(BaseModel):
|
|
role: Literal["system", "assistant"]
|
|
content: str
|
|
|
|
|
|
class ChatCompletionMessageContentTextPart(BaseModel):
|
|
type: Literal["text"]
|
|
text: str
|
|
|
|
|
|
class ChatCompletionMessageContentImageURL(BaseModel):
|
|
url: str
|
|
detail: Optional[Literal["auto", "low", "high"]] = "auto"
|
|
|
|
|
|
class ChatCompletionMessageContentImagePart(BaseModel):
|
|
type: Literal["image_url"]
|
|
image_url: ChatCompletionMessageContentImageURL
|
|
|
|
|
|
ChatCompletionMessageContentPart = Union[
|
|
ChatCompletionMessageContentTextPart, ChatCompletionMessageContentImagePart
|
|
]
|
|
|
|
|
|
class ChatCompletionMessageUserParam(BaseModel):
|
|
role: Literal["user"]
|
|
content: Union[str, List[ChatCompletionMessageContentPart]]
|
|
|
|
|
|
ChatCompletionMessageParam = Union[
|
|
ChatCompletionMessageGenericParam, ChatCompletionMessageUserParam
|
|
]
|
|
|
|
|
|
class ResponseFormat(BaseModel):
|
|
# type must be "json_object" or "text"
|
|
type: Literal["text", "json_object"]
|
|
|
|
|
|
class ChatCompletionRequest(BaseModel):
|
|
# Ordered by official OpenAI API documentation
|
|
# https://platform.openai.com/docs/api-reference/chat/create
|
|
messages: List[ChatCompletionMessageParam]
|
|
model: str
|
|
frequency_penalty: Optional[float] = 0.0
|
|
logit_bias: Optional[Dict[str, float]] = None
|
|
logprobs: Optional[bool] = False
|
|
top_logprobs: Optional[int] = None
|
|
max_tokens: Optional[int] = None
|
|
n: Optional[int] = 1
|
|
presence_penalty: Optional[float] = 0.0
|
|
response_format: Optional[ResponseFormat] = None
|
|
seed: Optional[int] = None
|
|
stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
|
|
stream: Optional[bool] = False
|
|
temperature: Optional[float] = 0.7
|
|
top_p: Optional[float] = 1.0
|
|
user: Optional[str] = None
|
|
|
|
# Extra parameters for SRT backend only and will be ignored by OpenAI models.
|
|
regex: Optional[str] = None
|
|
|
|
|
|
class ChatMessage(BaseModel):
|
|
role: Optional[str] = None
|
|
content: Optional[str] = None
|
|
|
|
|
|
class ChatCompletionResponseChoice(BaseModel):
|
|
index: int
|
|
message: ChatMessage
|
|
logprobs: Optional[Union[LogProbs, ChoiceLogprobs]] = None
|
|
finish_reason: str
|
|
|
|
|
|
class ChatCompletionResponse(BaseModel):
|
|
id: str
|
|
object: str = "chat.completion"
|
|
created: int = Field(default_factory=lambda: int(time.time()))
|
|
model: str
|
|
choices: List[ChatCompletionResponseChoice]
|
|
usage: UsageInfo
|
|
|
|
|
|
class DeltaMessage(BaseModel):
|
|
role: Optional[str] = None
|
|
content: Optional[str] = None
|
|
|
|
|
|
class ChatCompletionResponseStreamChoice(BaseModel):
|
|
index: int
|
|
delta: DeltaMessage
|
|
logprobs: Optional[Union[LogProbs, ChoiceLogprobs]] = None
|
|
finish_reason: Optional[str] = None
|
|
|
|
|
|
class ChatCompletionStreamResponse(BaseModel):
|
|
id: str
|
|
object: str = "chat.completion.chunk"
|
|
created: int = Field(default_factory=lambda: int(time.time()))
|
|
model: str
|
|
choices: List[ChatCompletionResponseStreamChoice]
|