161 lines
5.7 KiB
Python
161 lines
5.7 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
"""The request function for API endpoints."""
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
import traceback
|
|
from dataclasses import dataclass, field
|
|
from typing import Optional
|
|
|
|
import aiohttp
|
|
from tqdm.asyncio import tqdm
|
|
|
|
AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=6 * 60 * 60)
|
|
|
|
|
|
@dataclass
|
|
class RequestFuncInput:
|
|
"""The input for the request function."""
|
|
prompt: str
|
|
api_url: str
|
|
prompt_len: int
|
|
output_len: int
|
|
model: str
|
|
model_name: Optional[str] = None
|
|
best_of: int = 1
|
|
logprobs: Optional[int] = None
|
|
extra_body: Optional[dict] = None
|
|
multi_modal_content: Optional[dict] = None
|
|
ignore_eos: bool = False
|
|
|
|
|
|
@dataclass
|
|
class RequestFuncOutput:
|
|
"""The output of the request function including metrics."""
|
|
generated_text: str = ""
|
|
success: bool = False
|
|
latency: float = 0.0
|
|
output_tokens: int = 0
|
|
ttft: float = 0.0 # Time to first token
|
|
itl: list[float] = field(
|
|
default_factory=list) # list of inter-token latencies
|
|
tpot: float = 0.0 # avg next-token latencies
|
|
prompt_len: int = 0
|
|
error: str = ""
|
|
|
|
|
|
async def async_request_openai_completions(
|
|
request_func_input: RequestFuncInput,
|
|
pbar: Optional[tqdm] = None,
|
|
) -> RequestFuncOutput:
|
|
"""The async request function for the OpenAI Completions API.
|
|
|
|
Args:
|
|
request_func_input: The input for the request function.
|
|
pbar: The progress bar to display the progress.
|
|
|
|
Returns:
|
|
The output of the request function.
|
|
"""
|
|
api_url = request_func_input.api_url
|
|
assert api_url.endswith(
|
|
("completions", "profile")
|
|
), "OpenAI Completions API URL must end with 'completions' or 'profile'."
|
|
|
|
async with aiohttp.ClientSession(trust_env=True,
|
|
timeout=AIOHTTP_TIMEOUT) as session:
|
|
payload = {
|
|
"model": request_func_input.model_name \
|
|
if request_func_input.model_name else request_func_input.model,
|
|
"prompt": request_func_input.prompt,
|
|
"temperature": 0.0,
|
|
"best_of": request_func_input.best_of,
|
|
"max_tokens": request_func_input.output_len,
|
|
"logprobs": request_func_input.logprobs,
|
|
"stream": True,
|
|
"stream_options": {
|
|
"include_usage": True,
|
|
},
|
|
}
|
|
if request_func_input.ignore_eos:
|
|
payload["ignore_eos"] = request_func_input.ignore_eos
|
|
if request_func_input.extra_body:
|
|
payload.update(request_func_input.extra_body)
|
|
headers = {
|
|
"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"
|
|
}
|
|
|
|
output = RequestFuncOutput()
|
|
output.prompt_len = request_func_input.prompt_len
|
|
|
|
generated_text = ""
|
|
st = time.perf_counter()
|
|
most_recent_timestamp = st
|
|
try:
|
|
async with session.post(url=api_url, json=payload,
|
|
headers=headers) as response:
|
|
if response.status == 200:
|
|
first_chunk_received = False
|
|
async for chunk_bytes in response.content:
|
|
chunk_bytes = chunk_bytes.strip()
|
|
if not chunk_bytes:
|
|
continue
|
|
|
|
chunk = chunk_bytes.decode("utf-8").removeprefix(
|
|
"data: ")
|
|
if chunk != "[DONE]":
|
|
data = json.loads(chunk)
|
|
|
|
# NOTE: Some completion API might have a last
|
|
# usage summary response without a token so we
|
|
# want to check a token was generated
|
|
if choices := data.get("choices"):
|
|
# Note that text could be empty here
|
|
# e.g. for special tokens
|
|
text = choices[0].get("text")
|
|
timestamp = time.perf_counter()
|
|
# First token
|
|
if not first_chunk_received:
|
|
first_chunk_received = True
|
|
ttft = time.perf_counter() - st
|
|
output.ttft = ttft
|
|
|
|
# Decoding phase
|
|
else:
|
|
output.itl.append(timestamp -
|
|
most_recent_timestamp)
|
|
|
|
most_recent_timestamp = timestamp
|
|
generated_text += text or ""
|
|
elif usage := data.get("usage"):
|
|
output.output_tokens = usage.get(
|
|
"completion_tokens")
|
|
if first_chunk_received:
|
|
output.success = True
|
|
else:
|
|
output.success = False
|
|
output.error = (
|
|
"Never received a valid chunk to calculate TTFT."
|
|
"This response will be marked as failed!")
|
|
output.generated_text = generated_text
|
|
output.latency = most_recent_timestamp - st
|
|
else:
|
|
output.error = response.reason or ""
|
|
output.success = False
|
|
except Exception:
|
|
output.success = False
|
|
exc_info = sys.exc_info()
|
|
output.error = "".join(traceback.format_exception(*exc_info))
|
|
|
|
if pbar:
|
|
pbar.update(1)
|
|
return output
|
|
|
|
|
|
# TODO: Add more request functions for different API protocols.
|
|
ASYNC_REQUEST_FUNCS = {
|
|
"openai-comp": async_request_openai_completions,
|
|
}
|