Test openai vision api (#925)

This commit is contained in:
Ying Sheng
2024-08-04 20:51:55 -07:00
committed by GitHub
parent ebf69964cd
commit 3bc99e6fe4
13 changed files with 102 additions and 228 deletions

View File

@@ -390,8 +390,13 @@ class TokenizerManager:
obj.return_text_in_logprobs,
)
# Log requests
if self.server_args.log_requests and state.finished:
logger.info(f"in={obj.text}, out={out}")
if obj.text is None:
in_obj = {"text": self.tokenizer.decode(obj.input_ids)}
else:
in_obj = {"text": obj.text}
logger.info(f"in={in_obj}, out={out}")
state.out_list = []
if state.finished:

View File

@@ -18,7 +18,7 @@ from sglang.lang.backend.openai import OpenAI
from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
from sglang.utils import get_exception_traceback
MODEL_NAME_FOR_TEST = "meta-llama/Meta-Llama-3.1-8B-Instruct"
DEFAULT_MODEL_NAME_FOR_TEST = "meta-llama/Meta-Llama-3.1-8B-Instruct"
def call_generate_lightllm(prompt, temperature, max_tokens, stop=None, url=None):