support speculative execution for openai API (#48)

Co-authored-by: Ying Sheng <sqy1415@gmail.com>
This commit is contained in:
parasol-aser
2024-01-25 03:57:06 -06:00
committed by GitHub
parent 93414c8238
commit 23950056f0
10 changed files with 178 additions and 12 deletions

View File

@@ -2,13 +2,14 @@ import argparse
import random
import string
import sglang as sgl
from sglang.test.test_utils import (
add_common_sglang_args_and_parse,
select_sglang_backend,
)
from vllm.transformers_utils.tokenizer import get_tokenizer
import sglang as sgl
TOKENIZER = None
RANDOM_PREFILL_LEN = None
RANDOM_DECODE_LEN = None