support speculative execution for openai API (#48)

Co-authored-by: Ying Sheng <sqy1415@gmail.com>
This commit is contained in:
parasol-aser
2024-01-25 03:57:06 -06:00
committed by GitHub
parent 93414c8238
commit 23950056f0
10 changed files with 178 additions and 12 deletions

View File

@@ -1,7 +1,6 @@
import argparse
from enum import Enum
import sglang as sgl
from pydantic import BaseModel, constr
from sglang.srt.constrained.json_schema import build_regex_from_object
from sglang.test.test_utils import (
@@ -9,6 +8,8 @@ from sglang.test.test_utils import (
select_sglang_backend,
)
import sglang as sgl
IP_REGEX = r"((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)"
ip_fast_forward = (