support speculative execution for openai API (#48)

Co-authored-by: Ying Sheng <sqy1415@gmail.com>
2024-01-25 03:57:06 -06:00
parent 93414c8238
commit 23950056f0
10 changed files with 178 additions and 12 deletions
--- a/python/sglang/srt/managers/tokenizer_manager.py
+++ b/python/sglang/srt/managers/tokenizer_manager.py
@@ -200,6 +200,7 @@ class TokenizerManager:
                    )
                tokenized_obj = TokenizedGenerateReqInput(
                    rid=rid,
+                    input_text=obj.text[i],
                    input_ids=input_ids,
                    pixel_values=pixel_values,
                    image_hash=image_hash,