support speculative execution for openai API (#48)

Co-authored-by: Ying Sheng <sqy1415@gmail.com>
2024-01-25 03:57:06 -06:00
parent 93414c8238
commit 23950056f0
10 changed files with 178 additions and 12 deletions
--- a/python/sglang/srt/managers/detokenizer_manager.py
+++ b/python/sglang/srt/managers/detokenizer_manager.py
@@ -60,7 +60,9 @@ class DetokenizerManager:
                        if first_token.startswith("▁"):
                            output_strs[i] = " " + output_strs[i]

-                    output_strs[i] = recv_obj.output_and_fast_forward_strs[i] + output_strs[i]
+                    output_strs[i] = (
+                        recv_obj.output_and_fast_forward_strs[i] + output_strs[i]
+                    )

                self.send_to_tokenizer.send_pyobj(
                    BatchStrOut(