fast regex decode

Auto-detect constant str path in regex FSM, then extend instead.
2024-01-25 01:16:25 +08:00
parent 711d343530
commit 01ee0fbc05
16 changed files with 968 additions and 16 deletions
--- a/python/sglang/srt/managers/detokenizer_manager.py
+++ b/python/sglang/srt/managers/detokenizer_manager.py
@@ -60,6 +60,8 @@ class DetokenizerManager:
                        if first_token.startswith("▁"):
                            output_strs[i] = " " + output_strs[i]

+                    output_strs[i] = recv_obj.output_and_fast_forward_strs[i] + output_strs[i]
+
                self.send_to_tokenizer.send_pyobj(
                    BatchStrOut(
                        recv_obj.rids,