Abort disconnected requests (#457)

2024-05-20 18:41:21 -07:00
parent 3e684be7a3
commit 8dbdc018a3
8 changed files with 202 additions and 132 deletions
--- a/python/sglang/lang/interpreter.py
+++ b/python/sglang/lang/interpreter.py
@@ -580,8 +580,8 @@ class StreamExecutor:
    def _execute_role_end(self, expr: SglRoleEnd):
        if (
            self.cur_role == "assistant"
-            and self.backend.is_chat_model
            and self.api_num_spec_tokens is not None
+            and self.backend.is_chat_model
        ):
            # Execute the stored lazy generation calls
            self.backend.role_end_generate(self)