Higher priority for user input of max_prefill_tokens & format (#540)

2024-06-12 21:48:40 -07:00
parent 1374334d38
commit fb9296f0ed
50 changed files with 817 additions and 569 deletions
--- a/python/sglang/lang/interpreter.py
+++ b/python/sglang/lang/interpreter.py
@@ -507,7 +507,7 @@ class StreamExecutor:
                    )
                    return

-                else: # Speculative execution on models with completion interface
+                else:  # Speculative execution on models with completion interface
                    comp, meta_info = self._spec_gen(sampling_params)

            self.text_ += comp