Fix the deadlock in multi-node tp (#1122)

2024-08-16 01:39:24 -07:00
parent 6aa8ad14f8
commit 5a261bd055
7 changed files with 54 additions and 16 deletions
--- a/benchmark/gsm8k/bench_sglang.py
+++ b/benchmark/gsm8k/bench_sglang.py
@@ -64,7 +64,9 @@ def main(args):
    @sgl.function
    def few_shot_gsm8k(s, question):
        s += few_shot_examples + question
-        s += sgl.gen("answer", max_tokens=512, stop=["Question", "Assistant:"])
+        s += sgl.gen(
+            "answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"]
+        )

    #####################################
    ########## SGL Program End ##########