Support stop_token_ids in sglang API (#1092)

2024-08-14 17:31:39 -07:00
parent 1c2b5f5240
commit 73cf6834f2
6 changed files with 34 additions and 12 deletions
--- a/test/srt/test_moe_serving_throughput.py
+++ b/test/srt/test_moe_serving_throughput.py
@@ -84,7 +84,7 @@ class TestServingThroughput(unittest.TestCase):

        if os.getenv("SGLANG_IS_IN_CI", "false") == "true":
            # A100 (PCIE) performance
-            assert res["output_throughput"] > 940
+            assert res["output_throughput"] > 930

    def test_default_with_chunked_prefill(self):
        res = self.run_test(