Update CI threshold & Improve code style (#2159)

2024-11-24 06:29:38 -08:00
parent e3938b2f9c
commit 5652c56535
8 changed files with 126 additions and 41 deletions
--- a/python/sglang/bench_one_batch.py
+++ b/python/sglang/bench_one_batch.py
@@ -212,6 +212,7 @@ def extend(reqs, model_runner):
        token_to_kv_pool=model_runner.token_to_kv_pool,
        tree_cache=None,
        model_config=model_runner.model_config,
+        enable_overlap=False,
    )
    batch.prepare_for_extend()
    model_worker_batch = batch.get_model_worker_batch()