[Enhancement] Custom Logit Processor Improvement (#2998)

Signed-off-by: Hongpeng Guo <hpguo@anyscale.com>
2025-01-20 02:00:35 -08:00
parent 2584f6d944
commit 583697cd71
6 changed files with 79 additions and 28 deletions
--- a/python/sglang/bench_one_batch.py
+++ b/python/sglang/bench_one_batch.py
@@ -232,6 +232,7 @@ def extend(reqs, model_runner):
        model_config=model_runner.model_config,
        enable_overlap=False,
        spec_algorithm=SpeculativeAlgorithm.NONE,
+        enable_custom_logit_processor=False,
    )
    batch.prepare_for_extend()
    model_worker_batch = batch.get_model_worker_batch()