Fix cuda illegal memory access in overlap mode (#2070)

2024-11-17 21:29:30 -08:00
parent a9e90b4bce
commit 116685337e
3 changed files with 10 additions and 11 deletions
--- a/python/sglang/srt/managers/scheduler.py
+++ b/python/sglang/srt/managers/scheduler.py
@@ -390,6 +390,9 @@ class Scheduler:
            batch = self.get_next_batch_to_run()
            self.cur_batch = batch
            if batch:
+                # We need a stream synchronization here. Otherwise, there will be cuda illegal memory access errors.
+                _ = batch.seq_lens[0].item()
+
                result = self.run_batch(batch)
                result_queue.append((batch.copy(), result))