Add fp4 quantize before all-gather for Flashinfer cutlass MoE DP (max throughput) (#7667)

2025-08-15 22:08:11 -07:00
parent 87dab54824
commit eff4eb3fdd
16 changed files with 360 additions and 52 deletions
--- a/python/sglang/srt/model_executor/forward_batch_info.py
+++ b/python/sglang/srt/model_executor/forward_batch_info.py
@@ -649,7 +649,7 @@ class ForwardBatch:
            num_tokens = global_num_tokens[0]

        self.global_dp_buffer_len = buffer_len
-        set_dp_buffer_len(buffer_len, num_tokens)
+        set_dp_buffer_len(buffer_len, num_tokens, global_num_tokens)

        bs = self.batch_size