Remove cuda graph batch size adjustment for dp attention (#2484)
This commit is contained in:
@@ -221,12 +221,10 @@ class ServerArgs:
|
||||
if self.enable_dp_attention:
|
||||
self.dp_size = self.tp_size
|
||||
self.chunked_prefill_size = self.chunked_prefill_size // 2
|
||||
self.cuda_graph_max_bs = min(self.cuda_graph_max_bs, 96)
|
||||
self.schedule_conservativeness = self.schedule_conservativeness * 0.3
|
||||
self.disable_overlap_schedule = True
|
||||
logger.warning(
|
||||
f"DP attention is enabled. The chunked prefill size is adjusted to {self.chunked_prefill_size} to avoid MoE kernel issues. "
|
||||
f"The CUDA graph max batch size is adjusted to {self.cuda_graph_max_bs}. "
|
||||
f"The schedule conservativeness is adjusted to {self.schedule_conservativeness}. "
|
||||
"Data parallel size is adjusted to be the same as tensor parallel size. "
|
||||
"Overlap scheduler is disabled."
|
||||
|
||||
Reference in New Issue
Block a user