EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

fix: flashinfer_cutlass_moe: Use max of global expert scales instead of local for input scale (#10296)

Browse Source
This commit is contained in:
Trevor Morris
2025-09-11 20:19:17 -07:00
committed by GitHub
parent 3df05f4d6a
commit c7e85f5378
2 changed files with 9 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
python/sglang/srt/layers/moe/fused_moe_triton/layer.py
View File

@@ -503,8 +503,14 @@ class FusedMoE(torch.nn.Module):
param.data[:, :dim1, :dim2].copy_(loaded_weight) param.data[:, :dim1, :dim2].copy_(loaded_weight)
return return
# ModelOptNvFp4FusedMoEMethod uses max of global expert scaling factors for input scaling factor
load_global_experts = (
isinstance(self.quant_method, ModelOptNvFp4FusedMoEMethod)
and "input_scale" in weight_name
)
global_expert_location_metadata = get_global_expert_location_metadata() global_expert_location_metadata = get_global_expert_location_metadata()
if global_expert_location_metadata is None: if global_expert_location_metadata is None or load_global_experts:
self._weight_loader_impl( self._weight_loader_impl(
param=param, param=param,
loaded_weight=loaded_weight, loaded_weight=loaded_weight,

4
python/sglang/srt/layers/quantization/modelopt_quant.py
View File

@@ -996,13 +996,13 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
) )
w13_input_scale = PerTensorScaleParameter( w13_input_scale = PerTensorScaleParameter(
data=torch.empty(layer.num_local_experts, 2, dtype=torch.float32), data=torch.empty(layer.num_experts, 2, dtype=torch.float32),
weight_loader=weight_loader, weight_loader=weight_loader,
) )
layer.register_parameter("w13_input_scale", w13_input_scale) layer.register_parameter("w13_input_scale", w13_input_scale)
w2_input_scale = PerTensorScaleParameter( w2_input_scale = PerTensorScaleParameter(
data=torch.empty(layer.num_local_experts, dtype=torch.float32), data=torch.empty(layer.num_experts, dtype=torch.float32),
weight_loader=weight_loader, weight_loader=weight_loader,
) )
layer.register_parameter("w2_input_scale", w2_input_scale) layer.register_parameter("w2_input_scale", w2_input_scale)