EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix block wise fp8 torch compile (#3232)

Browse Source
This commit is contained in:
Ke Bao
2025-01-31 19:56:02 +08:00
committed by GitHub
parent 734daedd8f
commit c02e313914
1 changed files with 7 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
python/sglang/srt/layers/quantization/fp8.py
View File

@@ -290,6 +290,13 @@ class Fp8LinearMethod(LinearMethodBase):
weight_scale, requires_grad=False weight_scale, requires_grad=False
) )
layer.input_scale = None layer.input_scale = None
else:
layer.weight = torch.nn.Parameter(
layer.weight.data, requires_grad=False
)
layer.weight_scale_inv = torch.nn.Parameter(
layer.weight_scale_inv.data, requires_grad=False
)
return return
layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False) layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False)
# If checkpoint not serialized fp8, quantize the weights. # If checkpoint not serialized fp8, quantize the weights.