EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[Minor] improve kill scripts and torchao import (#1375)

Browse Source
This commit is contained in:
Lianmin Zheng
2024-09-10 11:27:03 -07:00
committed by GitHub
parent dff2860a69
commit 6c7cb90365
2 changed files with 9 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
python/sglang/srt/layers/torchao_utils.py
View File

@@ -3,15 +3,17 @@ Common utilities for torchao.
""" """
import torch import torch
from torchao.quantization import (
def torchao_quantize_param_data(param, torchao_config):
# Lazy import to suppress some warnings
from torchao.quantization import (
int4_weight_only, int4_weight_only,
int8_dynamic_activation_int8_weight, int8_dynamic_activation_int8_weight,
int8_weight_only, int8_weight_only,
quantize_, quantize_,
) )
def torchao_quantize_param_data(param, torchao_config):
dummy_linear = torch.nn.Linear(param.shape[1], param.shape[0], bias=False) dummy_linear = torch.nn.Linear(param.shape[1], param.shape[0], bias=False)
dummy_linear.weight = param dummy_linear.weight = param
if "int8wo" in torchao_config: if "int8wo" in torchao_config:

2
test/killall_sglang.sh
View File

@@ -1 +1 @@
kill -9 $(ps aux | grep 'sglang' | grep -v 'grep' | awk '{print $2}') kill -9 $(ps aux | grep 'sglang.launch_server' | grep -v 'grep' | awk '{print $2}')