EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Error out when torchao-config option is not recognized (#2107)

Browse Source
This commit is contained in:
Jerry Zhang
2024-11-20 17:37:28 -08:00
committed by GitHub
parent 722530fa01
commit 5c6a41facf
2 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
python/sglang/srt/layers/torchao_utils.py
View File

@@ -62,6 +62,8 @@ def torchao_quantize_param_data(param: torch.Tensor, torchao_config: str):
granularity=GRANULARITY_MAP[granularity] granularity=GRANULARITY_MAP[granularity]
), ),
) )
else:
raise ValueError(f"Unexpected config: {torchao_config}")
return dummy_linear.weight return dummy_linear.weight

2
python/sglang/srt/server_args.py
View File

@@ -661,7 +661,7 @@ class ServerArgs:
"--torchao-config", "--torchao-config",
type=str, type=str,
default=ServerArgs.torchao_config, default=ServerArgs.torchao_config,
help="Optimize the model with torchao. Experimental feature. Current choices are: int8dq, int8wo, int4wo-<group_size>, fp8wo", help="Optimize the model with torchao. Experimental feature. Current choices are: int8dq, int8wo, int4wo-<group_size>, fp8wo, fp8dq-per_tensor, fp8dq-per_row",
) )
parser.add_argument( parser.add_argument(
"--enable-nan-detection", "--enable-nan-detection",