bdc66972db
### What this PR does / why we need it? While using the LLM Compressor quantization tool from the VLLM community to generate quantized weights, the VLLM Ascend engine needs to be adapted to support the compressed tensors quantization format. 1. Add AscendCompressedTensorsConfig to replace CompressedTensorsConfig in vllm. 2. Support CompressedTensorsW8A8 static weight. - weight: per-channel, int8, symmetric; activation: per-tensor, int8, symmetric. 4. Support CompressedTensorsW8A8Dynamic weight. - weight: per-channel, int8, symmetric; activation: per-token, int8, symmetric, dynamic. 5. Modify the override_quantization_method in AscendQuantConfig. Co-authored-by: taoqun110 taoqun@huawei.com Co-authored-by: chenxi-hh chen464822955@163.com - vLLM version: v0.11.2 --------- Signed-off-by: LHXuuu <scut_xlh@163.com> Signed-off-by: chenxi-hh <chen464822955@163.com> Signed-off-by: chenxi-hh <32731611+chenxi-hh@users.noreply.github.com> Co-authored-by: chenxi-hh <chen464822955@163.com> Co-authored-by: chenxi-hh <32731611+chenxi-hh@users.noreply.github.com>
40 lines
1.0 KiB
TOML
40 lines
1.0 KiB
TOML
[build-system]
|
|
# Should be mirrored in requirements.txt
|
|
requires = [
|
|
"cmake>=3.26",
|
|
"decorator",
|
|
"einops",
|
|
"numpy<2.0.0",
|
|
"packaging",
|
|
"pip",
|
|
"pybind11",
|
|
"pyyaml",
|
|
"scipy",
|
|
"pandas",
|
|
"pandas-stubs",
|
|
"setuptools>=64",
|
|
"setuptools-scm>=8",
|
|
"transformers<=4.57.1",
|
|
"torch-npu==2.7.1",
|
|
"torch==2.7.1",
|
|
"torchvision",
|
|
"wheel",
|
|
"msgpack",
|
|
"quart",
|
|
"numba",
|
|
"opencv-python-headless<=4.11.0.86", # Required to avoid numpy version conflict with vllm
|
|
"compressed_tensors>=0.11.0"
|
|
]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[tool.pymarkdown]
|
|
plugins.md004.style = "sublist" # ul-style
|
|
plugins.md007.indent = 4 # ul-indent
|
|
plugins.md007.start_indented = true # ul-indent
|
|
plugins.md013.enabled = false # line-length
|
|
plugins.md041.enabled = false # first-line-h1
|
|
plugins.md033.enabled = false # inline-html
|
|
plugins.md046.enabled = false # code-block-style
|
|
plugins.md024.allow_different_nesting = true # no-duplicate-headers
|
|
plugins.md029.enabled = false # ol-prefix
|