EngineX/xc-llm-ascend
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[300I][Bugfix] fix unquant model weight nd2nz error (#6851)

Browse Source 
### What this PR does / why we need it?
- This PR fixes an issue with weight format conversion for unquantized
models running on Ascend 310P devices.

- The changes refactor the logic for converting weights to the
FRACTAL_NZ format. Previously, this was handled in a 310P-specific
linear layer implementation (`AscendUnquantizedLinearMethod310`). This
implementation has been removed, and the logic is now centralized in the
`maybe_trans_nz` utility function. This function now checks if the
device is a 310P and applies the NZ format cast accordingly for
`float16`/`bfloat16` weights.

- This refactoring simplifies the code by removing platform-specific
duplication and ensures correct weight handling for unquantized models
on 310P.

### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
ut and local test
- vLLM version: v0.15.0
- vLLM main:
83b47f67b1

---------

Signed-off-by: Tflowers-0129 <2906339855@qq.com>
This commit is contained in:
Shaoxu Cheng
2026-03-03 15:57:26 +08:00
committed by GitHub
parent f19f7b1fe2
commit 2064afe380
8 changed files with 214 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
vllm_ascend/_310p/ops/linear.py
View File

@@ -1,65 +0,0 @@
#
# Copyright (c) 2026 Huawei Technologies Co., Ltd. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import torch
import torch.nn as nn
import torch_npu
from vllm.model_executor.layers.linear import (
LinearBase,
QuantizeMethodBase,
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm_ascend.utils import ACL_FORMAT_FRACTAL_NZ
class AscendUnquantizedLinearMethod310(UnquantizedLinearMethod):
def process_weights_after_loading(self, layer: nn.Module) -> None:
super().process_weights_after_loading(layer)
if "conv1d" not in getattr(layer, "prefix", ""):
layer.weight.data = torch_npu.npu_format_cast(layer.weight.data, ACL_FORMAT_FRACTAL_NZ)
class AscendLinearBase310(LinearBase):
def __init__(
self,
input_size: int,
output_size: int,
skip_bias_add: bool = False,
params_dtype: object | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
*,
return_bias: bool = True,
disable_tp: bool = False,
):
nn.Module.__init__(self)
self.input_size = int(input_size)
self.output_size = int(output_size)
self.skip_bias_add = skip_bias_add
self.params_dtype = torch.float16
self.quant_config = quant_config
self.prefix = prefix
self.return_bias = return_bias
self.disable_tp = disable_tp
if quant_config is None:
self.quant_method: QuantizeMethodBase | None = AscendUnquantizedLinearMethod310()
else:
self.quant_method = quant_config.get_quant_method(self, prefix=prefix)

82
vllm_ascend/_310p/ops/vocab_parallel_embedding.py Normal file
View File

@@ -0,0 +1,82 @@
#
# Copyright (c) 2026 Huawei Technologies Co., Ltd. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import torch
import torch.nn.functional as F
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import (
DEFAULT_VOCAB_PADDING_SIZE,
UnquantizedEmbeddingMethod,
)
from vllm_ascend.ops.vocab_parallel_embedding import AscendParallelLMHead, AscendVocabParallelEmbedding
from vllm_ascend.utils import maybe_trans_nz
class AscendUnquantizedEmbeddingMethod310(UnquantizedEmbeddingMethod):
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight_nz = maybe_trans_nz(layer.weight)
def apply(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return F.linear(x, layer.weight_nz, bias)
class AscendVocabParallelEmbedding310(AscendVocabParallelEmbedding):
def __init__(
self,
num_embeddings: int,
embedding_dim: int,
params_dtype: torch.dtype | None = None,
org_num_embeddings: int | None = None,
padding_size: int = DEFAULT_VOCAB_PADDING_SIZE,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__(
num_embeddings, embedding_dim, params_dtype, org_num_embeddings, padding_size, quant_config, prefix
)
if quant_config is None:
self.quant_method = AscendUnquantizedEmbeddingMethod310()
class AscendParallelLMHead310(AscendParallelLMHead):
"""
Register ParallelLMHead as a custom op for Atlas 310p.
"""
def __init__(
self,
num_embeddings: int,
embedding_dim: int,
bias: bool = False,
params_dtype: torch.dtype | None = None,
org_num_embeddings: int | None = None,
padding_size: int = DEFAULT_VOCAB_PADDING_SIZE,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__(
num_embeddings, embedding_dim, bias, params_dtype, org_num_embeddings, padding_size, quant_config, prefix
)
if quant_config is None:
self.quant_method = AscendUnquantizedEmbeddingMethod310()