support qwen2 running on ascend npu device (#7022)
Co-authored-by: 刁莹煜 <diaoyingyu1@hisilicon.com>
This commit is contained in:
@@ -6,11 +6,12 @@ from typing import List, Mapping, Tuple, Union
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.quantization.fp8_kernel import scaled_fp8_quant
|
||||
from sglang.srt.utils import is_cuda
|
||||
from sglang.srt.utils import is_cuda, is_npu
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
_is_npu = is_npu()
|
||||
|
||||
if not _is_cuda:
|
||||
if not _is_cuda and not _is_npu:
|
||||
from vllm._custom_ops import scaled_fp8_quant
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user