mlapo add qdown output (#4707)
### What this PR does / why we need it?
This PR adds mlapo operation support qdown of output.
### Does this PR introduce _any_ user-facing change?
mlapo operation add enable_inner_out of input
### How was this patch tested?
CI passed with new added/existing test.
- vLLM version: v0.12.0
- vLLM main:
ad32e3e19c
---------
Signed-off-by: h1074112368 <h1074112368@gmail.com>
Co-authored-by: wangxiyuan <wangxiyuan1007@gmail.com>
This commit is contained in:
@@ -81,7 +81,7 @@ at::Tensor sgmv_expand_meta(at::Tensor &x, at::Tensor &weight, at::Tensor &lora_
|
||||
return y_out;
|
||||
}
|
||||
|
||||
std::tuple<at::Tensor &, at::Tensor &, at::Tensor &, at::Tensor &> mla_preprocess(
|
||||
std::tuple<at::Tensor &, at::Tensor &, at::Tensor &, at::Tensor &, at::Tensor &> mla_preprocess(
|
||||
const at::Tensor &hiddenState,
|
||||
const at::Tensor &wdqkv,
|
||||
const at::Tensor &descale0,
|
||||
@@ -106,12 +106,15 @@ std::tuple<at::Tensor &, at::Tensor &, at::Tensor &, at::Tensor &> mla_preproces
|
||||
const c10::optional<at::Tensor> &q_nope_scale,
|
||||
c10::optional<c10::string_view> cache_mode,
|
||||
c10::optional<c10::string_view> quant_mode,
|
||||
c10::optional<bool> enable_inner_out,
|
||||
at::Tensor &q_out0,
|
||||
at::Tensor &kv_cache_out0,
|
||||
at::Tensor &q_out1,
|
||||
at::Tensor &kv_cache_out1)
|
||||
at::Tensor &kv_cache_out1,
|
||||
at::Tensor &inner_out
|
||||
)
|
||||
{
|
||||
return {q_out0, kv_cache_out0, q_out1, kv_cache_out1};
|
||||
return {q_out0, kv_cache_out0, q_out1, kv_cache_out1, inner_out};
|
||||
}
|
||||
|
||||
std::tuple<at::Tensor, at::Tensor, at::Tensor> grouped_matmul_swiglu_quant(
|
||||
|
||||
Reference in New Issue
Block a user