[CI] Fix ngram & suffix test oom (#4755)
### What this PR does / why we need it?
Avoid oom during CI by using `with VllmRunner` instead of `LLM()`, and
enable `test_ngram_correctness`
### How was this patch tested?
CI passed.
- vLLM version: v0.12.0
- vLLM main:
ad32e3e19c
---------
Signed-off-by: fluctlux <38945811+fluctlux@users.noreply.github.com>
Co-authored-by: wangxiyuan <wangxiyuan1007@gmail.com>
This commit is contained in:
3
.github/workflows/_e2e_test.yaml
vendored
3
.github/workflows/_e2e_test.yaml
vendored
@@ -108,8 +108,7 @@ jobs:
|
|||||||
# ------------------------------------ v1 spec decode test ------------------------------------ #
|
# ------------------------------------ v1 spec decode test ------------------------------------ #
|
||||||
pytest -sv tests/e2e/singlecard/spec_decode_v1/test_v1_mtp_correctness.py
|
pytest -sv tests/e2e/singlecard/spec_decode_v1/test_v1_mtp_correctness.py
|
||||||
pytest -sv tests/e2e/singlecard/spec_decode_v1/test_v1_mtp_torchair_correctness.py
|
pytest -sv tests/e2e/singlecard/spec_decode_v1/test_v1_mtp_torchair_correctness.py
|
||||||
# Fix me: test_eagle_correctness OOM error
|
pytest -sv tests/e2e/singlecard/spec_decode_v1/test_v1_spec_decode.py
|
||||||
#pytest -sv tests/e2e/singlecard/spec_decode_v1/test_v1_spec_decode.py
|
|
||||||
|
|
||||||
e2e-2-cards:
|
e2e-2-cards:
|
||||||
name: multicard-2
|
name: multicard-2
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
import random
|
import random
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
@@ -9,6 +10,8 @@ from vllm import LLM, SamplingParams
|
|||||||
|
|
||||||
from tests.e2e.conftest import VllmRunner
|
from tests.e2e.conftest import VllmRunner
|
||||||
|
|
||||||
|
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def test_prompts():
|
def test_prompts():
|
||||||
@@ -61,7 +64,6 @@ def eagle3_model_name():
|
|||||||
return "vllm-ascend/EAGLE3-LLaMA3.1-Instruct-8B"
|
return "vllm-ascend/EAGLE3-LLaMA3.1-Instruct-8B"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skip("TODO: Revert me after ngram oom issue on ci is fixed")
|
|
||||||
def test_ngram_correctness(
|
def test_ngram_correctness(
|
||||||
test_prompts: list[list[dict[str, Any]]],
|
test_prompts: list[list[dict[str, Any]]],
|
||||||
sampling_config: SamplingParams,
|
sampling_config: SamplingParams,
|
||||||
@@ -71,9 +73,11 @@ def test_ngram_correctness(
|
|||||||
Compare the outputs of a original LLM and a speculative LLM
|
Compare the outputs of a original LLM and a speculative LLM
|
||||||
should be the same when using ngram speculative decoding.
|
should be the same when using ngram speculative decoding.
|
||||||
'''
|
'''
|
||||||
ref_llm = LLM(model=model_name, max_model_len=1024, enforce_eager=False)
|
|
||||||
ref_outputs = ref_llm.chat(test_prompts, sampling_config)
|
with VllmRunner(model_name, max_model_len=1024,
|
||||||
del ref_llm
|
enforce_eager=False) as ref_llm:
|
||||||
|
ref_outputs = ref_llm.model.chat(test_prompts, sampling_config)
|
||||||
|
|
||||||
with VllmRunner(model_name,
|
with VllmRunner(model_name,
|
||||||
speculative_config={
|
speculative_config={
|
||||||
"method": "ngram",
|
"method": "ngram",
|
||||||
@@ -156,9 +160,10 @@ def test_suffix_correctness(
|
|||||||
Compare the outputs of a original LLM and a speculative LLM
|
Compare the outputs of a original LLM and a speculative LLM
|
||||||
should be the same when using ngram speculative decoding.
|
should be the same when using ngram speculative decoding.
|
||||||
'''
|
'''
|
||||||
ref_llm = LLM(model=model_name, max_model_len=1024, enforce_eager=False)
|
with VllmRunner(model_name, max_model_len=1024,
|
||||||
ref_outputs = ref_llm.chat(test_prompts, sampling_config)
|
enforce_eager=False) as ref_llm:
|
||||||
del ref_llm
|
ref_outputs = ref_llm.model.chat(test_prompts, sampling_config)
|
||||||
|
|
||||||
with VllmRunner(model_name,
|
with VllmRunner(model_name,
|
||||||
speculative_config={
|
speculative_config={
|
||||||
"method": "suffix",
|
"method": "suffix",
|
||||||
|
|||||||
Reference in New Issue
Block a user