From 2cee0c32e55d8685248843c529917c4b81de792b Mon Sep 17 00:00:00 2001
From: ZT-AIA <63220130+ZT-AIA@users.noreply.github.com>
Date: Sat, 25 Apr 2026 19:05:33 +0800
Subject: [PATCH] [CI] Repair custom op nightly (#8707)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

<!--  Thanks for sending a pull request!

BEFORE SUBMITTING, PLEASE READ
https://docs.vllm.ai/en/latest/contributing/overview.html

-->
### What this PR does / why we need it?
<!--
- Please clarify what changes you are proposing. The purpose of this
section is to outline the changes and how this PR fixes the issue.
If possible, please consider writing useful notes for better and faster
reviews in your PR.

- Please clarify why the changes are needed. For instance, the use case
and bug description.

- Fixes #
-->
#### Fixed:
1. The function name in test_moe_init_routing_custom.py is incorrect; it
is not named as a test case function starting with 'test'.
2.In Night ops singlecard_ops add the printing of timestamps for use
cases, making it easier to quickly locate issues after a timeout occurs.
#### To be repaired:
1. The test_penality.py test case partially fails. It takes one hour.
The owner has been notified to fix the case after the 5.1 holiday.
——Yang Cheng
3. The csrc/copy_and_expand_eagle_inputs operator invoked by
test_copy_and_expand_eagle_inputs.py supports only 910b.——HF001
4. The test_causal_conv1d.py test case is incorrect. The triton operator
`causal_conv1d_fn` invoked by the test_causal_conv1d.py test case uses
`get_forward_context`, but the operator case does not use
`set_forward_context` (which is normal in the model). ——Zeng Tian
5. The test_causal_conv1d.py case is incorrect. In this scenario,
uboverflow occurs when the triton invoked ——Zeng Tian
### Does this PR introduce _any_ user-facing change?
<!--
Note that it means *any* user-facing change including all aspects such
as API, interface or other behavior changes.
Documentation-only updates are not considered user-facing changes.
-->
no
### How was this patch tested?
<!--
CI passed with new added/existing test.
If it was tested in a way different from regular unit tests, please
clarify how you tested step by step, ideally copy and paste-able, so
that other reviewers can test and check, and descendants can verify in
the future.
If tests were not added, please describe why they were not added and/or
why it was difficult to add.
-->
nightly

Signed-off-by: ZT-AIA <1028681969@qq.com>
---
 tests/e2e/nightly/single_node/ops/conftest.py   | 17 +++++++++++++++++
 .../test_copy_and_expand_eagle_inputs.py        | 16 ++++++++++++----
 .../test_moe_init_routing_custom.py             |  4 ++--
 .../singlecard_ops/triton/test_causal_conv1d.py |  8 ++++++++
 .../ops/singlecard_ops/triton/test_penality.py  |  5 ++++-
 5 files changed, 43 insertions(+), 7 deletions(-)
 create mode 100644 tests/e2e/nightly/single_node/ops/conftest.py

diff --git a/tests/e2e/nightly/single_node/ops/conftest.py b/tests/e2e/nightly/single_node/ops/conftest.py
new file mode 100644
index 00000000..681cfff8
--- /dev/null
+++ b/tests/e2e/nightly/single_node/ops/conftest.py
@@ -0,0 +1,17 @@
+import time
+from datetime import datetime
+import pytest
+
+
+@pytest.hookimpl(tryfirst=True, hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+    """Hook to add timestamp to test reports"""
+    start_time = datetime.now().strftime("[%H:%M:%S]")
+    
+    outcome = yield
+    
+    report = outcome.get_result()
+    
+    if report.when == 'call':
+        
+        print(f"{start_time}")
\ No newline at end of file
diff --git a/tests/e2e/nightly/single_node/ops/singlecard_ops/test_copy_and_expand_eagle_inputs.py b/tests/e2e/nightly/single_node/ops/singlecard_ops/test_copy_and_expand_eagle_inputs.py
index 0a3cc3d9..7446d082 100644
--- a/tests/e2e/nightly/single_node/ops/singlecard_ops/test_copy_and_expand_eagle_inputs.py
+++ b/tests/e2e/nightly/single_node/ops/singlecard_ops/test_copy_and_expand_eagle_inputs.py
@@ -238,7 +238,8 @@ def generate_test_case(rng, num_reqs, num_padding_slots, shift_input_ids,
 # Parametrized tests
 # ---------------------------------------------------------------------------
 @pytest.mark.skip(    
-    reason="Failure of an individual operator use case causes failures of other operators."
+    reason="Only one type of machine is supported. It is necessary to consult \
+        with him to confirm whether it can be adapted to other machines."
 )
 @pytest.mark.parametrize("num_reqs", [1, 2, 4, 8, 16])
 @pytest.mark.parametrize("num_padding_slots", [1, 2, 3, 5])
@@ -304,7 +305,8 @@ def test_copy_and_expand_eagle_inputs(num_reqs, num_padding_slots,
                                    msg="out_hidden_state_mapping mismatch")
 
 @pytest.mark.skip(    
-    reason="Failure of an individual operator use case causes failures of other operators."
+    reason="Only one type of machine is supported. It is necessary to consult \
+        with him to confirm whether it can be adapted to other machines."
 )
 @pytest.mark.parametrize("num_reqs", [1])
 @pytest.mark.parametrize("num_padding_slots", [1])
@@ -348,6 +350,10 @@ def test_minimal_case(num_reqs, num_padding_slots, shift_input_ids):
     torch.testing.assert_close(n_nti, torch.from_numpy(g_nti), atol=0, rtol=0)
 
 
+@pytest.mark.skip(    
+    reason="Only one type of machine is supported. It is necessary to consult \
+        with him to confirm whether it can be adapted to other machines."
+)
 @pytest.mark.parametrize("num_reqs", [3, 7, 13])
 def test_large_tokens_per_request(num_reqs):
     """Test with larger token counts per request."""
@@ -390,7 +396,8 @@ def test_large_tokens_per_request(num_reqs):
     torch.testing.assert_close(n_nti, torch.from_numpy(g_nti), atol=0, rtol=0)
 
 @pytest.mark.skip(    
-    reason="Failure of an individual operator use case causes failures of other operators."
+    reason="Only one type of machine is supported. It is necessary to consult \
+        with him to confirm whether it can be adapted to other machines."
 )
 @pytest.mark.parametrize("num_reqs", [3, 7, 13])
 def test_large_tokens_shift_true(num_reqs):
@@ -435,7 +442,8 @@ def test_large_tokens_shift_true(num_reqs):
     torch.testing.assert_close(n_hsm, torch.from_numpy(g_hsm), atol=0, rtol=0)
 
 @pytest.mark.skip(    
-    reason="Failure of an individual operator use case causes failures of other operators."
+    reason="Only one type of machine ascend910b is supported. It is necessary to consult \
+        with him to confirm whether it can be adapted to other machines."
 )
 @pytest.mark.parametrize("num_reqs", [1, 4, 8])
 def test_no_rejected_tokens(num_reqs):
diff --git a/tests/e2e/nightly/single_node/ops/singlecard_ops/test_moe_init_routing_custom.py b/tests/e2e/nightly/single_node/ops/singlecard_ops/test_moe_init_routing_custom.py
index d01596ec..4e88c46e 100644
--- a/tests/e2e/nightly/single_node/ops/singlecard_ops/test_moe_init_routing_custom.py
+++ b/tests/e2e/nightly/single_node/ops/singlecard_ops/test_moe_init_routing_custom.py
@@ -233,7 +233,7 @@ def cmp_out_golden(x_golden, x_out, dtype):
     return np.all(cmp)
 
 
-def test_moe_npu(x, expert_idx, scale, offset, active_num, expert_capacity,
+def run_moe_npu(x, expert_idx, scale, offset, active_num, expert_capacity,
                  expert_num, drop_pad_mode, expert_tokens_num_type,
                  expert_tokens_num_flag, quant_mode, active_expert_range,
                  row_idx_type):
@@ -339,7 +339,7 @@ def test_moe_init_routing_custom():
                                      dtype=torch.float)
             offset_ = None
 
-        result_pta = test_moe_npu(x_, expert_idx_, scale_, offset_,
+        result_pta = run_moe_npu(x_, expert_idx_, scale_, offset_,
                                   active_num_, expert_capacity_, expert_num_,
                                   drop_pad_mode_, expert_tokens_num_type_,
                                   expert_tokens_num_flag_, quant_mode_,
diff --git a/tests/e2e/nightly/single_node/ops/singlecard_ops/triton/test_causal_conv1d.py b/tests/e2e/nightly/single_node/ops/singlecard_ops/triton/test_causal_conv1d.py
index 655d18b5..eb359400 100644
--- a/tests/e2e/nightly/single_node/ops/singlecard_ops/triton/test_causal_conv1d.py
+++ b/tests/e2e/nightly/single_node/ops/singlecard_ops/triton/test_causal_conv1d.py
@@ -250,6 +250,10 @@ def test_ascend_causal_conv1d(dim, width, extra_state_len, seq_len, has_bias,
     validate_cmp(conv_states, conv_states_ref, itype)
 
 
+@pytest.mark.skip(    
+    reason="To use this tirton ops:causal_conv1d_fn, you need to set `get_forward_context`. After\
+          the model side dumps the data, Zeng Tian has made the necessary fixes."
+)
 @pytest.mark.parametrize('has_initial_state', [False, True])
 @pytest.mark.parametrize('itype', [torch.bfloat16])
 @pytest.mark.parametrize('silu_activation', [True])
@@ -378,6 +382,10 @@ def causal_conv1d_update_ref(x,
     return (out if activation is None else F.silu(out)).to(dtype=dtype_in)
 
 
+@pytest.mark.skip(    
+    reason="In this scenario, using tirton ops:causal_conv1d_update will cause an overflow. \
+        Later, Zeng Tian was responsible for fixing this issue."
+)
 @pytest.mark.parametrize("itype", [torch.bfloat16])
 @pytest.mark.parametrize("silu_activation", [True])
 @pytest.mark.parametrize("has_bias", [False, True])
diff --git a/tests/e2e/nightly/single_node/ops/singlecard_ops/triton/test_penality.py b/tests/e2e/nightly/single_node/ops/singlecard_ops/triton/test_penality.py
index fbad25f6..4420cade 100644
--- a/tests/e2e/nightly/single_node/ops/singlecard_ops/triton/test_penality.py
+++ b/tests/e2e/nightly/single_node/ops/singlecard_ops/triton/test_penality.py
@@ -183,7 +183,10 @@ def create_test_data(
         num_speculative_tokens,
     )
 
-
+@pytest.mark.skip(    
+    reason="The test case failed and took one hour. Yang Cheng \
+        has been notified to fix it after the holiday."
+)
 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
 @pytest.mark.parametrize("vocab_size", VOCAB_SIZE)
 @pytest.mark.parametrize("num_status", NUM_STATUS)