xc-llm-ascend/.github/workflows/_e2e_nightly_single_node.yaml

#
# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This file is a part of the vllm-ascend project.
#

name: 'e2e nightly test'

on:
  workflow_call:
    inputs:
      runner:
        required: true
        type: string
      image:
        required: false
        type: string
        default: "swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.5.1-910b-ubuntu22.04-py3.11"
      tests:
        required: false
        type: string
      config_file_path:
        required: false
        type: string
      name:
        required: false
        type: string
      vllm_version:
        required: false
        type: string
        default: "v0.18.0"
      should_run:
        required: true
        type: boolean

# Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly
# declared as "shell: bash -el {0}" on steps that need to be properly activated.
# It's used to activate ascend-toolkit environment variables.
defaults:
  run:
    shell: bash -el {0}

# only cancel in-progress runs of the same workflow
# and ignore the lint / 1 card / 4 cards test type
concurrency:
  group: ascend-nightly-${{ github.workflow_ref }}-${{ github.ref }}-${{ inputs.config_file_path || inputs.tests }}
  cancel-in-progress: true

jobs:
  e2e-nightly:
    name: ${{ inputs.name || inputs.config_file_path || inputs.tests }}
    runs-on: ${{ inputs.runner }}
    if: ${{ inputs.should_run }}
    timeout-minutes: 600
    container:
      image: ${{ inputs.image }}
    env:
      HF_HUB_OFFLINE: 1
      VLLM_USE_MODELSCOPE: True
      UV_INDEX_URL: http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple
      UV_EXTRA_INDEX_URL: https://mirrors.huaweicloud.com/ascend/repos/pypi
      UV_INDEX_STRATEGY: unsafe-best-match
      UV_NO_CACHE: 1
      UV_SYSTEM_PYTHON: 1
      VLLM_ENGINE_READY_TIMEOUT_S: 1800
    steps:
      - name: Check npu and CANN info
        run: |
          npu-smi info
          cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
          pip install uv
      
      - name: uninstall vlm vllm-ascend and remove code (if pr test)
        if: ${{ github.event_name == 'pull_request' }}
        run: |
          pip uninstall -y vllm vllm-ascend || true
          cp -r /vllm-workspace/vllm-ascend/benchmark /tmp/aisbench-backup || true
          rm -rf /vllm-workspace/vllm /vllm-workspace/vllm-ascend

      - name: Checkout vllm-project/vllm repo
        if: ${{ github.event_name == 'pull_request' }}
        uses: actions/checkout@v6
        with:
          repository: vllm-project/vllm
          ref: ${{ inputs.vllm_version }}
          path: ./temp-vllm 
          fetch-depth: 1

      - name: Checkout vllm-project/vllm-ascend repo
        if: ${{ github.event_name == 'pull_request' }}
        uses: actions/checkout@v6
        with:
          path: ./temp-vllm-ascend
          fetch-depth: 1

      - name: Move code to /vllm-workspace
        if: ${{ github.event_name == 'pull_request' }}
        run: |
          mv ./temp-vllm /vllm-workspace/vllm
          mv ./temp-vllm-ascend /vllm-workspace/vllm-ascend
          ls -R /vllm-workspace

      - name: Install vllm-project/vllm from source
        if: ${{ github.event_name == 'pull_request' }}
        working-directory: /vllm-workspace/vllm
        run: |
          VLLM_TARGET_DEVICE=empty uv pip install -e .
      
      - name: Install vllm-project/vllm-ascend
        if: ${{ github.event_name == 'pull_request' }}
        working-directory: /vllm-workspace/vllm-ascend
        env:
          PIP_EXTRA_INDEX_URL: https://mirrors.huaweicloud.com/ascend/repos/pypi
        run: |
          git config --global --add safe.directory /vllm-workspace/vllm-ascend
          pip install uc-manager
          uv pip install -r requirements-dev.txt
          uv pip install -v -e .

      - name: Install aisbench
        if: ${{ github.event_name == 'pull_request' }}
        shell: bash -l {0}
        run: |
          cp -r /tmp/aisbench-backup /vllm-workspace/vllm-ascend/benchmark
          cd /vllm-workspace/vllm-ascend/benchmark
          pip install pytest asyncio pytest-asyncio
          pip install -e . -r requirements/api.txt -r requirements/extra.txt
          python3 -m pip cache purge

      - name: Show vLLM and vLLM-Ascend version
        working-directory: /vllm-workspace
        run: |
          echo "Installed vLLM-related Python packages:"
          pip list | grep vllm || echo "No vllm packages found."

          echo ""
          echo "============================"
          echo "vLLM Git information"
          echo "============================"
          cd vllm
          if [ -d .git ]; then
            echo "Branch:      $(git rev-parse --abbrev-ref HEAD)"
            echo "Commit hash: $(git rev-parse HEAD)"
            echo "Author:      $(git log -1 --pretty=format:'%an <%ae>')"
            echo "Date:        $(git log -1 --pretty=format:'%ad' --date=iso)"
            echo "Message:     $(git log -1 --pretty=format:'%s')"
            echo "Tags:        $(git tag --points-at HEAD || echo 'None')"
            echo "Remote:      $(git remote -v | head -n1)"
            echo ""
          else
            echo "No .git directory found in vllm"
          fi
          cd ..

          echo ""
          echo "============================"
          echo "vLLM-Ascend Git information"
          echo "============================"
          cd vllm-ascend
          if [ -d .git ]; then
            echo "Branch:      $(git rev-parse --abbrev-ref HEAD)"
            echo "Commit hash: $(git rev-parse HEAD)"
            echo "Author:      $(git log -1 --pretty=format:'%an <%ae>')"
            echo "Date:        $(git log -1 --pretty=format:'%ad' --date=iso)"
            echo "Message:     $(git log -1 --pretty=format:'%s')"
            echo "Tags:        $(git tag --points-at HEAD || echo 'None')"
            echo "Remote:      $(git remote -v | head -n1)"
            echo ""
          else
            echo "No .git directory found in vllm-ascend"
          fi
          cd ..

      - name: Install clang
        shell: bash -l {0}
        run: |
          apt-get update && apt-get -y install clang-15
          update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 20
          update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-15 20

      - name: Validate Inputs
        run: |
          if [[ -z "${{ inputs.tests }}" && -z "${{ inputs.config_file_path }}" ]]; then
            echo "Error: Either 'tests' or 'config_file_path' must be provided."
            exit 1
          fi

      - name: Run Pytest (py-driven)
        if: ${{ inputs.tests != '' }}
        env:
          VLLM_WORKER_MULTIPROC_METHOD: spawn
          VLLM_USE_MODELSCOPE: True
          VLLM_CI_RUNNER: ${{ inputs.runner }}
        working-directory: /vllm-workspace/vllm-ascend
        run: |
          export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
          echo "Running pytest with tests path: ${{ inputs.tests }}"
          pytest -sv "${{ inputs.tests }}" \
          --ignore=tests/e2e/nightly/single_node/ops/singlecard_ops/test_fused_moe.py

      - name: Run Pytest (YAML-driven)
        if: ${{ always() && inputs.config_file_path != '' }} 
        env:
          VLLM_WORKER_MULTIPROC_METHOD: spawn
          VLLM_USE_MODELSCOPE: True
          VLLM_CI_RUNNER: ${{ inputs.runner }}
          CONFIG_YAML_PATH: ${{ inputs.config_file_path }}
        working-directory: /vllm-workspace/vllm-ascend
        run: |
          export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
          echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib" >> ~/.bashrc
          echo "Running YAML-driven test with config: ${{ inputs.config_file_path }}"
          pytest -sv tests/e2e/nightly/single_node/models/scripts/test_single_node.py