diff --git a/.github/workflows/nightly-test.yml b/.github/workflows/nightly-test.yml index 468147454..6caa16846 100644 --- a/.github/workflows/nightly-test.yml +++ b/.github/workflows/nightly-test.yml @@ -77,7 +77,6 @@ jobs: cd test/srt python3 test_nightly_vlms_mmmu_eval.py - nightly-test-perf-vlms: if: github.repository == 'sgl-project/sglang' runs-on: 1-gpu-runner @@ -105,3 +104,20 @@ jobs: GITHUB_RUN_NUMBER: ${{ github.run_number }} run: | python3 scripts/ci/publish_traces.py --vlm + + nightly-test-1-gpu: + if: github.repository == 'sgl-project/sglang' + runs-on: 1-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 10 + run: | + cd test/srt + python3 run_suite.py --suite nightly-1-gpu diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 90c19ba5f..5838ce09b 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -255,7 +255,7 @@ jobs: timeout-minutes: 30 run: | cd test/srt - python3 run_suite.py --suite per-commit --auto-partition-id ${{ matrix.part }} --auto-partition-size 11 + python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 11 unit-test-backend-2-gpu: needs: [check-changes, sgl-kernel-build-wheels] @@ -706,7 +706,7 @@ jobs: performance-test-2-gpu, accuracy-test-1-gpu, accuracy-test-2-gpu, unit-test-deepep-4-gpu, unit-test-deepep-8-gpu, - unit-test-backend-4-gpu-b200, + # unit-test-backend-4-gpu-b200, ] if: always() runs-on: ubuntu-latest diff --git a/scripts/ci/ci_install_dependency.sh b/scripts/ci/ci_install_dependency.sh index d91084b6e..b9b2c2311 100755 --- a/scripts/ci/ci_install_dependency.sh +++ b/scripts/ci/ci_install_dependency.sh @@ -24,7 +24,7 @@ if [ "$IS_BLACKWELL" = "1" ]; then PIP_INSTALL_SUFFIX="--break-system-packages" # Clean up existing installations - $PIP_CMD uninstall -y flashinfer_python sgl-kernel sglang vllm torch torchaudio $PIP_INSTALL_SUFFIX || true + $PIP_CMD uninstall -y flashinfer_python sgl-kernel sglang vllm $PIP_INSTALL_SUFFIX || true else # In normal cases, we use uv, which is much faster than pip. pip install --upgrade pip @@ -35,7 +35,7 @@ else PIP_INSTALL_SUFFIX="--index-strategy unsafe-best-match" # Clean up existing installations - $PIP_CMD uninstall flashinfer_python sgl-kernel sglang vllm torch torchaudio || true + $PIP_CMD uninstall flashinfer_python sgl-kernel sglang vllm || true fi # Install the main package diff --git a/test/README.md b/test/README.md index 1854ec955..8149617ff 100644 --- a/test/README.md +++ b/test/README.md @@ -10,7 +10,7 @@ cd sglang/test/srt python3 test_srt_endpoint.py # Run a single test -python3 -m unittest test_srt_endpoint.TestSRTEndpoint.test_simple_decode +python3 test_srt_endpoint.py TestSRTEndpoint.test_simple_decode # Run a suite with multiple files python3 run_suite.py --suite per-commit @@ -21,21 +21,23 @@ python3 run_suite.py --suite per-commit cd sglang/test/lang # Run a single file -python3 test_srt_backend.py +python3 test_choices.py ``` ## Adding or Updating Tests in CI - Create new test files under `test/srt` or `test/lang` depending on the type of test. -- Ensure they are referenced in the respective `run_suite.py` (e.g., `test/srt/run_suite.py`) so they’re picked up in CI. For most small test cases, they can be added to the `per-commit` suite. Sort the test cases alphabetically. -- The CI will run the `per-commit` and `nightly` automatically. If you need special setup or custom test groups, you may modify the workflows in [`.github/workflows/`](https://github.com/sgl-project/sglang/tree/main/.github/workflows). - +- Ensure they are referenced in the respective `run_suite.py` (e.g., `test/srt/run_suite.py`) so they are picked up in CI. For most small test cases, they can be added to the `per-commit-1-gpu` suite. Sort the test cases alphabetically by name. +- Ensure you added `unittest.main()` for unittest and `pytest.main([__file__])` for pytest in the scripts. The CI run them via `python3 test_file.py`. +- The CI will run some suites such as `per-commit-1-gpu`, `per-commit-2-gpu`, and `nightly-1-gpu` automatically. If you need special setup or custom test groups, you may modify the workflows in [`.github/workflows/`](https://github.com/sgl-project/sglang/tree/main/.github/workflows). ## Writing Elegant Test Cases -- Examine existing tests in [sglang/test](https://github.com/sgl-project/sglang/tree/main/test) for practical examples. +- Learn from existing examples in [sglang/test/srt](https://github.com/sgl-project/sglang/tree/main/test/srt). +- Reduce the test time by using smaller models and reusing the server for multiple test cases. Launching a server takes a lot of time. +- Use as few GPUs as possible. Do not run long tests with 8-gpu runners. +- If the test cases take too long, considering adding them to nightly tests instead of per-commit tests. - Keep each test function focused on a single scenario or piece of functionality. - Give tests descriptive names reflecting their purpose. - Use robust assertions (e.g., assert, unittest methods) to validate outcomes. - Clean up resources to avoid side effects and preserve test independence. -- Reduce the test time by using smaller models and reusing the server for multiple test cases. diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index e16d94421..1e2d9e022 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -13,7 +13,7 @@ class TestFile: # NOTE: please sort the test cases alphabetically by the test file name suites = { - "per-commit": [ + "per-commit-1-gpu": [ TestFile("function_call/test_json_schema_constraint.py", 30), TestFile("hicache/test_hicache.py", 116), TestFile("hicache/test_hicache_eagle.py", 150), @@ -179,6 +179,9 @@ suites = { TestFile("test_vllm_dependency.py", 185), # TestFile("test_gguf.py", 96), ], + # If the test cases take too long, considering adding them to nightly tests instead of per-commit tests + "nightly-1-gpu": [], + "nightly-8-gpu": [], } # Add AMD tests diff --git a/test/srt/test_mla_int8_deepseek_v3.py b/test/srt/test_mla_int8_deepseek_v3.py index b0ec49714..ceea88351 100644 --- a/test/srt/test_mla_int8_deepseek_v3.py +++ b/test/srt/test_mla_int8_deepseek_v3.py @@ -72,7 +72,8 @@ class TestDeepseekV3MTPChannelInt8(CustomTestCase): "16", "--enable-torch-compile", "--torch-compile-max-bs", - "2" "--speculative-algorithm", + "2", + "--speculative-algorithm", "EAGLE", "--speculative-draft-model-path", "sgl-project/sglang-ci-dsv3-channel-int8-test-NextN", @@ -177,7 +178,8 @@ class TestDeepseekV3MTPBlockInt8(CustomTestCase): "16", "--enable-torch-compile", "--torch-compile-max-bs", - "2" "--speculative-algorithm", + "2", + "--speculative-algorithm", "EAGLE", "--speculative-num-steps", "2",