diff --git a/.github/workflows/pr-benchmark-rust.yml b/.github/workflows/pr-benchmark-rust.yml index e039cba23..937fbbea1 100644 --- a/.github/workflows/pr-benchmark-rust.yml +++ b/.github/workflows/pr-benchmark-rust.yml @@ -15,6 +15,11 @@ on: concurrency: group: pr-benchmark-rust-${{ github.ref }} cancel-in-progress: true + +env: + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + permissions: contents: read pull-requests: write @@ -33,9 +38,10 @@ jobs: run: | bash scripts/ci/ci_install_rust.sh - - name: Setup sccache - uses: mozilla-actions/sccache-action@v0.0.3 - continue-on-error: true + - name: Configure sccache + uses: mozilla-actions/sccache-action@v0.0.9 + with: + version: "v0.10.0" - name: Rust cache uses: Swatinem/rust-cache@v2 @@ -45,29 +51,19 @@ jobs: shared-key: "rust-cache" # Save cache even on failure save-if: true + cache-all-crates: true + cache-on-failure: true - name: Check benchmarks compile run: | source "$HOME/.cargo/env" cd sgl-router/ - # Try to use sccache, but disable if it fails - if command -v sccache &> /dev/null; then - echo "Testing sccache availability..." - # Try to start sccache and check if it works - export RUSTC_WRAPPER=sccache - export SCCACHE_GHA_ENABLED="true" - if sccache --start-server 2>/dev/null && sccache --show-stats 2>/dev/null; then - echo "sccache is working, using it for compilation" - else - echo "sccache failed to start, falling back to regular cargo" - unset RUSTC_WRAPPER - unset SCCACHE_GHA_ENABLED - fi - else - echo "sccache not available, using regular cargo" - fi cargo check --benches + - name: Show sccache stats + if: always() + run: sccache --show-stats + # Full benchmark jobs that only run with label or on main branch benchmark-request-processing: name: Request Processing Benchmark @@ -88,9 +84,10 @@ jobs: run: | bash scripts/ci/ci_install_rust.sh - - name: Setup sccache - uses: mozilla-actions/sccache-action@v0.0.3 - continue-on-error: true + - name: Configure sccache + uses: mozilla-actions/sccache-action@v0.0.9 + with: + version: "v0.10.0" - name: Rust cache uses: Swatinem/rust-cache@v2 @@ -98,6 +95,8 @@ jobs: workspaces: sgl-router # Share cache across all benchmark jobs shared-key: "rust-cache" + cache-all-crates: true + cache-on-failure: true # Save cache even on failure save-if: true @@ -134,6 +133,10 @@ jobs: sgl-router/target/criterion/benchmark_summary/ retention-days: 30 + - name: Show sccache stats + if: always() + run: sccache --show-stats + benchmark-tokenizer: name: Tokenizer Benchmark if: | @@ -152,9 +155,10 @@ jobs: run: | bash scripts/ci/ci_install_rust.sh - - name: Setup sccache - uses: mozilla-actions/sccache-action@v0.0.3 - continue-on-error: true + - name: Configure sccache + uses: mozilla-actions/sccache-action@v0.0.9 + with: + version: "v0.10.0" - name: Rust cache uses: Swatinem/rust-cache@v2 @@ -162,6 +166,8 @@ jobs: workspaces: sgl-router # Share cache across all benchmark jobs shared-key: "rust-cache" + cache-all-crates: true + cache-on-failure: true # Save cache even on failure save-if: true @@ -215,9 +221,10 @@ jobs: run: | bash scripts/ci/ci_install_rust.sh - - name: Setup sccache - uses: mozilla-actions/sccache-action@v0.0.3 - continue-on-error: true + - name: Configure sccache + uses: mozilla-actions/sccache-action@v0.0.9 + with: + version: "v0.10.0" - name: Rust cache uses: Swatinem/rust-cache@v2 @@ -225,6 +232,8 @@ jobs: workspaces: sgl-router # Share cache across all benchmark jobs shared-key: "rust-cache" + cache-all-crates: true + cache-on-failure: true # Save cache even on failure save-if: true @@ -260,6 +269,10 @@ jobs: sgl-router/target/criterion/tool_parser*/ retention-days: 30 + - name: Show sccache stats + if: always() + run: sccache --show-stats + benchmark-summary: name: Benchmark Summary needs: [benchmark-request-processing, benchmark-tokenizer, benchmark-tool-parser] diff --git a/.github/workflows/pr-test-rust.yml b/.github/workflows/pr-test-rust.yml index 1f737c794..99c3c51d8 100644 --- a/.github/workflows/pr-test-rust.yml +++ b/.github/workflows/pr-test-rust.yml @@ -15,6 +15,10 @@ concurrency: group: pr-test-rust-${{ github.ref }} cancel-in-progress: true +env: + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + jobs: unit-test-rust: if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' @@ -27,10 +31,17 @@ jobs: run: | bash scripts/ci/ci_install_rust.sh + - name: Configure sccache + uses: mozilla-actions/sccache-action@v0.0.9 + with: + version: "v0.10.0" + - name: Rust cache uses: Swatinem/rust-cache@v2 with: workspaces: sgl-router + cache-all-crates: true + cache-on-failure: true - name: Run lint run: | @@ -65,6 +76,11 @@ jobs: # Run quick benchmarks to ensure they work using Python script python3 scripts/run_benchmarks.py --quick + - name: Show sccache stats + if: always() + run: sccache --show-stats + + e2e-python: pytest-rust: if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' runs-on: BM.A10.4 @@ -77,6 +93,18 @@ jobs: run: | bash scripts/ci/ci_install_rust.sh + - name: Configure sccache + uses: mozilla-actions/sccache-action@v0.0.9 + with: + version: "v0.10.0" + + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: sgl-router + cache-all-crates: true + cache-on-failure: true + - name: Install SGLang dependencies run: | sudo bash scripts/ci/ci_install_dependency.sh @@ -84,6 +112,7 @@ jobs: - name: Build python binding run: | source "$HOME/.cargo/env" + export RUSTC_WRAPPER=sccache cd sgl-router pip install setuptools-rust wheel build python3 -m build diff --git a/sgl-router/Makefile b/sgl-router/Makefile index feda1ee63..c220b26f1 100644 --- a/sgl-router/Makefile +++ b/sgl-router/Makefile @@ -1,6 +1,15 @@ # SGLang Router Makefile # Provides convenient shortcuts for common development tasks +# Check if sccache is available and set RUSTC_WRAPPER accordingly +SCCACHE := $(shell which sccache 2>/dev/null) +ifdef SCCACHE + export RUSTC_WRAPPER := $(SCCACHE) + $(info Using sccache for compilation caching) +else + $(info sccache not found. Install it for faster builds: cargo install sccache) +endif + .PHONY: help bench bench-quick bench-baseline bench-compare test build clean help: ## Show this help message @@ -90,3 +99,33 @@ perf-monitor: ## Run continuous performance monitoring else \ echo "Warning: 'watch' command not found. Install it or run 'make bench-quick' manually."; \ fi + +# sccache management targets +setup-sccache: ## Install and configure sccache + @echo "Setting up sccache..." + @./scripts/setup-sccache.sh + +sccache-stats: ## Show sccache statistics + @if [ -n "$(SCCACHE)" ]; then \ + echo "sccache statistics:"; \ + sccache -s; \ + else \ + echo "sccache not installed. Run 'make setup-sccache' to install it."; \ + fi + +sccache-clean: ## Clear sccache cache + @if [ -n "$(SCCACHE)" ]; then \ + echo "Clearing sccache cache..."; \ + sccache -C; \ + echo "sccache cache cleared"; \ + else \ + echo "sccache not installed"; \ + fi + +sccache-stop: ## Stop the sccache server + @if [ -n "$(SCCACHE)" ]; then \ + echo "Stopping sccache server..."; \ + sccache --stop-server || true; \ + else \ + echo "sccache not installed"; \ + fi diff --git a/sgl-router/scripts/setup-sccache.sh b/sgl-router/scripts/setup-sccache.sh new file mode 100755 index 000000000..2cdfb1e58 --- /dev/null +++ b/sgl-router/scripts/setup-sccache.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +IFS=$'\n\t' + +echo "Setting up sccache for faster Rust compilation..." + +has_cmd() { command -v "$1" >/dev/null 2>&1; } + +install_sccache() { + echo "sccache not found." + if [[ "${AUTO_INSTALL:-0}" != "1" ]]; then + read -r -p "Install sccache now? [y/N] " response + response=${response:-N} + if [[ ! "$response" =~ ^[Yy]$ ]]; then + echo "Skipping installation. Please install sccache manually:" + echo " cargo install sccache" + echo " or" + echo " brew install sccache (macOS)" + echo " or" + echo " sudo apt-get install -y sccache (Debian/Ubuntu)" + echo " or" + echo " sudo dnf install -y sccache (RHEL/Fedora)" + echo " or" + echo " sudo pacman -S sccache (Arch)" + exit 0 + fi + fi + + if has_cmd cargo; then + echo "Installing via cargo..." + cargo install sccache --locked + elif has_cmd brew; then + echo "Installing via Homebrew..." + brew install sccache + elif has_cmd apt-get; then + echo "Installing via apt-get..." + sudo apt-get update -y && sudo apt-get install -y sccache + elif has_cmd dnf; then + echo "Installing via dnf..." + sudo dnf install -y sccache + elif has_cmd pacman; then + echo "Installing via pacman..." + sudo pacman -S --noconfirm sccache + else + echo "No supported package manager detected. Install manually:" + echo " cargo install sccache" + exit 1 + fi +} + +if ! has_cmd sccache; then + install_sccache +fi + +echo "Configuring sccache..." + +export SCCACHE_CACHE_SIZE="${SCCACHE_CACHE_SIZE:-10G}" +export SCCACHE_STATS="${SCCACHE_STATS:-1}" + +# Set RUSTC_WRAPPER to sccache for this shell session. +SCCACHE_BIN="$(command -v sccache)" +if [[ -z "${SCCACHE_BIN}" ]]; then + echo "Unexpected: sccache still not on PATH after install. Check your environment." + exit 1 +fi +export RUSTC_WRAPPER="${SCCACHE_BIN}" + +echo "sccache version: $(sccache --version || echo 'unknown')" +echo "Current cache stats:" +sccache -s || true + +# If script not sourced, remind user about persistence. +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + echo + echo "Environment variables exported for this process only." + echo "To persist, add to your shell profile (e.g., ~/.bashrc or ~/.zshrc):" + echo ' export RUSTC_WRAPPER="$(command -v sccache 2>/dev/null || echo "")"' + echo ' export SCCACHE_CACHE_SIZE="10G"' + # echo ' export SCCACHE_DIR="$HOME/.cache/sccache"' + echo ' export SCCACHE_STATS="1"' +fi + +echo "sccache is configured." diff --git a/sgl-router/src/core/worker.rs b/sgl-router/src/core/worker.rs index f25fc6eea..51c3cdd65 100644 --- a/sgl-router/src/core/worker.rs +++ b/sgl-router/src/core/worker.rs @@ -986,7 +986,7 @@ pub fn start_health_checker( // Periodically reset load counters to prevent drift // Only do this when we believe all workers should be idle - if check_count.is_multiple_of(LOAD_RESET_INTERVAL) { + if check_count % LOAD_RESET_INTERVAL == 0 { let max_load = workers_to_check.iter().map(|w| w.load()).max().unwrap_or(0); // Only reset if load appears to be very low (likely drift) if max_load <= 2 {