fastpath oversized basic test outputs

This commit is contained in:
2026-07-15 17:18:39 +08:00
parent 2b7880efa7
commit 9fc7e98af4
4 changed files with 315 additions and 0 deletions

View File

@@ -0,0 +1,112 @@
#!/usr/bin/env bash
set -euo pipefail
OUT="${1:-/root/work/logs/platform_245k_repro_20260715_01}"
PORT="${PORT:-1111}"
MODEL="${MODEL:-/root/public-storage/models/Qwen/Qwen3.6-35B-A3B}"
mkdir -p "$OUT"
cd /root
export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages
export LD_LIBRARY_PATH=/usr/local/corex/lib64:/usr/local/iluvatar/lib64:/usr/local/openmpi/lib
export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/openmpi/bin
export VLLM_ENGINE_ITERATION_TIMEOUT_S=3600
export MOE_NATIVE=1
export CHUNK_PARALLEL=1
export PREFIX_FLASH=1
export RMSNORM_NATIVE=1
export LOOP1_NATIVE=1
export PROF_TRACE=0
export PROF_MOE_SUMMARY=0
export MOE_DECODE_NATIVE=0
python3 - <<'PY'
import os
import signal
import subprocess
import time
out = subprocess.check_output(["ps", "-eo", "pid,args"], text=True)
pids = []
for line in out.splitlines():
if "vllm.entrypoints.openai.api_server" in line or "VllmWorkerProcess" in line:
pid = int(line.strip().split(None, 1)[0])
if pid != os.getpid():
pids.append(pid)
print("stop_pids", pids)
for pid in pids:
try:
os.kill(pid, signal.SIGTERM)
except ProcessLookupError:
pass
time.sleep(8)
out = subprocess.check_output(["ps", "-eo", "pid,args"], text=True)
left = []
for line in out.splitlines():
if "vllm.entrypoints.openai.api_server" in line or "VllmWorkerProcess" in line:
pid = int(line.strip().split(None, 1)[0])
left.append(pid)
for pid in left:
try:
os.kill(pid, signal.SIGKILL)
except ProcessLookupError:
pass
print("sigkill_pids", left)
PY
echo "[repro] gpu before start" | tee "$OUT/driver.log"
LD_LIBRARY_PATH=/usr/local/corex-3.2.3/lib64:/usr/local/corex/lib64:/usr/local/corex/lib:/usr/local/iluvatar/lib64 \
/usr/local/corex/bin/ixsmi | tee -a "$OUT/driver.log"
nohup python3 -m vllm.entrypoints.openai.api_server \
--model "$MODEL" \
--served-model-name llm \
--host 0.0.0.0 \
--port "$PORT" \
--max-model-len 245000 \
--enforce-eager \
--gpu-memory-utilization 0.9 \
--trust-remote-code \
-tp 4 \
--max-num-seqs 1 \
--disable-log-requests \
--disable-frontend-multiprocessing \
--max-num-batched-tokens 16384 \
--enable-chunked-prefill \
--max-seq-len-to-capture 245000 \
--enable-auto-tool-choice \
--tool-call-parser qwen3_coder \
--reasoning-parser qwen3 \
--enable-prefix-caching \
--chat-template /workspace/chat_template_multi_system.jinja \
> "$OUT/server.log" 2>&1 &
echo $! > "$OUT/server.pid"
echo "[repro] server_pid=$(cat "$OUT/server.pid")" | tee -a "$OUT/driver.log"
for i in $(seq 1 90); do
if python3 - <<PY >/dev/null 2>&1
import urllib.request
urllib.request.urlopen("http://127.0.0.1:${PORT}/health", timeout=2).read()
PY
then
echo "[repro] health_ok_after_checks=$i" | tee -a "$OUT/driver.log"
exit 0
fi
if ! kill -0 "$(cat "$OUT/server.pid")" 2>/dev/null; then
echo "[repro] server_exited_at_check=$i" | tee -a "$OUT/driver.log"
tail -160 "$OUT/server.log" | tee -a "$OUT/driver.log"
exit 2
fi
if [ $((i % 6)) -eq 0 ]; then
echo "[repro] waiting_check=$i" | tee -a "$OUT/driver.log"
tail -10 "$OUT/server.log" | tee -a "$OUT/driver.log"
LD_LIBRARY_PATH=/usr/local/corex-3.2.3/lib64:/usr/local/corex/lib64:/usr/local/corex/lib:/usr/local/iluvatar/lib64 \
/usr/local/corex/bin/ixsmi | head -45 | tee -a "$OUT/driver.log"
fi
sleep 10
done
echo "[repro] health_timeout" | tee -a "$OUT/driver.log"
tail -220 "$OUT/server.log" | tee -a "$OUT/driver.log"
exit 3