fastpath oversized basic test outputs
This commit is contained in:
112
worklogs/remote_scripts/start_platform_245k_repro.sh
Normal file
112
worklogs/remote_scripts/start_platform_245k_repro.sh
Normal file
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
OUT="${1:-/root/work/logs/platform_245k_repro_20260715_01}"
|
||||
PORT="${PORT:-1111}"
|
||||
MODEL="${MODEL:-/root/public-storage/models/Qwen/Qwen3.6-35B-A3B}"
|
||||
mkdir -p "$OUT"
|
||||
|
||||
cd /root
|
||||
export PYTHONPATH=/usr/local/corex/lib64/python3/dist-packages
|
||||
export LD_LIBRARY_PATH=/usr/local/corex/lib64:/usr/local/iluvatar/lib64:/usr/local/openmpi/lib
|
||||
export PATH=/usr/local/corex/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/openmpi/bin
|
||||
export VLLM_ENGINE_ITERATION_TIMEOUT_S=3600
|
||||
export MOE_NATIVE=1
|
||||
export CHUNK_PARALLEL=1
|
||||
export PREFIX_FLASH=1
|
||||
export RMSNORM_NATIVE=1
|
||||
export LOOP1_NATIVE=1
|
||||
export PROF_TRACE=0
|
||||
export PROF_MOE_SUMMARY=0
|
||||
export MOE_DECODE_NATIVE=0
|
||||
|
||||
python3 - <<'PY'
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
out = subprocess.check_output(["ps", "-eo", "pid,args"], text=True)
|
||||
pids = []
|
||||
for line in out.splitlines():
|
||||
if "vllm.entrypoints.openai.api_server" in line or "VllmWorkerProcess" in line:
|
||||
pid = int(line.strip().split(None, 1)[0])
|
||||
if pid != os.getpid():
|
||||
pids.append(pid)
|
||||
print("stop_pids", pids)
|
||||
for pid in pids:
|
||||
try:
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
time.sleep(8)
|
||||
out = subprocess.check_output(["ps", "-eo", "pid,args"], text=True)
|
||||
left = []
|
||||
for line in out.splitlines():
|
||||
if "vllm.entrypoints.openai.api_server" in line or "VllmWorkerProcess" in line:
|
||||
pid = int(line.strip().split(None, 1)[0])
|
||||
left.append(pid)
|
||||
for pid in left:
|
||||
try:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
print("sigkill_pids", left)
|
||||
PY
|
||||
|
||||
echo "[repro] gpu before start" | tee "$OUT/driver.log"
|
||||
LD_LIBRARY_PATH=/usr/local/corex-3.2.3/lib64:/usr/local/corex/lib64:/usr/local/corex/lib:/usr/local/iluvatar/lib64 \
|
||||
/usr/local/corex/bin/ixsmi | tee -a "$OUT/driver.log"
|
||||
|
||||
nohup python3 -m vllm.entrypoints.openai.api_server \
|
||||
--model "$MODEL" \
|
||||
--served-model-name llm \
|
||||
--host 0.0.0.0 \
|
||||
--port "$PORT" \
|
||||
--max-model-len 245000 \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.9 \
|
||||
--trust-remote-code \
|
||||
-tp 4 \
|
||||
--max-num-seqs 1 \
|
||||
--disable-log-requests \
|
||||
--disable-frontend-multiprocessing \
|
||||
--max-num-batched-tokens 16384 \
|
||||
--enable-chunked-prefill \
|
||||
--max-seq-len-to-capture 245000 \
|
||||
--enable-auto-tool-choice \
|
||||
--tool-call-parser qwen3_coder \
|
||||
--reasoning-parser qwen3 \
|
||||
--enable-prefix-caching \
|
||||
--chat-template /workspace/chat_template_multi_system.jinja \
|
||||
> "$OUT/server.log" 2>&1 &
|
||||
|
||||
echo $! > "$OUT/server.pid"
|
||||
echo "[repro] server_pid=$(cat "$OUT/server.pid")" | tee -a "$OUT/driver.log"
|
||||
|
||||
for i in $(seq 1 90); do
|
||||
if python3 - <<PY >/dev/null 2>&1
|
||||
import urllib.request
|
||||
urllib.request.urlopen("http://127.0.0.1:${PORT}/health", timeout=2).read()
|
||||
PY
|
||||
then
|
||||
echo "[repro] health_ok_after_checks=$i" | tee -a "$OUT/driver.log"
|
||||
exit 0
|
||||
fi
|
||||
if ! kill -0 "$(cat "$OUT/server.pid")" 2>/dev/null; then
|
||||
echo "[repro] server_exited_at_check=$i" | tee -a "$OUT/driver.log"
|
||||
tail -160 "$OUT/server.log" | tee -a "$OUT/driver.log"
|
||||
exit 2
|
||||
fi
|
||||
if [ $((i % 6)) -eq 0 ]; then
|
||||
echo "[repro] waiting_check=$i" | tee -a "$OUT/driver.log"
|
||||
tail -10 "$OUT/server.log" | tee -a "$OUT/driver.log"
|
||||
LD_LIBRARY_PATH=/usr/local/corex-3.2.3/lib64:/usr/local/corex/lib64:/usr/local/corex/lib:/usr/local/iluvatar/lib64 \
|
||||
/usr/local/corex/bin/ixsmi | head -45 | tee -a "$OUT/driver.log"
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
|
||||
echo "[repro] health_timeout" | tee -a "$OUT/driver.log"
|
||||
tail -220 "$OUT/server.log" | tee -a "$OUT/driver.log"
|
||||
exit 3
|
||||
Reference in New Issue
Block a user