add moe bucket profiling and record bottlenecks

worklogs/remote_scripts/run_batched_tokens_experiment.sh

@@ -7,7 +7,9 @@ LABEL="${2:?usage: run_batched_tokens_experiment.sh MAX_BATCHED_TOKENS LABEL}"
 OUT_DIR="/root/work/logs/${LABEL}"
 MODEL_PATH="/root/public-storage/models/Qwen/Qwen3.6-35B-A3B"
 DATASET="/root/work/logs/synthetic_cumulative_8_s3_cap40k.jsonl"
-RESULT_JSON="${OUT_DIR}/eager_100k_b${MAX_BATCHED_TOKENS}_c1_r8_t256_cap40k.json"
+MAX_REQUESTS="${MAX_REQUESTS:-8}"
+MAX_TOKENS="${MAX_TOKENS:-256}"
+RESULT_JSON="${OUT_DIR}/eager_100k_b${MAX_BATCHED_TOKENS}_c1_r${MAX_REQUESTS}_t${MAX_TOKENS}_cap40k.json"
 SERVER_LOG="${OUT_DIR}/server.log"
 DRIVER_LOG="${OUT_DIR}/driver.log"
 
@@ -35,7 +37,10 @@ export CHUNK_PARALLEL=1
 export PREFIX_FLASH=1
 export RMSNORM_NATIVE=1
 export LOOP1_NATIVE=1
-export PROF_TRACE=0
+export PROF_TRACE="${PROF_TRACE:-0}"
+export PROF_MOE_SUMMARY="${PROF_MOE_SUMMARY:-0}"
+export PROF_MOE_SYNC="${PROF_MOE_SYNC:-1}"
+export PROF_MOE_INTERVAL="${PROF_MOE_INTERVAL:-400}"
 export VLLM_ENGINE_ITERATION_TIMEOUT_S=3600
 
 echo "[exp] gpu before start"
@@ -96,8 +101,8 @@ python3 /root/work/formal_perf_bench.py \
   --model llm \
   --dataset "${DATASET}" \
   --concurrency 1 \
-  --max-requests 8 \
-  --max-tokens 256 \
+  --max-requests "${MAX_REQUESTS}" \
+  --max-tokens "${MAX_TOKENS}" \
   --timeout 1800 \
   --out "${RESULT_JSON}"
 BENCH_RC="$?"