From 1c2b5f524041752442856428db451510a75add96 Mon Sep 17 00:00:00 2001
From: Yineng Zhang <me@zhyncs.com>
Date: Thu, 15 Aug 2024 01:39:15 +0800
Subject: [PATCH] docs: update nsys usage (#1103)

---
 benchmark/latency_throughput/README.md | 36 +++++++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/benchmark/latency_throughput/README.md b/benchmark/latency_throughput/README.md
index b6c2e6797..b1061793a 100644
--- a/benchmark/latency_throughput/README.md
+++ b/benchmark/latency_throughput/README.md
@@ -33,8 +33,42 @@ python3 bench_serving.py --backend srt --port 30000 --tokenizer meta-llama/Llama
 ```
 
 ### Profile with Nsight
+0. Prerequisite
+```bash
+# install nsys
+# https://docs.nvidia.com/nsight-systems/InstallationGuide/index.html
+apt update
+apt install -y --no-install-recommends gnupg
+echo "deb http://developer.download.nvidia.com/devtools/repos/ubuntu$(source /etc/lsb-release; echo "$DISTRIB_RELEASE" | tr -d .)/$(dpkg --print-architecture) /" | tee /etc/apt/sources.list.d/nvidia-devtools.list
+apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub
+apt update
+apt install nsight-systems-cli
+```
+
 1. To profile a single batch, use `nsys profile --cuda-graph-trace=node python3 -m sglang.bench_latency --model meta-llama/Meta-Llama-3-8B --batch-size 64 --input-len 512`
-2. To profile a server, use `nsys profile --trace-fork-before-exec=true --cuda-graph-trace=node python3 -m sglang.launch_server --model meta-llama/Meta-Llama-3-8B`.
+
+2. To profile a server, e.g.
+
+```bash
+# server
+# set the delay and duration times according to needs
+nsys profile --trace-fork-before-exec=true --cuda-graph-trace=node -o sglang.out --delay 60 --duration 70 python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --disable-radix-cache
+
+# client
+python3 -m sglang.bench_serving --backend sglang --num-prompts 6000 --dataset-name random --random-input 4096 --random-output 2048
+```
+
+3. Use NVTX, e.g.
+
+```bash
+# install nvtx
+pip install nvtx
+
+# code snippets
+import nvtx
+with nvtx.annotate("description", color="color"):
+    # some critical code
+```
 
 
 ## Other baselines