add cumulative prefix benchmark generator

worklogs/2026-07-13-initial-run.md

@@ -147,3 +147,32 @@
   - Long-context uncached prefill is the immediate bottleneck.
   - Current synthetic generator must be improved to make same-session adjacent requests share exact prefix, otherwise cache behavior is not representative of the official 65.6% cached-token workload.
   - Current runtime also needs template compatibility work for multiple/irregular system messages if official requests include them unmodified.
+
+## 2026-07-14 Cumulative Prefix Synthetic Benchmark Round 2
+
+- Added `--cumulative` mode to `generate_official_like_dataset.py`.
+- New mode shares exact system/tool prefix and grows each session history in place, so same-session later requests actually reuse earlier token prefixes.
+- Generated dataset:
+  - `/root/work/logs/synthetic_cumulative_8_s3_cap40k.jsonl`
+  - 8 requests, 3 sessions, approximate prompt average `27452.75`.
+  - Prompt cap `40000` to keep test runtime manageable under current `--max-model-len 100000`.
+- Benchmark:
+  - Result file: `/root/work/logs/perf_cumulative8_s3_cap40k_c1_r4.json`.
+  - Concurrency `1`, requests `4`, max output tokens `64`.
+- Results:
+  - Success rate: `100%` (`4/4`).
+  - TTFT P90: `58.49s`.
+  - Output TPS P10 per request: `5.94 tok/s`.
+  - Aggregate output TPS: `1.55 tok/s`.
+  - Aggregate uncached input TPS: `284.96 tok/s`.
+  - Aggregate cache TPS: `296.87 tok/s`.
+  - Cached tokens: `49152 / 96333`, cache hit rate `51.02%`.
+  - Weighted token throughput: `989.82`.
+- Target comparison:
+  - Cache hit rate and success rate pass.
+  - TTFT, Output TPS P10, and weighted throughput are far below target.
+- Interpretation:
+  - Prefix cache is now being exercised correctly, but cached prefill is still too slow at this configuration.
+  - Current serving parameter `--max-num-seqs 1` prevents useful concurrent batching and likely caps weighted throughput.
+  - Decode path is also slow: request-level output TPS only `5.9-8.5 tok/s`.
+  - Next experiment should restart service with higher `--max-num-seqs`, higher `--max-num-batched-tokens`, and possibly `gpu-memory-utilization 0.95`, then rerun the same cumulative dataset.

worklogs/generate_official_like_dataset.py

@@ -237,6 +237,118 @@ def build_request(rng, req_id, session_id, shared_prefix_tokens, total_prompt_to
     }
 
 
+def estimate_tokens(messages, tools):
+    chars = 0
+    for msg in messages:
+        chars += len(msg.get("role") or "") + len(msg.get("content") or "")
+        if msg.get("tool_calls"):
+            chars += len(json.dumps(msg["tool_calls"], ensure_ascii=False))
+        if msg.get("name"):
+            chars += len(msg["name"])
+    chars += len(json.dumps(tools, ensure_ascii=False))
+    return max(1, chars // 4)
+
+
+def make_history_turn(rng, state, turn_id, approx_tool_tokens):
+    user = {
+        "role": "user",
+        "content": (
+            f"Continue session {state['session_id']} turn {turn_id}. "
+            "Inspect files, reason about logs, and call the next tool if needed."
+        ),
+    }
+    call = make_tool_call(rng, turn_id)
+    assistant = {
+        "role": "assistant",
+        "content": "",
+        "tool_calls": [call],
+    }
+    tool = {
+        "role": "tool",
+        "tool_call_id": call["id"],
+        "name": call["function"]["name"],
+        "content": rand_text(rng, approx_tool_tokens),
+    }
+    return [user, assistant, tool]
+
+
+def generate_cumulative_requests(rng, count, session_count, max_prompt_tokens):
+    """Generate requests whose prefixes are actually reusable.
+
+    The first version of this generator only stored an intended cache ratio in
+    metadata. This mode makes the byte/text prefix identical across requests by
+    sharing tools/system content and by growing each session history in place.
+    """
+    common_tools = [make_tool(name) for name in TOOL_NAMES]
+    # Keep a large, fixed global prefix. It imitates common system prompt,
+    # agent framework instructions, and mounted tool schemas.
+    common_system = {
+        "role": "system",
+        "content": SYSTEM_PROMPT + "\n" + COMMON_PROJECT_CONTEXT + rand_text(rng, 6000),
+    }
+    sessions = {}
+    session_order = []
+    requests = []
+
+    def get_session(sid):
+        if sid not in sessions:
+            sessions[sid] = {
+                "session_id": sid,
+                "messages": [common_system],
+                "turn_id": 0,
+            }
+            session_order.append(sid)
+        return sessions[sid]
+
+    recent = []
+    for req_id in range(count):
+        # For small benchmark sets, force meaningful reuse. For larger sets,
+        # this still creates short-distance affinity similar to the official
+        # median gap of a few requests.
+        if recent and rng.random() < 0.65:
+            sid = rng.choice(recent[-32:])
+        else:
+            sid = f"sess_{len(session_order) % max(1, session_count):04d}"
+        recent.append(sid)
+        state = get_session(sid)
+        target = target_prompt_tokens(rng, max_prompt_tokens)
+
+        # Grow the historical context until this request has the target scale.
+        while estimate_tokens(state["messages"], common_tools) < max(1024, target - 256):
+            remaining = target - estimate_tokens(state["messages"], common_tools)
+            chunk = min(max(256, remaining), rng.randint(800, 3500))
+            state["messages"].extend(make_history_turn(rng, state, state["turn_id"], chunk))
+            state["turn_id"] += 1
+
+        current_user = {
+            "role": "user",
+            "content": "基于以上工具结果，继续完成当前任务。需要时发起下一步 tool_call。",
+        }
+        req_messages = list(state["messages"]) + [current_user]
+        requests.append({
+            "id": f"synthetic_cumulative_{req_id:04d}",
+            "model": "llm",
+            "messages": req_messages,
+            "tools": common_tools,
+            "tool_choice": "auto",
+            "stream": True,
+            "temperature": 0,
+            "max_tokens": target_output_tokens(rng),
+            "metadata": {
+                "session_id": sid,
+                "target_prompt_tokens_approx": estimate_tokens(req_messages, common_tools),
+            },
+        })
+
+        # Pretend the model made a tool call and the environment returned a
+        # result, so the next request in this session contains this full prefix.
+        state["messages"].append(current_user)
+        state["messages"].extend(make_history_turn(rng, state, state["turn_id"], rng.randint(500, 2500))[1:])
+        state["turn_id"] += 1
+
+    return requests
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("--out", required=True)
@@ -244,6 +356,7 @@ def main():
     parser.add_argument("--sessions", type=int, default=768)
     parser.add_argument("--seed", type=int, default=20260714)
     parser.add_argument("--jsonl", action="store_true")
+    parser.add_argument("--cumulative", action="store_true", help="Generate exact reusable session prefixes.")
     parser.add_argument(
         "--max-prompt-tokens",
         type=int,
@@ -253,26 +366,29 @@ def main():
     args = parser.parse_args()
 
     rng = random.Random(args.seed)
-    session_ids = [f"sess_{i:04d}" for i in range(args.sessions)]
+    if args.cumulative:
-    # Bias toward near-neighbor session reuse by reusing some recent sessions.
+        requests = generate_cumulative_requests(rng, args.requests, args.sessions, args.max_prompt_tokens)
-    requests = []
+    else:
-    recent = []
+        session_ids = [f"sess_{i:04d}" for i in range(args.sessions)]
-    for i in range(args.requests):
+        # Bias toward near-neighbor session reuse by reusing some recent sessions.
-        if recent and rng.random() < 0.22:
+        requests = []
-            session_id = rng.choice(recent[-32:])
+        recent = []
-        else:
+        for i in range(args.requests):
-            session_id = session_ids[i % len(session_ids)]
+            if recent and rng.random() < 0.22:
-        recent.append(session_id)
+                session_id = rng.choice(recent[-32:])
-        total = target_prompt_tokens(rng, args.max_prompt_tokens)
+            else:
-        # Official weighted cache hit is about 65.6%; request-level varies.
+                session_id = session_ids[i % len(session_ids)]
-        if total >= 128000:
+            recent.append(session_id)
-            cache_ratio = rng.uniform(0.1, 0.35)
+            total = target_prompt_tokens(rng, args.max_prompt_tokens)
-        elif total >= 16000:
+            # Official weighted cache hit is about 65.6%; request-level varies.
-            cache_ratio = rng.uniform(0.55, 0.72)
+            if total >= 128000:
-        else:
+                cache_ratio = rng.uniform(0.1, 0.35)
-            cache_ratio = rng.uniform(0.75, 0.92)
+            elif total >= 16000:
-        shared = int(total * cache_ratio)
+                cache_ratio = rng.uniform(0.55, 0.72)
-        requests.append(build_request(rng, i, session_id, shared, total))
+            else:
+                cache_ratio = rng.uniform(0.75, 0.92)
+            shared = int(total * cache_ratio)
+            requests.append(build_request(rng, i, session_id, shared, total))
 
     out = Path(args.out)
     out.parent.mkdir(parents=True, exist_ok=True)
@@ -282,13 +398,14 @@ def main():
         out.write_text(json.dumps(requests, ensure_ascii=False, indent=2), encoding="utf-8")
 
     total_prompt = sum(x["metadata"]["target_prompt_tokens_approx"] for x in requests)
-    total_shared = sum(x["metadata"]["shared_prefix_tokens_approx"] for x in requests)
+    total_shared = sum(x["metadata"].get("shared_prefix_tokens_approx", 0) for x in requests)
     print(json.dumps({
         "out": str(out),
         "requests": len(requests),
         "sessions": len({x["metadata"]["session_id"] for x in requests}),
         "approx_prompt_tokens_avg": total_prompt / len(requests),
-        "approx_cache_ratio_weighted": total_shared / total_prompt,
+        "approx_cache_ratio_weighted": total_shared / total_prompt if total_shared else None,
+        "cumulative": args.cumulative,
     }, ensure_ascii=False, indent=2))