diff --git a/worklogs/2026-07-13-initial-run.md b/worklogs/2026-07-13-initial-run.md index 73ba5c5..ec42998 100644 --- a/worklogs/2026-07-13-initial-run.md +++ b/worklogs/2026-07-13-initial-run.md @@ -147,3 +147,32 @@ - Long-context uncached prefill is the immediate bottleneck. - Current synthetic generator must be improved to make same-session adjacent requests share exact prefix, otherwise cache behavior is not representative of the official 65.6% cached-token workload. - Current runtime also needs template compatibility work for multiple/irregular system messages if official requests include them unmodified. + +## 2026-07-14 Cumulative Prefix Synthetic Benchmark Round 2 + +- Added `--cumulative` mode to `generate_official_like_dataset.py`. +- New mode shares exact system/tool prefix and grows each session history in place, so same-session later requests actually reuse earlier token prefixes. +- Generated dataset: + - `/root/work/logs/synthetic_cumulative_8_s3_cap40k.jsonl` + - 8 requests, 3 sessions, approximate prompt average `27452.75`. + - Prompt cap `40000` to keep test runtime manageable under current `--max-model-len 100000`. +- Benchmark: + - Result file: `/root/work/logs/perf_cumulative8_s3_cap40k_c1_r4.json`. + - Concurrency `1`, requests `4`, max output tokens `64`. +- Results: + - Success rate: `100%` (`4/4`). + - TTFT P90: `58.49s`. + - Output TPS P10 per request: `5.94 tok/s`. + - Aggregate output TPS: `1.55 tok/s`. + - Aggregate uncached input TPS: `284.96 tok/s`. + - Aggregate cache TPS: `296.87 tok/s`. + - Cached tokens: `49152 / 96333`, cache hit rate `51.02%`. + - Weighted token throughput: `989.82`. +- Target comparison: + - Cache hit rate and success rate pass. + - TTFT, Output TPS P10, and weighted throughput are far below target. +- Interpretation: + - Prefix cache is now being exercised correctly, but cached prefill is still too slow at this configuration. + - Current serving parameter `--max-num-seqs 1` prevents useful concurrent batching and likely caps weighted throughput. + - Decode path is also slow: request-level output TPS only `5.9-8.5 tok/s`. + - Next experiment should restart service with higher `--max-num-seqs`, higher `--max-num-batched-tokens`, and possibly `gpu-memory-utilization 0.95`, then rerun the same cumulative dataset. diff --git a/worklogs/generate_official_like_dataset.py b/worklogs/generate_official_like_dataset.py index 5db4de9..5d8cb19 100644 --- a/worklogs/generate_official_like_dataset.py +++ b/worklogs/generate_official_like_dataset.py @@ -237,6 +237,118 @@ def build_request(rng, req_id, session_id, shared_prefix_tokens, total_prompt_to } +def estimate_tokens(messages, tools): + chars = 0 + for msg in messages: + chars += len(msg.get("role") or "") + len(msg.get("content") or "") + if msg.get("tool_calls"): + chars += len(json.dumps(msg["tool_calls"], ensure_ascii=False)) + if msg.get("name"): + chars += len(msg["name"]) + chars += len(json.dumps(tools, ensure_ascii=False)) + return max(1, chars // 4) + + +def make_history_turn(rng, state, turn_id, approx_tool_tokens): + user = { + "role": "user", + "content": ( + f"Continue session {state['session_id']} turn {turn_id}. " + "Inspect files, reason about logs, and call the next tool if needed." + ), + } + call = make_tool_call(rng, turn_id) + assistant = { + "role": "assistant", + "content": "", + "tool_calls": [call], + } + tool = { + "role": "tool", + "tool_call_id": call["id"], + "name": call["function"]["name"], + "content": rand_text(rng, approx_tool_tokens), + } + return [user, assistant, tool] + + +def generate_cumulative_requests(rng, count, session_count, max_prompt_tokens): + """Generate requests whose prefixes are actually reusable. + + The first version of this generator only stored an intended cache ratio in + metadata. This mode makes the byte/text prefix identical across requests by + sharing tools/system content and by growing each session history in place. + """ + common_tools = [make_tool(name) for name in TOOL_NAMES] + # Keep a large, fixed global prefix. It imitates common system prompt, + # agent framework instructions, and mounted tool schemas. + common_system = { + "role": "system", + "content": SYSTEM_PROMPT + "\n" + COMMON_PROJECT_CONTEXT + rand_text(rng, 6000), + } + sessions = {} + session_order = [] + requests = [] + + def get_session(sid): + if sid not in sessions: + sessions[sid] = { + "session_id": sid, + "messages": [common_system], + "turn_id": 0, + } + session_order.append(sid) + return sessions[sid] + + recent = [] + for req_id in range(count): + # For small benchmark sets, force meaningful reuse. For larger sets, + # this still creates short-distance affinity similar to the official + # median gap of a few requests. + if recent and rng.random() < 0.65: + sid = rng.choice(recent[-32:]) + else: + sid = f"sess_{len(session_order) % max(1, session_count):04d}" + recent.append(sid) + state = get_session(sid) + target = target_prompt_tokens(rng, max_prompt_tokens) + + # Grow the historical context until this request has the target scale. + while estimate_tokens(state["messages"], common_tools) < max(1024, target - 256): + remaining = target - estimate_tokens(state["messages"], common_tools) + chunk = min(max(256, remaining), rng.randint(800, 3500)) + state["messages"].extend(make_history_turn(rng, state, state["turn_id"], chunk)) + state["turn_id"] += 1 + + current_user = { + "role": "user", + "content": "基于以上工具结果，继续完成当前任务。需要时发起下一步 tool_call。", + } + req_messages = list(state["messages"]) + [current_user] + requests.append({ + "id": f"synthetic_cumulative_{req_id:04d}", + "model": "llm", + "messages": req_messages, + "tools": common_tools, + "tool_choice": "auto", + "stream": True, + "temperature": 0, + "max_tokens": target_output_tokens(rng), + "metadata": { + "session_id": sid, + "target_prompt_tokens_approx": estimate_tokens(req_messages, common_tools), + }, + }) + + # Pretend the model made a tool call and the environment returned a + # result, so the next request in this session contains this full prefix. + state["messages"].append(current_user) + state["messages"].extend(make_history_turn(rng, state, state["turn_id"], rng.randint(500, 2500))[1:]) + state["turn_id"] += 1 + + return requests + + def main(): parser = argparse.ArgumentParser() parser.add_argument("--out", required=True) @@ -244,6 +356,7 @@ def main(): parser.add_argument("--sessions", type=int, default=768) parser.add_argument("--seed", type=int, default=20260714) parser.add_argument("--jsonl", action="store_true") + parser.add_argument("--cumulative", action="store_true", help="Generate exact reusable session prefixes.") parser.add_argument( "--max-prompt-tokens", type=int, @@ -253,26 +366,29 @@ def main(): args = parser.parse_args() rng = random.Random(args.seed) - session_ids = [f"sess_{i:04d}" for i in range(args.sessions)] - # Bias toward near-neighbor session reuse by reusing some recent sessions. - requests = [] - recent = [] - for i in range(args.requests): - if recent and rng.random() < 0.22: - session_id = rng.choice(recent[-32:]) - else: - session_id = session_ids[i % len(session_ids)] - recent.append(session_id) - total = target_prompt_tokens(rng, args.max_prompt_tokens) - # Official weighted cache hit is about 65.6%; request-level varies. - if total >= 128000: - cache_ratio = rng.uniform(0.1, 0.35) - elif total >= 16000: - cache_ratio = rng.uniform(0.55, 0.72) - else: - cache_ratio = rng.uniform(0.75, 0.92) - shared = int(total * cache_ratio) - requests.append(build_request(rng, i, session_id, shared, total)) + if args.cumulative: + requests = generate_cumulative_requests(rng, args.requests, args.sessions, args.max_prompt_tokens) + else: + session_ids = [f"sess_{i:04d}" for i in range(args.sessions)] + # Bias toward near-neighbor session reuse by reusing some recent sessions. + requests = [] + recent = [] + for i in range(args.requests): + if recent and rng.random() < 0.22: + session_id = rng.choice(recent[-32:]) + else: + session_id = session_ids[i % len(session_ids)] + recent.append(session_id) + total = target_prompt_tokens(rng, args.max_prompt_tokens) + # Official weighted cache hit is about 65.6%; request-level varies. + if total >= 128000: + cache_ratio = rng.uniform(0.1, 0.35) + elif total >= 16000: + cache_ratio = rng.uniform(0.55, 0.72) + else: + cache_ratio = rng.uniform(0.75, 0.92) + shared = int(total * cache_ratio) + requests.append(build_request(rng, i, session_id, shared, total)) out = Path(args.out) out.parent.mkdir(parents=True, exist_ok=True) @@ -282,13 +398,14 @@ def main(): out.write_text(json.dumps(requests, ensure_ascii=False, indent=2), encoding="utf-8") total_prompt = sum(x["metadata"]["target_prompt_tokens_approx"] for x in requests) - total_shared = sum(x["metadata"]["shared_prefix_tokens_approx"] for x in requests) + total_shared = sum(x["metadata"].get("shared_prefix_tokens_approx", 0) for x in requests) print(json.dumps({ "out": str(out), "requests": len(requests), "sessions": len({x["metadata"]["session_id"] for x in requests}), "approx_prompt_tokens_avg": total_prompt / len(requests), - "approx_cache_ratio_weighted": total_shared / total_prompt, + "approx_cache_ratio_weighted": total_shared / total_prompt if total_shared else None, + "cumulative": args.cumulative, }, ensure_ascii=False, indent=2))