baseline working model service and perf tooling

2026-07-14 13:57:40 +08:00
parent 1902c81fdd
commit c8cc9401a4
5 changed files with 837 additions and 1 deletions
--- a/worklogs/generate_official_like_dataset.py
+++ b/worklogs/generate_official_like_dataset.py
@@ -0,0 +1,296 @@
+import argparse
+import json
+import random
+from pathlib import Path
+
+
+TOOL_NAMES = [
+    "terminal_exec",
+    "process_list",
+    "read_file",
+    "edit_file",
+    "write_file",
+    "glob_files",
+    "grep_code",
+    "web_search",
+    "web_fetch",
+    "browser_open",
+    "mcp_codebase_memory_search",
+    "mcp_codebase_memory_graph",
+    "sessions_spawn",
+    "sessions_send",
+    "subagents_run",
+    "memory_search",
+]
+
+
+SYSTEM_PROMPT = """You are Codex, a coding agent. You help the user modify, test, and reason about code.
+Follow the available tools carefully. Prefer precise shell commands, inspect files before editing,
+and return concise progress updates. When tool calls are needed, produce valid tool call arguments."""
+
+
+COMMON_PROJECT_CONTEXT = """
+Repository: /home/user/workspace/project
+Task: implement features, inspect logs, run tests, patch code, summarize results.
+Conventions:
+- Use rg for search.
+- Avoid destructive commands.
+- Keep a worklog.
+- Preserve unrelated user changes.
+"""
+
+
+CODE_SNIPPET = """
+def fetch_prices(symbols, start_date, end_date, retries=3):
+    results = []
+    for symbol in symbols:
+        payload = {
+            "symbol": symbol,
+            "start": start_date,
+            "end": end_date,
+            "adjust": "qfq",
+        }
+        # retry network request with backoff
+        for attempt in range(retries):
+            try:
+                results.append(client.get("/prices", params=payload))
+                break
+            except TimeoutError:
+                time.sleep(1.5 * (attempt + 1))
+    return results
+"""
+
+
+def make_tool(name):
+    if "terminal" in name or "process" in name:
+        props = {"command": {"type": "string", "description": "Shell command to execute."}}
+        required = ["command"]
+    elif "read" in name:
+        props = {"path": {"type": "string"}}
+        required = ["path"]
+    elif "edit" in name:
+        props = {
+            "file_path": {"type": "string"},
+            "old_string": {"type": "string"},
+            "new_string": {"type": "string"},
+        }
+        required = ["file_path", "old_string", "new_string"]
+    elif "web" in name:
+        props = {"query": {"type": "string"}, "count": {"type": "integer"}}
+        required = ["query"]
+    else:
+        props = {"input": {"type": "string"}, "limit": {"type": "integer"}}
+        required = ["input"]
+    return {
+        "type": "function",
+        "function": {
+            "name": name,
+            "description": f"Synthetic benchmark tool: {name}",
+            "parameters": {
+                "type": "object",
+                "properties": props,
+                "required": required,
+            },
+        },
+    }
+
+
+def rand_text(rng, approx_tokens):
+    # Mostly ASCII/code-like text keeps the chars/token ratio simple enough for
+    # synthetic load generation. This is approximate; server usage is authoritative.
+    unit = (
+        "tool output line: status=ok path=/home/user/project/src/module.py "
+        "grep result includes function names, stack frames, JSON fields, "
+        "中文说明：这里包含工具返回、日志片段、代码上下文和转义字符。 "
+        + CODE_SNIPPET.replace("\n", "\\n")
+        + "\n"
+    )
+    chars = max(1, approx_tokens * 4)
+    return (unit * (chars // len(unit) + 1))[:chars]
+
+
+def make_tool_call(rng, idx):
+    name = rng.choice(TOOL_NAMES)
+    if "terminal" in name:
+        args = {
+            "command": "cd /home/user/project && rg -n \"TODO|FIXME|error\" src tests 2>/dev/null | head -50"
+        }
+    elif "read" in name:
+        args = {"path": "/home/user/workspace/.agents/skills/data-analysis/SKILL.md"}
+    elif "edit" in name:
+        args = {
+            "file_path": "/home/user/scripts/fetch_prices.py",
+            "old_string": "# 全局频率限制器实例\n_rate_limiter = RateLimiter(min_interval=2.0, max_per_minute=20)",
+            "new_string": "# 全局频率限制器实例 - 根据配额上调\n_rate_limiter = RateLimiter(min_interval=1.5, max_per_minute=40)",
+        }
+    elif "web" in name:
+        args = {"query": "深圳 周末活动 推荐 2026年7月", "count": 10}
+    else:
+        args = {"input": "inspect project memory and summarize related files", "limit": 20}
+    return {
+        "id": f"call_{idx}_{rng.randrange(10**8)}",
+        "type": "function",
+        "function": {"name": name, "arguments": json.dumps(args, ensure_ascii=False)},
+    }
+
+
+def target_prompt_tokens(rng, max_prompt_tokens=None):
+    # Approximate official distribution: many 4K-16K, meaningful 32K+ tail,
+    # small number of 128K+ requests.
+    x = rng.random()
+    if x < 0.08:
+        value = rng.randint(1000, 4000)
+        return min(value, max_prompt_tokens) if max_prompt_tokens else value
+    if x < 0.50:
+        value = rng.randint(4000, 16000)
+        return min(value, max_prompt_tokens) if max_prompt_tokens else value
+    if x < 0.74:
+        value = rng.randint(16000, 32000)
+        return min(value, max_prompt_tokens) if max_prompt_tokens else value
+    if x < 0.95:
+        high = min(128000, max_prompt_tokens) if max_prompt_tokens else 128000
+        value = rng.randint(32000, max(32000, high))
+        return value
+    high = min(235000, max_prompt_tokens) if max_prompt_tokens else 235000
+    return rng.randint(128000, max(128000, high)) if high >= 128000 else high
+
+
+def target_output_tokens(rng):
+    x = rng.random()
+    if x < 0.66:
+        return rng.randint(64, 256)
+    if x < 0.96:
+        return rng.randint(256, 2000)
+    return rng.randint(2000, 8192)
+
+
+def build_request(rng, req_id, session_id, shared_prefix_tokens, total_prompt_tokens):
+    tool_count = max(1, int(rng.lognormvariate(2.8, 0.8)))
+    tool_count = min(tool_count, 92)
+    tools = [make_tool(rng.choice(TOOL_NAMES)) for _ in range(tool_count)]
+
+    messages = [
+        {
+            "role": "system",
+            "content": SYSTEM_PROMPT + "\n" + COMMON_PROJECT_CONTEXT + rand_text(rng, shared_prefix_tokens),
+        },
+    ]
+
+    remaining = max(512, total_prompt_tokens - shared_prefix_tokens)
+    turns = rng.randint(12, 65)
+    per_turn = max(32, remaining // max(1, turns))
+    call_idx = 0
+    for turn in range(turns):
+        messages.append({
+            "role": "user",
+            "content": (
+                f"Session {session_id}, turn {turn}: inspect the codebase and continue the task. "
+                "Need exact commands, possible edits, and concise next action."
+            ),
+        })
+        if rng.random() < 0.78:
+            calls = [make_tool_call(rng, call_idx)]
+            call_idx += 1
+            messages.append({
+                "role": "assistant",
+                # This runtime validates that every message has content or
+                # reasoning_content, even when assistant emits tool_calls.
+                "content": "",
+                "tool_calls": calls,
+            })
+            tool_len = int(per_turn * rng.uniform(0.5, 2.0))
+            if rng.random() > 0.99:
+                tool_len = rng.randint(12000, 50000)
+            elif rng.random() > 0.90:
+                tool_len = rng.randint(2500, 16000)
+            messages.append({
+                "role": "tool",
+                "tool_call_id": calls[0]["id"],
+                "name": calls[0]["function"]["name"],
+                "content": rand_text(rng, tool_len),
+            })
+        else:
+            messages.append({
+                "role": "assistant",
+                "content": "我会先检查相关文件和日志，再给出下一步修改建议。",
+            })
+
+    messages.append({
+        "role": "user",
+        "content": "基于以上工具结果，继续完成当前任务。需要时发起下一步 tool_call。",
+    })
+
+    return {
+        "id": f"synthetic_{req_id:04d}",
+        "model": "llm",
+        "messages": messages,
+        "tools": tools,
+        "tool_choice": "auto",
+        "stream": True,
+        "temperature": 0,
+        "max_tokens": target_output_tokens(rng),
+        "metadata": {
+            "session_id": session_id,
+            "target_prompt_tokens_approx": total_prompt_tokens,
+            "shared_prefix_tokens_approx": shared_prefix_tokens,
+        },
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--out", required=True)
+    parser.add_argument("--requests", type=int, default=881)
+    parser.add_argument("--sessions", type=int, default=768)
+    parser.add_argument("--seed", type=int, default=20260714)
+    parser.add_argument("--jsonl", action="store_true")
+    parser.add_argument(
+        "--max-prompt-tokens",
+        type=int,
+        default=None,
+        help="Cap synthetic prompt-token target. Use 90000 for a server started with --max-model-len 100000.",
+    )
+    args = parser.parse_args()
+
+    rng = random.Random(args.seed)
+    session_ids = [f"sess_{i:04d}" for i in range(args.sessions)]
+    # Bias toward near-neighbor session reuse by reusing some recent sessions.
+    requests = []
+    recent = []
+    for i in range(args.requests):
+        if recent and rng.random() < 0.22:
+            session_id = rng.choice(recent[-32:])
+        else:
+            session_id = session_ids[i % len(session_ids)]
+        recent.append(session_id)
+        total = target_prompt_tokens(rng, args.max_prompt_tokens)
+        # Official weighted cache hit is about 65.6%; request-level varies.
+        if total >= 128000:
+            cache_ratio = rng.uniform(0.1, 0.35)
+        elif total >= 16000:
+            cache_ratio = rng.uniform(0.55, 0.72)
+        else:
+            cache_ratio = rng.uniform(0.75, 0.92)
+        shared = int(total * cache_ratio)
+        requests.append(build_request(rng, i, session_id, shared, total))
+
+    out = Path(args.out)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    if args.jsonl:
+        out.write_text("\n".join(json.dumps(x, ensure_ascii=False) for x in requests) + "\n", encoding="utf-8")
+    else:
+        out.write_text(json.dumps(requests, ensure_ascii=False, indent=2), encoding="utf-8")
+
+    total_prompt = sum(x["metadata"]["target_prompt_tokens_approx"] for x in requests)
+    total_shared = sum(x["metadata"]["shared_prefix_tokens_approx"] for x in requests)
+    print(json.dumps({
+        "out": str(out),
+        "requests": len(requests),
+        "sessions": len({x["metadata"]["session_id"] for x in requests}),
+        "approx_prompt_tokens_avg": total_prompt / len(requests),
+        "approx_cache_ratio_weighted": total_shared / total_prompt,
+    }, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    main()