diff --git a/qwen3_6_scripts/patch_ops.sh b/qwen3_6_scripts/patch_ops.sh index 1035b2f..82d23c1 100755 --- a/qwen3_6_scripts/patch_ops.sh +++ b/qwen3_6_scripts/patch_ops.sh @@ -42,7 +42,8 @@ cp ./paged_attn.py /usr/local/corex/lib/python3/dist-packages/vllm/attention/ops python3 ./patch_model_runner.py # --- transformers: Qwen3_5 tokenizer / model files -------------------------- -pip install transformers==4.55.3 -i https://pypi.tuna.tsinghua.edu.cn/simple +pip install transformers==4.55.3 -i https://pypi.tuna.tsinghua.edu.cn/simple || \ + echo "[patch_ops] WARN: transformers==4.55.3 install failed; continuing with installed transformers." cp -r ./qwen3_5 /usr/local/lib/python3.10/site-packages/transformers/models/ cp -r ./qwen3_5_moe /usr/local/lib/python3.10/site-packages/transformers/models/ python3 ./patch_transformers_qwen3_5.py diff --git a/worklogs/2026-07-13-initial-run.md b/worklogs/2026-07-13-initial-run.md new file mode 100644 index 0000000..73ba5c5 --- /dev/null +++ b/worklogs/2026-07-13-initial-run.md @@ -0,0 +1,149 @@ +# 2026-07-13 Initial Run Worklog + +## Working Rules + +- Local repository is the source of truth for code changes. +- Remote Phanthy GPU server is used for build, runtime, and validation. +- Every meaningful experiment records: code version, command, environment, result, issue, and next action. +- Changes should be committed with git after a coherent milestone or before risky experiments. + +## Remote Target + +- Host: `ssh-55c3b0b3.default.gpu.phanthy.com` +- SSH requires TLS ProxyCommand on port `32222`. +- User: `root` + +## Status + +- Local repository inspected. +- SSH connectivity confirmed with `whoami` and `hostname`. +- Remote environment inspected. + +## Remote Environment Findings + +- Remote shell user: `root` +- Remote hostname: `cc-55c3b0b3-8c03-4fe2-8ef8-7109a6aff0d6-0` +- Remote appears to already be inside a container. +- `docker` is not installed in the remote runtime container. +- CoreX is installed under `/usr/local/corex -> /usr/local/corex-3.2.3`. +- Iluvatar devices are visible as `/dev/iluvatar0` through `/dev/iluvatar3`. +- PyTorch is available only when `PYTHONPATH` and `LD_LIBRARY_PATH` include CoreX paths. +- PyTorch reports `torch.cuda.is_available() == True` and `torch.cuda.device_count() == 4`. +- Model path found: `/root/public-storage/models/Qwen/Qwen3.6-35B-A3B`. +- Model `config.json` already has `architectures: ["Qwen3_5MoeForCausalLM"]`. +- Because remote has no Docker, first run will patch the current runtime directly instead of building an image. + +## First Run Plan + +1. Sync only lightweight submission files to `/root/work/enginex-vllm-bi100-qwen36`. +2. Backup target runtime files before applying `qwen3_6_scripts/patch_ops.sh`. +3. Apply patches in the remote CoreX container. +4. Start OpenAI-compatible API server against `/root/public-storage/models/Qwen/Qwen3.6-35B-A3B`. +5. Run minimal smoke tests before deeper compatibility tests. + +## 2026-07-14 First Patch Attempt Diagnosis + +- User-provided remote log showed repeated `$'\r': command not found` in `patch_ops.sh`. +- Root cause: the shell script, and likely copied `.py` patch helpers, arrived on Linux with Windows CRLF line endings. +- Consequence: `python3 ./patch_model_runner.py\r` and similar patch commands did not execute correctly. +- Later server startup failed with `unrecognized arguments: --reasoning-parser qwen3`. +- Interpretation: OpenAI API runtime files such as `cli_args.py` and `api_server.py` were not patched into `/usr/local/corex/lib/python3/dist-packages/vllm/...`. +- Next action: normalize CRLF to LF on the remote copy, rerun `patch_ops.sh`, verify `--reasoning-parser` exists in the installed vLLM CLI, then restart the API server. + +## 2026-07-14 First Server Start After CRLF Fix + +- User verified `python3 -m vllm.entrypoints.openai.api_server --help | grep reasoning-parser` now prints `--reasoning-parser`. +- Startup log shows `reasoning_parser='qwen3'`, so OpenAI API argument patch is active. +- `ixsmi` shows no vLLM GPU process and `/health` returns `Connection refused`, so the API server process exited before serving. +- Need inspect the tail of `/root/work/logs/server_first_run.log` after `Downcasting torch.float32 to torch.float16`. +- Additional suspicion to verify: some patches target `/usr/local/corex/lib/python3/dist-packages` while the Qwen3_5 registry patch targets `/usr/local/corex/lib64/python3/dist-packages`; confirm the actual imported registry path and whether both `lib` and `lib64` contain `Qwen3_5MoeForCausalLM`. + +## 2026-07-14 Server Still Initializing + +- User checked `jobs -l` and `ps`; master process PID 2052 is still running. +- Log has no `Traceback`, `ERROR`, or common fatal exceptions. +- Log reached `Worker ready; awaiting tasks` for three worker processes, so multiprocess executor startup is progressing. +- Both `/usr/local/corex/lib/.../vllm` and `/usr/local/corex/lib64/.../vllm` contain `registry.py` patched with `Qwen3_5MoeForCausalLM`, and `qwen3_5.py` exists in both. +- Earlier `/health` failure was likely checked before API server finished engine initialization and started listening. +- Next action: wait and monitor model loading/GPU memory, then retry `/health`; only diagnose hang if no new log/GPU memory movement for several minutes. + +## 2026-07-14 Port Occupied By Stale First Process + +- User started a second server while PID 2052 from the first run was still alive. +- Second run exited with `OSError: [Errno 98] Address already in use` at `sock.bind(("", args.port))`. +- `ps` still shows PID 2052 holding the API server command, but `ixsmi` shows no model GPU memory/processes. +- Interpretation: PID 2052 is a stale or stuck master process occupying port 1111, not a healthy loaded model service. +- Next action: stop PID 2052 and any VllmWorkerProcess children, verify port 1111 is free, restart once with a fresh log filename, then wait for explicit `Uvicorn running` / startup-complete logs before testing `/health`. + +## 2026-07-14 New Instance Startup Fix + +- New host: `ssh-8d2ae743.default.gpu.phanthy.com`. +- Initial failure: `api_server.py: error: unrecognized arguments: --reasoning-parser qwen3`. +- Cause 1: new instance runtime had not yet applied `qwen3_6_scripts/patch_ops.sh`. +- Applied `patch_ops.sh`; `cli_args.py` in both `/usr/local/corex/lib/...` and `/usr/local/corex/lib64/...` then contained `--reasoning-parser`. +- Cause 2: service was started from repository root `/root/data-disk-1/enginex-vllm-bi100-qwen36`, whose local `./vllm` package shadowed the patched CoreX runtime vLLM. +- Fix: start service from `/root` and set CoreX `PYTHONPATH`/`LD_LIBRARY_PATH`, using `python3 -B` to avoid stale bytecode. +- Working log: `/root/work/logs/server_from_root_B.log`. +- Health check reached `GET /health HTTP/1.1" 200 OK`. +- Minimal `/v1/chat/completions` request reached `POST /v1/chat/completions HTTP/1.1" 200 OK`. +- `ixsmi` shows model loaded at roughly 29GB per BI-V100 card. + +## 2026-07-14 Baseline Smoke And Mini Benchmark + +- Remote benchmark script copied to `/root/work/remote_smoke_bench.py`. +- Result file: `/root/work/logs/baseline_smoke_20260713_195201.json`. +- `/health` status: `200`. +- Non-stream smoke request: `200`, 16 completion tokens in `2.41s`, about `6.63 tok/s`. +- Streaming 3-run summary: + - TTFT average: `1.03s`. + - TTFT P90 from 3 samples: about `0.80s` using the simple small-sample estimator. + - Output TPS after TTFT: about `8.91 tok/s`. +- Prefix-cache probe: + - First long-prefix request: `cached_tokens=0`, 16 completion tokens in `6.59s`. + - Second related long-prefix request: `cached_tokens=1056 / 1069 prompt tokens`, 16 completion tokens in `3.61s`. + - Server metrics also reported GPU prefix cache hit rate around `50.36%`. +- Current status: model is fully runnable and API-compatible enough for smoke tests, but generation throughput is far below the contest target of Output TPS P10 >= 20. +- First performance direction: reduce thinking-token waste, profile decode path, verify XFormers fallback cost, then tune serving parameters after a larger benchmark. + +## 2026-07-14 Official-Like Benchmark Dataset Preparation + +- Current repository dataset `chat_dataset_v0.json` is only a 4-conversation smoke dataset. +- It has short multi-turn chat/role-play prompts and no realistic `tools`, assistant `tool_calls`, or `tool` role messages. +- It is not representative of the official benchmark, which is an 881-request long-context Agent coding workload. +- Added `worklogs/formal_perf_bench.py` support for OpenAI-style request items with `messages`, `tools`, `tool_choice`, `stream`, and per-request `max_tokens`. +- Added `worklogs/generate_official_like_dataset.py` to synthesize official-like long-context, stream=true, tool-heavy requests. +- Uploaded updated scripts to: + - `/root/work/formal_perf_bench.py` + - `/root/work/generate_official_like_dataset.py` +- Server smoke generation succeeded: + - `/root/work/logs/synthetic_agent_perf_16.jsonl` + - 16 requests, 13 sessions, approx prompt average `26611.5`, approximate weighted cache ratio `0.638`. + - File size `4.5M`. + +## 2026-07-14 Official-Like Synthetic Benchmark Round 1 + +- Fixed synthetic request format issues: + - Assistant tool-call messages now use `content: ""` instead of `content: null`. + - Synthetic system/context content is merged into a single leading system message because the current runtime rejects multiple system messages with `System message must be at the beginning`. + - Added `--max-prompt-tokens` to cap generated prompt sizes under the current server `--max-model-len 100000`. +- Generated dataset: + - `/root/work/logs/synthetic_agent_perf_16_fixed2_cap90k.jsonl` + - 16 requests, approximate average prompt tokens `25712`, approximate intended cache ratio `0.637`. +- Benchmark: + - Command output file: `/root/work/logs/perf_synth16_fixed2_cap90k_c1_r4.json`. + - Concurrency `1`, requests `4`, max output tokens `64`. +- Results: + - Success rate: `100%` (`4/4`). + - Prompt tokens total: `166080`. + - Completion tokens total: `256`. + - Cached tokens total: `96`. + - Actual cache hit rate: `0.058%`, far below official-like target; generator does not yet create exact cumulative session-prefix reuse. + - TTFT P90: `129.07s`. + - Output TPS P10 per request: `5.23 tok/s`. + - Aggregate output TPS: `0.607 tok/s`. + - Aggregate uncached input TPS: `393.52 tok/s`. + - Weighted token throughput: `1111.78`, far below `8000`. +- Interpretation: + - Long-context uncached prefill is the immediate bottleneck. + - Current synthetic generator must be improved to make same-session adjacent requests share exact prefix, otherwise cache behavior is not representative of the official 65.6% cached-token workload. + - Current runtime also needs template compatibility work for multiple/irregular system messages if official requests include them unmodified. diff --git a/worklogs/formal_perf_bench.py b/worklogs/formal_perf_bench.py new file mode 100644 index 0000000..dc4f631 --- /dev/null +++ b/worklogs/formal_perf_bench.py @@ -0,0 +1,255 @@ +import argparse +import concurrent.futures +import json +import math +import statistics +import time +import urllib.error +import urllib.request +from dataclasses import dataclass, asdict +from datetime import datetime +from pathlib import Path + + +@dataclass +class RequestResult: + ok: bool + elapsed_sec: float + ttft_sec: float | None + prompt_tokens: int + cached_tokens: int + completion_tokens: int + reasoning_tokens: int + output_tps: float | None + error: str | None = None + + +def percentile(values, pct): + if not values: + return None + values = sorted(values) + if len(values) == 1: + return values[0] + pos = (len(values) - 1) * pct / 100.0 + lo = math.floor(pos) + hi = math.ceil(pos) + if lo == hi: + return values[lo] + return values[lo] * (hi - pos) + values[hi] * (pos - lo) + + +def load_dataset(path): + text = Path(path).read_text(encoding="utf-8") + if path.endswith(".jsonl"): + data = [json.loads(line) for line in text.splitlines() if line.strip()] + else: + data = json.loads(text) + + # Official-like format: each item is already an OpenAI chat completion + # request with messages/tools/tool_choice/stream/etc. + if data and isinstance(data[0], dict) and "messages" in data[0]: + return data + + requests = [] + for item in data: + system_prompt = item.get("system_prompt") or "You are a helpful assistant." + history = [{"role": "system", "content": system_prompt}] + for question in item.get("user_questions", []): + messages = history + [{"role": "user", "content": question}] + requests.append({"messages": messages}) + # Synthetic assistant placeholder keeps later prompts multi-turn. + history = messages + [{"role": "assistant", "content": "好的,我们继续。"}] + return requests + + +def post_stream(url, model, request_item, max_tokens, timeout): + allowed_fields = { + "messages", + "tools", + "tool_choice", + "model", + "max_tokens", + "temperature", + "top_p", + "stop", + "presence_penalty", + "frequency_penalty", + "n", + "response_format", + } + payload = {k: v for k, v in dict(request_item).items() if k in allowed_fields} + payload["model"] = payload.get("model") or model + # The benchmark CLI controls output length, even if the synthetic dataset + # stores a larger official-like max_tokens value. + payload["max_tokens"] = max_tokens + payload["temperature"] = payload.get("temperature", 0) + payload["stream"] = True + payload["stream_options"] = {"include_usage": True} + req = urllib.request.Request( + url.rstrip("/") + "/v1/chat/completions", + data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + + start = time.perf_counter() + first_token_at = None + usage = {} + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + for raw in resp: + line = raw.decode("utf-8", errors="replace").strip() + if not line or not line.startswith("data:"): + continue + data = line[5:].strip() + if data == "[DONE]": + break + obj = json.loads(data) + if obj.get("usage"): + usage = obj["usage"] + for choice in obj.get("choices") or []: + delta = choice.get("delta") or {} + text = delta.get("content") or delta.get("reasoning_content") or "" + if text and first_token_at is None: + first_token_at = time.perf_counter() + elapsed = time.perf_counter() - start + ttft = first_token_at - start if first_token_at else None + completion_tokens = int(usage.get("completion_tokens") or 0) + prompt_tokens = int(usage.get("prompt_tokens") or 0) + reasoning_tokens = int(usage.get("reasoning_tokens") or 0) + details = usage.get("prompt_tokens_details") or {} + cached_tokens = int(details.get("cached_tokens") or 0) + decode_sec = elapsed - ttft if ttft is not None else elapsed + output_tps = completion_tokens / decode_sec if completion_tokens and decode_sec > 0 else None + return RequestResult( + ok=True, + elapsed_sec=elapsed, + ttft_sec=ttft, + prompt_tokens=prompt_tokens, + cached_tokens=cached_tokens, + completion_tokens=completion_tokens, + reasoning_tokens=reasoning_tokens, + output_tps=output_tps, + ) + except urllib.error.HTTPError as exc: + body = "" + try: + body = exc.read().decode("utf-8", errors="replace") + except Exception: + pass + elapsed = time.perf_counter() - start + return RequestResult( + ok=False, + elapsed_sec=elapsed, + ttft_sec=None, + prompt_tokens=0, + cached_tokens=0, + completion_tokens=0, + reasoning_tokens=0, + output_tps=None, + error=f"HTTPError {exc.code}: {body[:1000]}", + ) + except Exception as exc: + elapsed = time.perf_counter() - start + return RequestResult( + ok=False, + elapsed_sec=elapsed, + ttft_sec=None, + prompt_tokens=0, + cached_tokens=0, + completion_tokens=0, + reasoning_tokens=0, + output_tps=None, + error=f"{type(exc).__name__}: {exc}", + ) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--url", default="http://127.0.0.1:1111") + parser.add_argument("--model", default="llm") + parser.add_argument("--dataset", required=True) + parser.add_argument("--concurrency", type=int, default=1) + parser.add_argument("--max-requests", type=int, default=20) + parser.add_argument("--max-tokens", type=int, default=128) + parser.add_argument("--timeout", type=int, default=600) + parser.add_argument("--out", default="/root/work/logs/formal_perf_result.json") + args = parser.parse_args() + + reqs = load_dataset(args.dataset) + if not reqs: + raise SystemExit("empty dataset") + scheduled = [reqs[i % len(reqs)] for i in range(args.max_requests)] + + wall_start = time.perf_counter() + results = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as pool: + futures = [ + pool.submit(post_stream, args.url, args.model, request_item, args.max_tokens, args.timeout) + for request_item in scheduled + ] + for fut in concurrent.futures.as_completed(futures): + results.append(fut.result()) + print(f"done {len(results)}/{len(scheduled)} ok={results[-1].ok}", flush=True) + wall_sec = time.perf_counter() - wall_start + + ok_results = [r for r in results if r.ok] + success_rate = len(ok_results) / len(results) if results else 0.0 + ttfts = [r.ttft_sec for r in ok_results if r.ttft_sec is not None] + output_tps_values = [r.output_tps for r in ok_results if r.output_tps is not None] + + prompt_tokens = sum(r.prompt_tokens for r in ok_results) + cached_tokens = sum(r.cached_tokens for r in ok_results) + input_tokens_uncached = max(prompt_tokens - cached_tokens, 0) + output_tokens = sum(r.completion_tokens for r in ok_results) + + aggregate_output_tps = output_tokens / wall_sec if wall_sec > 0 else 0.0 + aggregate_input_tps = input_tokens_uncached / wall_sec if wall_sec > 0 else 0.0 + aggregate_cache_tps = cached_tokens / wall_sec if wall_sec > 0 else 0.0 + weighted = aggregate_output_tps * 16.796 + aggregate_input_tps * 2.799 + aggregate_cache_tps * 0.56 + cache_hit_rate = cached_tokens / prompt_tokens if prompt_tokens else 0.0 + + summary = { + "created_at": datetime.now().isoformat(timespec="seconds"), + "url": args.url, + "model": args.model, + "dataset": args.dataset, + "concurrency": args.concurrency, + "max_requests": args.max_requests, + "max_tokens": args.max_tokens, + "wall_sec": wall_sec, + "success_rate": success_rate, + "ttft_p90_sec": percentile(ttfts, 90), + "output_tps_p10_per_request": percentile(output_tps_values, 10), + "aggregate_output_tps": aggregate_output_tps, + "aggregate_input_tps_uncached": aggregate_input_tps, + "aggregate_cache_tps": aggregate_cache_tps, + "cache_hit_rate": cache_hit_rate, + "weighted_token_throughput": weighted, + "totals": { + "requests": len(results), + "success": len(ok_results), + "prompt_tokens": prompt_tokens, + "input_tokens_uncached": input_tokens_uncached, + "cached_tokens": cached_tokens, + "completion_tokens": output_tokens, + "reasoning_tokens": sum(r.reasoning_tokens for r in ok_results), + }, + "targets": { + "output_tps_p10_per_request_gte_20": (percentile(output_tps_values, 10) or 0) >= 20, + "ttft_p90_lte_5": (percentile(ttfts, 90) or 999) <= 5, + "cache_hit_rate_gte_50pct": cache_hit_rate >= 0.5, + "success_rate_gte_99pct": success_rate >= 0.99, + "weighted_token_throughput_gte_8000": weighted >= 8000, + }, + "results": [asdict(r) for r in results], + } + + Path(args.out).parent.mkdir(parents=True, exist_ok=True) + Path(args.out).write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8") + print(json.dumps({k: v for k, v in summary.items() if k != "results"}, ensure_ascii=False, indent=2)) + print("RESULT_FILE", args.out) + + +if __name__ == "__main__": + main() diff --git a/worklogs/generate_official_like_dataset.py b/worklogs/generate_official_like_dataset.py new file mode 100644 index 0000000..5db4de9 --- /dev/null +++ b/worklogs/generate_official_like_dataset.py @@ -0,0 +1,296 @@ +import argparse +import json +import random +from pathlib import Path + + +TOOL_NAMES = [ + "terminal_exec", + "process_list", + "read_file", + "edit_file", + "write_file", + "glob_files", + "grep_code", + "web_search", + "web_fetch", + "browser_open", + "mcp_codebase_memory_search", + "mcp_codebase_memory_graph", + "sessions_spawn", + "sessions_send", + "subagents_run", + "memory_search", +] + + +SYSTEM_PROMPT = """You are Codex, a coding agent. You help the user modify, test, and reason about code. +Follow the available tools carefully. Prefer precise shell commands, inspect files before editing, +and return concise progress updates. When tool calls are needed, produce valid tool call arguments.""" + + +COMMON_PROJECT_CONTEXT = """ +Repository: /home/user/workspace/project +Task: implement features, inspect logs, run tests, patch code, summarize results. +Conventions: +- Use rg for search. +- Avoid destructive commands. +- Keep a worklog. +- Preserve unrelated user changes. +""" + + +CODE_SNIPPET = """ +def fetch_prices(symbols, start_date, end_date, retries=3): + results = [] + for symbol in symbols: + payload = { + "symbol": symbol, + "start": start_date, + "end": end_date, + "adjust": "qfq", + } + # retry network request with backoff + for attempt in range(retries): + try: + results.append(client.get("/prices", params=payload)) + break + except TimeoutError: + time.sleep(1.5 * (attempt + 1)) + return results +""" + + +def make_tool(name): + if "terminal" in name or "process" in name: + props = {"command": {"type": "string", "description": "Shell command to execute."}} + required = ["command"] + elif "read" in name: + props = {"path": {"type": "string"}} + required = ["path"] + elif "edit" in name: + props = { + "file_path": {"type": "string"}, + "old_string": {"type": "string"}, + "new_string": {"type": "string"}, + } + required = ["file_path", "old_string", "new_string"] + elif "web" in name: + props = {"query": {"type": "string"}, "count": {"type": "integer"}} + required = ["query"] + else: + props = {"input": {"type": "string"}, "limit": {"type": "integer"}} + required = ["input"] + return { + "type": "function", + "function": { + "name": name, + "description": f"Synthetic benchmark tool: {name}", + "parameters": { + "type": "object", + "properties": props, + "required": required, + }, + }, + } + + +def rand_text(rng, approx_tokens): + # Mostly ASCII/code-like text keeps the chars/token ratio simple enough for + # synthetic load generation. This is approximate; server usage is authoritative. + unit = ( + "tool output line: status=ok path=/home/user/project/src/module.py " + "grep result includes function names, stack frames, JSON fields, " + "中文说明:这里包含工具返回、日志片段、代码上下文和转义字符。 " + + CODE_SNIPPET.replace("\n", "\\n") + + "\n" + ) + chars = max(1, approx_tokens * 4) + return (unit * (chars // len(unit) + 1))[:chars] + + +def make_tool_call(rng, idx): + name = rng.choice(TOOL_NAMES) + if "terminal" in name: + args = { + "command": "cd /home/user/project && rg -n \"TODO|FIXME|error\" src tests 2>/dev/null | head -50" + } + elif "read" in name: + args = {"path": "/home/user/workspace/.agents/skills/data-analysis/SKILL.md"} + elif "edit" in name: + args = { + "file_path": "/home/user/scripts/fetch_prices.py", + "old_string": "# 全局频率限制器实例\n_rate_limiter = RateLimiter(min_interval=2.0, max_per_minute=20)", + "new_string": "# 全局频率限制器实例 - 根据配额上调\n_rate_limiter = RateLimiter(min_interval=1.5, max_per_minute=40)", + } + elif "web" in name: + args = {"query": "深圳 周末活动 推荐 2026年7月", "count": 10} + else: + args = {"input": "inspect project memory and summarize related files", "limit": 20} + return { + "id": f"call_{idx}_{rng.randrange(10**8)}", + "type": "function", + "function": {"name": name, "arguments": json.dumps(args, ensure_ascii=False)}, + } + + +def target_prompt_tokens(rng, max_prompt_tokens=None): + # Approximate official distribution: many 4K-16K, meaningful 32K+ tail, + # small number of 128K+ requests. + x = rng.random() + if x < 0.08: + value = rng.randint(1000, 4000) + return min(value, max_prompt_tokens) if max_prompt_tokens else value + if x < 0.50: + value = rng.randint(4000, 16000) + return min(value, max_prompt_tokens) if max_prompt_tokens else value + if x < 0.74: + value = rng.randint(16000, 32000) + return min(value, max_prompt_tokens) if max_prompt_tokens else value + if x < 0.95: + high = min(128000, max_prompt_tokens) if max_prompt_tokens else 128000 + value = rng.randint(32000, max(32000, high)) + return value + high = min(235000, max_prompt_tokens) if max_prompt_tokens else 235000 + return rng.randint(128000, max(128000, high)) if high >= 128000 else high + + +def target_output_tokens(rng): + x = rng.random() + if x < 0.66: + return rng.randint(64, 256) + if x < 0.96: + return rng.randint(256, 2000) + return rng.randint(2000, 8192) + + +def build_request(rng, req_id, session_id, shared_prefix_tokens, total_prompt_tokens): + tool_count = max(1, int(rng.lognormvariate(2.8, 0.8))) + tool_count = min(tool_count, 92) + tools = [make_tool(rng.choice(TOOL_NAMES)) for _ in range(tool_count)] + + messages = [ + { + "role": "system", + "content": SYSTEM_PROMPT + "\n" + COMMON_PROJECT_CONTEXT + rand_text(rng, shared_prefix_tokens), + }, + ] + + remaining = max(512, total_prompt_tokens - shared_prefix_tokens) + turns = rng.randint(12, 65) + per_turn = max(32, remaining // max(1, turns)) + call_idx = 0 + for turn in range(turns): + messages.append({ + "role": "user", + "content": ( + f"Session {session_id}, turn {turn}: inspect the codebase and continue the task. " + "Need exact commands, possible edits, and concise next action." + ), + }) + if rng.random() < 0.78: + calls = [make_tool_call(rng, call_idx)] + call_idx += 1 + messages.append({ + "role": "assistant", + # This runtime validates that every message has content or + # reasoning_content, even when assistant emits tool_calls. + "content": "", + "tool_calls": calls, + }) + tool_len = int(per_turn * rng.uniform(0.5, 2.0)) + if rng.random() > 0.99: + tool_len = rng.randint(12000, 50000) + elif rng.random() > 0.90: + tool_len = rng.randint(2500, 16000) + messages.append({ + "role": "tool", + "tool_call_id": calls[0]["id"], + "name": calls[0]["function"]["name"], + "content": rand_text(rng, tool_len), + }) + else: + messages.append({ + "role": "assistant", + "content": "我会先检查相关文件和日志,再给出下一步修改建议。", + }) + + messages.append({ + "role": "user", + "content": "基于以上工具结果,继续完成当前任务。需要时发起下一步 tool_call。", + }) + + return { + "id": f"synthetic_{req_id:04d}", + "model": "llm", + "messages": messages, + "tools": tools, + "tool_choice": "auto", + "stream": True, + "temperature": 0, + "max_tokens": target_output_tokens(rng), + "metadata": { + "session_id": session_id, + "target_prompt_tokens_approx": total_prompt_tokens, + "shared_prefix_tokens_approx": shared_prefix_tokens, + }, + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--out", required=True) + parser.add_argument("--requests", type=int, default=881) + parser.add_argument("--sessions", type=int, default=768) + parser.add_argument("--seed", type=int, default=20260714) + parser.add_argument("--jsonl", action="store_true") + parser.add_argument( + "--max-prompt-tokens", + type=int, + default=None, + help="Cap synthetic prompt-token target. Use 90000 for a server started with --max-model-len 100000.", + ) + args = parser.parse_args() + + rng = random.Random(args.seed) + session_ids = [f"sess_{i:04d}" for i in range(args.sessions)] + # Bias toward near-neighbor session reuse by reusing some recent sessions. + requests = [] + recent = [] + for i in range(args.requests): + if recent and rng.random() < 0.22: + session_id = rng.choice(recent[-32:]) + else: + session_id = session_ids[i % len(session_ids)] + recent.append(session_id) + total = target_prompt_tokens(rng, args.max_prompt_tokens) + # Official weighted cache hit is about 65.6%; request-level varies. + if total >= 128000: + cache_ratio = rng.uniform(0.1, 0.35) + elif total >= 16000: + cache_ratio = rng.uniform(0.55, 0.72) + else: + cache_ratio = rng.uniform(0.75, 0.92) + shared = int(total * cache_ratio) + requests.append(build_request(rng, i, session_id, shared, total)) + + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + if args.jsonl: + out.write_text("\n".join(json.dumps(x, ensure_ascii=False) for x in requests) + "\n", encoding="utf-8") + else: + out.write_text(json.dumps(requests, ensure_ascii=False, indent=2), encoding="utf-8") + + total_prompt = sum(x["metadata"]["target_prompt_tokens_approx"] for x in requests) + total_shared = sum(x["metadata"]["shared_prefix_tokens_approx"] for x in requests) + print(json.dumps({ + "out": str(out), + "requests": len(requests), + "sessions": len({x["metadata"]["session_id"] for x in requests}), + "approx_prompt_tokens_avg": total_prompt / len(requests), + "approx_cache_ratio_weighted": total_shared / total_prompt, + }, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/worklogs/remote_smoke_bench.py b/worklogs/remote_smoke_bench.py new file mode 100644 index 0000000..e53315c --- /dev/null +++ b/worklogs/remote_smoke_bench.py @@ -0,0 +1,135 @@ +import json +import statistics +import sys +import time +import urllib.request +from datetime import datetime +from pathlib import Path + + +BASE_URL = sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:1111" +OUT_DIR = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("/root/work/logs") +MODEL = sys.argv[3] if len(sys.argv) > 3 else "llm" + + +def post_json(path, payload, timeout=300): + req = urllib.request.Request( + BASE_URL + path, + data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.status, resp.read().decode("utf-8") + + +def health(): + with urllib.request.urlopen(BASE_URL + "/health", timeout=10) as resp: + return resp.status + + +def chat_once(prompt, max_tokens=32): + payload = { + "model": MODEL, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, + "temperature": 0, + } + start = time.perf_counter() + status, body = post_json("/v1/chat/completions", payload) + elapsed = time.perf_counter() - start + obj = json.loads(body) + usage = obj.get("usage") or {} + completion_tokens = usage.get("completion_tokens") or 0 + return { + "status": status, + "elapsed_sec": elapsed, + "usage": usage, + "output_tps": completion_tokens / elapsed if elapsed and completion_tokens else None, + "content_preview": (obj["choices"][0]["message"].get("content") or "")[:200], + } + + +def chat_stream(prompt, max_tokens=64): + payload = { + "model": MODEL, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, + "temperature": 0, + "stream": True, + "stream_options": {"include_usage": True}, + } + req = urllib.request.Request( + BASE_URL + "/v1/chat/completions", + data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + start = time.perf_counter() + first_token_at = None + usage = None + pieces = [] + with urllib.request.urlopen(req, timeout=300) as resp: + for raw in resp: + line = raw.decode("utf-8", errors="replace").strip() + if not line or not line.startswith("data:"): + continue + data = line[5:].strip() + if data == "[DONE]": + break + obj = json.loads(data) + if obj.get("usage"): + usage = obj["usage"] + choices = obj.get("choices") or [] + if choices: + delta = choices[0].get("delta") or {} + text = delta.get("content") or delta.get("reasoning_content") or "" + if text: + if first_token_at is None: + first_token_at = time.perf_counter() + pieces.append(text) + elapsed = time.perf_counter() - start + completion_tokens = (usage or {}).get("completion_tokens") or 0 + gen_time = elapsed - (first_token_at - start) if first_token_at else elapsed + return { + "elapsed_sec": elapsed, + "ttft_sec": (first_token_at - start) if first_token_at else None, + "usage": usage, + "output_tps_after_ttft": completion_tokens / gen_time if gen_time and completion_tokens else None, + "content_preview": "".join(pieces)[:200], + } + + +def main(): + OUT_DIR.mkdir(parents=True, exist_ok=True) + results = { + "base_url": BASE_URL, + "model": MODEL, + "created_at": datetime.now().isoformat(timespec="seconds"), + } + results["health_status"] = health() + results["smoke_nonstream"] = chat_once("你好,请用一句话介绍你自己。", max_tokens=16) + + short_prompt = "请用中文简要说明什么是模型推理服务。" + stream_runs = [chat_stream(short_prompt, max_tokens=64) for _ in range(3)] + results["stream_runs"] = stream_runs + ttfts = [x["ttft_sec"] for x in stream_runs if x["ttft_sec"] is not None] + tps = [x["output_tps_after_ttft"] for x in stream_runs if x["output_tps_after_ttft"] is not None] + results["stream_summary"] = { + "ttft_avg_sec": statistics.mean(ttfts) if ttfts else None, + "ttft_p90_sec": sorted(ttfts)[int(0.9 * (len(ttfts) - 1))] if ttfts else None, + "output_tps_avg": statistics.mean(tps) if tps else None, + } + + repeated = "以下是一段用于测试前缀缓存的公共上下文:" + ("模型部署竞赛关注吞吐、延迟、缓存命中和稳定性。" * 80) + results["prefix_cache_probe_1"] = chat_once(repeated + "\n请总结一句话。", max_tokens=16) + results["prefix_cache_probe_2"] = chat_once(repeated + "\n请换一种说法总结一句话。", max_tokens=16) + + out = OUT_DIR / f"baseline_smoke_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" + out.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8") + print(json.dumps(results, ensure_ascii=False, indent=2), flush=True) + print("RESULT_FILE", out, flush=True) + + +if __name__ == "__main__": + main()