From 3adf13ad22fcbae53eb01a4f18f8ca59c918b090 Mon Sep 17 00:00:00 2001 From: Victor Cao Date: Fri, 16 Oct 2026 22:24:54 +0800 Subject: [PATCH] live gate: default-on queue-gated task/add (<95), fail-closed, cap 3/cycle, banned+dedupe, kill switch (54 tests) --- .gitignore | 1 + Dockerfile | 1 + Makefile | 2 +- README.md | 27 +++-- docs/platform-contract-agent.md | 23 ++-- reports/live-gate-summary.md | 27 +++++ src/banned.py | 15 +++ src/gate.py | 155 +++++++++------------------ src/live_cycle.py | 137 ++++++++++++++++++++++++ src/live_queue.py | 65 ++++++++++++ src/server.py | 19 ++++ tests/test_gate.py | 183 ++++++-------------------------- tests/test_live_cycle.py | 109 +++++++++++++++++++ tests/test_live_queue.py | 55 ++++++++++ tests/test_sigterm.py | 1 + 15 files changed, 548 insertions(+), 272 deletions(-) create mode 100644 reports/live-gate-summary.md create mode 100644 src/banned.py create mode 100644 src/live_cycle.py create mode 100644 src/live_queue.py create mode 100644 tests/test_live_cycle.py create mode 100644 tests/test_live_queue.py diff --git a/.gitignore b/.gitignore index 8f01d46..182242f 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ build/ # Dry-run / local artifacts (keep directory) reports/* !reports/.gitkeep +!reports/live-gate-summary.md # Editor / OS .DS_Store diff --git a/Dockerfile b/Dockerfile index 733fe08..4a1fa0d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,7 @@ WORKDIR /app # Copy only what the runtime needs (see .dockerignore) COPY src/ ./src/ +COPY templates/ ./templates/ ENV PORT=8080 ENV PYTHONUNBUFFERED=1 diff --git a/Makefile b/Makefile index 7eb9d72..75427c9 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ help: @echo "build Docker build using platform base (needs Docker + TCR access)" @echo "smoke-health local HTTP smoke (needs sockets; default test STRATEGY_ID)" @echo "dry-run read-only pending report (example fallback)" - @echo "No register/submit/deploy/push — default live off." + @echo "No register/submit/deploy/push — live defaults on; ALLOW_LIVE_SUBMIT=0 stops live." test: $(PYTHON) -m unittest discover -s tests -v diff --git a/README.md b/README.md index 41f8f90..2428aa0 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ ModelHub XC(信创模盒)**适配智能体** Phase1 本地骨架。 -> **不代注册、不代提交。** 发布智能体本身 **不得分**。积分来自成功适配,数字一律 **待平台核实**。 -> 冲突裁决:**适配 W1 优先**;本仓默认 dry-run,不抢验证队列。 +> **不代注册;Victor 已批准队列闸门 live 代提。** 发布智能体本身 **不得分**。积分来自成功适配,数字一律 **待平台核实**。 +> 冲突裁决:**适配 W1 优先**;本仓默认 live 开启,仅在实时验证队列低于阈值时提交。 ## 契约要点 @@ -12,13 +12,18 @@ ModelHub XC(信创模盒)**适配智能体** Phase1 本地骨架。 | 端口 | 8080 | | 健康检查 | `GET /health` → `{"status":"ok"}` | | 启动 | **必须**有非空 `STRATEGY_ID`,否则 exit ≠0 | -| Live | `ALLOW_LIVE_SUBMIT=1` **且** `LIVE_CONFIRM=I_CONFIRM_LIVE_SUBMIT`;Phase1 仍无真实 `task/add` 客户端 | -| Base 镜像 | 平台 `xc_agent_platform/python:3.11-slim`(勿擅自改默认 FROM) | +| Live | 默认开启;`ALLOW_LIVE_SUBMIT=0` 为停交开关;必须提供 `STRATEGY_ID` | +| Base 镜像 | 平台 `xc_agent_platform/python:3.11-slim`(保留 FROM) | -闸门以 `src/gate.py` 实现为准:仅 `ALLOW_LIVE_SUBMIT=1` 与 -`LIVE_CONFIRM=I_CONFIRM_LIVE_SUBMIT` 同时满足才开逻辑闸门。PRD 示例 -`ALLOW_LIVE_SUBMIT=true` / `CONFIRM_LIVE=YES` 不受支持;本仓不读取确认文件。 -即使开闸,无真实客户端仍返回 blocked;请保持默认关闭。 +Victor 的确认已内置,`LIVE_CONFIRM` 仍接受但不再必需。每次 task/add 前, +通过 Xc-Token 全量分页读取 onlyMine task/page;非终态计入队列,读取失败停交。 +`LIVE_QUEUE_THRESHOLD=95`(严格小于),`LIVE_MAX_SUBMITS_PER_CYCLE=3`, +`LIVE_CYCLE_SECONDS=600`(最小 60 秒)。环境队列提示仅可额外阻塞。 +候选来自 `PENDING_SUBMIT_MD` / `PENDING_SUBMIT_TXT`,否则使用 +`src/data/pending-submit.md`(若存在);A verified_reuse → 其他 A → B。 +禁提、重复、GPU 未映射、占位符候选跳过。`LIVE_STATE_FILE` 默认 +`/tmp/live-submitted.json`,在请求前持久化去重;失败请求也不自动重试。 +配置优先 build-config,失败使用 sample 并记录来源及原因。token 缺失只 hold。 ## 快速开始 @@ -34,6 +39,7 @@ make build # 需 Docker + 可达平台 base;失败见 docs/build-not ```bash export STRATEGY_ID=local-dev +export ALLOW_LIVE_SUBMIT=0 # 本地开发停交 python3 -m src.server # → http://127.0.0.1:8080/health ``` @@ -71,6 +77,5 @@ tests/ # stdlib unittest ## 许可与边界 -本仓库用于 Victor 本地/Gitea 交付骨架;任何 ModelHub 写操作须 Victor 本人在控制台或显式批准后执行。 - -Phase2:`make dry-run-pending` 只读生成 A→B 草稿,名称硬门与禁提过滤已接入;需 Victor 人工提交/确认。task/add 客户端默认不调用,必须显式注入并通过双闸门、队列及单次预算;测试仅 mock HTTP。 +本仓库用于 Victor 本地/Gitea 交付。Victor 已批准上述队列闸门自动提交;不自动注册。 +`make dry-run-pending` 仍仅生成草稿;测试全部 mock 平台接口。 diff --git a/docs/platform-contract-agent.md b/docs/platform-contract-agent.md index b352e37..171ffa2 100644 --- a/docs/platform-contract-agent.md +++ b/docs/platform-contract-agent.md @@ -2,7 +2,7 @@ - 日期:2026-10-16 Asia/Shanghai - 依据:产品规划 `agent-product-plan-2026-10-16.md` §1 + PRD `02-agent-contribution-prd.md` -- **冲突裁决:适配 W1 优先**(队列 ≥95 禁 live;本骨架默认 dry-run) +- **冲突裁决:适配 W1 优先**(队列 ≥95 禁 live;默认 live 开启,实时队列闸门控制) ## 核实等级 @@ -29,7 +29,7 @@ | **STRATEGY_ID 缺失** | **Phase1:进程启动失败(exit ≠0)** | 产品规划 Phase1 + PRD | | 资源 | 约 100m–1 CPU / 256–512Mi | B | | **发布智能体得分** | **0 分**(发布 ≠ 得分) | B | -| Live 默认 | **关**;双重闸门 `ALLOW_LIVE_SUBMIT=1` + `LIVE_CONFIRM=I_CONFIRM_LIVE_SUBMIT` | 本仓 | +| Live 默认 | **开**(Victor 已批准);`ALLOW_LIVE_SUBMIT=0` 停交,`LIVE_CONFIRM` 可选 | 本仓 | | 积分 | 来自**成功适配**;智能体代提成功计同一成功分;公式/周报 **待平台核实** | B / D | ## 鉴权双轨(文档索引,Phase1 不代调) @@ -41,7 +41,7 @@ ## 禁止 -- 代注册、代 live 提交、Makefile 含 register/submit/deploy/push +- 代注册、绕过实时队列闸门、Makefile 含 register/submit/deploy/push - 日志/响应/镜像层写入 token 或 STRATEGY_ID **值** - 把「智能体已发布」写入千分进度 @@ -51,12 +51,17 @@ |------|------| | 异步验证队列 | 适配 W1 优先;≥95 全体停交(含智能体 live) | | Codex | hold 期扩池/归因 > 智能体脚手架 | -| 本骨架 | 默认 dry-run;本轮无真实 `task/add` HTTP 客户端 | +| 本骨架 | 默认 live;每次 task/add 前实时读取 onlyMine 队列 | ## 闸门配置以实现为准 -`src/gate.py` 只接受 `ALLOW_LIVE_SUBMIT=1` 与 -`LIVE_CONFIRM=I_CONFIRM_LIVE_SUBMIT`(去除首尾空白后精确匹配)。 -PRD 的 `ALLOW_LIVE_SUBMIT=true` / `CONFIRM_LIVE=YES` 仅为需求示例, -本仓不支持这些值/别名,也不读取确认文件。开闸无客户端仍 blocked。 -这只是本地逻辑闸门,不代表已获 live 授权或已校验队列与候选硬门。 +`ALLOW_LIVE_SUBMIT` 默认 1,0 为 kill switch;`STRATEGY_ID` 必需。 +Victor 确认已内置,`LIVE_CONFIRM` 仍接受但不必需。 +`LIVE_QUEUE_THRESHOLD` 默认 95,只有完整分页的 onlyMine task/page 非终态数严格小于阈值才提交。 +缺凭证、HTTP/平台错误、JSON 错误、分页不完整或无效配置均停交。 +ASYNC_QUEUE_DEPTH / QUEUE_SNAPSHOT_JSON 仅为额外阻塞提示。 +每周期最多 `LIVE_MAX_SUBMITS_PER_CYCLE`(默认 3)次;周期 +`LIVE_CYCLE_SECONDS` 默认 600、最小 60 秒,通过 shutdown Event 等待。 +候选 A verified_reuse 优先,禁提过滤、GPU enum、占位符检查及请求前持久化去重。 +`LIVE_STATE_FILE` 默认 /tmp/live-submitted.json;保留该文件可跨周期/重启去重。 +build-config 失败回退 sample 并记录原因;日志不记录响应正文或凭证。 diff --git a/reports/live-gate-summary.md b/reports/live-gate-summary.md new file mode 100644 index 0000000..b2e86dd --- /dev/null +++ b/reports/live-gate-summary.md @@ -0,0 +1,27 @@ +# Queue-gated live submit + +Victor-approved live defaults on. ALLOW_LIVE_SUBMIT=0 stops submission; +STRATEGY_ID remains mandatory. LIVE_CONFIRM is accepted but optional. + +Each add requires a fresh authenticated onlyMine task/page snapshot, complete +through data.total (maximum 20 pages). All nonterminal records count; +queue >= LIVE_QUEUE_THRESHOLD (default 95) stops the cycle. Authentication, +transport, HTTP, platform, JSON and pagination errors fail closed. Environment +queue hints can only add blockers. + +Cycles prioritize Section A verified_reuse, other A, then B; reject banned +families, unmapped GPUs and placeholders; use build-config with recorded sample +fallback; reserve model identity in memory and LIVE_STATE_FILE before calling +TaskAddClient. Remote task identities also block duplicates. Default cap is 3 +attempts per cycle. Failed calls retain their reservations to prevent retries. +The state file defaults to /tmp/live-submitted.json; preserve or mount it for +restart durability beyond ephemeral container storage. + +Server runs a daemon worker every 600 seconds (minimum 60), using the shutdown +Event for interruptible waits. Missing credentials hold instead of crashing. +Docker FROM is unchanged; templates are copied for runtime fallback. + +Validation: make test passes 54 tests, including mocked platform calls and local +SIGTERM integration. Sandbox initially denied local sockets; the suite passed +with local socket permission. No real ModelHub calls, token writes, commits, +tags or pushes were performed. diff --git a/src/banned.py b/src/banned.py new file mode 100644 index 0000000..95d73af --- /dev/null +++ b/src/banned.py @@ -0,0 +1,15 @@ +"""Case-insensitive live model exclusions.""" +import re + +RULES = ( + ('gpt2', r'gpt-?2|dialogpt|distilgpt2'), + ('phi', r'phi[-_. ]?(?:3(?:\.5)?|4)(?:\D|$)'), + ('nemotron-h', r'nemotron-h'), ('olmo2', r'olmo-?2'), + ('lfm2', r'lfm2'), ('spark2.5', r'spark2_5|spark-x2\.5'), + ('breeze', r'breeze'), ('gemma2', r'gemma-?2'), + ('mistral-v0.1', r'mistral.*v0\.1'), ('tinyllama', r'tinyllama'), +) + + +def banned_rule(model): + return next((name for name, pattern in RULES if re.search(pattern, model, re.I)), None) diff --git a/src/gate.py b/src/gate.py index f7f2a55..9f2e02c 100644 --- a/src/gate.py +++ b/src/gate.py @@ -1,123 +1,74 @@ -"""Dual live-submit gate. Default: dry-run / blocked. +"""Live defaults on with baked-in approval. LIVE_CONFIRM remains optional. -Live is allowed only when ALL of: - - ALLOW_LIVE_SUBMIT=1 - - LIVE_CONFIRM=I_CONFIRM_LIVE_SUBMIT - - STRATEGY_ID present - -Phase2 client exists but requires explicit injection. No automatic wiring. -Queue ≥95 blocks; at most one injected submit per process. +Environment hints only block; a fresh authenticated API snapshot authorizes each add. """ - -from __future__ import annotations - import os -import threading -from .queue_depth import queue_depth from dataclasses import dataclass -from typing import Any, Callable, Dict, Optional, Protocol - from .strategy import strategy_id_present -_submit_count = 0 -_submit_lock = threading.Lock() - -LIVE_CONFIRM_VALUE = "I_CONFIRM_LIVE_SUBMIT" +LIVE_CONFIRM_VALUE = 'I_CONFIRM_LIVE_SUBMIT' -class SubmitClient(Protocol): - """Minimal interface for a submit client (injected fake in tests).""" +def live_enabled(): + return os.environ.get('ALLOW_LIVE_SUBMIT', '1').strip() == '1' - def submit(self, payload: Dict[str, Any]) -> Dict[str, Any]: - ... + +def queue_threshold(): + value = int(os.environ.get('LIVE_QUEUE_THRESHOLD', '95')) + if value < 1: + raise ValueError('invalid threshold') + return value @dataclass(frozen=True) class GateDecision: - mode: str # "dry-run" | "live" | "blocked" + mode: str allowed: bool reason: str -def _allow_live_flag() -> bool: - return os.environ.get("ALLOW_LIVE_SUBMIT", "0").strip() == "1" - - -def _live_confirm_ok() -> bool: - return os.environ.get("LIVE_CONFIRM", "").strip() == LIVE_CONFIRM_VALUE - - -def evaluate_gate() -> GateDecision: - """Evaluate whether live submit is permitted.""" +def evaluate_gate(): + if not live_enabled(): + return GateDecision('dry-run', False, 'kill_switch') if not strategy_id_present(): - return GateDecision( - mode="blocked", - allowed=False, - reason="STRATEGY_ID missing or empty", - ) - if not _allow_live_flag(): - return GateDecision( - mode="dry-run", - allowed=False, - reason="ALLOW_LIVE_SUBMIT is not 1 (default dry-run)", - ) - if not _live_confirm_ok(): - return GateDecision( - mode="blocked", - allowed=False, - reason=f"LIVE_CONFIRM must be exactly {LIVE_CONFIRM_VALUE!r}", - ) - depth = queue_depth() - if depth is None and ("ASYNC_QUEUE_DEPTH" in os.environ or "QUEUE_SNAPSHOT_JSON" in os.environ): - return GateDecision("blocked", False, "queue depth invalid/unknown; human confirmation required") - if depth is not None and depth >= 95: - return GateDecision("blocked", False, "queue ≥95: 禁 live") - return GateDecision( - mode="live", - allowed=True, - reason="dual gate open (ALLOW_LIVE_SUBMIT=1 and LIVE_CONFIRM set)", - ) + return GateDecision('blocked', False, 'STRATEGY_ID missing or empty') + try: + threshold = queue_threshold() + except ValueError: + return GateDecision('blocked', False, 'invalid_threshold') + # Evaluate each configured hint independently; neither can override the other. + for key in ('ASYNC_QUEUE_DEPTH', 'QUEUE_SNAPSHOT_JSON'): + if key not in os.environ: + continue + if key == 'ASYNC_QUEUE_DEPTH': + try: + depth = int(os.environ[key]) + except ValueError: + depth = None + else: + import json + from pathlib import Path + try: + data = json.loads(Path(os.environ[key]).read_text()) + value = next(data[k] for k in ('depth', 'queue_depth', 'async_queue_depth', 'ASYNC_QUEUE_DEPTH') if k in data) + depth = None if isinstance(value, bool) else int(str(value)) + except (OSError, ValueError, TypeError, StopIteration): + depth = None + if depth is None or depth < 0: + return GateDecision('blocked', False, 'queue_hint_invalid') + if depth >= threshold: + return GateDecision('blocked', False, f'queue>={threshold} hint_count={depth}') + return GateDecision('live', True, 'live_enabled') -def attempt_live_submit( - payload: Dict[str, Any], - client: Optional[SubmitClient] = None, -) -> Dict[str, Any]: - """Attempt a live submit only if the gate opens and a client is injected. - - Phase2 TaskAddClient must be explicitly injected and dual-gated. - Tests must use fake clients or mock HTTP only. - """ - global _submit_count - decision = evaluate_gate() - if not decision.allowed: - return { - "ok": False, - "mode": decision.mode, - "reason": decision.reason, - "submitted": False, - } +def attempt_live_submit(payload, client=None, queue_reader=None): + """Compatibility entry point; delegates to the same fresh queue cycle.""" if client is None: - return { - "ok": False, - "mode": "blocked", - "reason": ( - "Phase2 task/add client exists; explicit client injection required; " - "no real request without injection and dual gate" - ), - "submitted": False, - } - with _submit_lock: - if _submit_count >= 1: - return dict(ok=False, mode="blocked", submitted=False, - reason="process limit: at most 1 real add") - # Reserve before invocation: a transport failure may still have created a task. - _submit_count += 1 - result = client.submit(payload) - return { - "ok": True, - "mode": "live", - "reason": decision.reason, - "submitted": True, - "result": result, - } + return dict(ok=False, submitted=False, mode='blocked', reason='no real request without explicit client') + from .live_cycle import run_cycle + decisions = run_cycle([dict(model_address=payload.get('modelAddress'), targetGpu=payload.get('targetGpu'), + framework=payload.get('framework', 'vllm'), configParams=payload.get('configParams'))], + client=client, queue_reader=queue_reader) + result = decisions[-1] + return dict(result, ok=result['action'] == 'submitted', submitted=result['action'] == 'submitted', + mode='live' if result['action'] == 'submitted' else 'blocked') diff --git a/src/live_cycle.py b/src/live_cycle.py new file mode 100644 index 0000000..3aa9f65 --- /dev/null +++ b/src/live_cycle.py @@ -0,0 +1,137 @@ +"""Queue-gated live cycles; all remote collaborators are injectable.""" +import json +import os +import sys +import threading +from pathlib import Path +from .banned import banned_rule +from .config_params import GPU_ENUMS, build_config_params_with_meta +from .gate import evaluate_gate, queue_threshold +from .live_queue import read_queue, model_key +from .pending_pool import build_ordered_candidates +from .task_add_client import TaskAddClient, build_payload + +_submitted = set() +_cycle_lock = threading.Lock() + + +def run_cycle(candidates=None, client=None, queue_reader=None, now=None): + with _cycle_lock: + return _run_cycle(candidates, client, queue_reader, now) + + +def _run_cycle(candidates, client, queue_reader, now): + decisions = [] + + def emit(action, reason, **fields): + decision = dict(action=action, reason=reason, **fields) + decisions.append(decision) + print('[live] ' + action + ' ' + ' '.join(f'{k}={str(v).replace(chr(10), " ").replace(chr(13), " ")}' for k, v in dict(fields, reason=reason).items()), file=sys.stderr) + + gate = evaluate_gate() + if not gate.allowed: + emit('hold', gate.reason) + return decisions + try: + cap = int(os.environ.get('LIVE_MAX_SUBMITS_PER_CYCLE', '3')) + if cap < 1: + raise ValueError + except ValueError: + emit('hold', 'invalid_cycle_cap') + return decisions + if candidates is None: + fallback = Path(__file__).parent / 'data/pending-submit.md' + try: + candidates = build_ordered_candidates( + os.environ.get('PENDING_SUBMIT_MD', str(fallback)), + os.environ.get('PENDING_SUBMIT_TXT', str(fallback.with_suffix('.txt')))) + except (OSError, ValueError, IndexError): + emit('hold', 'candidate_read_failed') + return decisions + candidates = sorted(candidates, key=lambda c: (c.get('section', 'B') != 'A', c.get('foothold_class') != 'verified_reuse')) + if not candidates: + emit('hold', 'no_candidates') + return decisions + state = Path(os.environ.get('LIVE_STATE_FILE', '/tmp/live-submitted.json')) + try: + saved = json.loads(state.read_text()) if state.exists() else [] + if not isinstance(saved, list) or any(not isinstance(s, str) for s in saved): + raise ValueError + known = set(map(model_key, saved)) | _submitted + except (OSError, ValueError): + emit('hold', 'state_read_failed') + return decisions + reader = queue_reader or read_queue + client = client if client is not None else TaskAddClient() + attempts = 0 + for candidate in candidates: + model = candidate.get('model_address') or candidate.get('modelAddress') or '' + rule = banned_rule(model + ' ' + str(candidate.get('model_name', candidate.get('modelName', '')))) + if rule: + emit('skip', 'banned:' + rule, model=model) + continue + key = model_key(model) + if key in known: + emit('skip', 'duplicate', model=model) + continue + if candidate.get('skip_reason'): + emit('skip', 'pending_filter', model=model) + continue + if attempts >= cap: + emit('hold', 'cycle_cap') + break + gpu = next(iter(candidate.get('try_next_gpus') or []), candidate.get('targetGpu')) + if gpu not in GPU_ENUMS: + emit('skip', 'gpu_unmapped', model=model) + continue + framework = candidate.get('framework') or 'vllm' + try: + yaml, meta = build_config_params_with_meta(gpu, framework) + except Exception: + emit('skip', 'config_failed', model=model) + continue + payload = build_payload(model, GPU_ENUMS[gpu], framework, os.environ['STRATEGY_ID'].strip(), yaml) + if any(not value or '<' in str(value) or '>' in str(value) for value in payload.values()): + emit('skip', 'placeholder', model=model) + continue + # Fresh complete snapshot immediately before reserving and sending each add. + try: + count, why, remote = reader() + except Exception: + count, why, remote = None, 'reader_error', set() + if count is None: + # Only internal fixed reasons are emitted; injected errors can contain secrets. + safe = why if why in {'missing/invalid_token', 'http_error', 'malformed_json', 'transport/http_error', 'platform_error', 'malformed_data', 'malformed_records', 'pagination_total_changed', 'pagination_inconsistent', 'incomplete_pagination'} else 'reader_error' + emit('hold', 'queue_read_failed:' + safe) + break + if type(count) is not int or count < 0: + emit('hold', 'queue_read_failed:invalid_count') + break + if count >= queue_threshold(): + emit('hold', f'queue>={queue_threshold()}', count=count) + break + if key in {model_key(m) for m in remote}: + emit('skip', 'duplicate', model=model) + continue + known.add(key) + _submitted.add(key) + try: + state.parent.mkdir(parents=True, exist_ok=True) + temporary = state.with_name(state.name + '.tmp') + temporary.write_text(json.dumps(sorted(known))) + temporary.replace(state) + except OSError: + emit('hold', 'state_write_failed') + break + attempts += 1 + try: + result = client.submit(payload) + status, code = result.get('http_status'), result.get('platform_code') + status = status if type(status) is int else None + code = code if type(code) is int else None + reason = 'task_add_response' if status is not None else 'transport_error' + except Exception: + status, code, reason = None, None, 'submit_error' + emit('submitted', reason, model=model, gpu=GPU_ENUMS[gpu], queue_before=count, + http=status, code=code, **meta) + return decisions diff --git a/src/live_queue.py b/src/live_queue.py new file mode 100644 index 0000000..f93c3bc --- /dev/null +++ b/src/live_queue.py @@ -0,0 +1,65 @@ +"""Fresh authenticated queue snapshots. Never expose response bodies or credentials.""" +import json +import os +import urllib.request +from .task_add_client import resolve_token + +TERMINAL = {'success', 'failed', 'fail', 'error', 'cancelled', 'canceled'} + + +def model_key(value): + value = str(value or '').strip().lower().rstrip('/') + for prefix in ('https://huggingface.co/', 'http://huggingface.co/', 'https://modelscope.cn/models/'): + if value.startswith(prefix): + value = value[len(prefix):] + return value + + +def read_queue(): + """Return (count, reason, model identities); count None means fail closed.""" + try: + token = resolve_token() + except (ValueError, OSError, UnicodeError): + return None, 'missing/invalid_token', set() + count, models, seen, total = 0, set(), 0, None + for page in range(1, 21): + try: + request = urllib.request.Request( + os.environ.get('MODELHUB_API_BASE', 'https://modelhub.org.cn').rstrip('/') + + f'/api/adapt/task/page?onlyMine=true&pageSize=100¤t={page}', + headers={'Xc-Token': token, 'Accept': 'application/json'}, method='GET') + with urllib.request.urlopen(request, timeout=30) as response: + if not 200 <= response.status < 300: + return None, 'http_error', set() + body = json.loads(response.read()) + except (ValueError, UnicodeError): + return None, 'malformed_json', set() + except OSError: + return None, 'transport/http_error', set() + if not isinstance(body, dict) or type(body.get('code')) not in (int, str) or str(body['code']) != '0': + return None, 'platform_error', set() + data = body.get('data') + if not isinstance(data, dict) or type(data.get('total')) is not int or data['total'] < 0: + return None, 'malformed_data', set() + records = data.get('records') + if not isinstance(records, list) or any(not isinstance(r, dict) for r in records): + return None, 'malformed_records', set() + if total is not None and total != data['total']: + return None, 'pagination_total_changed', set() + total = data['total'] + seen += len(records) + if seen > total: + return None, 'pagination_inconsistent', set() + for record in records: + count += str(record.get('status', '')).strip().lower() not in TERMINAL + for field in ('modelAddress', 'model_address'): + if record.get(field): + models.add(model_key(record[field])) + if record.get('modelAuthorName') and record.get('modelId'): + models.add(model_key(f"{record['modelAuthorName']}/{record['modelId']}")) + if seen == total: + # Do not retain arbitrary response fields, which may echo credentials. + return count, 'ok', {m for m in models if token.lower() not in m} + if not records: + break + return None, 'incomplete_pagination', set() diff --git a/src/server.py b/src/server.py index e076722..9269170 100644 --- a/src/server.py +++ b/src/server.py @@ -17,6 +17,8 @@ from typing import Optional from .health import handle_get from .strategy import require_strategy_id +from .gate import live_enabled +from .live_cycle import run_cycle _shutdown = threading.Event() _httpd: Optional[ThreadingHTTPServer] = None @@ -56,6 +58,20 @@ def _on_sigterm(signum: int, frame) -> None: # noqa: ARG001 threading.Thread(target=httpd.shutdown, daemon=True).start() +def _live_worker(): + try: + interval = max(60, int(os.environ.get('LIVE_CYCLE_SECONDS', '600'))) + except ValueError: + interval = 600 + while not _shutdown.is_set(): + try: + run_cycle() + except Exception: + sys.stderr.write('[live] hold reason=cycle_error\n') + if _shutdown.wait(interval): + break + + def main() -> None: global _httpd @@ -71,6 +87,9 @@ def main() -> None: httpd.daemon_threads = True _httpd = httpd + if live_enabled(): + threading.Thread(target=_live_worker, daemon=True).start() + sys.stderr.write(f"Listening on {host}:{port} (GET /health, GET /status)\n") try: httpd.serve_forever(poll_interval=0.5) diff --git a/tests/test_gate.py b/tests/test_gate.py index 7a2d9e2..1bac647 100644 --- a/tests/test_gate.py +++ b/tests/test_gate.py @@ -1,159 +1,44 @@ -"""Gate defaults to dry-run / blocked; live only with dual confirm + STRATEGY_ID.""" - -from __future__ import annotations - +"""Default live approval, kill switch, strategy and conservative hint gates.""" import os +import tempfile import unittest -from unittest import mock - -from src.gate import LIVE_CONFIRM_VALUE, attempt_live_submit, evaluate_gate - - -class FakeClient: - def __init__(self) -> None: - self.calls = [] - - def submit(self, payload): - self.calls.append(payload) - return {"task_id": "fake-1"} +from pathlib import Path +from unittest.mock import patch +from src.gate import evaluate_gate, attempt_live_submit, LIVE_CONFIRM_VALUE class TestGate(unittest.TestCase): def setUp(self): - self.counter = mock.patch('src.gate._submit_count', 0) - self.counter.start() - self.addCleanup(self.counter.stop) - self.env = mock.patch.dict(os.environ, {}, clear=True) - self.env.start() - self.addCleanup(self.env.stop) + env = patch.dict(os.environ, {'STRATEGY_ID': 'sid'}, clear=True) + env.start() + self.addCleanup(env.stop) - def test_queue_and_process_limit(self): - client = FakeClient() - with mock.patch.dict(os.environ, { - 'STRATEGY_ID': 'sid', 'ALLOW_LIVE_SUBMIT': '1', - 'LIVE_CONFIRM': LIVE_CONFIRM_VALUE, 'ASYNC_QUEUE_DEPTH': '95' - }): - self.assertIn('≥95', attempt_live_submit({}, client)['reason']) - self.assertEqual(client.calls, []) - os.environ['ASYNC_QUEUE_DEPTH'] = '94' - self.assertTrue(attempt_live_submit({}, client)['submitted']) - self.assertFalse(attempt_live_submit({}, client)['submitted']) - self.assertEqual(len(client.calls), 1) + def test_default_live_and_optional_confirmation(self): + self.assertTrue(evaluate_gate().allowed) + for value in ('', 'YES', LIVE_CONFIRM_VALUE): + with patch.dict(os.environ, {'LIVE_CONFIRM': value}): + self.assertTrue(evaluate_gate().allowed) - def _clear_gate_env(self): - return mock.patch.dict( - os.environ, - { - "STRATEGY_ID": "sid-1", - "ALLOW_LIVE_SUBMIT": "0", - "LIVE_CONFIRM": "", - }, - clear=False, - ) + def test_kill_switch_and_missing_strategy(self): + for env in ({'ALLOW_LIVE_SUBMIT': '0'}, {'ALLOW_LIVE_SUBMIT': 'true'}, {'STRATEGY_ID': ' '}): + with patch.dict(os.environ, env): + self.assertFalse(evaluate_gate().allowed) - def test_default_dry_run(self) -> None: - with self._clear_gate_env(): - d = evaluate_gate() - self.assertFalse(d.allowed) - self.assertEqual(d.mode, "dry-run") + def test_hints_and_threshold(self): + for value in ('95', 'invalid', '-1'): + with patch.dict(os.environ, {'ASYNC_QUEUE_DEPTH': value}): + self.assertFalse(evaluate_gate().allowed) + with patch.dict(os.environ, {'ASYNC_QUEUE_DEPTH': '94'}): + self.assertTrue(evaluate_gate().allowed) + with patch.dict(os.environ, {'LIVE_QUEUE_THRESHOLD': 'bad'}): + self.assertFalse(evaluate_gate().allowed) + with tempfile.TemporaryDirectory() as d: + p = Path(d) / 'hint.json' + p.write_text('{"depth":95}') + with patch.dict(os.environ, {'QUEUE_SNAPSHOT_JSON': str(p), 'ASYNC_QUEUE_DEPTH': '1'}): + self.assertFalse(evaluate_gate().allowed) - def test_missing_strategy_blocked(self) -> None: - env = { - "ALLOW_LIVE_SUBMIT": "1", - "LIVE_CONFIRM": LIVE_CONFIRM_VALUE, - } - # Remove STRATEGY_ID - base = {k: v for k, v in os.environ.items() if k != "STRATEGY_ID"} - base.update(env) - with mock.patch.dict(os.environ, base, clear=True): - d = evaluate_gate() - self.assertFalse(d.allowed) - self.assertEqual(d.mode, "blocked") - - def test_allow_without_confirm_blocked(self) -> None: - with mock.patch.dict( - os.environ, - { - "STRATEGY_ID": "sid-1", - "ALLOW_LIVE_SUBMIT": "1", - "LIVE_CONFIRM": "YES", - }, - clear=False, - ): - d = evaluate_gate() - self.assertFalse(d.allowed) - self.assertEqual(d.mode, "blocked") - - def test_dual_gate_opens(self) -> None: - with mock.patch.dict( - os.environ, - { - "STRATEGY_ID": "sid-1", - "ALLOW_LIVE_SUBMIT": "1", - "LIVE_CONFIRM": LIVE_CONFIRM_VALUE, - }, - clear=False, - ): - d = evaluate_gate() - self.assertTrue(d.allowed) - self.assertEqual(d.mode, "live") - - def test_open_gate_without_client_still_blocked(self) -> None: - """Phase2: an open gate never constructs a network client.""" - with mock.patch.dict( - os.environ, - { - "STRATEGY_ID": "sid-1", - "ALLOW_LIVE_SUBMIT": "1", - "LIVE_CONFIRM": LIVE_CONFIRM_VALUE, - }, - clear=False, - ): - result = attempt_live_submit({"modelAddress": "x/y"}) - self.assertFalse(result["submitted"]) - self.assertIn("no real", result["reason"].lower()) - self.assertEqual(result["mode"], "blocked") - self.assertFalse(result["ok"]) - - def test_prd_example_and_whitespace_id_never_submit(self) -> None: - cases = [ - {"ALLOW_LIVE_SUBMIT": "true", "LIVE_CONFIRM": LIVE_CONFIRM_VALUE}, - {"ALLOW_LIVE_SUBMIT": "1", "LIVE_CONFIRM": "", "CONFIRM_LIVE": "YES"}, - {"ALLOW_LIVE_SUBMIT": "1", "LIVE_CONFIRM": LIVE_CONFIRM_VALUE, "STRATEGY_ID": " "}, - ] - for overrides in cases: - with self.subTest(overrides=overrides): - client = FakeClient() - env = {"STRATEGY_ID": "sid-test", **overrides} - with mock.patch.dict(os.environ, env, clear=True): - result = attempt_live_submit({}, client=client) - self.assertFalse(result["submitted"]) - self.assertFalse(result["ok"]) - self.assertEqual(client.calls, []) - self.assertTrue(result["reason"]) - - def test_injected_fake_client(self) -> None: - client = FakeClient() - with mock.patch.dict( - os.environ, - { - "STRATEGY_ID": "sid-1", - "ALLOW_LIVE_SUBMIT": "1", - "LIVE_CONFIRM": LIVE_CONFIRM_VALUE, - }, - clear=False, - ): - result = attempt_live_submit({"modelAddress": "x/y"}, client=client) - self.assertTrue(result["submitted"]) - self.assertEqual(len(client.calls), 1) - - def test_default_attempt_rejects(self) -> None: - client = FakeClient() - with self._clear_gate_env(): - result = attempt_live_submit({"modelAddress": "x/y"}, client=client) - self.assertFalse(result["submitted"]) - self.assertEqual(len(client.calls), 0) - - -if __name__ == "__main__": - unittest.main() + def test_no_client_no_network(self): + with patch('urllib.request.urlopen') as network: + self.assertFalse(attempt_live_submit({})['submitted']) + network.assert_not_called() diff --git a/tests/test_live_cycle.py b/tests/test_live_cycle.py new file mode 100644 index 0000000..68134e6 --- /dev/null +++ b/tests/test_live_cycle.py @@ -0,0 +1,109 @@ +import io +import os +import tempfile +import unittest +from unittest.mock import Mock, patch +from src.live_cycle import run_cycle, _submitted + + +def candidate(name, section='B', foothold='unknown'): + return dict(model_address='org/' + name, section=section, foothold_class=foothold, try_next_gpus=['壁砺']) + + +class TestLiveCycle(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.env = patch.dict(os.environ, {'STRATEGY_ID': 'sid', 'LIVE_STATE_FILE': directory.name + '/state.json', 'CONFIG_PARAMS_SOURCE': 'sample'}, clear=True) + self.env.start() + self.addCleanup(self.env.stop) + _submitted.clear() + self.client = Mock() + self.client.submit.return_value = dict(http_status=200, platform_code=0) + network = patch('urllib.request.urlopen', side_effect=AssertionError('network forbidden')) + network.start() + self.addCleanup(network.stop) + logs = patch('sys.stderr', new_callable=io.StringIO) + self.logs = logs.start() + self.addCleanup(logs.stop) + + def cycle(self, rows=None, counts=(94,)): + reader = Mock(side_effect=[(n, 'platform_error' if n is None else 'ok', set()) for n in counts]) + return run_cycle(rows or [candidate('a')], self.client, reader) + + def test_high_and_failed_queue(self): + for n in (95, None): + decisions = self.cycle(counts=(n,)) + self.assertEqual(decisions[-1]['action'], 'hold') + self.client.submit.assert_not_called() + self.assertIn('[live] hold', self.logs.getvalue()) + + def test_order_and_cap(self): + rows = [candidate('b'), candidate('a', 'A'), candidate('verified', 'A', 'verified_reuse'), candidate('c'), candidate('d')] + self.cycle(rows, (90, 90, 90)) + self.assertEqual([c.args[0]['modelAddress'] for c in self.client.submit.call_args_list], ['org/verified', 'org/a', 'org/b']) + self.assertIn('cycle_cap', self.logs.getvalue()) + + def test_midcycle(self): + self.cycle([candidate(str(i)) for i in range(5)], (93, 94, 95)) + self.assertEqual(self.client.submit.call_count, 2) + + def test_kill_and_threshold(self): + for env in ({'ALLOW_LIVE_SUBMIT': '0'}, {'LIVE_QUEUE_THRESHOLD': 'bad'}, {'LIVE_QUEUE_THRESHOLD': '20'}): + with patch.dict(os.environ, env): + self.cycle(counts=(20,)) + self.client.submit.assert_not_called() + with patch.dict(os.environ, {'LIVE_QUEUE_THRESHOLD': '100'}): + self.cycle(counts=(99,)) + self.client.submit.assert_called_once() + + def test_banned_duplicate_persistence(self): + rows = [candidate('GPT-2'), candidate('ok'), candidate('ok')] + self.cycle(rows) + _submitted.clear() + self.cycle([candidate('ok')], ()) + self.client.submit.assert_called_once() + self.assertIn('banned:', self.logs.getvalue()) + self.assertIn('duplicate', self.logs.getvalue()) + + def test_remote_duplicate_and_transport_reservation(self): + run_cycle([candidate('remote')], self.client, lambda: (1, 'ok', {'org/remote'})) + self.client.submit.assert_not_called() + self.client.submit.side_effect = OSError('do not log') + self.cycle() + self.cycle(counts=()) + self.client.submit.assert_called_once() + self.assertNotIn('do not log', self.logs.getvalue()) + + def test_missing_token_real_reader(self): + run_cycle([candidate('a')], self.client) + self.client.submit.assert_not_called() + self.assertIn('missing/invalid_token', self.logs.getvalue()) + + def test_worker_waits_and_stops(self): + from src.server import _live_worker + with patch('src.server._shutdown') as shutdown, patch('src.server.run_cycle') as cycle: + shutdown.is_set.return_value = False + shutdown.wait.return_value = True + with patch.dict(os.environ, {'LIVE_CYCLE_SECONDS': '1'}): + _live_worker() + cycle.assert_called_once() + shutdown.wait.assert_called_once_with(60) + + def test_unauthenticated_platform_zero_submits(self): + import json + response = Mock() + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=False) + response.status = 200 + response.read.return_value = json.dumps({'code': 40100}).encode() + with patch.dict(os.environ, {'XC_TOKEN': 'synthetic-secret'}), patch('urllib.request.urlopen', return_value=response): + run_cycle([candidate('a')], self.client) + self.client.submit.assert_not_called() + + def test_unmapped_and_placeholder(self): + rows = [dict(model_address='org/unmapped', targetGpu='unknown'), candidate('')] + self.cycle(rows, ()) + self.client.submit.assert_not_called() + self.assertIn('gpu_unmapped', self.logs.getvalue()) + self.assertIn('placeholder', self.logs.getvalue()) diff --git a/tests/test_live_queue.py b/tests/test_live_queue.py new file mode 100644 index 0000000..2bf4d71 --- /dev/null +++ b/tests/test_live_queue.py @@ -0,0 +1,55 @@ +import json +import os +import unittest +from unittest.mock import patch, MagicMock +from src.live_queue import read_queue + + +class TestLiveQueue(unittest.TestCase): + def response(self, body): + r = MagicMock() + r.__enter__.return_value = r + r.status = 200 + r.read.return_value = json.dumps(body).encode() + return r + + def test_pagination_and_status(self): + pages = [self.response({'code': 0, 'data': {'total': 3, 'records': [ + {'status': 'waiting', 'modelAddress': 'org/a'}, {'status': 'success', 'verifyResult': -1}]}}), + self.response({'code': 0, 'data': {'total': 3, 'records': [{'status': '等待中'}]}})] + with patch.dict(os.environ, {'XC_TOKEN': 'synthetic-secret'}, clear=True), patch('urllib.request.urlopen', side_effect=pages) as call: + count, why, models = read_queue() + self.assertEqual(count, 2) + self.assertIn('org/a', models) + self.assertIn('current=2', call.call_args.args[0].full_url) + + def test_fail_closed(self): + for body in ({'code': 40100}, {'code': 40101}, {'code': 0, 'data': {'total': 1, 'records': []}}, []): + with patch.dict(os.environ, {'XC_TOKEN': 'synthetic-secret'}, clear=True), patch('urllib.request.urlopen', return_value=self.response(body)): + self.assertIsNone(read_queue()[0]) + with patch.dict(os.environ, {}, clear=True), patch('urllib.request.urlopen') as call: + self.assertIsNone(read_queue()[0]) + call.assert_not_called() + + def test_page_limit_and_transport(self): + with patch.dict(os.environ, {'XC_TOKEN': 'synthetic-secret'}, clear=True): + with patch('urllib.request.urlopen', return_value=self.response({'code': 0, 'data': {'total': 21, 'records': [{'status': 'waiting'}]}})) as call: + self.assertIsNone(read_queue()[0]) + self.assertEqual(call.call_count, 20) + with patch('urllib.request.urlopen', side_effect=OSError('synthetic-secret')): + self.assertNotIn('synthetic-secret', str(read_queue())) + + def test_invalid_json_http_and_auth_header(self): + with patch.dict(os.environ, {'XC_TOKEN': 'synthetic-secret'}, clear=True): + r = self.response({}) + r.read.return_value = b'invalid synthetic-secret' + with patch('urllib.request.urlopen', return_value=r): + self.assertEqual(read_queue()[1], 'malformed_json') + r.status = 503 + with patch('urllib.request.urlopen', return_value=r): + self.assertIsNone(read_queue()[0]) + with patch('urllib.request.urlopen', return_value=self.response({'code': 0, 'data': {'total': 0, 'records': []}})) as call: + self.assertEqual(read_queue()[0], 0) + req = call.call_args.args[0] + self.assertEqual(req.get_header('Xc-token'), 'synthetic-secret') + self.assertEqual(req.method, 'GET') diff --git a/tests/test_sigterm.py b/tests/test_sigterm.py index dc9f22c..24b82a8 100644 --- a/tests/test_sigterm.py +++ b/tests/test_sigterm.py @@ -20,6 +20,7 @@ class TestSigterm(unittest.TestCase): port = "19090" env = os.environ.copy() env["STRATEGY_ID"] = "sigterm-test" + env["ALLOW_LIVE_SUBMIT"] = "0" env["PORT"] = port env["HOST"] = "127.0.0.1"