diff --git a/docs/gpu-perf-order.md b/docs/gpu-perf-order.md index d9a9557..e252d1a 100644 --- a/docs/gpu-perf-order.md +++ b/docs/gpu-perf-order.md @@ -45,5 +45,7 @@ - `POST /api/adapt/task/build-config`，query 为 `gpuType` / `framework` / `taskType`，空 body；成功 JSON `code=0`，`data` 为 YAML。 - UI 无「自动配置」标签；选择框架触发 effect 填入 configParams。模型地址不进入该请求。 - 抓包观察 Authorization（值未记录）；容器默认 Xc-Token 社区实践，可显式切换 Authorization，尚无 token 鉴权实证。 -- 默认 dry-run 离线 sample；CONFIG_PARAMS_SOURCE=build-config 或 ALLOW_BUILD_CONFIG=1 且 token 可解析时调用 API，失败不可静默回退。 +- 默认 dry-run：`resolve_token()` 成功时优先 build-config；失败回退 sample，逐条报告 configParams_source、GAP 和 fallback reason。无 token → sample。CONFIG_PARAMS_SOURCE=sample 强制离线；=build-config 强制 API 路径（dry-run with_meta 仍可回退）。make build-config-dry 保持失败非零退出。 +- API gpuType 仅映射壁砺/壁砺166M/Biren_166m 与曦望/曦望 S2/Sunrise_pt-200-x1；未知卡不请求 API，报告 GAP: gpu enum unmapped。 +- 选策优化后仍为 experience_order，perf_order=unset；order_source=experience；无正式性能表。Section A 仅按 foothold 文本将 verified_reuse 提前，其余 A 保持文件序，再接 B。API foothold evidence: GAP (not wired)。 - **live 前必须有 token + API 调用实证**；sample 单独不是 production-ready。队列 ≥95 继续禁 live，本轮无 live。 diff --git a/docs/tag-dryrun-notes-draft.md b/docs/tag-dryrun-notes-draft.md index 1cc2464..784c18d 100644 --- a/docs/tag-dryrun-notes-draft.md +++ b/docs/tag-dryrun-notes-draft.md @@ -1,13 +1,14 @@ -# Tag 文案草稿（仍默认 dry-run）— 未授权勿 push +# Tag 文案草稿（仍默认 dry-run） -- 建议 Tag 名（待 Victor 确认）：`v0.2.0-phase2-dryrun` 或 `v0.2.0-submit-ready-dryrun` -- 相对 Phase1 `v0.1.0-phase1@0d0f3b6`：pending A 池 + 经验序排序、只读 task/page 探针、task/add 客户端仍双重闸门默认关、configParams sample 仅 dry-run -- **不含**：live 开闸、真实自动配置、开闸环境变量默认开启 -- 推 Tag 条件：仅当 Victor **明确**说「先上线 dry-run 客户端」；产品当前：**先别 push Tag** -- 队列满 / configParams 缺口期间：**不开闸** +- 建议 Tag：`v0.2.0-phase2-dryrun` +- 基线：控制台当前 `v0.1.0-phase1` @ `0d0f3b6` +- 相对 Phase1：pending **Section A** + 经验序；只读 `task/page` 探针；`build-config` 客户端（默认离线 sample，opt-in 才调 API）；`task/add` 双重闸门默认关 +- **不含**：live 开闸、默认真实提交 +- 推送：仅 Victor 本人；Agent 不代 push +- 队列≥95 / 无书面批准：**不开闸** -推送检查清单（人工）： -1. `make test` 绿 -2. ALLOW_LIVE_SUBMIT 默认非 1；LIVE_CONFIRM 未设 -3. 控制台重建后日志仍为 dry-run / Phase2 格式 +检查清单： +1. `make test` 42/42 +2. `ALLOW_LIVE_SUBMIT` 默认非 1 +3. 重建后 `/health` 200；`/status` token/strategy 布尔 4. 不代注册、不代 live diff --git a/scripts/dry_run.py b/scripts/dry_run.py index bf0426f..04c200f 100755 --- a/scripts/dry_run.py +++ b/scripts/dry_run.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Read-only Phase2 pending draft report. Offline by default; opt-in configuration generation only.""" +"""Read-only Phase2 pending draft report. Token-resolvable build-config first; explicit sample fallback.""" import json import os import sys @@ -10,7 +10,7 @@ from zoneinfo import ZoneInfo ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) from src.pending_pool import build_ordered_candidates, input_path, experience_order -from src.config_params import build_config_params, config_params_source, SOURCE, GAP +from src.config_params import build_config_params_with_meta, SOURCE, GAP from src.queue_depth import queue_depth from src.task_add_client import build_payload @@ -27,9 +27,9 @@ def _render(candidates, ts, src): c.get('section', 'B') == os.environ['PENDING_SECTION'].strip().upper()] depth = queue_depth() lines = [f'# 代提候选 / 需 Victor 确认 — Dry-run — {ts} (Asia/Shanghai)', '', - '> 模式：dry-run；零网络写；需 Victor 人工提交/确认', + '> 模式：dry-run；仅配置生成 API；无任务提交；需 Victor 人工提交/确认', '> 积分：待平台核实', '> perf_order=unset；order_source=experience（经验序 / 待平台核实）', - f'> configParams source: {SOURCE}', f'> configParams_source={config_params_source()}', f'> {GAP}', '', + f'> configParams source: {SOURCE}', '> API foothold evidence: GAP (not wired)', f'> {GAP}', '', '## A→B 顺序草稿', '', f'输入：{src}；候选条数：{len(candidates)}', f'队列深度：{depth if depth is not None else "未知，需人工确认"}', '队列≥95：禁 live；适配 W1 优先，不抢空位。', ''] @@ -45,8 +45,11 @@ def _render(candidates, ts, src): f'过滤：{c.get("skip_reason") or "名称硬门未命中；元数据/唯一性待人工核实"}', '']) lines.extend([f'未适配卡（仅 pending 缺口）：{", ".join(gpus) or "未提供，需确认"}', 'perf_order=unset；order_source=experience；天垓100 一般不扇出', '']) + yaml, meta = build_config_params_with_meta(target) + lines.extend([f'foothold_class={c.get("foothold_class", "unknown")}; evidence_source={c.get("evidence_source", "GAP")}; no_download={c.get("no_download", False)}', + '; '.join(f'{key}={value}' for key, value in meta.items()), '']) draft = build_payload(model, target, 'vllm', strategy_id='', - config_params=build_config_params(target)) + config_params=yaml) # Display only; this truncated draft must never be submitted. draft['configParams'] = draft['configParams'][:400] + '\n… [truncated; dry-run only]' lines.extend(['build-config query: gpuType=&framework=vllm&taskType=text-generation; POST empty body', 'POST 字段草稿（skipped 行禁止提交）：', '```json', json.dumps(draft, ensure_ascii=False, indent=2), '```', diff --git a/src/config_params.py b/src/config_params.py index f366521..fa365dd 100644 --- a/src/config_params.py +++ b/src/config_params.py @@ -1,31 +1,66 @@ -"""Offline sample by default; explicit opt-in enables captured build-config API.""" +"""Dry-run configuration: token-resolvable build-config first; failures fall back to sample with GAP.""" import os from pathlib import Path from src.build_config_client import build_config, BuildConfigError from src.task_add_client import resolve_token -SOURCE = 'POST /api/adapt/task/build-config (discovered via capture); default templates/config_params.sample.yaml' -GAP = 'GAP: build-config API discovered via capture (autoconfig-capture-2026-10-16.md); live/实锤 needs token + API call and auth proof. Sample is dry-run draft only, NOT production-ready.' +SOURCE = 'POST /api/adapt/task/build-config (discovered via capture); fallback templates/config_params.sample.yaml' +GAP = 'GAP: build-config API discovered via capture; sample is dry-run draft only, NOT production-ready. API foothold evidence: GAP (not wired).' +GPU_ENUMS = { + '壁砺': 'Biren_166m', '壁砺166M': 'Biren_166m', 'Biren_166m': 'Biren_166m', + '曦望': 'Sunrise_pt-200-x1', '曦望 S2': 'Sunrise_pt-200-x1', + 'Sunrise_pt-200-x1': 'Sunrise_pt-200-x1', +} def config_params_source(): + """Requested source preference for dry-run. + + - CONFIG_PARAMS_SOURCE=sample → force offline sample + - CONFIG_PARAMS_SOURCE=build-config → force API path (with soft fallback in with_meta) + - default (empty): prefer build-config when resolve_token() succeeds; else sample + - ALLOW_BUILD_CONFIG=1 remains a legacy opt-in synonym when source empty and token ok + """ source = os.environ.get('CONFIG_PARAMS_SOURCE', '').strip() if source not in ('', 'sample', 'build-config'): raise BuildConfigError('unsupported CONFIG_PARAMS_SOURCE') - if source == 'build-config': - return 'build-config' - if os.environ.get('ALLOW_BUILD_CONFIG') == '1': + if source: + return source + try: + resolve_token() + except (ValueError, OSError): + return 'sample' + # Token resolvable → prefer build-config (kickoff select-opt default). + return 'build-config' + + +def build_config_params_with_meta(gpu_type=None, framework='vllm', task_type='text-generation'): + """Return (YAML, metadata); API failures explicitly fall back for dry-run. + + scripts/build_config_dry.py calls build_config directly and remains strict. + """ + source = config_params_source() + meta = {'configParams_source': 'sample'} + try: + resolve_token() + except (ValueError, OSError): + meta['auth'] = 'missing' + else: + meta['auth'] = 'ok' + gpu_enum = GPU_ENUMS.get(gpu_type) + if source == 'build-config' and gpu_enum is None: + meta['configParams_fallback_reason'] = 'GAP: gpu enum unmapped' + elif source == 'build-config': try: - resolve_token() - except (ValueError, OSError): - pass + yaml = build_config(gpu_enum, framework, task_type) + except BuildConfigError as error: + meta['configParams_fallback_reason'] = 'GAP: ' + error.error_attribution else: - return 'build-config' - return 'sample' + meta['configParams_source'] = 'build-config' + return yaml, meta + return (Path(__file__).resolve().parents[1] / 'templates/config_params.sample.yaml').read_text(), meta def build_config_params(gpu_type=None, framework='vllm', task_type='text-generation'): - """Return YAML; opted-in API failures propagate, never fall back to sample.""" - if config_params_source() == 'build-config': - return build_config(gpu_type, framework, task_type) - return (Path(__file__).resolve().parents[1] / 'templates/config_params.sample.yaml').read_text() + """Compatible YAML-only dry-run wrapper, including explicit fallback.""" + return build_config_params_with_meta(gpu_type, framework, task_type)[0] diff --git a/src/pending_pool.py b/src/pending_pool.py index eeb9594..4c07d2f 100644 --- a/src/pending_pool.py +++ b/src/pending_pool.py @@ -42,8 +42,31 @@ def experience_order(gpus): return sorted(dict.fromkeys(gpus), key=tier) -def _candidate(model, section, gpu_text, notes): +def classify_foothold(text): + """Classify literal pending evidence; mixed markers prefer verified.""" + for label, markers in ( + ('verified_reuse', ('已验证成功', '验证通过')), + ('in_flight', ('正在验证', '验证排队中', '验证进行中')), + ('failed_foothold', ('验证失败',)), + ): + if any(marker in text for marker in markers): + return label + return 'unknown' + + +def has_no_download_marker(text): + """True only for explicit 免下载/已下载; never inferred from verified alone.""" + return any(marker in text for marker in ('免下载', '已下载')) + + +def _candidate(model, section, gpu_text, notes, foothold_text=''): + evidence_text = foothold_text if section == 'A' else '' + foothold_class = classify_foothold(evidence_text) if section == 'A' else 'unknown' + no_download = has_no_download_marker(evidence_text + ' ' + notes) + evidence = 'pending-foothold' if (foothold_class != 'unknown' or no_download) else 'GAP' return dict(model_address=model, section=section, + foothold_text=foothold_text, foothold_class=foothold_class, + evidence_source=evidence, no_download=no_download, try_next_gpus=experience_order(re.findall(GPUS, gpu_text)), perf_order="unset", order_source="experience", notes=notes, comment=notes, skip_reason=None, @@ -62,7 +85,7 @@ def parse_pending_md(path): cols = [s.strip() for s in line.strip('|').split('|')] model = re.search(r'`([^`]+)`', cols[1]) if model: - rows.append(_candidate(model[1], section, cols[3] if section == 'A' else cols[-1], cols[-1])) + rows.append(_candidate(model[1], section, cols[3] if section == 'A' else cols[-1], cols[-1], cols[2] if section == 'A' else '')) return rows @@ -76,7 +99,7 @@ def parse_pending_txt(path): match = re.match(r'#([AB])', comment) section = match[1] if match else 'B' gpu_text = re.split(r'(?:still\s+)?try\s*:', comment, maxsplit=1)[-1] if 'try:' in comment else (comment if section == 'B' else '') - row = _candidate(parts[0].strip(), section, gpu_text, comment) + row = _candidate(parts[0].strip(), section, gpu_text, comment, comment if section == 'A' else '') if not match: row.update(needs_human=True, status='needs_human') rows.append(row) @@ -103,4 +126,4 @@ def build_ordered_candidates(md_path=None, txt_path=None, dnr_path=None, banned_ # Metadata gates (weights, context, riskTag, uniqueness) remain unverified. row['needs_human'] = not bool(reason) row['status'] = 'skipped' if reason else 'needs_human' - return sorted(rows, key=lambda r: r['section'] != 'A') + return sorted(rows, key=lambda r: (r['section'] != 'A', not (r['foothold_class'] == 'verified_reuse' or r.get('no_download')))) diff --git a/tests/test_config_params.py b/tests/test_config_params.py index 42d222c..588ac28 100644 --- a/tests/test_config_params.py +++ b/tests/test_config_params.py @@ -4,7 +4,7 @@ import unittest import os from unittest.mock import patch from pathlib import Path -from src.config_params import build_config_params, config_params_source, GAP +from src.config_params import build_config_params, config_params_source, build_config_params_with_meta, GAP from src.build_config_client import BuildConfigError from scripts.dry_run import _render @@ -16,32 +16,68 @@ class TestConfigParams(unittest.TestCase): self.addCleanup(env.stop) def test_source_selection_and_failure(self): - with patch("urllib.request.urlopen") as network: - self.assertEqual(config_params_source(), "sample") - build_config_params() - network.assert_not_called() - self.assertIn("discovered via capture", GAP) - with patch.dict(os.environ, {"CONFIG_PARAMS_SOURCE": "build-config"}): - with self.assertRaisesRegex(BuildConfigError, "missing/invalid credential"): - build_config_params("gpu") - with patch.dict(os.environ, {"ALLOW_BUILD_CONFIG": "1", "XC_TOKEN": "test-secret"}), patch("src.config_params.build_config", return_value="api: yaml") as client: - self.assertEqual(config_params_source(), "build-config") - self.assertEqual(build_config_params("gpu"), "api: yaml") - client.assert_called_once_with("gpu", "vllm", "text-generation") - with patch.dict(os.environ, {"CONFIG_PARAMS_SOURCE": "build-config"}), patch("src.config_params.build_config", side_effect=BuildConfigError("transport/timeout")): - with self.assertRaisesRegex(BuildConfigError, "transport/timeout"): - build_config_params("gpu") + # No token → sample, no network + with patch('src.config_params.resolve_token', side_effect=ValueError), patch('src.config_params.build_config') as client: + self.assertEqual(config_params_source(), 'sample') + yaml, meta = build_config_params_with_meta('壁砺') + self.assertEqual(meta['auth'], 'missing') + self.assertEqual(meta['configParams_source'], 'sample') + client.assert_not_called() + # Token alone → prefer build-config (kickoff default) + for gpu, enum in [('壁砺', 'Biren_166m'), ('壁砺166M', 'Biren_166m'), + ('Biren_166m', 'Biren_166m'), ('曦望', 'Sunrise_pt-200-x1'), + ('曦望 S2', 'Sunrise_pt-200-x1'), ('Sunrise_pt-200-x1', 'Sunrise_pt-200-x1')]: + with self.subTest(gpu=gpu), patch('src.config_params.resolve_token', return_value='synthetic'), patch('src.config_params.build_config', return_value='api: yaml') as client: + self.assertEqual(config_params_source(), 'build-config') + yaml, meta = build_config_params_with_meta(gpu) + self.assertEqual(yaml, 'api: yaml') + self.assertEqual(meta['configParams_source'], 'build-config') + self.assertEqual(meta['auth'], 'ok') + client.assert_called_once_with(enum, 'vllm', 'text-generation') + # Failure → sample + GAP reason + with patch('src.config_params.resolve_token', return_value='synthetic'), patch('src.config_params.build_config', side_effect=BuildConfigError('transport/timeout')): + yaml, meta = build_config_params_with_meta('壁砺') + self.assertEqual(meta['configParams_source'], 'sample') + self.assertEqual(meta['configParams_fallback_reason'], 'GAP: transport/timeout') + self.assertIn('max_model_len: 4096', yaml) + # Forced build-config also soft-falls back in with_meta + with patch.dict(os.environ, {'CONFIG_PARAMS_SOURCE': 'build-config'}), patch('src.config_params.resolve_token', return_value='synthetic'), patch('src.config_params.build_config', side_effect=BuildConfigError('transport/timeout')): + yaml, meta = build_config_params_with_meta('壁砺') + self.assertEqual(meta['configParams_fallback_reason'], 'GAP: transport/timeout') + # Unmapped GPU → no API call + for gpu in ('天垓150', '智铠 mrv-100', 'MetaX_c-500', '', None): + with patch('src.config_params.resolve_token', return_value='synthetic'), patch('src.config_params.build_config') as client: + _, meta = build_config_params_with_meta(gpu) + self.assertIn('GAP: gpu enum unmapped', meta['configParams_fallback_reason']) + client.assert_not_called() + # Force sample even with token + with patch.dict(os.environ, {'CONFIG_PARAMS_SOURCE': 'sample'}), patch('src.config_params.resolve_token', return_value='synthetic'), patch('src.config_params.build_config') as client: + self.assertEqual(config_params_source(), 'sample') + build_config_params('壁砺') + client.assert_not_called() + + def test_report_api_failure_metadata(self): + with patch('src.config_params.resolve_token', return_value='synthetic'), patch('src.config_params.build_config', side_effect=BuildConfigError('transport/timeout')): + report = _render([{'model_address': 'org/a', 'try_next_gpus': ['壁砺'], + 'foothold_class': 'verified_reuse', 'evidence_source': 'pending-foothold', + 'no_download': False}], 'test', 'fixture') + for value in ('foothold_class=verified_reuse', 'evidence_source=pending-foothold', 'no_download=False', + 'configParams_source=sample', 'configParams_fallback_reason=GAP: transport/timeout'): + self.assertIn(value, report) def test_official_sample_exact(self): scrape = Path('/workspace/modelhub-adapt/logs/api-docs-scrape-2026-10-16.md') if scrape.is_file(): body = json.loads(re.search(r'```json\n(.*?)\n```', scrape.read_text(), re.S)[1]) - self.assertEqual(build_config_params(), body['configParams']) - self.assertIn('max_model_len: 4096', build_config_params()) - self.assertIn('framework: vllm', build_config_params()) + with patch('src.config_params.resolve_token', side_effect=ValueError): + self.assertEqual(build_config_params(), body['configParams']) + with patch('src.config_params.resolve_token', side_effect=ValueError): + self.assertIn('max_model_len: 4096', build_config_params()) + self.assertIn('framework: vllm', build_config_params()) self.assertIn('NOT production-ready', GAP) def test_report_draft_and_gap(self): - report = _render([{'model_address': 'org/a', 'try_next_gpus': ['壁砺']}], 'test', 'fixture') - for value in ('代提候选 / 需 Victor 确认', 'configParams source', 'configParams_source=sample', 'GAP:', 'truncated', 'max_model_len: 4096', 'perf_order=unset', 'order_source=experience'): + with patch('src.config_params.resolve_token', side_effect=ValueError): + report = _render([{'model_address': 'org/a', 'try_next_gpus': ['壁砺']}], 'test', 'fixture') + for value in ('代提候选 / 需 Victor 确认', 'configParams source', 'configParams_source=sample', 'foothold_class=unknown', 'evidence_source=GAP', 'no_download=False', 'GAP:', 'truncated', 'max_model_len: 4096', 'perf_order=unset', 'order_source=experience'): self.assertIn(value, report) diff --git a/tests/test_pending_pool.py b/tests/test_pending_pool.py index 375ceac..2088cb3 100644 --- a/tests/test_pending_pool.py +++ b/tests/test_pending_pool.py @@ -47,3 +47,40 @@ class TestPendingPool(unittest.TestCase): self.assertEqual(len(rows), 1) self.assertEqual(rows[0]['try_next_gpus'], ['天垓150']) self.assertEqual(rows[0]['perf_order'], 'unset') + + def test_foothold_evidence_and_stable_verified_priority(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + md = root / 'pending.md' + md.write_text('## Section B\n| 1 | `org/b` | 0 | 壁砺 |\n## Section A\n| 1 | `org/unknown` | 曦望 | 壁砺 | 已验证成功 |\n| 2 | `org/flight` | 验证失败; 正在验证 | 壁砺 | — |\n| 3 | `org/verified` | 他人:已验证成功; 正在验证 | 壁砺 | — |\n| 4 | `org/fail` | 验证失败 | 曦望 | — |\n| 5 | `org/pass` | 既有:验证通过 | 曦望 | — |\n| 6 | `org/queue` | 验证排队中 | 壁砺 | — |\n| 7 | `org/progress` | 验证进行中 | 壁砺 | — |\n') + rows = build_ordered_candidates(md, dnr_path=root/'none', banned_path=root/'none') + self.assertEqual([r['model_address'] for r in rows], + ['org/verified', 'org/pass', 'org/unknown', 'org/flight', + 'org/fail', 'org/queue', 'org/progress', 'org/b']) + self.assertEqual([r['foothold_class'] for r in rows], + ['verified_reuse', 'verified_reuse', 'unknown', 'in_flight', + 'failed_foothold', 'in_flight', 'in_flight', 'unknown']) + for row in rows: + self.assertEqual(row['evidence_source'], 'GAP' if row['foothold_class'] == 'unknown' else 'pending-foothold') + self.assertEqual(rows[0]['foothold_text'], '他人:已验证成功; 正在验证') + txt = root / 'pending.txt' + txt.write_text('org/a\t#A done:曦望; try:壁砺\norg/v\t#A 验证通过; try:壁砺\n') + parsed = parse_pending_txt(txt) + self.assertEqual(parsed[0]['foothold_class'], 'unknown') + self.assertEqual(parsed[0]['evidence_source'], 'GAP') + self.assertEqual(parsed[1]['foothold_class'], 'verified_reuse') + + def test_no_download_marker_without_inventing(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + md = root / 'pending.md' + md.write_text('## Section A\n| 1 | `org/plain` | 曦望:验证排队中 | 壁砺 | — |\n| 2 | `org/cached` | 壁砺:已下载; 曦望:免下载 | 天垓150 | — |\n| 3 | `org/verified` | 壁砺:已验证成功 | 天垓150 | — |\n') + rows = build_ordered_candidates(md, dnr_path=root/'none', banned_path=root/'none') + by = {r['model_address']: r for r in rows} + self.assertFalse(by['org/plain']['no_download']) + self.assertEqual(by['org/plain']['evidence_source'], 'pending-foothold') + self.assertTrue(by['org/cached']['no_download']) + self.assertEqual(by['org/cached']['evidence_source'], 'pending-foothold') + self.assertFalse(by['org/verified']['no_download']) # verified alone does not invent no_download + self.assertEqual([r['model_address'] for r in rows], + ['org/cached', 'org/verified', 'org/plain'])