xc_agent_platform_demo/src/pending_pool.py

"""Read-only pending pool. Name checks cannot establish metadata/uniqueness PASS."""
import os
import re
from pathlib import Path

BASE = Path('/workspace/modelhub-adapt')
GPUS = r'曦望(?: S2)?|壁砺(?:166M)?|天垓150|天垓100|智铠(?: mrv-100)?|Iluvatar_mrv-100|Biren_166m|Sunrise_pt-200-x1'
FAMILIES = ('lfm2', 'olmo-2', 'olmo2', 'spark-x2.5', 'spark2_5', 'nemotron-h')


def input_path(env, filename):
    return Path(os.environ.get(env) or BASE / filename)


def load_do_not_resubmit(path):
    # Exact ids only; includes collision bullets and overnight tables.
    return {s for s in re.findall(r'`([^`]+)`', Path(path).read_text())
            if re.fullmatch(r'[\w.-]+/[\w.\-/]+', s)}


def load_banned_name_tokens(path):
    text = Path(path).read_text().lower()
    return {s for s in ('abliterated', 'heretic', 'uncensored', 'unsensored') if s in text}


def experience_order(gpus):
    """Stable tier sort of missing chips only; tier-one pending order wins.

    docs/gpu-perf-order.md Section B is experience, never official performance.
    Unknown names retain their relative order after known tiers; no chips added.
    """
    def tier(gpu):
        if any(name in gpu for name in ('壁砺', '曦望', 'Biren_166m', 'Sunrise_pt-200-x1')):
            return 1
        if '天垓150' in gpu:
            return 2
        if '天垓100' in gpu:
            return 3
        if '智铠' in gpu or 'mrv-100' in gpu:
            return 4
        return 5
    return sorted(dict.fromkeys(gpus), key=tier)


def classify_foothold(text):
    """Classify literal pending evidence; mixed markers prefer verified."""
    for label, markers in (
        ('verified_reuse', ('已验证成功', '验证通过')),
        ('in_flight', ('正在验证', '验证排队中', '验证进行中')),
        ('failed_foothold', ('验证失败',)),
    ):
        if any(marker in text for marker in markers):
            return label
    return 'unknown'


def has_no_download_marker(text):
    """True only for explicit 免下载/已下载; never inferred from verified alone."""
    return any(marker in text for marker in ('免下载', '已下载'))


def _candidate(model, section, gpu_text, notes, foothold_text=''):
    evidence_text = foothold_text if section == 'A' else ''
    foothold_class = classify_foothold(evidence_text) if section == 'A' else 'unknown'
    no_download = has_no_download_marker(evidence_text + ' ' + notes)
    evidence = 'pending-foothold' if (foothold_class != 'unknown' or no_download) else 'GAP'
    return dict(model_address=model, section=section,
                foothold_text=foothold_text, foothold_class=foothold_class,
                evidence_source=evidence, no_download=no_download,
                try_next_gpus=experience_order(re.findall(GPUS, gpu_text)),
                perf_order="unset", order_source="experience",
                notes=notes, comment=notes, skip_reason=None,
                needs_human=False, status='candidate')


def parse_pending_md(path):
    rows, section = [], None
    for line in Path(path).read_text().splitlines():
        heading = re.match(r'##\s+Section\s+([AB])\b', line)
        if heading:
            section = heading[1]
        elif line.startswith('## '):
            section = None
        if section and re.match(r'\|\s*\d+\s*\|', line):
            cols = [s.strip() for s in line.strip('|').split('|')]
            model = re.search(r'`([^`]+)`', cols[1])
            if model:
                rows.append(_candidate(model[1], section, cols[3] if section == 'A' else cols[-1], cols[-1], cols[2] if section == 'A' else ''))
    return rows


def parse_pending_txt(path):
    rows = []
    for line in Path(path).read_text().splitlines():
        if not line.strip() or line.startswith('#'):
            continue
        parts = re.split(r'\t|\\t', line, maxsplit=1)
        comment = parts[1] if len(parts) == 2 else ''
        match = re.match(r'#([AB])', comment)
        section = match[1] if match else 'B'
        gpu_text = re.split(r'(?:still\s+)?try\s*:', comment, maxsplit=1)[-1] if 'try:' in comment else (comment if section == 'B' else '')
        row = _candidate(parts[0].strip(), section, gpu_text, comment, comment if section == 'A' else '')
        if not match:
            row.update(needs_human=True, status='needs_human')
        rows.append(row)
    return rows


def build_ordered_candidates(md_path=None, txt_path=None, dnr_path=None, banned_path=None):
    md = Path(md_path) if md_path is not None else input_path('PENDING_SUBMIT_MD', 'pending-submit.md')
    txt = Path(txt_path) if txt_path is not None else input_path('PENDING_SUBMIT_TXT', 'pending-submit.txt')
    rows = parse_pending_md(md) if md.is_file() else parse_pending_txt(txt) if txt.is_file() else []
    dnr = Path(dnr_path) if dnr_path is not None else input_path('DO_NOT_RESUBMIT_MD', 'do-not-resubmit.md')
    banned = Path(banned_path) if banned_path is not None else input_path('BANNED_PATTERNS_MD', 'banned-patterns.md')
    excluded = load_do_not_resubmit(dnr) if dnr.is_file() else set()
    tokens = load_banned_name_tokens(banned) if banned.is_file() else set()
    section = os.environ.get('PENDING_SECTION', '').strip().upper()
    if section:
        rows = [row for row in rows if row['section'] == section]
    for row in rows:
        name = row['model_address']
        reason = ('do-not-resubmit' if name in excluded else
                  'hard-gate: banned name token' if any(t in name.lower() for t in tokens) else
                  'hard-gate: banned family' if any(t in name.lower() for t in FAMILIES) else None)
        row['skip_reason'] = reason
        # Metadata gates (weights, context, riskTag, uniqueness) remain unverified.
        row['needs_human'] = not bool(reason)
        row['status'] = 'skipped' if reason else 'needs_human'
    return sorted(rows, key=lambda r: (r['section'] != 'A', not (r['foothold_class'] == 'verified_reuse' or r.get('no_download'))))