"""Read-only pending pool. Name checks cannot establish metadata/uniqueness PASS.""" import os import re from pathlib import Path BASE = Path('/workspace/modelhub-adapt') GPUS = r'曦望(?: S2)?|壁砺(?:166M)?|天垓150|天垓100|智铠(?: mrv-100)?|Iluvatar_mrv-100|Biren_166m|Sunrise_pt-200-x1' FAMILIES = ('lfm2', 'olmo-2', 'olmo2', 'spark-x2.5', 'spark2_5', 'nemotron-h') def input_path(env, filename): return Path(os.environ.get(env) or BASE / filename) def load_do_not_resubmit(path): # Exact ids only; includes collision bullets and overnight tables. return {s for s in re.findall(r'`([^`]+)`', Path(path).read_text()) if re.fullmatch(r'[\w.-]+/[\w.\-/]+', s)} def load_banned_name_tokens(path): text = Path(path).read_text().lower() return {s for s in ('abliterated', 'heretic', 'uncensored', 'unsensored') if s in text} def experience_order(gpus): """Stable tier sort of missing chips only; tier-one pending order wins. docs/gpu-perf-order.md Section B is experience, never official performance. Unknown names retain their relative order after known tiers; no chips added. """ def tier(gpu): if any(name in gpu for name in ('壁砺', '曦望', 'Biren_166m', 'Sunrise_pt-200-x1')): return 1 if '天垓150' in gpu: return 2 if '天垓100' in gpu: return 3 if '智铠' in gpu or 'mrv-100' in gpu: return 4 return 5 return sorted(dict.fromkeys(gpus), key=tier) def _candidate(model, section, gpu_text, notes): return dict(model_address=model, section=section, try_next_gpus=experience_order(re.findall(GPUS, gpu_text)), perf_order="unset", order_source="experience", notes=notes, comment=notes, skip_reason=None, needs_human=False, status='candidate') def parse_pending_md(path): rows, section = [], None for line in Path(path).read_text().splitlines(): heading = re.match(r'##\s+Section\s+([AB])\b', line) if heading: section = heading[1] elif line.startswith('## '): section = None if section and re.match(r'\|\s*\d+\s*\|', line): cols = [s.strip() for s in line.strip('|').split('|')] model = re.search(r'`([^`]+)`', cols[1]) if model: rows.append(_candidate(model[1], section, cols[3] if section == 'A' else cols[-1], cols[-1])) return rows def parse_pending_txt(path): rows = [] for line in Path(path).read_text().splitlines(): if not line.strip() or line.startswith('#'): continue parts = re.split(r'\t|\\t', line, maxsplit=1) comment = parts[1] if len(parts) == 2 else '' match = re.match(r'#([AB])', comment) section = match[1] if match else 'B' gpu_text = re.split(r'(?:still\s+)?try\s*:', comment, maxsplit=1)[-1] if 'try:' in comment else (comment if section == 'B' else '') row = _candidate(parts[0].strip(), section, gpu_text, comment) if not match: row.update(needs_human=True, status='needs_human') rows.append(row) return rows def build_ordered_candidates(md_path=None, txt_path=None, dnr_path=None, banned_path=None): md = Path(md_path) if md_path is not None else input_path('PENDING_SUBMIT_MD', 'pending-submit.md') txt = Path(txt_path) if txt_path is not None else input_path('PENDING_SUBMIT_TXT', 'pending-submit.txt') rows = parse_pending_md(md) if md.is_file() else parse_pending_txt(txt) if txt.is_file() else [] dnr = Path(dnr_path) if dnr_path is not None else input_path('DO_NOT_RESUBMIT_MD', 'do-not-resubmit.md') banned = Path(banned_path) if banned_path is not None else input_path('BANNED_PATTERNS_MD', 'banned-patterns.md') excluded = load_do_not_resubmit(dnr) if dnr.is_file() else set() tokens = load_banned_name_tokens(banned) if banned.is_file() else set() section = os.environ.get('PENDING_SECTION', '').strip().upper() if section: rows = [row for row in rows if row['section'] == section] for row in rows: name = row['model_address'] reason = ('do-not-resubmit' if name in excluded else 'hard-gate: banned name token' if any(t in name.lower() for t in tokens) else 'hard-gate: banned family' if any(t in name.lower() for t in FAMILIES) else None) row['skip_reason'] = reason # Metadata gates (weights, context, riskTag, uniqueness) remain unverified. row['needs_human'] = not bool(reason) row['status'] = 'skipped' if reason else 'needs_human' return sorted(rows, key=lambda r: r['section'] != 'A')