CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: learn framework architecture incompatibilities

Browse Source
This commit is contained in:
CoolBoy
2026-08-12 08:02:51 +08:00
parent d908706f9a
commit 615bcad124
10 changed files with 503 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
modelhub_submmit_api/README.md
View File

@@ -23,6 +23,7 @@ It currently supports:
- `history_stats.py`: online history aggregation, ranking, and warnings
- `candidate_preflight.py`: deterministic repository, memory, context, and compatibility gates
- `failure_taxonomy.py`: deterministic/platform/semantic failure routing
- `architecture_compatibility.py`: exact architecture identities and learned compatibility keys
- `llm_classifier.py`: offline-only experimental ambiguity-analysis helper
- `template_selector.py`: template lookup and GPU normalization
- `task_registry.py`: task-type and framework selection rules
@@ -93,6 +94,11 @@ bash run_poll.sh --dry-run
validation and a confidence score at least 10% above the best incumbent.
- Five consecutive local failures pause a GPU/framework pair for 12 hours; a
sub-20% rate over the latest 20 terminal tasks pauses it for 6 hours.
- An explicit "framework does not support this model/architecture" failure learns
a 30-day GPU + framework + task + architecture block. Architecture identity
comes from candidate `config.json` (`architectures`, with `model_type` only as
fallback), never from repository names. A newer success clears the block, and
generic unsupported backend/operator messages cannot create one.
- A strategy generation lasts exactly 200 platform-accepted submissions. Rejected API calls and
duplicates do not advance it. The next cycle refreshes platform history before submitting again.
- Strategy state is stored in `.modelhub_state/gpu_strategy.json`; a generation never recalculates

65
modelhub_submmit_api/architecture_compatibility.py Normal file
View File

@@ -0,0 +1,65 @@
from __future__ import annotations
from collections.abc import Iterable
from typing import Any
EXPLICIT_ARCHITECTURE_FAILURE_CATEGORY = "framework_architecture_unsupported"
EXPLICIT_ARCHITECTURE_FAILURE_ACTION = "block_gpu_framework_architecture"
EXPLICIT_ARCHITECTURE_FAILURE_REASON = "explicit_framework_model_unsupported"
DEFAULT_ARCHITECTURE_BLOCK_TTL_DAYS = 30
def architecture_profile(
model_type: Any,
architectures: Any,
) -> dict[str, Any] | None:
"""Build a stable, conservative architecture identity for feedback matching."""
normalized_architectures = _normalize_architectures(architectures)
normalized_model_type = _normalize(model_type)
if normalized_architectures:
return {
"matchType": "architectures",
"signature": "architectures:" + ",".join(normalized_architectures),
"architectures": normalized_architectures,
"modelType": normalized_model_type or None,
}
if normalized_model_type:
return {
"matchType": "model_type",
"signature": f"model_type:{normalized_model_type}",
"architectures": [],
"modelType": normalized_model_type,
}
return None
def architecture_compatibility_key(
target_gpu: Any,
framework: Any,
task_type: Any,
signature: Any,
) -> str | None:
parts = (
_normalize(target_gpu),
_normalize(framework),
_normalize(task_type),
_normalize(signature),
)
if not all(parts):
return None
return "|".join(parts)
def _normalize_architectures(value: Any) -> list[str]:
if isinstance(value, str):
values: Iterable[Any] = [value]
elif isinstance(value, (list, tuple, set)):
values = value
else:
values = []
return sorted({_normalize(item) for item in values if _normalize(item)})
def _normalize(value: Any) -> str:
return str(value or "").strip().casefold()

56
modelhub_submmit_api/candidate_preflight.py
View File

@@ -8,6 +8,7 @@ from dataclasses import dataclass
from datetime import timedelta
from typing import Any
from architecture_compatibility import architecture_compatibility_key, architecture_profile
from llm_classifier import LLMAssistedClassifier
from models import ModelInspection
from common import parse_datetime, utc_now
@@ -150,10 +151,16 @@ class CandidatePreflightAdvisor:
self._llm_blocks = 0
self._context_clamps = 0
self._ambiguous = 0
self._architecture_blocks_applied = 0
self._feedback_stats: dict[str, Any] = {}
self._architecture_compatibility_blocks: dict[str, dict[str, Any]] = {}
def set_feedback_stats(self, report: dict[str, Any] | None) -> None:
self._feedback_stats = report if isinstance(report, dict) else {}
raw_blocks = self._feedback_stats.get("architectureCompatibilityBlocks") or {}
self._architecture_compatibility_blocks = (
raw_blocks if isinstance(raw_blocks, dict) else {}
)
for gpu, value in (self._feedback_stats.get("observedGpuMemoryGiB") or {}).items():
try:
memory_gib = float(value)
@@ -190,6 +197,24 @@ class CandidatePreflightAdvisor:
with self._lock:
self._assessed += 1
learned_block = self._matching_architecture_block(
inspection=inspection,
target_gpu=target_gpu,
framework=framework,
task_type=task_type,
)
if learned_block is not None:
metadata["architectureCompatibilityBlock"] = learned_block
with self._lock:
self._architecture_blocks_applied += 1
return self._hard_block(
config_params,
"preflight_learned_architecture_incompatible",
warnings,
ambiguous,
metadata,
)
# Empty file_paths means an injected/test inspection lacks structural
# metadata. Real discoveries with an empty tree already fail the weight
# compatibility gate, so do not make this test/fallback state a blocker.
@@ -389,6 +414,33 @@ class CandidatePreflightAdvisor:
metadata=metadata,
)
def _matching_architecture_block(
self,
*,
inspection: ModelInspection,
target_gpu: str,
framework: str,
task_type: str,
) -> dict[str, Any] | None:
profile = architecture_profile(inspection.model_type, inspection.architectures)
if profile is None:
return None
key = architecture_compatibility_key(
target_gpu,
framework,
task_type,
profile["signature"],
)
if key is None:
return None
block = self._architecture_compatibility_blocks.get(key)
if not isinstance(block, dict):
return None
expires_at = parse_datetime(block.get("expiresAt"))
if expires_at is None or expires_at <= utc_now():
return None
return dict(block)
def summary(self) -> dict[str, Any]:
with self._lock:
summary = {
@@ -397,6 +449,10 @@ class CandidatePreflightAdvisor:
"hardBlocks": self._hard_blocks,
"llmBlocks": self._llm_blocks,
"ambiguousCandidates": self._ambiguous,
"architectureCompatibilityBlocksLoaded": len(
self._architecture_compatibility_blocks
),
"architectureCompatibilityBlocksApplied": self._architecture_blocks_applied,
"contextLengthClamps": self._context_clamps,
"knownGpuMemoryGiB": dict(self.gpu_memory_gib),
"gpuMemoryEvidence": dict(self.gpu_memory_evidence),

11
modelhub_submmit_api/failure_log_inspector.py
View File

@@ -18,7 +18,8 @@ MAX_ERROR_REPORT_BYTES = 1_000_000
ERROR_LINE_PATTERN = re.compile(
r"(?:\b(?:[A-Za-z_]*(?:Error|Exception)|PREFLIGHT_[A-Z_]+|OOM)\b|"
r"out of memory|not supported|unsupported|does not recognize|cannot|can.t|"
r"not found|no such file|failed to|invalid|traceback|找不到空闲卡)",
r"not found|no such file|failed to|invalid|traceback|"
r"找不到空闲卡|不支持|暂不支持|不兼容|请换用|请更换)",
re.IGNORECASE,
)
@@ -74,10 +75,14 @@ def classify_failure_archive(
error_lines = _extract_error_lines(runtime_log)
report_code = str(report.get("code") or "").strip() or None
classification = classify_failure_report(report_code, error_lines)
suggestion = str(report.get("suggestion") or "")[:500] or None
classification_inputs = [*error_lines]
if suggestion:
classification_inputs.append(suggestion)
classification = classify_failure_report(report_code, classification_inputs)
result: dict[str, Any] = {
"failureCode": report_code,
"failureSuggestion": str(report.get("suggestion") or "")[:500] or None,
"failureSuggestion": suggestion,
"failureCategory": classification.category,
"failureScope": classification.scope,
"failureAction": classification.action,

54
modelhub_submmit_api/failure_taxonomy.py
View File

@@ -4,6 +4,12 @@ import re
from dataclasses import asdict, dataclass
from typing import Iterable
from architecture_compatibility import (
EXPLICIT_ARCHITECTURE_FAILURE_ACTION,
EXPLICIT_ARCHITECTURE_FAILURE_CATEGORY,
EXPLICIT_ARCHITECTURE_FAILURE_REASON,
)
@dataclass(frozen=True)
class FailureClassification:
@@ -70,6 +76,42 @@ DETERMINISTIC_LOG_PATTERNS = (
(re.compile(r"config\.json.*(?:not found|no config)|Invalid repository ID or local directory", re.I), "MODEL_FILE_NOT_FOUND"),
)
# These patterns deliberately require both framework and model/architecture
# semantics. Generic messages such as "attention backend not supported" or
# "GPU type not supported" must not create an architecture blacklist.
EXPLICIT_FRAMEWORK_MODEL_UNSUPPORTED_PATTERNS = (
re.compile(
r"(?:当前|该|此)(?:推理)?框架.{0,100}(?:不支持|暂不支持|无法支持|不兼容)"
r".{0,100}(?:该|此|当前)?(?:模型架构|模型|架构)",
re.I,
),
re.compile(
r"(?:该|此|当前)?(?:模型架构|模型|架构).{0,100}"
r"(?:不被|不受).{0,50}(?:当前|该|此)?(?:推理)?框架.{0,30}支持",
re.I,
),
re.compile(
r"\b(?:this|the|current)\s+framework\b.{0,100}"
r"\b(?:does\s+not|doesn't|cannot|can't)\s+support\b.{0,100}"
r"\b(?:model|model\s+architecture|architecture)\b",
re.I,
),
re.compile(
r"\b(?:model|model\s+architecture|architecture)\b.{0,100}"
r"\b(?:is\s+)?not\s+supported\s+by\b.{0,80}\bframework\b",
re.I,
),
)
MODEL_NOT_SUPPORTED_SUGGESTION_PATTERNS = (
re.compile(r"请(?:更换|换用|使用|选择).{0,50}(?:受支持|支持的)(?:模型|模型架构)", re.I),
re.compile(
r"\b(?:please\s+)?(?:change|switch|use|choose).{0,60}"
r"\b(?:a\s+)?supported\s+(?:model|model\s+architecture)\b",
re.I,
),
)
def classify_failure_report(report_code: str | None, log_lines: Iterable[str] = ()) -> FailureClassification:
code = str(report_code or "").strip().upper()
@@ -93,6 +135,18 @@ def classify_failure_report(report_code: str | None, log_lines: Iterable[str] =
for pattern, inferred_code in DETERMINISTIC_LOG_PATTERNS:
if pattern.search(text):
return DETERMINISTIC_POLICIES[inferred_code]
if any(pattern.search(text) for pattern in EXPLICIT_FRAMEWORK_MODEL_UNSUPPORTED_PATTERNS) or (
code == "MODEL_NOT_SUPPORTED"
and any(pattern.search(text) for pattern in MODEL_NOT_SUPPORTED_SUGGESTION_PATTERNS)
):
return FailureClassification(
EXPLICIT_ARCHITECTURE_FAILURE_CATEGORY,
"model_gpu_framework",
EXPLICIT_ARCHITECTURE_FAILURE_ACTION,
True,
False,
EXPLICIT_ARCHITECTURE_FAILURE_REASON,
)
if code in SEMANTIC_POLICIES:
return SEMANTIC_POLICIES[code]
return FailureClassification(

140
modelhub_submmit_api/outcome_tracker.py
View File

@@ -3,9 +3,17 @@ from __future__ import annotations
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timedelta
import os
from pathlib import Path
from typing import Any
from architecture_compatibility import (
DEFAULT_ARCHITECTURE_BLOCK_TTL_DAYS,
EXPLICIT_ARCHITECTURE_FAILURE_CATEGORY,
EXPLICIT_ARCHITECTURE_FAILURE_REASON,
architecture_compatibility_key,
architecture_profile,
)
from common import append_jsonl, parse_datetime, read_jsonl, update_jsonl, utc_now
from failure_log_inspector import fetch_and_classify_failure_log
from history_stats import classify_failure, is_failure, is_success
@@ -245,7 +253,8 @@ class OutcomeTracker:
return failed_at >= now - timedelta(hours=max(0, int(cooldown_hours)))
def get_stats_report(self) -> dict[str, Any]:
now = _now_iso()
now_datetime = utc_now()
now = now_datetime.isoformat()
terminal = [r for r in self._records if r.get("outcome") in {"success", "failed"}]
gpu_groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
@@ -344,6 +353,17 @@ class OutcomeTracker:
previous = observed_gpu_memory.get(gpu)
observed_gpu_memory[gpu] = min(previous, memory_gib) if previous else memory_gib
architecture_block_ttl_days = _architecture_block_ttl_days()
architecture_blocks = _build_architecture_compatibility_blocks(
terminal,
now=now_datetime,
ttl_days=architecture_block_ttl_days,
)
architecture_blocks_by_gpu_framework: dict[str, int] = defaultdict(int)
for block in architecture_blocks.values():
combination = f"{block['targetGpu']}|{block['framework']}"
architecture_blocks_by_gpu_framework[combination] += 1
return {
"generatedAt": now,
"totalRecords": len(self._records),
@@ -356,6 +376,12 @@ class OutcomeTracker:
"recentCombinationStats": recent_combination_stats,
"profileCombinationStats": profile_combination_stats,
"recentProfileCombinationStats": recent_profile_combination_stats,
"architectureCompatibilityBlocks": architecture_blocks,
"architectureCompatibilitySummary": {
"activeBlockCount": len(architecture_blocks),
"ttlDays": architecture_block_ttl_days,
"byGpuFramework": dict(architecture_blocks_by_gpu_framework),
},
"observedGpuMemoryGiB": observed_gpu_memory,
"totals": _summarize(terminal),
"warnings": warnings,
@@ -489,6 +515,118 @@ def _summarize(records: list[dict[str, Any]]) -> dict[str, Any]:
}
def _architecture_block_ttl_days() -> int:
raw = os.getenv("MODELHUB_ARCHITECTURE_BLOCK_TTL_DAYS")
if raw is None:
return DEFAULT_ARCHITECTURE_BLOCK_TTL_DAYS
try:
value = int(raw)
except ValueError:
return DEFAULT_ARCHITECTURE_BLOCK_TTL_DAYS
return min(365, max(1, value))
def _build_architecture_compatibility_blocks(
records: list[dict[str, Any]],
*,
now: datetime,
ttl_days: int,
) -> dict[str, dict[str, Any]]:
successes: dict[str, list[tuple[datetime, dict[str, Any]]]] = defaultdict(list)
failures: dict[str, list[tuple[datetime, dict[str, Any], dict[str, Any]]]] = defaultdict(list)
cutoff = now - timedelta(days=max(1, int(ttl_days)))
for record in records:
profile_data = record.get("modelProfile")
if not isinstance(profile_data, dict):
continue
profile = architecture_profile(
profile_data.get("modelType"),
profile_data.get("architectures"),
)
if profile is None:
continue
target_gpu = str(record.get("targetGpu") or "").strip()
framework = str(record.get("framework") or "").strip()
task_type = str(record.get("taskType") or "").strip()
key = architecture_compatibility_key(
target_gpu,
framework,
task_type,
profile["signature"],
)
event_time = (
parse_datetime(record.get("submitTime"))
or parse_datetime(record.get("lastSyncTime"))
)
if key is None or event_time is None:
continue
if record.get("outcome") == "success":
successes[key].append((event_time, record))
continue
if not _is_explicit_architecture_failure(record) or event_time < cutoff:
continue
failures[key].append((event_time, record, profile))
blocks: dict[str, dict[str, Any]] = {}
for key, failure_events in failures.items():
failure_events.sort(key=lambda item: item[0])
latest_failure_at, latest_failure, latest_profile = failure_events[-1]
success_events = successes.get(key) or []
latest_success_at = max((item[0] for item in success_events), default=None)
if latest_success_at is not None and latest_success_at >= latest_failure_at:
continue
effective_failures = [
item
for item in failure_events
if latest_success_at is None or item[0] > latest_success_at
]
expires_at = latest_failure_at + timedelta(days=max(1, int(ttl_days)))
if expires_at <= now:
continue
blocks[key] = {
"targetGpu": str(latest_failure.get("targetGpu") or ""),
"framework": str(latest_failure.get("framework") or ""),
"taskType": str(latest_failure.get("taskType") or ""),
"matchType": latest_profile["matchType"],
"architectureSignature": latest_profile["signature"],
"architectures": latest_profile["architectures"],
"modelType": latest_profile["modelType"],
"evidenceCount": len(effective_failures),
"latestFailureAt": latest_failure_at.isoformat(),
"latestSuccessfulAt": latest_success_at.isoformat() if latest_success_at else None,
"expiresAt": expires_at.isoformat(),
"sourceTaskIds": _bounded_unique(
record.get("taskId") for _, record, _ in reversed(effective_failures)
),
"sourceModelIds": _bounded_unique(
record.get("modelId") for _, record, _ in reversed(effective_failures)
),
}
return blocks
def _is_explicit_architecture_failure(record: dict[str, Any]) -> bool:
return bool(
record.get("outcome") == "failed"
and record.get("failureCategory") == EXPLICIT_ARCHITECTURE_FAILURE_CATEGORY
and record.get("failureDeterministic") is True
and record.get("failureClassificationReason") == EXPLICIT_ARCHITECTURE_FAILURE_REASON
)
def _bounded_unique(values: Any, limit: int = 5) -> list[str]:
result: list[str] = []
for value in values:
rendered = str(value or "").strip()
if not rendered or rendered in result:
continue
result.append(rendered)
if len(result) >= limit:
break
return result
def _is_platform_failure(record: dict[str, Any]) -> bool:
if record.get("outcome") != "failed":
return False

3
modelhub_submmit_api/poll_runner.py
View File

@@ -428,7 +428,8 @@ def run_poll_loop(
f"success={totals.get('successCount', 0)} "
f"failed={totals.get('failureCount', 0)} "
f"success_rate={totals.get('successRate', 0):.3f} "
f"failure_rate={totals.get('failureRate', 0):.3f}"
f"failure_rate={totals.get('failureRate', 0):.3f} "
f"architecture_blocks={len(stats.get('architectureCompatibilityBlocks') or {})}"
)
except Exception as exc:
log(f"[poll] cycle={cycles} outcome_stats_error={exc}")

2
modelhub_submmit_api/version.py
View File

@@ -1 +1 @@
AGENT_VERSION = "2026.08.12.1"
AGENT_VERSION = "2026.08.12.2"