feat: learn framework architecture incompatibilities

modelhub_submmit_api/outcome_tracker.py

@@ -3,9 +3,17 @@ from __future__ import annotations
 from collections import defaultdict
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime, timedelta
+import os
 from pathlib import Path
 from typing import Any
 
+from architecture_compatibility import (
+    DEFAULT_ARCHITECTURE_BLOCK_TTL_DAYS,
+    EXPLICIT_ARCHITECTURE_FAILURE_CATEGORY,
+    EXPLICIT_ARCHITECTURE_FAILURE_REASON,
+    architecture_compatibility_key,
+    architecture_profile,
+)
 from common import append_jsonl, parse_datetime, read_jsonl, update_jsonl, utc_now
 from failure_log_inspector import fetch_and_classify_failure_log
 from history_stats import classify_failure, is_failure, is_success
@@ -245,7 +253,8 @@ class OutcomeTracker:
         return failed_at >= now - timedelta(hours=max(0, int(cooldown_hours)))
 
     def get_stats_report(self) -> dict[str, Any]:
-        now = _now_iso()
+        now_datetime = utc_now()
+        now = now_datetime.isoformat()
         terminal = [r for r in self._records if r.get("outcome") in {"success", "failed"}]
 
         gpu_groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
@@ -344,6 +353,17 @@ class OutcomeTracker:
                 previous = observed_gpu_memory.get(gpu)
                 observed_gpu_memory[gpu] = min(previous, memory_gib) if previous else memory_gib
 
+        architecture_block_ttl_days = _architecture_block_ttl_days()
+        architecture_blocks = _build_architecture_compatibility_blocks(
+            terminal,
+            now=now_datetime,
+            ttl_days=architecture_block_ttl_days,
+        )
+        architecture_blocks_by_gpu_framework: dict[str, int] = defaultdict(int)
+        for block in architecture_blocks.values():
+            combination = f"{block['targetGpu']}|{block['framework']}"
+            architecture_blocks_by_gpu_framework[combination] += 1
+
         return {
             "generatedAt": now,
             "totalRecords": len(self._records),
@@ -356,6 +376,12 @@ class OutcomeTracker:
             "recentCombinationStats": recent_combination_stats,
             "profileCombinationStats": profile_combination_stats,
             "recentProfileCombinationStats": recent_profile_combination_stats,
+            "architectureCompatibilityBlocks": architecture_blocks,
+            "architectureCompatibilitySummary": {
+                "activeBlockCount": len(architecture_blocks),
+                "ttlDays": architecture_block_ttl_days,
+                "byGpuFramework": dict(architecture_blocks_by_gpu_framework),
+            },
             "observedGpuMemoryGiB": observed_gpu_memory,
             "totals": _summarize(terminal),
             "warnings": warnings,
@@ -489,6 +515,118 @@ def _summarize(records: list[dict[str, Any]]) -> dict[str, Any]:
     }
 
 
+def _architecture_block_ttl_days() -> int:
+    raw = os.getenv("MODELHUB_ARCHITECTURE_BLOCK_TTL_DAYS")
+    if raw is None:
+        return DEFAULT_ARCHITECTURE_BLOCK_TTL_DAYS
+    try:
+        value = int(raw)
+    except ValueError:
+        return DEFAULT_ARCHITECTURE_BLOCK_TTL_DAYS
+    return min(365, max(1, value))
+
+
+def _build_architecture_compatibility_blocks(
+    records: list[dict[str, Any]],
+    *,
+    now: datetime,
+    ttl_days: int,
+) -> dict[str, dict[str, Any]]:
+    successes: dict[str, list[tuple[datetime, dict[str, Any]]]] = defaultdict(list)
+    failures: dict[str, list[tuple[datetime, dict[str, Any], dict[str, Any]]]] = defaultdict(list)
+    cutoff = now - timedelta(days=max(1, int(ttl_days)))
+
+    for record in records:
+        profile_data = record.get("modelProfile")
+        if not isinstance(profile_data, dict):
+            continue
+        profile = architecture_profile(
+            profile_data.get("modelType"),
+            profile_data.get("architectures"),
+        )
+        if profile is None:
+            continue
+        target_gpu = str(record.get("targetGpu") or "").strip()
+        framework = str(record.get("framework") or "").strip()
+        task_type = str(record.get("taskType") or "").strip()
+        key = architecture_compatibility_key(
+            target_gpu,
+            framework,
+            task_type,
+            profile["signature"],
+        )
+        event_time = (
+            parse_datetime(record.get("submitTime"))
+            or parse_datetime(record.get("lastSyncTime"))
+        )
+        if key is None or event_time is None:
+            continue
+        if record.get("outcome") == "success":
+            successes[key].append((event_time, record))
+            continue
+        if not _is_explicit_architecture_failure(record) or event_time < cutoff:
+            continue
+        failures[key].append((event_time, record, profile))
+
+    blocks: dict[str, dict[str, Any]] = {}
+    for key, failure_events in failures.items():
+        failure_events.sort(key=lambda item: item[0])
+        latest_failure_at, latest_failure, latest_profile = failure_events[-1]
+        success_events = successes.get(key) or []
+        latest_success_at = max((item[0] for item in success_events), default=None)
+        if latest_success_at is not None and latest_success_at >= latest_failure_at:
+            continue
+        effective_failures = [
+            item
+            for item in failure_events
+            if latest_success_at is None or item[0] > latest_success_at
+        ]
+        expires_at = latest_failure_at + timedelta(days=max(1, int(ttl_days)))
+        if expires_at <= now:
+            continue
+        blocks[key] = {
+            "targetGpu": str(latest_failure.get("targetGpu") or ""),
+            "framework": str(latest_failure.get("framework") or ""),
+            "taskType": str(latest_failure.get("taskType") or ""),
+            "matchType": latest_profile["matchType"],
+            "architectureSignature": latest_profile["signature"],
+            "architectures": latest_profile["architectures"],
+            "modelType": latest_profile["modelType"],
+            "evidenceCount": len(effective_failures),
+            "latestFailureAt": latest_failure_at.isoformat(),
+            "latestSuccessfulAt": latest_success_at.isoformat() if latest_success_at else None,
+            "expiresAt": expires_at.isoformat(),
+            "sourceTaskIds": _bounded_unique(
+                record.get("taskId") for _, record, _ in reversed(effective_failures)
+            ),
+            "sourceModelIds": _bounded_unique(
+                record.get("modelId") for _, record, _ in reversed(effective_failures)
+            ),
+        }
+    return blocks
+
+
+def _is_explicit_architecture_failure(record: dict[str, Any]) -> bool:
+    return bool(
+        record.get("outcome") == "failed"
+        and record.get("failureCategory") == EXPLICIT_ARCHITECTURE_FAILURE_CATEGORY
+        and record.get("failureDeterministic") is True
+        and record.get("failureClassificationReason") == EXPLICIT_ARCHITECTURE_FAILURE_REASON
+    )
+
+
+def _bounded_unique(values: Any, limit: int = 5) -> list[str]:
+    result: list[str] = []
+    for value in values:
+        rendered = str(value or "").strip()
+        if not rendered or rendered in result:
+            continue
+        result.append(rendered)
+        if len(result) >= limit:
+            break
+    return result
+
+
 def _is_platform_failure(record: dict[str, Any]) -> bool:
     if record.get("outcome") != "failed":
         return False