submmit/modelhub_submmit_api/outcome_tracker.py

from __future__ import annotations

from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timedelta
import os
from pathlib import Path
from typing import Any

from architecture_compatibility import (
    DEFAULT_ARCHITECTURE_BLOCK_TTL_DAYS,
    EXPLICIT_ARCHITECTURE_FAILURE_CATEGORY,
    EXPLICIT_ARCHITECTURE_FAILURE_REASON,
    architecture_compatibility_key,
    architecture_profile,
    architecture_profiles,
)
from common import append_jsonl, parse_datetime, read_jsonl, update_jsonl, utc_now
from failure_log_inspector import fetch_and_classify_failure_log
from history_stats import classify_failure, is_failure, is_success
from llm_classifier import LLMAssistedClassifier
from modelhub_client import ModelHubClient, ModelHubClientPool
from task_registry import task_type_from_history_task


DEFAULT_OUTCOMES_PATH = Path("outcomes/submissions.jsonl")
FAILURE_ENRICHMENT_LIMIT = 40
FAILURE_ENRICHMENT_WORKERS = 4
FAILURE_ENRICHMENT_MAX_ATTEMPTS = 3
TERMINAL_TASK_STATUSES = {"success", "failed", "error", "cancelled", "completed"}


def _now_iso() -> str:
    return utc_now().isoformat()


class OutcomeTracker:
    def __init__(self, path: Path | str) -> None:
        self.path = Path(path)
        self._records: list[dict[str, Any]] = []
        self._by_task_id: dict[str, dict[str, Any]] = {}
        self._by_model_gpu: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
        self._failed_model_gpus: dict[tuple[str, str], datetime] = {}
        self._failure_llm_classifier: LLMAssistedClassifier | None = None

        self._records = read_jsonl(self.path)
        self._rebuild_indexes()

        last_sync_times = [
            parse_datetime(record.get("lastSyncTime"))
            for record in self._records
            if record.get("lastSyncTime")
        ]
        self._last_sync_time: datetime = max(last_sync_times) if last_sync_times else utc_now() - timedelta(days=7)

    def set_failure_llm_classifier(self, classifier: LLMAssistedClassifier | None) -> None:
        self._failure_llm_classifier = classifier

    def get_task_compatibility_contexts(self) -> dict[str, dict[str, Any]]:
        """Return locally known submit metadata needed for account queue cleanup."""
        contexts: dict[str, dict[str, Any]] = {}
        for task_id, record in self._by_task_id.items():
            framework = str(record.get("framework") or "").strip()
            task_type = str(record.get("taskType") or "").strip()
            if not framework or not task_type:
                continue
            profile = record.get("modelProfile")
            contexts[task_id] = {
                "taskId": task_id,
                "modelId": str(record.get("modelId") or ""),
                "targetGpu": str(record.get("targetGpu") or ""),
                "framework": framework,
                "taskType": task_type,
                "modelProfile": dict(profile) if isinstance(profile, dict) else {},
                "submitTime": record.get("submitTime"),
            }
        return contexts

    def get_strategy_history_records(self) -> list[dict[str, Any]]:
        """Expose only successes and evidence-attributable failures for GPU ranking."""
        records: list[dict[str, Any]] = []
        for record in self._records:
            outcome = record.get("outcome")
            if outcome == "success":
                verify_result = 1
            elif _is_attributable_failure(record):
                verify_result = -1
            else:
                continue
            records.append(
                {
                    **record,
                    "gpuType": record.get("targetGpu"),
                    "status": "success",
                    "verifyResult": verify_result,
                    "updateTime": record.get("lastSyncTime") or record.get("submitTime"),
                }
            )
        return records

    def _rebuild_indexes(self) -> None:
        self._by_task_id.clear()
        self._by_model_gpu.clear()
        for record in self._records:
            task_id = record.get("taskId")
            if task_id:
                self._by_task_id[str(task_id)] = record
            model_id = record.get("modelId") or ""
            target_gpu = record.get("targetGpu") or ""
            self._by_model_gpu[(model_id, target_gpu)].append(record)

        self._rebuild_failed_index()

    def record_submission(
        self,
        model_id: str,
        target_gpu: str,
        framework: str,
        task_type: str,
        task_id: str | None,
        submit_time: str,
        model_profile: dict[str, Any] | None = None,
    ) -> None:
        record: dict[str, Any] = {
            "modelId": model_id,
            "targetGpu": target_gpu,
            "framework": framework,
            "taskType": task_type,
            "taskId": task_id,
            "submitTime": submit_time,
            "lastSyncTime": None,
            "status": "pending",
            "verifyResult": None,
            "outcome": "pending",
            "failReason": None,
            "modelProfile": dict(model_profile or {}),
        }
        self._records.append(record)
        if task_id:
            self._by_task_id[task_id] = record
        self._by_model_gpu[(model_id, target_gpu)].append(record)
        append_jsonl(self.path, record)

    def mark_policy_cancellations(self, decisions: list[dict[str, Any]]) -> int:
        """Persist intentional task stops so they never become failure evidence."""
        marked = 0
        marked_at = _now_iso()
        for decision in decisions:
            task_id_value = decision.get("taskId")
            if task_id_value is None:
                continue
            task_id = str(task_id_value)
            existing = self._by_task_id.get(task_id)
            if existing is not None and existing.get("outcome") in {"success", "failed"}:
                continue
            reasons = list(decision.get("cleanupReasons") or [decision.get("reason")])
            reasons = [str(reason) for reason in reasons if reason]
            if existing is None:
                existing = {
                    "modelId": decision.get("modelId") or "",
                    "targetGpu": decision.get("gpuType") or decision.get("targetGpu") or "",
                    "framework": decision.get("framework") or "",
                    "taskType": decision.get("taskType") or "",
                    "taskId": task_id,
                    "submitTime": decision.get("submitTime") or marked_at,
                    "verifyResult": None,
                }
                self._records.append(existing)
                self._by_task_id[task_id] = existing
                self._by_model_gpu[(existing["modelId"], existing["targetGpu"])].append(existing)
            existing.update(
                {
                    "lastSyncTime": marked_at,
                    "status": "cancellation_requested",
                    "outcome": "policy_cancelled",
                    "failReason": None,
                    "policyCancelled": True,
                    "policyCancellationReasons": reasons,
                    "policyCancelledAt": marked_at,
                }
            )
            marked += 1
        if marked:
            self._rebuild_failed_index()
        return marked

    def sync_from_api(self, client: ModelHubClient | ModelHubClientPool) -> int:
        try:
            begin = self._last_sync_time
            end = utc_now()
            # Use fanout for pools to sync outcomes across all accounts
            list_kwargs: dict[str, Any] = {"begin_time": begin, "end_time": end, "page_size": 100, "only_mine": True}
            if isinstance(client, ModelHubClientPool):
                list_kwargs["_fanout_all"] = True
            tasks = client.list_tasks(**list_kwargs)
        except Exception:
            return 0

        updated_count = 0
        enrichment_candidates: list[dict[str, Any]] = []
        for task in tasks:
            task_id = str(task.get("taskId")) if task.get("taskId") is not None else None
            if not task_id:
                continue

            existing = self._by_task_id.get(task_id)
            if existing is not None:
                if existing.get("outcome") in {"pending", "policy_cancelled"}:
                    self._update_record_from_task(existing, task)
                    if existing.get("outcome") == "failed" and existing.get("logCosUrl"):
                        enrichment_candidates.append(existing)
                    updated_count += 1
            else:
                status = str(task.get("status") or "").lower()
                if status in TERMINAL_TASK_STATUSES:
                    record = self._create_record_from_task(task)
                    self._records.append(record)
                    self._by_task_id[task_id] = record
                    model_id = record["modelId"]
                    target_gpu = record["targetGpu"]
                    self._by_model_gpu[(model_id, target_gpu)].append(record)
                    updated_count += 1

        # Retry a small bounded set of our own failed submissions. Newer logs
        # contain the target image, so enrichment can recover the framework even
        # when the history API omits it.
        candidate_ids = {id(record) for record in enrichment_candidates}
        for record in self._records:
            if len(enrichment_candidates) >= FAILURE_ENRICHMENT_LIMIT:
                break
            if id(record) in candidate_ids:
                continue
            if (
                record.get("outcome") == "failed"
                and record.get("logCosUrl")
                and not record.get("failureCategory")
                and int(record.get("failureEnrichmentAttempts") or 0) < FAILURE_ENRICHMENT_MAX_ATTEMPTS
            ):
                enrichment_candidates.append(record)
                candidate_ids.add(id(record))

        enrichment_attempts = self._enrich_failure_records(
            enrichment_candidates[:FAILURE_ENRICHMENT_LIMIT]
        )

        if updated_count or enrichment_attempts:
            self._last_sync_time = end
            self._rebuild_failed_index()
            self.save()

        return updated_count

    def bootstrap_from_history_tasks(
        self,
        tasks: list[dict[str, Any]],
        *,
        task_contexts: dict[str, dict[str, Any]] | None = None,
        enrichment_limit: int = 0,
        enrichment_workers: int = 8,
        log: Any = None,
    ) -> dict[str, int]:
        """Import terminal history and classify every usable historical failure."""
        contexts = task_contexts if isinstance(task_contexts, dict) else {}
        imported = 0
        refreshed = 0
        terminal_seen = 0
        seen_task_ids: set[str] = set()

        for raw_task in tasks:
            if not isinstance(raw_task, dict):
                continue
            task_id_value = raw_task.get("taskId")
            if task_id_value is None:
                continue
            task_id = str(task_id_value)
            if not task_id or task_id in seen_task_ids:
                continue
            seen_task_ids.add(task_id)
            status = str(raw_task.get("status") or "").strip().lower()
            if status not in TERMINAL_TASK_STATUSES:
                continue
            terminal_seen += 1

            task = self._enrich_history_task(raw_task, contexts.get(task_id))
            existing = self._by_task_id.get(task_id)
            if existing is None:
                record = self._create_record_from_task(task)
                self._records.append(record)
                self._by_task_id[task_id] = record
                self._by_model_gpu[(record["modelId"], record["targetGpu"])].append(record)
                imported += 1
                continue

            self._merge_record_metadata(existing, task)
            if existing.get("outcome") in {"pending", "policy_cancelled"}:
                self._update_record_from_task(existing, task)
                refreshed += 1
            elif task.get("logCosUrl") and not existing.get("failureCategory"):
                existing["logCosUrl"] = task.get("logCosUrl")

        candidates = [
            record
            for record in self._records
            if (
                record.get("outcome") == "failed"
                and record.get("logCosUrl")
                and not record.get("failureCategory")
                and int(record.get("failureEnrichmentAttempts") or 0)
                < FAILURE_ENRICHMENT_MAX_ATTEMPTS
            )
        ]
        candidates.sort(key=_outcome_record_timestamp, reverse=True)
        if enrichment_limit > 0:
            candidates = candidates[:enrichment_limit]
        self._last_sync_time = utc_now()
        self._rebuild_failed_index()
        if imported or refreshed:
            self.save()

        candidate_task_ids = [
            str(record["taskId"])
            for record in candidates
            if record.get("taskId") is not None
        ]
        enrichment_attempts = 0
        classified_total = 0
        errors_total = 0
        batch_size = 200
        total_candidates = len(candidate_task_ids)
        for offset in range(0, total_candidates, batch_size):
            batch_ids = candidate_task_ids[offset : offset + batch_size]
            batch_records = [
                self._by_task_id[task_id]
                for task_id in batch_ids
                if task_id in self._by_task_id
            ]

            def progress(
                completed: int,
                _total: int,
                classified: int,
                errors: int,
            ) -> None:
                if log is None:
                    return
                global_completed = offset + completed
                if global_completed == total_candidates or global_completed % 50 == 0:
                    log(
                        f"[architecture-bootstrap] failure_logs={global_completed}/{total_candidates} "
                        f"classified={classified_total + classified} "
                        f"errors={errors_total + errors}"
                    )

            enrichment_attempts += self._enrich_failure_records(
                batch_records,
                workers=max(1, int(enrichment_workers)),
                progress=progress,
            )
            classified_total += sum(
                1 for record in batch_records if record.get("failureCategory")
            )
            errors_total += sum(
                1 for record in batch_records if record.get("failureEnrichmentError")
            )
            self._rebuild_failed_index()
            self.save()

        explicit_architecture_failures = sum(
            1 for record in self._records if _is_explicit_architecture_failure(record)
        )
        recovered_frameworks = sum(
            1
            for record in self._records
            if record.get("failureDetectedFramework") and record.get("framework")
        )
        return {
            "recordsScanned": len(seen_task_ids),
            "terminalRecords": terminal_seen,
            "importedRecords": imported,
            "refreshedRecords": refreshed,
            "eligibleFailureLogs": total_candidates,
            "enrichmentAttempts": enrichment_attempts,
            "explicitArchitectureFailures": explicit_architecture_failures,
            "recoveredFrameworks": recovered_frameworks,
        }

    @staticmethod
    def _enrich_history_task(
        task: dict[str, Any],
        context: dict[str, Any] | None,
    ) -> dict[str, Any]:
        enriched = dict(task)
        context = context if isinstance(context, dict) else {}
        if not enriched.get("modelId") and context.get("modelId"):
            enriched["modelId"] = context.get("modelId")
        if not (enriched.get("gpuType") or enriched.get("targetGpu")) and context.get("targetGpu"):
            enriched["targetGpu"] = context.get("targetGpu")
        if not enriched.get("framework") and context.get("framework"):
            enriched["framework"] = context.get("framework")
        if not enriched.get("taskType"):
            enriched["taskType"] = context.get("taskType") or task_type_from_history_task(enriched)
        if context.get("modelProfile") and not enriched.get("modelProfile"):
            enriched["modelProfile"] = context.get("modelProfile")
        if context.get("submitTime") and not (
            enriched.get("submitTime") or enriched.get("createTime")
        ):
            enriched["submitTime"] = context.get("submitTime")
        return enriched

    @staticmethod
    def _merge_record_metadata(record: dict[str, Any], task: dict[str, Any]) -> None:
        field_sources = {
            "modelId": task.get("modelId") or task.get("model_id"),
            "targetGpu": task.get("gpuType") or task.get("targetGpu"),
            "framework": task.get("framework"),
            "taskType": task.get("taskType") or task_type_from_history_task(task),
            "modelProfile": task.get("modelProfile"),
        }
        for field, value in field_sources.items():
            if not record.get(field) and value not in (None, "", "unknown"):
                record[field] = dict(value) if field == "modelProfile" and isinstance(value, dict) else value
        if task.get("logCosUrl") and not record.get("failureCategory"):
            record["logCosUrl"] = task.get("logCosUrl")

    def _enrich_failure_records(
        self,
        records: list[dict[str, Any]],
        *,
        workers: int = FAILURE_ENRICHMENT_WORKERS,
        progress: Any = None,
    ) -> int:
        if not records:
            return 0

        def inspect(record: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any] | None, str | None]:
            try:
                result = fetch_and_classify_failure_log(
                    str(record["logCosUrl"]),
                    task_context=record,
                    llm_classifier=self._failure_llm_classifier,
                )
                return record, result, None
            except Exception as exc:
                return record, None, f"{type(exc).__name__}: {exc}"

        attempted = 0
        classified = 0
        errors = 0
        with ThreadPoolExecutor(max_workers=min(max(1, int(workers)), len(records))) as executor:
            futures = [executor.submit(inspect, record) for record in records]
            for future in as_completed(futures):
                record, result, error = future.result()
                attempted += 1
                record["failureEnrichmentAttempts"] = int(record.get("failureEnrichmentAttempts") or 0) + 1
                if result is None:
                    errors += 1
                    record["failureEnrichmentError"] = error
                    if progress is not None:
                        progress(attempted, len(records), classified, errors)
                    continue
                record.update(result)
                if not record.get("framework") and result.get("failureDetectedFramework"):
                    record["framework"] = result.get("failureDetectedFramework")
                record["failReason"] = result.get("failureCategory") or record.get("failReason")
                record["failureEnrichmentError"] = None
                record.pop("logCosUrl", None)
                classified += 1
                if progress is not None:
                    progress(attempted, len(records), classified, errors)
        return attempted

    def is_model_gpu_failed(
        self,
        model_id: str,
        target_gpu: str,
        *,
        cooldown_hours: int = 24,
        now: datetime | None = None,
    ) -> bool:
        failed_at = self._failed_model_gpus.get((model_id, target_gpu))
        if failed_at is None:
            return False
        now = now or utc_now()
        return failed_at >= now - timedelta(hours=max(0, int(cooldown_hours)))

    def get_stats_report(self) -> dict[str, Any]:
        now_datetime = utc_now()
        now = now_datetime.isoformat()
        terminal = [r for r in self._records if r.get("outcome") in {"success", "failed"}]

        gpu_groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
        framework_groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
        combo_groups: dict[tuple[str, str, str], list[dict[str, Any]]] = defaultdict(list)
        profile_groups: dict[tuple[str, str, str, str, str], list[dict[str, Any]]] = defaultdict(list)
        sized_profile_groups: dict[tuple[str, str, str, str, str, int], list[dict[str, Any]]] = defaultdict(list)

        for record in terminal:
            gpu = record.get("targetGpu") or "unknown"
            fw = record.get("framework") or "unknown"
            tt = record.get("taskType") or "unknown"
            gpu_groups[gpu].append(record)
            framework_groups[f"{fw}"].append(record)
            combo_groups[(gpu, fw, tt)].append(record)
            profile = record.get("modelProfile") or {}
            model_type = str(profile.get("modelType") or "").strip()
            if model_type:
                quantization = str(profile.get("quantizationMethod") or "none").strip()
                profile_groups[(gpu, fw, tt, model_type, quantization)].append(record)
                try:
                    load_bytes = max(1, int(profile.get("estimatedLoadBytes") or 0))
                except (TypeError, ValueError):
                    load_bytes = 0
                if load_bytes > 0:
                    size_bucket = int(load_bytes).bit_length() - 1
                    sized_profile_groups[(gpu, fw, tt, model_type, quantization, size_bucket)].append(record)

        gpu_summaries = {gpu: _summarize(records) for gpu, records in gpu_groups.items()}
        framework_summaries = {fw: _summarize(records) for fw, records in framework_groups.items()}
        combination_stats = {
            f"{gpu}|{fw}|{tt}": {"targetGpu": gpu, "framework": fw, "taskType": tt, **_summarize(records)}
            for (gpu, fw, tt), records in combo_groups.items()
        }
        recent_combination_stats: dict[str, dict[str, Any]] = {}
        for (gpu, fw, tt), records in combo_groups.items():
            recent = sorted(records, key=_outcome_record_timestamp, reverse=True)[:20]
            consecutive_failures = _consecutive_attributable_failures(recent)
            consecutive_platform_failures = _consecutive_platform_failures(recent)
            last_terminal_at = None
            if recent:
                last_terminal_at = (
                    parse_datetime(recent[0].get("lastSyncTime"))
                    or parse_datetime(recent[0].get("submitTime"))
                )
            recent_combination_stats[f"{gpu}|{fw}|{tt}"] = {
                "targetGpu": gpu,
                "framework": fw,
                "taskType": tt,
                **_summarize(recent),
                "consecutiveFailures": consecutive_failures,
                "consecutivePlatformFailures": consecutive_platform_failures,
                "lastPlatformFailureAt": _latest_platform_failure_at(recent),
                "lastTerminalAt": last_terminal_at.isoformat() if last_terminal_at else None,
            }

        profile_combination_stats: dict[str, dict[str, Any]] = {}
        recent_profile_combination_stats: dict[str, dict[str, Any]] = {}
        for (gpu, fw, tt, model_type, quantization), records in profile_groups.items():
            key = f"{gpu}|{fw}|{tt}|{model_type}|{quantization}"
            profile_combination_stats[key] = {
                "targetGpu": gpu,
                "framework": fw,
                "taskType": tt,
                "modelType": model_type,
                "quantizationMethod": quantization,
                **_summarize(records),
            }
            recent = sorted(records, key=_outcome_record_timestamp, reverse=True)[:20]
            consecutive_failures = _consecutive_attributable_failures(recent)
            last_terminal_at = None
            if recent:
                last_terminal_at = (
                    parse_datetime(recent[0].get("lastSyncTime"))
                    or parse_datetime(recent[0].get("submitTime"))
                )
            recent_profile_combination_stats[key] = {
                **profile_combination_stats[key],
                **_summarize(recent),
                "consecutiveFailures": consecutive_failures,
                "lastTerminalAt": last_terminal_at.isoformat() if last_terminal_at else None,
            }

        sized_profile_combination_stats: dict[str, dict[str, Any]] = {}
        for (gpu, fw, tt, model_type, quantization, size_bucket), records in sized_profile_groups.items():
            key = f"{gpu}|{fw}|{tt}|{model_type}|{quantization}|{size_bucket}"
            sized_profile_combination_stats[key] = {
                "targetGpu": gpu,
                "framework": fw,
                "taskType": tt,
                "modelType": model_type,
                "quantizationMethod": quantization,
                "loadSizeLog2Bucket": size_bucket,
                **_summarize(records),
            }

        warnings: list[str] = []
        for gpu, summary in gpu_summaries.items():
            if summary["decisionTotal"] >= 4 and summary["decisionFailureRate"] >= 0.5:
                warnings.append(f"GPU {gpu} 本地统计失败率偏高（≥50%），建议重点关注。")
        for key, stat in combination_stats.items():
            if stat["decisionTotal"] >= 3 and stat["decisionFailureRate"] >= 0.6:
                warnings.append(f"组合 {key} 近期失败集中，建议降低该 GPU+框架的提交优先级。")

        pending_count = sum(1 for r in self._records if r.get("outcome") == "pending")
        policy_cancelled_count = sum(
            1 for record in self._records if record.get("outcome") == "policy_cancelled"
        )
        observed_gpu_memory: dict[str, float] = {}
        for record in self._records:
            gpu = str(record.get("targetGpu") or "")
            try:
                memory_gib = float(record.get("failureObservedGpuMemoryGiB"))
            except (TypeError, ValueError):
                continue
            if gpu and 0 < memory_gib <= 1024:
                previous = observed_gpu_memory.get(gpu)
                observed_gpu_memory[gpu] = min(previous, memory_gib) if previous else memory_gib

        architecture_block_ttl_days = _architecture_block_ttl_days()
        architecture_blocks = _build_architecture_compatibility_blocks(
            terminal,
            now=now_datetime,
            ttl_days=architecture_block_ttl_days,
        )
        architecture_blocks_by_gpu_framework: dict[str, int] = defaultdict(int)
        for block in architecture_blocks.values():
            combination = f"{block['targetGpu']}|{block['framework']}"
            architecture_blocks_by_gpu_framework[combination] += 1

        return {
            "generatedAt": now,
            "totalRecords": len(self._records),
            "pendingRecords": pending_count,
            "policyCancelledRecords": policy_cancelled_count,
            "terminalRecords": len(terminal),
            "gpuSummaries": gpu_summaries,
            "frameworkSummaries": framework_summaries,
            "combinationStats": combination_stats,
            "recentCombinationStats": recent_combination_stats,
            "profileCombinationStats": profile_combination_stats,
            "recentProfileCombinationStats": recent_profile_combination_stats,
            "sizedProfileCombinationStats": sized_profile_combination_stats,
            "architectureCompatibilityBlocks": architecture_blocks,
            "architectureCompatibilitySummary": {
                "activeBlockCount": len(architecture_blocks),
                "ttlDays": architecture_block_ttl_days,
                "byGpuFramework": dict(architecture_blocks_by_gpu_framework),
            },
            "observedGpuMemoryGiB": observed_gpu_memory,
            "totals": _summarize(terminal),
            "warnings": warnings,
        }

    def save(self) -> None:
        local_records = list(self._records)

        def merge(existing: list[dict[str, Any]]) -> list[dict[str, Any]]:
            merged = list(existing)
            index_by_key = {_outcome_record_key(record): index for index, record in enumerate(merged)}
            for record in local_records:
                key = _outcome_record_key(record)
                existing_index = index_by_key.get(key)
                if existing_index is None:
                    index_by_key[key] = len(merged)
                    merged.append(record)
                    continue
                merged[existing_index] = _prefer_newer_outcome(merged[existing_index], record)
            return merged

        self._records = update_jsonl(self.path, merge)
        self._rebuild_indexes()

    def _rebuild_failed_index(self) -> None:
        self._failed_model_gpus.clear()
        latest_by_combo: dict[tuple[str, str], tuple[datetime, dict[str, Any]]] = {}
        for record in self._records:
            # Platform, unresolved, and policy outcomes neither clear nor
            # create a model/GPU cooldown. The full-history audit showed that
            # more than half of failures lack enough evidence for attribution.
            if _is_policy_cancelled(record) or (
                record.get("outcome") == "failed"
                and not _is_attributable_failure(record)
            ):
                continue
            model_id = record.get("modelId") or ""
            target_gpu = record.get("targetGpu") or ""
            event_time = parse_datetime(record.get("submitTime")) or parse_datetime(record.get("lastSyncTime"))
            if not model_id or not target_gpu or event_time is None:
                continue
            key = (model_id, target_gpu)
            current = latest_by_combo.get(key)
            if current is None or event_time >= current[0]:
                latest_by_combo[key] = (event_time, record)

        for key, (event_time, record) in latest_by_combo.items():
            if record.get("outcome") == "failed":
                self._failed_model_gpus[key] = event_time

    @staticmethod
    def _update_record_from_task(record: dict[str, Any], task: dict[str, Any]) -> None:
        record["status"] = task.get("status")
        record["verifyResult"] = task.get("verifyResult")
        record["lastSyncTime"] = _now_iso()
        if task.get("logCosUrl"):
            record["logCosUrl"] = task.get("logCosUrl")
        status = str(task.get("status") or "").strip().lower()
        if is_success(task):
            record["outcome"] = "success"
            record["failReason"] = None
            if record.get("policyCancelled"):
                record["policyCancellationResolvedAsSuccess"] = True
                record["policyCancelled"] = False
        elif record.get("policyCancelled"):
            # A successful stop may be observed as waiting/running briefly before
            # the platform publishes its terminal cancellation state. None of
            # those intermediate states should become failure evidence.
            record["outcome"] = "policy_cancelled"
            record["failReason"] = None
        elif is_failure(task):
            record["outcome"] = "failed"
            record["failReason"] = classify_failure(task)
        else:
            record["outcome"] = "pending"

    @staticmethod
    def _create_record_from_task(task: dict[str, Any]) -> dict[str, Any]:
        create_time = (
            parse_datetime(task.get("submitTime"))
            or parse_datetime(task.get("createTime"))
            or parse_datetime(task.get("updateTime"))
        )
        model_profile = task.get("modelProfile")
        record: dict[str, Any] = {
            "modelId": task.get("modelId") or task.get("model_id") or "",
            "targetGpu": task.get("gpuType") or task.get("targetGpu") or "",
            "framework": task.get("framework") or "",
            "taskType": task.get("taskType") or task_type_from_history_task(task) or "",
            "taskId": str(task.get("taskId")) if task.get("taskId") is not None else None,
            "submitTime": create_time.isoformat() if create_time else _now_iso(),
            "lastSyncTime": _now_iso(),
            "status": task.get("status"),
            "verifyResult": task.get("verifyResult"),
            "outcome": "pending",
            "failReason": None,
            "logCosUrl": task.get("logCosUrl"),
            "modelProfile": dict(model_profile) if isinstance(model_profile, dict) else {},
        }
        if is_success(task):
            record["outcome"] = "success"
        elif is_failure(task):
            record["outcome"] = "failed"
            record["failReason"] = classify_failure(task)
        return record


def _summarize(records: list[dict[str, Any]]) -> dict[str, Any]:
    total = len(records)
    success_count = sum(1 for r in records if r.get("outcome") == "success")
    failure_count = sum(1 for r in records if r.get("outcome") == "failed")
    pending_count = sum(1 for r in records if r.get("outcome") == "pending")
    attributable_failure_count = sum(
        1 for record in records if _is_attributable_failure(record)
    )
    platform_failure_count = sum(1 for record in records if _is_platform_failure(record))
    unresolved_failure_count = max(
        0,
        failure_count - attributable_failure_count - platform_failure_count,
    )
    decision_total = success_count + attributable_failure_count

    failure_breakdown: dict[str, int] = defaultdict(int)
    for r in records:
        reason = r.get("failureCategory") or r.get("failReason")
        if reason:
            failure_breakdown[reason] += 1

    return {
        "total": total,
        "successCount": success_count,
        "failureCount": failure_count,
        "attributableFailureCount": attributable_failure_count,
        "platformFailureCount": platform_failure_count,
        "unresolvedFailureCount": unresolved_failure_count,
        "decisionTotal": decision_total,
        "pendingCount": pending_count,
        "successRate": round(success_count / total, 4) if total > 0 else 0.0,
        "failureRate": round(failure_count / total, 4) if total > 0 else 0.0,
        "decisionSuccessRate": round(success_count / decision_total, 4) if decision_total > 0 else 0.0,
        "decisionFailureRate": round(attributable_failure_count / decision_total, 4) if decision_total > 0 else 0.0,
        "pendingRate": round(pending_count / total, 4) if total > 0 else 0.0,
        "failureBreakdown": dict(failure_breakdown),
    }


def _architecture_block_ttl_days() -> int:
    raw = os.getenv("MODELHUB_ARCHITECTURE_BLOCK_TTL_DAYS")
    if raw is None:
        return DEFAULT_ARCHITECTURE_BLOCK_TTL_DAYS
    try:
        value = int(raw)
    except ValueError:
        return DEFAULT_ARCHITECTURE_BLOCK_TTL_DAYS
    return min(365, max(1, value))


def _build_architecture_compatibility_blocks(
    records: list[dict[str, Any]],
    *,
    now: datetime,
    ttl_days: int,
) -> dict[str, dict[str, Any]]:
    successes: dict[str, list[tuple[datetime, dict[str, Any]]]] = defaultdict(list)
    failures: dict[str, list[tuple[datetime, dict[str, Any], dict[str, Any]]]] = defaultdict(list)
    cutoff = now - timedelta(days=max(1, int(ttl_days)))

    for record in records:
        target_gpu = str(record.get("targetGpu") or "").strip()
        framework = str(record.get("framework") or "").strip()
        task_type = str(record.get("taskType") or "").strip()
        event_time = (
            parse_datetime(record.get("submitTime"))
            or parse_datetime(record.get("lastSyncTime"))
        )
        if not target_gpu or not framework or not task_type or event_time is None:
            continue
        if record.get("outcome") == "success":
            for profile in _stored_architecture_profiles(record, include_model_type=True):
                key = architecture_compatibility_key(
                    target_gpu,
                    framework,
                    task_type,
                    profile["signature"],
                )
                if key is not None:
                    successes[key].append((event_time, record))
            continue
        if not _is_explicit_architecture_failure(record) or event_time < cutoff:
            continue
        for profile in _failure_architecture_profiles(record):
            key = architecture_compatibility_key(
                target_gpu,
                framework,
                task_type,
                profile["signature"],
            )
            if key is not None:
                failures[key].append((event_time, record, profile))

    blocks: dict[str, dict[str, Any]] = {}
    for key, failure_events in failures.items():
        failure_events.sort(key=lambda item: item[0])
        latest_failure_at, latest_failure, latest_profile = failure_events[-1]
        success_events = successes.get(key) or []
        latest_success_at = max((item[0] for item in success_events), default=None)
        if latest_success_at is not None and latest_success_at >= latest_failure_at:
            continue
        effective_failures = [
            item
            for item in failure_events
            if latest_success_at is None or item[0] > latest_success_at
        ]
        expires_at = latest_failure_at + timedelta(days=max(1, int(ttl_days)))
        if expires_at <= now:
            continue
        blocks[key] = {
            "targetGpu": str(latest_failure.get("targetGpu") or ""),
            "framework": str(latest_failure.get("framework") or ""),
            "taskType": str(latest_failure.get("taskType") or ""),
            "matchType": latest_profile["matchType"],
            "architectureSignature": latest_profile["signature"],
            "architectures": latest_profile["architectures"],
            "modelType": latest_profile["modelType"],
            "evidenceCount": len(effective_failures),
            "latestFailureAt": latest_failure_at.isoformat(),
            "latestSuccessfulAt": latest_success_at.isoformat() if latest_success_at else None,
            "expiresAt": expires_at.isoformat(),
            "sourceTaskIds": _bounded_unique(
                record.get("taskId") for _, record, _ in reversed(effective_failures)
            ),
            "sourceModelIds": _bounded_unique(
                record.get("modelId") for _, record, _ in reversed(effective_failures)
            ),
        }
    return blocks


def _stored_architecture_profiles(
    record: dict[str, Any],
    *,
    include_model_type: bool,
) -> list[dict[str, Any]]:
    profile_data = record.get("modelProfile")
    if not isinstance(profile_data, dict):
        return []
    if include_model_type:
        return architecture_profiles(
            profile_data.get("modelType"),
            profile_data.get("architectures"),
        )
    profile = architecture_profile(
        profile_data.get("modelType"),
        profile_data.get("architectures"),
    )
    return [profile] if profile is not None else []


def _failure_architecture_profiles(record: dict[str, Any]) -> list[dict[str, Any]]:
    profiles: list[dict[str, Any]] = []
    unsupported_architectures = record.get("failureUnsupportedArchitectures")
    if isinstance(unsupported_architectures, list) and unsupported_architectures:
        profile = architecture_profile(None, unsupported_architectures)
        if profile is not None:
            profiles.append(profile)
    unsupported_model_types = record.get("failureUnsupportedModelTypes")
    if isinstance(unsupported_model_types, list):
        for model_type in unsupported_model_types:
            profile = architecture_profile(model_type, None)
            if profile is not None:
                profiles.append(profile)
    if not profiles:
        profiles.extend(_stored_architecture_profiles(record, include_model_type=False))
    deduped: dict[str, dict[str, Any]] = {}
    for profile in profiles:
        deduped[profile["signature"]] = profile
    return list(deduped.values())


def _is_explicit_architecture_failure(record: dict[str, Any]) -> bool:
    return bool(
        record.get("outcome") == "failed"
        and record.get("failureCategory") == EXPLICIT_ARCHITECTURE_FAILURE_CATEGORY
        and record.get("failureDeterministic") is True
        and record.get("failureClassificationReason") == EXPLICIT_ARCHITECTURE_FAILURE_REASON
    )


def _bounded_unique(values: Any, limit: int = 5) -> list[str]:
    result: list[str] = []
    for value in values:
        rendered = str(value or "").strip()
        if not rendered or rendered in result:
            continue
        result.append(rendered)
        if len(result) >= limit:
            break
    return result


def _is_platform_failure(record: dict[str, Any]) -> bool:
    if record.get("outcome") != "failed":
        return False
    category = str(record.get("failureCategory") or "").lower()
    scope = str(record.get("failureScope") or "").lower()
    return scope == "platform" or category.startswith("platform_")


def _is_attributable_failure(record: dict[str, Any]) -> bool:
    """Require classified, non-platform evidence before penalizing a strategy."""
    if record.get("outcome") != "failed" or _is_platform_failure(record):
        return False
    category = str(record.get("failureCategory") or "").strip().lower()
    scope = str(record.get("failureScope") or "").strip().lower()
    return bool(category and scope not in {"", "unknown", "platform"})


def _is_policy_cancelled(record: dict[str, Any]) -> bool:
    return bool(record.get("policyCancelled")) or record.get("outcome") == "policy_cancelled"


def _consecutive_attributable_failures(records: list[dict[str, Any]]) -> int:
    count = 0
    for record in records:
        if record.get("outcome") == "success":
            break
        if _is_attributable_failure(record):
            count += 1
    return count


def _consecutive_platform_failures(records: list[dict[str, Any]]) -> int:
    count = 0
    for record in records:
        if not _is_platform_failure(record):
            break
        count += 1
    return count


def _latest_platform_failure_at(records: list[dict[str, Any]]) -> str | None:
    for record in records:
        if not _is_platform_failure(record):
            continue
        timestamp = parse_datetime(record.get("submitTime")) or parse_datetime(record.get("lastSyncTime"))
        return timestamp.isoformat() if timestamp else None
    return None


def _outcome_record_timestamp(record: dict[str, Any]) -> float:
    timestamp = parse_datetime(record.get("submitTime")) or parse_datetime(record.get("lastSyncTime"))
    return timestamp.timestamp() if timestamp else 0.0


def _outcome_record_key(record: dict[str, Any]) -> str:
    task_id = record.get("taskId")
    if task_id is not None:
        return f"task:{task_id}"
    return "fallback:{model}|{gpu}|{time}".format(
        model=record.get("modelId") or "",
        gpu=record.get("targetGpu") or "",
        time=record.get("submitTime") or "",
    )


def _outcome_version(record: dict[str, Any]) -> tuple[int, float, float]:
    last_sync = parse_datetime(record.get("lastSyncTime"))
    submit_time = parse_datetime(record.get("submitTime"))
    outcome_rank = 1 if record.get("outcome") in {"success", "failed", "policy_cancelled"} else 0
    return (
        outcome_rank,
        last_sync.timestamp() if last_sync else 0.0,
        submit_time.timestamp() if submit_time else 0.0,
    )


def _prefer_newer_outcome(existing: dict[str, Any], candidate: dict[str, Any]) -> dict[str, Any]:
    if _outcome_version(candidate) >= _outcome_version(existing):
        return candidate
    return existing