from __future__ import annotations from collections import defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timedelta import os from pathlib import Path from typing import Any from architecture_compatibility import ( DEFAULT_ARCHITECTURE_BLOCK_TTL_DAYS, EXPLICIT_ARCHITECTURE_FAILURE_CATEGORY, EXPLICIT_ARCHITECTURE_FAILURE_REASON, architecture_compatibility_key, architecture_profile, architecture_profiles, ) from common import append_jsonl, parse_datetime, read_jsonl, update_jsonl, utc_now from failure_log_inspector import fetch_and_classify_failure_log from history_stats import classify_failure, is_failure, is_success from llm_classifier import LLMAssistedClassifier from modelhub_client import ModelHubClient, ModelHubClientPool from task_registry import task_type_from_history_task DEFAULT_OUTCOMES_PATH = Path("outcomes/submissions.jsonl") FAILURE_ENRICHMENT_LIMIT = 40 FAILURE_ENRICHMENT_WORKERS = 4 FAILURE_ENRICHMENT_MAX_ATTEMPTS = 3 TERMINAL_TASK_STATUSES = {"success", "failed", "error", "cancelled", "completed"} def _now_iso() -> str: return utc_now().isoformat() class OutcomeTracker: def __init__(self, path: Path | str) -> None: self.path = Path(path) self._records: list[dict[str, Any]] = [] self._by_task_id: dict[str, dict[str, Any]] = {} self._by_model_gpu: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) self._failed_model_gpus: dict[tuple[str, str], datetime] = {} self._failure_llm_classifier: LLMAssistedClassifier | None = None self._records = read_jsonl(self.path) self._rebuild_indexes() last_sync_times = [ parse_datetime(record.get("lastSyncTime")) for record in self._records if record.get("lastSyncTime") ] self._last_sync_time: datetime = max(last_sync_times) if last_sync_times else utc_now() - timedelta(days=7) def set_failure_llm_classifier(self, classifier: LLMAssistedClassifier | None) -> None: self._failure_llm_classifier = classifier def get_task_compatibility_contexts(self) -> dict[str, dict[str, Any]]: """Return locally known submit metadata needed for account queue cleanup.""" contexts: dict[str, dict[str, Any]] = {} for task_id, record in self._by_task_id.items(): framework = str(record.get("framework") or "").strip() task_type = str(record.get("taskType") or "").strip() if not framework or not task_type: continue profile = record.get("modelProfile") contexts[task_id] = { "taskId": task_id, "modelId": str(record.get("modelId") or ""), "targetGpu": str(record.get("targetGpu") or ""), "framework": framework, "taskType": task_type, "modelProfile": dict(profile) if isinstance(profile, dict) else {}, "submitTime": record.get("submitTime"), } return contexts def _rebuild_indexes(self) -> None: self._by_task_id.clear() self._by_model_gpu.clear() for record in self._records: task_id = record.get("taskId") if task_id: self._by_task_id[str(task_id)] = record model_id = record.get("modelId") or "" target_gpu = record.get("targetGpu") or "" self._by_model_gpu[(model_id, target_gpu)].append(record) self._rebuild_failed_index() def record_submission( self, model_id: str, target_gpu: str, framework: str, task_type: str, task_id: str | None, submit_time: str, model_profile: dict[str, Any] | None = None, ) -> None: record: dict[str, Any] = { "modelId": model_id, "targetGpu": target_gpu, "framework": framework, "taskType": task_type, "taskId": task_id, "submitTime": submit_time, "lastSyncTime": None, "status": "pending", "verifyResult": None, "outcome": "pending", "failReason": None, "modelProfile": dict(model_profile or {}), } self._records.append(record) if task_id: self._by_task_id[task_id] = record self._by_model_gpu[(model_id, target_gpu)].append(record) append_jsonl(self.path, record) def mark_policy_cancellations(self, decisions: list[dict[str, Any]]) -> int: """Persist intentional task stops so they never become failure evidence.""" marked = 0 marked_at = _now_iso() for decision in decisions: task_id_value = decision.get("taskId") if task_id_value is None: continue task_id = str(task_id_value) existing = self._by_task_id.get(task_id) if existing is not None and existing.get("outcome") in {"success", "failed"}: continue reasons = list(decision.get("cleanupReasons") or [decision.get("reason")]) reasons = [str(reason) for reason in reasons if reason] if existing is None: existing = { "modelId": decision.get("modelId") or "", "targetGpu": decision.get("gpuType") or decision.get("targetGpu") or "", "framework": decision.get("framework") or "", "taskType": decision.get("taskType") or "", "taskId": task_id, "submitTime": decision.get("submitTime") or marked_at, "verifyResult": None, } self._records.append(existing) self._by_task_id[task_id] = existing self._by_model_gpu[(existing["modelId"], existing["targetGpu"])].append(existing) existing.update( { "lastSyncTime": marked_at, "status": "cancellation_requested", "outcome": "policy_cancelled", "failReason": None, "policyCancelled": True, "policyCancellationReasons": reasons, "policyCancelledAt": marked_at, } ) marked += 1 if marked: self._rebuild_failed_index() return marked def sync_from_api(self, client: ModelHubClient | ModelHubClientPool) -> int: try: begin = self._last_sync_time end = utc_now() # Use fanout for pools to sync outcomes across all accounts list_kwargs: dict[str, Any] = {"begin_time": begin, "end_time": end, "page_size": 100, "only_mine": True} if isinstance(client, ModelHubClientPool): list_kwargs["_fanout_all"] = True tasks = client.list_tasks(**list_kwargs) except Exception: return 0 updated_count = 0 enrichment_candidates: list[dict[str, Any]] = [] for task in tasks: task_id = str(task.get("taskId")) if task.get("taskId") is not None else None if not task_id: continue existing = self._by_task_id.get(task_id) if existing is not None: if existing.get("outcome") in {"pending", "policy_cancelled"}: self._update_record_from_task(existing, task) if existing.get("outcome") == "failed" and existing.get("logCosUrl"): enrichment_candidates.append(existing) updated_count += 1 else: status = str(task.get("status") or "").lower() if status in TERMINAL_TASK_STATUSES: record = self._create_record_from_task(task) self._records.append(record) self._by_task_id[task_id] = record model_id = record["modelId"] target_gpu = record["targetGpu"] self._by_model_gpu[(model_id, target_gpu)].append(record) updated_count += 1 # Retry a small bounded set of our own failed submissions. Newer logs # contain the target image, so enrichment can recover the framework even # when the history API omits it. candidate_ids = {id(record) for record in enrichment_candidates} for record in self._records: if len(enrichment_candidates) >= FAILURE_ENRICHMENT_LIMIT: break if id(record) in candidate_ids: continue if ( record.get("outcome") == "failed" and record.get("logCosUrl") and not record.get("failureCategory") and int(record.get("failureEnrichmentAttempts") or 0) < FAILURE_ENRICHMENT_MAX_ATTEMPTS ): enrichment_candidates.append(record) candidate_ids.add(id(record)) enrichment_attempts = self._enrich_failure_records( enrichment_candidates[:FAILURE_ENRICHMENT_LIMIT] ) if updated_count or enrichment_attempts: self._last_sync_time = end self._rebuild_failed_index() self.save() return updated_count def bootstrap_from_history_tasks( self, tasks: list[dict[str, Any]], *, task_contexts: dict[str, dict[str, Any]] | None = None, enrichment_limit: int = 0, enrichment_workers: int = 8, log: Any = None, ) -> dict[str, int]: """Import terminal history and classify every usable historical failure.""" contexts = task_contexts if isinstance(task_contexts, dict) else {} imported = 0 refreshed = 0 terminal_seen = 0 seen_task_ids: set[str] = set() for raw_task in tasks: if not isinstance(raw_task, dict): continue task_id_value = raw_task.get("taskId") if task_id_value is None: continue task_id = str(task_id_value) if not task_id or task_id in seen_task_ids: continue seen_task_ids.add(task_id) status = str(raw_task.get("status") or "").strip().lower() if status not in TERMINAL_TASK_STATUSES: continue terminal_seen += 1 task = self._enrich_history_task(raw_task, contexts.get(task_id)) existing = self._by_task_id.get(task_id) if existing is None: record = self._create_record_from_task(task) self._records.append(record) self._by_task_id[task_id] = record self._by_model_gpu[(record["modelId"], record["targetGpu"])].append(record) imported += 1 continue self._merge_record_metadata(existing, task) if existing.get("outcome") in {"pending", "policy_cancelled"}: self._update_record_from_task(existing, task) refreshed += 1 elif task.get("logCosUrl") and not existing.get("failureCategory"): existing["logCosUrl"] = task.get("logCosUrl") candidates = [ record for record in self._records if ( record.get("outcome") == "failed" and record.get("logCosUrl") and not record.get("failureCategory") and int(record.get("failureEnrichmentAttempts") or 0) < FAILURE_ENRICHMENT_MAX_ATTEMPTS ) ] candidates.sort(key=_outcome_record_timestamp, reverse=True) if enrichment_limit > 0: candidates = candidates[:enrichment_limit] self._last_sync_time = utc_now() self._rebuild_failed_index() if imported or refreshed: self.save() candidate_task_ids = [ str(record["taskId"]) for record in candidates if record.get("taskId") is not None ] enrichment_attempts = 0 classified_total = 0 errors_total = 0 batch_size = 200 total_candidates = len(candidate_task_ids) for offset in range(0, total_candidates, batch_size): batch_ids = candidate_task_ids[offset : offset + batch_size] batch_records = [ self._by_task_id[task_id] for task_id in batch_ids if task_id in self._by_task_id ] def progress( completed: int, _total: int, classified: int, errors: int, ) -> None: if log is None: return global_completed = offset + completed if global_completed == total_candidates or global_completed % 50 == 0: log( f"[architecture-bootstrap] failure_logs={global_completed}/{total_candidates} " f"classified={classified_total + classified} " f"errors={errors_total + errors}" ) enrichment_attempts += self._enrich_failure_records( batch_records, workers=max(1, int(enrichment_workers)), progress=progress, ) classified_total += sum( 1 for record in batch_records if record.get("failureCategory") ) errors_total += sum( 1 for record in batch_records if record.get("failureEnrichmentError") ) self._rebuild_failed_index() self.save() explicit_architecture_failures = sum( 1 for record in self._records if _is_explicit_architecture_failure(record) ) recovered_frameworks = sum( 1 for record in self._records if record.get("failureDetectedFramework") and record.get("framework") ) return { "recordsScanned": len(seen_task_ids), "terminalRecords": terminal_seen, "importedRecords": imported, "refreshedRecords": refreshed, "eligibleFailureLogs": total_candidates, "enrichmentAttempts": enrichment_attempts, "explicitArchitectureFailures": explicit_architecture_failures, "recoveredFrameworks": recovered_frameworks, } @staticmethod def _enrich_history_task( task: dict[str, Any], context: dict[str, Any] | None, ) -> dict[str, Any]: enriched = dict(task) context = context if isinstance(context, dict) else {} if not enriched.get("modelId") and context.get("modelId"): enriched["modelId"] = context.get("modelId") if not (enriched.get("gpuType") or enriched.get("targetGpu")) and context.get("targetGpu"): enriched["targetGpu"] = context.get("targetGpu") if not enriched.get("framework") and context.get("framework"): enriched["framework"] = context.get("framework") if not enriched.get("taskType"): enriched["taskType"] = context.get("taskType") or task_type_from_history_task(enriched) if context.get("modelProfile") and not enriched.get("modelProfile"): enriched["modelProfile"] = context.get("modelProfile") if context.get("submitTime") and not ( enriched.get("submitTime") or enriched.get("createTime") ): enriched["submitTime"] = context.get("submitTime") return enriched @staticmethod def _merge_record_metadata(record: dict[str, Any], task: dict[str, Any]) -> None: field_sources = { "modelId": task.get("modelId") or task.get("model_id"), "targetGpu": task.get("gpuType") or task.get("targetGpu"), "framework": task.get("framework"), "taskType": task.get("taskType") or task_type_from_history_task(task), "modelProfile": task.get("modelProfile"), } for field, value in field_sources.items(): if not record.get(field) and value not in (None, "", "unknown"): record[field] = dict(value) if field == "modelProfile" and isinstance(value, dict) else value if task.get("logCosUrl") and not record.get("failureCategory"): record["logCosUrl"] = task.get("logCosUrl") def _enrich_failure_records( self, records: list[dict[str, Any]], *, workers: int = FAILURE_ENRICHMENT_WORKERS, progress: Any = None, ) -> int: if not records: return 0 def inspect(record: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any] | None, str | None]: try: result = fetch_and_classify_failure_log( str(record["logCosUrl"]), task_context=record, llm_classifier=self._failure_llm_classifier, ) return record, result, None except Exception as exc: return record, None, f"{type(exc).__name__}: {exc}" attempted = 0 classified = 0 errors = 0 with ThreadPoolExecutor(max_workers=min(max(1, int(workers)), len(records))) as executor: futures = [executor.submit(inspect, record) for record in records] for future in as_completed(futures): record, result, error = future.result() attempted += 1 record["failureEnrichmentAttempts"] = int(record.get("failureEnrichmentAttempts") or 0) + 1 if result is None: errors += 1 record["failureEnrichmentError"] = error if progress is not None: progress(attempted, len(records), classified, errors) continue record.update(result) if not record.get("framework") and result.get("failureDetectedFramework"): record["framework"] = result.get("failureDetectedFramework") record["failReason"] = result.get("failureCategory") or record.get("failReason") record["failureEnrichmentError"] = None record.pop("logCosUrl", None) classified += 1 if progress is not None: progress(attempted, len(records), classified, errors) return attempted def is_model_gpu_failed( self, model_id: str, target_gpu: str, *, cooldown_hours: int = 24, now: datetime | None = None, ) -> bool: failed_at = self._failed_model_gpus.get((model_id, target_gpu)) if failed_at is None: return False now = now or utc_now() return failed_at >= now - timedelta(hours=max(0, int(cooldown_hours))) def get_stats_report(self) -> dict[str, Any]: now_datetime = utc_now() now = now_datetime.isoformat() terminal = [r for r in self._records if r.get("outcome") in {"success", "failed"}] gpu_groups: dict[str, list[dict[str, Any]]] = defaultdict(list) framework_groups: dict[str, list[dict[str, Any]]] = defaultdict(list) combo_groups: dict[tuple[str, str, str], list[dict[str, Any]]] = defaultdict(list) profile_groups: dict[tuple[str, str, str, str, str], list[dict[str, Any]]] = defaultdict(list) for record in terminal: gpu = record.get("targetGpu") or "unknown" fw = record.get("framework") or "unknown" tt = record.get("taskType") or "unknown" gpu_groups[gpu].append(record) framework_groups[f"{fw}"].append(record) combo_groups[(gpu, fw, tt)].append(record) profile = record.get("modelProfile") or {} model_type = str(profile.get("modelType") or "").strip() if model_type: quantization = str(profile.get("quantizationMethod") or "none").strip() profile_groups[(gpu, fw, tt, model_type, quantization)].append(record) gpu_summaries = {gpu: _summarize(records) for gpu, records in gpu_groups.items()} framework_summaries = {fw: _summarize(records) for fw, records in framework_groups.items()} combination_stats = { f"{gpu}|{fw}|{tt}": {"targetGpu": gpu, "framework": fw, "taskType": tt, **_summarize(records)} for (gpu, fw, tt), records in combo_groups.items() } recent_combination_stats: dict[str, dict[str, Any]] = {} for (gpu, fw, tt), records in combo_groups.items(): recent = sorted(records, key=_outcome_record_timestamp, reverse=True)[:20] consecutive_failures = _consecutive_attributable_failures(recent) consecutive_platform_failures = _consecutive_platform_failures(recent) last_terminal_at = None if recent: last_terminal_at = ( parse_datetime(recent[0].get("lastSyncTime")) or parse_datetime(recent[0].get("submitTime")) ) recent_combination_stats[f"{gpu}|{fw}|{tt}"] = { "targetGpu": gpu, "framework": fw, "taskType": tt, **_summarize(recent), "consecutiveFailures": consecutive_failures, "consecutivePlatformFailures": consecutive_platform_failures, "lastPlatformFailureAt": _latest_platform_failure_at(recent), "lastTerminalAt": last_terminal_at.isoformat() if last_terminal_at else None, } profile_combination_stats: dict[str, dict[str, Any]] = {} recent_profile_combination_stats: dict[str, dict[str, Any]] = {} for (gpu, fw, tt, model_type, quantization), records in profile_groups.items(): key = f"{gpu}|{fw}|{tt}|{model_type}|{quantization}" profile_combination_stats[key] = { "targetGpu": gpu, "framework": fw, "taskType": tt, "modelType": model_type, "quantizationMethod": quantization, **_summarize(records), } recent = sorted(records, key=_outcome_record_timestamp, reverse=True)[:20] consecutive_failures = _consecutive_attributable_failures(recent) last_terminal_at = None if recent: last_terminal_at = ( parse_datetime(recent[0].get("lastSyncTime")) or parse_datetime(recent[0].get("submitTime")) ) recent_profile_combination_stats[key] = { **profile_combination_stats[key], **_summarize(recent), "consecutiveFailures": consecutive_failures, "lastTerminalAt": last_terminal_at.isoformat() if last_terminal_at else None, } warnings: list[str] = [] for gpu, summary in gpu_summaries.items(): if summary["decisionTotal"] >= 4 and summary["decisionFailureRate"] >= 0.5: warnings.append(f"GPU {gpu} 本地统计失败率偏高（≥50%），建议重点关注。") for key, stat in combination_stats.items(): if stat["decisionTotal"] >= 3 and stat["decisionFailureRate"] >= 0.6: warnings.append(f"组合 {key} 近期失败集中，建议降低该 GPU+框架的提交优先级。") pending_count = sum(1 for r in self._records if r.get("outcome") == "pending") policy_cancelled_count = sum( 1 for record in self._records if record.get("outcome") == "policy_cancelled" ) observed_gpu_memory: dict[str, float] = {} for record in self._records: gpu = str(record.get("targetGpu") or "") try: memory_gib = float(record.get("failureObservedGpuMemoryGiB")) except (TypeError, ValueError): continue if gpu and 0 < memory_gib <= 1024: previous = observed_gpu_memory.get(gpu) observed_gpu_memory[gpu] = min(previous, memory_gib) if previous else memory_gib architecture_block_ttl_days = _architecture_block_ttl_days() architecture_blocks = _build_architecture_compatibility_blocks( terminal, now=now_datetime, ttl_days=architecture_block_ttl_days, ) architecture_blocks_by_gpu_framework: dict[str, int] = defaultdict(int) for block in architecture_blocks.values(): combination = f"{block['targetGpu']}|{block['framework']}" architecture_blocks_by_gpu_framework[combination] += 1 return { "generatedAt": now, "totalRecords": len(self._records), "pendingRecords": pending_count, "policyCancelledRecords": policy_cancelled_count, "terminalRecords": len(terminal), "gpuSummaries": gpu_summaries, "frameworkSummaries": framework_summaries, "combinationStats": combination_stats, "recentCombinationStats": recent_combination_stats, "profileCombinationStats": profile_combination_stats, "recentProfileCombinationStats": recent_profile_combination_stats, "architectureCompatibilityBlocks": architecture_blocks, "architectureCompatibilitySummary": { "activeBlockCount": len(architecture_blocks), "ttlDays": architecture_block_ttl_days, "byGpuFramework": dict(architecture_blocks_by_gpu_framework), }, "observedGpuMemoryGiB": observed_gpu_memory, "totals": _summarize(terminal), "warnings": warnings, } def save(self) -> None: local_records = list(self._records) def merge(existing: list[dict[str, Any]]) -> list[dict[str, Any]]: merged = list(existing) index_by_key = {_outcome_record_key(record): index for index, record in enumerate(merged)} for record in local_records: key = _outcome_record_key(record) existing_index = index_by_key.get(key) if existing_index is None: index_by_key[key] = len(merged) merged.append(record) continue merged[existing_index] = _prefer_newer_outcome(merged[existing_index], record) return merged self._records = update_jsonl(self.path, merge) self._rebuild_indexes() def _rebuild_failed_index(self) -> None: self._failed_model_gpus.clear() latest_by_combo: dict[tuple[str, str], tuple[datetime, dict[str, Any]]] = {} for record in self._records: # Infrastructure failures and our own policy cancellations neither # clear nor create a model/GPU cooldown. Look through them to the # latest attributable outcome. if _is_platform_failure(record) or _is_policy_cancelled(record): continue model_id = record.get("modelId") or "" target_gpu = record.get("targetGpu") or "" event_time = parse_datetime(record.get("submitTime")) or parse_datetime(record.get("lastSyncTime")) if not model_id or not target_gpu or event_time is None: continue key = (model_id, target_gpu) current = latest_by_combo.get(key) if current is None or event_time >= current[0]: latest_by_combo[key] = (event_time, record) for key, (event_time, record) in latest_by_combo.items(): if record.get("outcome") == "failed": self._failed_model_gpus[key] = event_time @staticmethod def _update_record_from_task(record: dict[str, Any], task: dict[str, Any]) -> None: record["status"] = task.get("status") record["verifyResult"] = task.get("verifyResult") record["lastSyncTime"] = _now_iso() if task.get("logCosUrl"): record["logCosUrl"] = task.get("logCosUrl") status = str(task.get("status") or "").strip().lower() if is_success(task): record["outcome"] = "success" record["failReason"] = None if record.get("policyCancelled"): record["policyCancellationResolvedAsSuccess"] = True record["policyCancelled"] = False elif record.get("policyCancelled"): # A successful stop may be observed as waiting/running briefly before # the platform publishes its terminal cancellation state. None of # those intermediate states should become failure evidence. record["outcome"] = "policy_cancelled" record["failReason"] = None elif is_failure(task): record["outcome"] = "failed" record["failReason"] = classify_failure(task) else: record["outcome"] = "pending" @staticmethod def _create_record_from_task(task: dict[str, Any]) -> dict[str, Any]: create_time = ( parse_datetime(task.get("submitTime")) or parse_datetime(task.get("createTime")) or parse_datetime(task.get("updateTime")) ) model_profile = task.get("modelProfile") record: dict[str, Any] = { "modelId": task.get("modelId") or task.get("model_id") or "", "targetGpu": task.get("gpuType") or task.get("targetGpu") or "", "framework": task.get("framework") or "", "taskType": task.get("taskType") or task_type_from_history_task(task) or "", "taskId": str(task.get("taskId")) if task.get("taskId") is not None else None, "submitTime": create_time.isoformat() if create_time else _now_iso(), "lastSyncTime": _now_iso(), "status": task.get("status"), "verifyResult": task.get("verifyResult"), "outcome": "pending", "failReason": None, "logCosUrl": task.get("logCosUrl"), "modelProfile": dict(model_profile) if isinstance(model_profile, dict) else {}, } if is_success(task): record["outcome"] = "success" elif is_failure(task): record["outcome"] = "failed" record["failReason"] = classify_failure(task) return record def _summarize(records: list[dict[str, Any]]) -> dict[str, Any]: total = len(records) success_count = sum(1 for r in records if r.get("outcome") == "success") failure_count = sum(1 for r in records if r.get("outcome") == "failed") pending_count = sum(1 for r in records if r.get("outcome") == "pending") attributable_failure_count = sum( 1 for record in records if record.get("outcome") == "failed" and not _is_platform_failure(record) ) platform_failure_count = failure_count - attributable_failure_count decision_total = success_count + attributable_failure_count failure_breakdown: dict[str, int] = defaultdict(int) for r in records: reason = r.get("failureCategory") or r.get("failReason") if reason: failure_breakdown[reason] += 1 return { "total": total, "successCount": success_count, "failureCount": failure_count, "attributableFailureCount": attributable_failure_count, "platformFailureCount": platform_failure_count, "decisionTotal": decision_total, "pendingCount": pending_count, "successRate": round(success_count / total, 4) if total > 0 else 0.0, "failureRate": round(failure_count / total, 4) if total > 0 else 0.0, "decisionSuccessRate": round(success_count / decision_total, 4) if decision_total > 0 else 0.0, "decisionFailureRate": round(attributable_failure_count / decision_total, 4) if decision_total > 0 else 0.0, "pendingRate": round(pending_count / total, 4) if total > 0 else 0.0, "failureBreakdown": dict(failure_breakdown), } def _architecture_block_ttl_days() -> int: raw = os.getenv("MODELHUB_ARCHITECTURE_BLOCK_TTL_DAYS") if raw is None: return DEFAULT_ARCHITECTURE_BLOCK_TTL_DAYS try: value = int(raw) except ValueError: return DEFAULT_ARCHITECTURE_BLOCK_TTL_DAYS return min(365, max(1, value)) def _build_architecture_compatibility_blocks( records: list[dict[str, Any]], *, now: datetime, ttl_days: int, ) -> dict[str, dict[str, Any]]: successes: dict[str, list[tuple[datetime, dict[str, Any]]]] = defaultdict(list) failures: dict[str, list[tuple[datetime, dict[str, Any], dict[str, Any]]]] = defaultdict(list) cutoff = now - timedelta(days=max(1, int(ttl_days))) for record in records: target_gpu = str(record.get("targetGpu") or "").strip() framework = str(record.get("framework") or "").strip() task_type = str(record.get("taskType") or "").strip() event_time = ( parse_datetime(record.get("submitTime")) or parse_datetime(record.get("lastSyncTime")) ) if not target_gpu or not framework or not task_type or event_time is None: continue if record.get("outcome") == "success": for profile in _stored_architecture_profiles(record, include_model_type=True): key = architecture_compatibility_key( target_gpu, framework, task_type, profile["signature"], ) if key is not None: successes[key].append((event_time, record)) continue if not _is_explicit_architecture_failure(record) or event_time < cutoff: continue for profile in _failure_architecture_profiles(record): key = architecture_compatibility_key( target_gpu, framework, task_type, profile["signature"], ) if key is not None: failures[key].append((event_time, record, profile)) blocks: dict[str, dict[str, Any]] = {} for key, failure_events in failures.items(): failure_events.sort(key=lambda item: item[0]) latest_failure_at, latest_failure, latest_profile = failure_events[-1] success_events = successes.get(key) or [] latest_success_at = max((item[0] for item in success_events), default=None) if latest_success_at is not None and latest_success_at >= latest_failure_at: continue effective_failures = [ item for item in failure_events if latest_success_at is None or item[0] > latest_success_at ] expires_at = latest_failure_at + timedelta(days=max(1, int(ttl_days))) if expires_at <= now: continue blocks[key] = { "targetGpu": str(latest_failure.get("targetGpu") or ""), "framework": str(latest_failure.get("framework") or ""), "taskType": str(latest_failure.get("taskType") or ""), "matchType": latest_profile["matchType"], "architectureSignature": latest_profile["signature"], "architectures": latest_profile["architectures"], "modelType": latest_profile["modelType"], "evidenceCount": len(effective_failures), "latestFailureAt": latest_failure_at.isoformat(), "latestSuccessfulAt": latest_success_at.isoformat() if latest_success_at else None, "expiresAt": expires_at.isoformat(), "sourceTaskIds": _bounded_unique( record.get("taskId") for _, record, _ in reversed(effective_failures) ), "sourceModelIds": _bounded_unique( record.get("modelId") for _, record, _ in reversed(effective_failures) ), } return blocks def _stored_architecture_profiles( record: dict[str, Any], *, include_model_type: bool, ) -> list[dict[str, Any]]: profile_data = record.get("modelProfile") if not isinstance(profile_data, dict): return [] if include_model_type: return architecture_profiles( profile_data.get("modelType"), profile_data.get("architectures"), ) profile = architecture_profile( profile_data.get("modelType"), profile_data.get("architectures"), ) return [profile] if profile is not None else [] def _failure_architecture_profiles(record: dict[str, Any]) -> list[dict[str, Any]]: profiles: list[dict[str, Any]] = [] unsupported_architectures = record.get("failureUnsupportedArchitectures") if isinstance(unsupported_architectures, list) and unsupported_architectures: profile = architecture_profile(None, unsupported_architectures) if profile is not None: profiles.append(profile) unsupported_model_types = record.get("failureUnsupportedModelTypes") if isinstance(unsupported_model_types, list): for model_type in unsupported_model_types: profile = architecture_profile(model_type, None) if profile is not None: profiles.append(profile) if not profiles: profiles.extend(_stored_architecture_profiles(record, include_model_type=False)) deduped: dict[str, dict[str, Any]] = {} for profile in profiles: deduped[profile["signature"]] = profile return list(deduped.values()) def _is_explicit_architecture_failure(record: dict[str, Any]) -> bool: return bool( record.get("outcome") == "failed" and record.get("failureCategory") == EXPLICIT_ARCHITECTURE_FAILURE_CATEGORY and record.get("failureDeterministic") is True and record.get("failureClassificationReason") == EXPLICIT_ARCHITECTURE_FAILURE_REASON ) def _bounded_unique(values: Any, limit: int = 5) -> list[str]: result: list[str] = [] for value in values: rendered = str(value or "").strip() if not rendered or rendered in result: continue result.append(rendered) if len(result) >= limit: break return result def _is_platform_failure(record: dict[str, Any]) -> bool: if record.get("outcome") != "failed": return False category = str(record.get("failureCategory") or "").lower() scope = str(record.get("failureScope") or "").lower() return scope == "platform" or category.startswith("platform_") def _is_policy_cancelled(record: dict[str, Any]) -> bool: return bool(record.get("policyCancelled")) or record.get("outcome") == "policy_cancelled" def _consecutive_attributable_failures(records: list[dict[str, Any]]) -> int: count = 0 for record in records: if record.get("outcome") == "success": break if record.get("outcome") == "failed" and not _is_platform_failure(record): count += 1 return count def _consecutive_platform_failures(records: list[dict[str, Any]]) -> int: count = 0 for record in records: if not _is_platform_failure(record): break count += 1 return count def _latest_platform_failure_at(records: list[dict[str, Any]]) -> str | None: for record in records: if not _is_platform_failure(record): continue timestamp = parse_datetime(record.get("submitTime")) or parse_datetime(record.get("lastSyncTime")) return timestamp.isoformat() if timestamp else None return None def _outcome_record_timestamp(record: dict[str, Any]) -> float: timestamp = parse_datetime(record.get("submitTime")) or parse_datetime(record.get("lastSyncTime")) return timestamp.timestamp() if timestamp else 0.0 def _outcome_record_key(record: dict[str, Any]) -> str: task_id = record.get("taskId") if task_id is not None: return f"task:{task_id}" return "fallback:{model}|{gpu}|{time}".format( model=record.get("modelId") or "", gpu=record.get("targetGpu") or "", time=record.get("submitTime") or "", ) def _outcome_version(record: dict[str, Any]) -> tuple[int, float, float]: last_sync = parse_datetime(record.get("lastSyncTime")) submit_time = parse_datetime(record.get("submitTime")) outcome_rank = 1 if record.get("outcome") in {"success", "failed", "policy_cancelled"} else 0 return ( outcome_rank, last_sync.timestamp() if last_sync else 0.0, submit_time.timestamp() if submit_time else 0.0, ) def _prefer_newer_outcome(existing: dict[str, Any], candidate: dict[str, Any]) -> dict[str, Any]: if _outcome_version(candidate) >= _outcome_version(existing): return candidate return existing