958 lines
40 KiB
Python
958 lines
40 KiB
Python
from __future__ import annotations
|
||
|
||
from collections import defaultdict
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
from datetime import datetime, timedelta
|
||
import os
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from architecture_compatibility import (
|
||
DEFAULT_ARCHITECTURE_BLOCK_TTL_DAYS,
|
||
EXPLICIT_ARCHITECTURE_FAILURE_CATEGORY,
|
||
EXPLICIT_ARCHITECTURE_FAILURE_REASON,
|
||
architecture_compatibility_key,
|
||
architecture_profile,
|
||
architecture_profiles,
|
||
)
|
||
from common import append_jsonl, parse_datetime, read_jsonl, update_jsonl, utc_now
|
||
from failure_log_inspector import fetch_and_classify_failure_log
|
||
from history_stats import classify_failure, is_failure, is_success
|
||
from llm_classifier import LLMAssistedClassifier
|
||
from modelhub_client import ModelHubClient, ModelHubClientPool
|
||
from task_registry import task_type_from_history_task
|
||
|
||
|
||
DEFAULT_OUTCOMES_PATH = Path("outcomes/submissions.jsonl")
|
||
FAILURE_ENRICHMENT_LIMIT = 40
|
||
FAILURE_ENRICHMENT_WORKERS = 4
|
||
FAILURE_ENRICHMENT_MAX_ATTEMPTS = 3
|
||
TERMINAL_TASK_STATUSES = {"success", "failed", "error", "cancelled", "completed"}
|
||
|
||
|
||
def _now_iso() -> str:
|
||
return utc_now().isoformat()
|
||
|
||
|
||
class OutcomeTracker:
|
||
def __init__(self, path: Path | str) -> None:
|
||
self.path = Path(path)
|
||
self._records: list[dict[str, Any]] = []
|
||
self._by_task_id: dict[str, dict[str, Any]] = {}
|
||
self._by_model_gpu: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
|
||
self._failed_model_gpus: dict[tuple[str, str], datetime] = {}
|
||
self._failure_llm_classifier: LLMAssistedClassifier | None = None
|
||
|
||
self._records = read_jsonl(self.path)
|
||
self._rebuild_indexes()
|
||
|
||
last_sync_times = [
|
||
parse_datetime(record.get("lastSyncTime"))
|
||
for record in self._records
|
||
if record.get("lastSyncTime")
|
||
]
|
||
self._last_sync_time: datetime = max(last_sync_times) if last_sync_times else utc_now() - timedelta(days=7)
|
||
|
||
def set_failure_llm_classifier(self, classifier: LLMAssistedClassifier | None) -> None:
|
||
self._failure_llm_classifier = classifier
|
||
|
||
def get_task_compatibility_contexts(self) -> dict[str, dict[str, Any]]:
|
||
"""Return locally known submit metadata needed for account queue cleanup."""
|
||
contexts: dict[str, dict[str, Any]] = {}
|
||
for task_id, record in self._by_task_id.items():
|
||
framework = str(record.get("framework") or "").strip()
|
||
task_type = str(record.get("taskType") or "").strip()
|
||
if not framework or not task_type:
|
||
continue
|
||
profile = record.get("modelProfile")
|
||
contexts[task_id] = {
|
||
"taskId": task_id,
|
||
"modelId": str(record.get("modelId") or ""),
|
||
"targetGpu": str(record.get("targetGpu") or ""),
|
||
"framework": framework,
|
||
"taskType": task_type,
|
||
"modelProfile": dict(profile) if isinstance(profile, dict) else {},
|
||
"submitTime": record.get("submitTime"),
|
||
}
|
||
return contexts
|
||
|
||
def _rebuild_indexes(self) -> None:
|
||
self._by_task_id.clear()
|
||
self._by_model_gpu.clear()
|
||
for record in self._records:
|
||
task_id = record.get("taskId")
|
||
if task_id:
|
||
self._by_task_id[str(task_id)] = record
|
||
model_id = record.get("modelId") or ""
|
||
target_gpu = record.get("targetGpu") or ""
|
||
self._by_model_gpu[(model_id, target_gpu)].append(record)
|
||
|
||
self._rebuild_failed_index()
|
||
|
||
def record_submission(
|
||
self,
|
||
model_id: str,
|
||
target_gpu: str,
|
||
framework: str,
|
||
task_type: str,
|
||
task_id: str | None,
|
||
submit_time: str,
|
||
model_profile: dict[str, Any] | None = None,
|
||
) -> None:
|
||
record: dict[str, Any] = {
|
||
"modelId": model_id,
|
||
"targetGpu": target_gpu,
|
||
"framework": framework,
|
||
"taskType": task_type,
|
||
"taskId": task_id,
|
||
"submitTime": submit_time,
|
||
"lastSyncTime": None,
|
||
"status": "pending",
|
||
"verifyResult": None,
|
||
"outcome": "pending",
|
||
"failReason": None,
|
||
"modelProfile": dict(model_profile or {}),
|
||
}
|
||
self._records.append(record)
|
||
if task_id:
|
||
self._by_task_id[task_id] = record
|
||
self._by_model_gpu[(model_id, target_gpu)].append(record)
|
||
append_jsonl(self.path, record)
|
||
|
||
def mark_policy_cancellations(self, decisions: list[dict[str, Any]]) -> int:
|
||
"""Persist intentional task stops so they never become failure evidence."""
|
||
marked = 0
|
||
marked_at = _now_iso()
|
||
for decision in decisions:
|
||
task_id_value = decision.get("taskId")
|
||
if task_id_value is None:
|
||
continue
|
||
task_id = str(task_id_value)
|
||
existing = self._by_task_id.get(task_id)
|
||
if existing is not None and existing.get("outcome") in {"success", "failed"}:
|
||
continue
|
||
reasons = list(decision.get("cleanupReasons") or [decision.get("reason")])
|
||
reasons = [str(reason) for reason in reasons if reason]
|
||
if existing is None:
|
||
existing = {
|
||
"modelId": decision.get("modelId") or "",
|
||
"targetGpu": decision.get("gpuType") or decision.get("targetGpu") or "",
|
||
"framework": decision.get("framework") or "",
|
||
"taskType": decision.get("taskType") or "",
|
||
"taskId": task_id,
|
||
"submitTime": decision.get("submitTime") or marked_at,
|
||
"verifyResult": None,
|
||
}
|
||
self._records.append(existing)
|
||
self._by_task_id[task_id] = existing
|
||
self._by_model_gpu[(existing["modelId"], existing["targetGpu"])].append(existing)
|
||
existing.update(
|
||
{
|
||
"lastSyncTime": marked_at,
|
||
"status": "cancellation_requested",
|
||
"outcome": "policy_cancelled",
|
||
"failReason": None,
|
||
"policyCancelled": True,
|
||
"policyCancellationReasons": reasons,
|
||
"policyCancelledAt": marked_at,
|
||
}
|
||
)
|
||
marked += 1
|
||
if marked:
|
||
self._rebuild_failed_index()
|
||
return marked
|
||
|
||
def sync_from_api(self, client: ModelHubClient | ModelHubClientPool) -> int:
|
||
try:
|
||
begin = self._last_sync_time
|
||
end = utc_now()
|
||
# Use fanout for pools to sync outcomes across all accounts
|
||
list_kwargs: dict[str, Any] = {"begin_time": begin, "end_time": end, "page_size": 100, "only_mine": True}
|
||
if isinstance(client, ModelHubClientPool):
|
||
list_kwargs["_fanout_all"] = True
|
||
tasks = client.list_tasks(**list_kwargs)
|
||
except Exception:
|
||
return 0
|
||
|
||
updated_count = 0
|
||
enrichment_candidates: list[dict[str, Any]] = []
|
||
for task in tasks:
|
||
task_id = str(task.get("taskId")) if task.get("taskId") is not None else None
|
||
if not task_id:
|
||
continue
|
||
|
||
existing = self._by_task_id.get(task_id)
|
||
if existing is not None:
|
||
if existing.get("outcome") in {"pending", "policy_cancelled"}:
|
||
self._update_record_from_task(existing, task)
|
||
if existing.get("outcome") == "failed" and existing.get("logCosUrl"):
|
||
enrichment_candidates.append(existing)
|
||
updated_count += 1
|
||
else:
|
||
status = str(task.get("status") or "").lower()
|
||
if status in TERMINAL_TASK_STATUSES:
|
||
record = self._create_record_from_task(task)
|
||
self._records.append(record)
|
||
self._by_task_id[task_id] = record
|
||
model_id = record["modelId"]
|
||
target_gpu = record["targetGpu"]
|
||
self._by_model_gpu[(model_id, target_gpu)].append(record)
|
||
updated_count += 1
|
||
|
||
# Retry a small bounded set of our own failed submissions. Newer logs
|
||
# contain the target image, so enrichment can recover the framework even
|
||
# when the history API omits it.
|
||
candidate_ids = {id(record) for record in enrichment_candidates}
|
||
for record in self._records:
|
||
if len(enrichment_candidates) >= FAILURE_ENRICHMENT_LIMIT:
|
||
break
|
||
if id(record) in candidate_ids:
|
||
continue
|
||
if (
|
||
record.get("outcome") == "failed"
|
||
and record.get("logCosUrl")
|
||
and not record.get("failureCategory")
|
||
and int(record.get("failureEnrichmentAttempts") or 0) < FAILURE_ENRICHMENT_MAX_ATTEMPTS
|
||
):
|
||
enrichment_candidates.append(record)
|
||
candidate_ids.add(id(record))
|
||
|
||
enrichment_attempts = self._enrich_failure_records(
|
||
enrichment_candidates[:FAILURE_ENRICHMENT_LIMIT]
|
||
)
|
||
|
||
if updated_count or enrichment_attempts:
|
||
self._last_sync_time = end
|
||
self._rebuild_failed_index()
|
||
self.save()
|
||
|
||
return updated_count
|
||
|
||
def bootstrap_from_history_tasks(
|
||
self,
|
||
tasks: list[dict[str, Any]],
|
||
*,
|
||
task_contexts: dict[str, dict[str, Any]] | None = None,
|
||
enrichment_limit: int = 0,
|
||
enrichment_workers: int = 8,
|
||
log: Any = None,
|
||
) -> dict[str, int]:
|
||
"""Import terminal history and classify every usable historical failure."""
|
||
contexts = task_contexts if isinstance(task_contexts, dict) else {}
|
||
imported = 0
|
||
refreshed = 0
|
||
terminal_seen = 0
|
||
seen_task_ids: set[str] = set()
|
||
|
||
for raw_task in tasks:
|
||
if not isinstance(raw_task, dict):
|
||
continue
|
||
task_id_value = raw_task.get("taskId")
|
||
if task_id_value is None:
|
||
continue
|
||
task_id = str(task_id_value)
|
||
if not task_id or task_id in seen_task_ids:
|
||
continue
|
||
seen_task_ids.add(task_id)
|
||
status = str(raw_task.get("status") or "").strip().lower()
|
||
if status not in TERMINAL_TASK_STATUSES:
|
||
continue
|
||
terminal_seen += 1
|
||
|
||
task = self._enrich_history_task(raw_task, contexts.get(task_id))
|
||
existing = self._by_task_id.get(task_id)
|
||
if existing is None:
|
||
record = self._create_record_from_task(task)
|
||
self._records.append(record)
|
||
self._by_task_id[task_id] = record
|
||
self._by_model_gpu[(record["modelId"], record["targetGpu"])].append(record)
|
||
imported += 1
|
||
continue
|
||
|
||
self._merge_record_metadata(existing, task)
|
||
if existing.get("outcome") in {"pending", "policy_cancelled"}:
|
||
self._update_record_from_task(existing, task)
|
||
refreshed += 1
|
||
elif task.get("logCosUrl") and not existing.get("failureCategory"):
|
||
existing["logCosUrl"] = task.get("logCosUrl")
|
||
|
||
candidates = [
|
||
record
|
||
for record in self._records
|
||
if (
|
||
record.get("outcome") == "failed"
|
||
and record.get("logCosUrl")
|
||
and not record.get("failureCategory")
|
||
and int(record.get("failureEnrichmentAttempts") or 0)
|
||
< FAILURE_ENRICHMENT_MAX_ATTEMPTS
|
||
)
|
||
]
|
||
candidates.sort(key=_outcome_record_timestamp, reverse=True)
|
||
if enrichment_limit > 0:
|
||
candidates = candidates[:enrichment_limit]
|
||
self._last_sync_time = utc_now()
|
||
self._rebuild_failed_index()
|
||
if imported or refreshed:
|
||
self.save()
|
||
|
||
candidate_task_ids = [
|
||
str(record["taskId"])
|
||
for record in candidates
|
||
if record.get("taskId") is not None
|
||
]
|
||
enrichment_attempts = 0
|
||
classified_total = 0
|
||
errors_total = 0
|
||
batch_size = 200
|
||
total_candidates = len(candidate_task_ids)
|
||
for offset in range(0, total_candidates, batch_size):
|
||
batch_ids = candidate_task_ids[offset : offset + batch_size]
|
||
batch_records = [
|
||
self._by_task_id[task_id]
|
||
for task_id in batch_ids
|
||
if task_id in self._by_task_id
|
||
]
|
||
|
||
def progress(
|
||
completed: int,
|
||
_total: int,
|
||
classified: int,
|
||
errors: int,
|
||
) -> None:
|
||
if log is None:
|
||
return
|
||
global_completed = offset + completed
|
||
if global_completed == total_candidates or global_completed % 50 == 0:
|
||
log(
|
||
f"[architecture-bootstrap] failure_logs={global_completed}/{total_candidates} "
|
||
f"classified={classified_total + classified} "
|
||
f"errors={errors_total + errors}"
|
||
)
|
||
|
||
enrichment_attempts += self._enrich_failure_records(
|
||
batch_records,
|
||
workers=max(1, int(enrichment_workers)),
|
||
progress=progress,
|
||
)
|
||
classified_total += sum(
|
||
1 for record in batch_records if record.get("failureCategory")
|
||
)
|
||
errors_total += sum(
|
||
1 for record in batch_records if record.get("failureEnrichmentError")
|
||
)
|
||
self._rebuild_failed_index()
|
||
self.save()
|
||
|
||
explicit_architecture_failures = sum(
|
||
1 for record in self._records if _is_explicit_architecture_failure(record)
|
||
)
|
||
recovered_frameworks = sum(
|
||
1
|
||
for record in self._records
|
||
if record.get("failureDetectedFramework") and record.get("framework")
|
||
)
|
||
return {
|
||
"recordsScanned": len(seen_task_ids),
|
||
"terminalRecords": terminal_seen,
|
||
"importedRecords": imported,
|
||
"refreshedRecords": refreshed,
|
||
"eligibleFailureLogs": total_candidates,
|
||
"enrichmentAttempts": enrichment_attempts,
|
||
"explicitArchitectureFailures": explicit_architecture_failures,
|
||
"recoveredFrameworks": recovered_frameworks,
|
||
}
|
||
|
||
@staticmethod
|
||
def _enrich_history_task(
|
||
task: dict[str, Any],
|
||
context: dict[str, Any] | None,
|
||
) -> dict[str, Any]:
|
||
enriched = dict(task)
|
||
context = context if isinstance(context, dict) else {}
|
||
if not enriched.get("modelId") and context.get("modelId"):
|
||
enriched["modelId"] = context.get("modelId")
|
||
if not (enriched.get("gpuType") or enriched.get("targetGpu")) and context.get("targetGpu"):
|
||
enriched["targetGpu"] = context.get("targetGpu")
|
||
if not enriched.get("framework") and context.get("framework"):
|
||
enriched["framework"] = context.get("framework")
|
||
if not enriched.get("taskType"):
|
||
enriched["taskType"] = context.get("taskType") or task_type_from_history_task(enriched)
|
||
if context.get("modelProfile") and not enriched.get("modelProfile"):
|
||
enriched["modelProfile"] = context.get("modelProfile")
|
||
if context.get("submitTime") and not (
|
||
enriched.get("submitTime") or enriched.get("createTime")
|
||
):
|
||
enriched["submitTime"] = context.get("submitTime")
|
||
return enriched
|
||
|
||
@staticmethod
|
||
def _merge_record_metadata(record: dict[str, Any], task: dict[str, Any]) -> None:
|
||
field_sources = {
|
||
"modelId": task.get("modelId") or task.get("model_id"),
|
||
"targetGpu": task.get("gpuType") or task.get("targetGpu"),
|
||
"framework": task.get("framework"),
|
||
"taskType": task.get("taskType") or task_type_from_history_task(task),
|
||
"modelProfile": task.get("modelProfile"),
|
||
}
|
||
for field, value in field_sources.items():
|
||
if not record.get(field) and value not in (None, "", "unknown"):
|
||
record[field] = dict(value) if field == "modelProfile" and isinstance(value, dict) else value
|
||
if task.get("logCosUrl") and not record.get("failureCategory"):
|
||
record["logCosUrl"] = task.get("logCosUrl")
|
||
|
||
def _enrich_failure_records(
|
||
self,
|
||
records: list[dict[str, Any]],
|
||
*,
|
||
workers: int = FAILURE_ENRICHMENT_WORKERS,
|
||
progress: Any = None,
|
||
) -> int:
|
||
if not records:
|
||
return 0
|
||
|
||
def inspect(record: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any] | None, str | None]:
|
||
try:
|
||
result = fetch_and_classify_failure_log(
|
||
str(record["logCosUrl"]),
|
||
task_context=record,
|
||
llm_classifier=self._failure_llm_classifier,
|
||
)
|
||
return record, result, None
|
||
except Exception as exc:
|
||
return record, None, f"{type(exc).__name__}: {exc}"
|
||
|
||
attempted = 0
|
||
classified = 0
|
||
errors = 0
|
||
with ThreadPoolExecutor(max_workers=min(max(1, int(workers)), len(records))) as executor:
|
||
futures = [executor.submit(inspect, record) for record in records]
|
||
for future in as_completed(futures):
|
||
record, result, error = future.result()
|
||
attempted += 1
|
||
record["failureEnrichmentAttempts"] = int(record.get("failureEnrichmentAttempts") or 0) + 1
|
||
if result is None:
|
||
errors += 1
|
||
record["failureEnrichmentError"] = error
|
||
if progress is not None:
|
||
progress(attempted, len(records), classified, errors)
|
||
continue
|
||
record.update(result)
|
||
if not record.get("framework") and result.get("failureDetectedFramework"):
|
||
record["framework"] = result.get("failureDetectedFramework")
|
||
record["failReason"] = result.get("failureCategory") or record.get("failReason")
|
||
record["failureEnrichmentError"] = None
|
||
record.pop("logCosUrl", None)
|
||
classified += 1
|
||
if progress is not None:
|
||
progress(attempted, len(records), classified, errors)
|
||
return attempted
|
||
|
||
def is_model_gpu_failed(
|
||
self,
|
||
model_id: str,
|
||
target_gpu: str,
|
||
*,
|
||
cooldown_hours: int = 24,
|
||
now: datetime | None = None,
|
||
) -> bool:
|
||
failed_at = self._failed_model_gpus.get((model_id, target_gpu))
|
||
if failed_at is None:
|
||
return False
|
||
now = now or utc_now()
|
||
return failed_at >= now - timedelta(hours=max(0, int(cooldown_hours)))
|
||
|
||
def get_stats_report(self) -> dict[str, Any]:
|
||
now_datetime = utc_now()
|
||
now = now_datetime.isoformat()
|
||
terminal = [r for r in self._records if r.get("outcome") in {"success", "failed"}]
|
||
|
||
gpu_groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||
framework_groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||
combo_groups: dict[tuple[str, str, str], list[dict[str, Any]]] = defaultdict(list)
|
||
profile_groups: dict[tuple[str, str, str, str, str], list[dict[str, Any]]] = defaultdict(list)
|
||
|
||
for record in terminal:
|
||
gpu = record.get("targetGpu") or "unknown"
|
||
fw = record.get("framework") or "unknown"
|
||
tt = record.get("taskType") or "unknown"
|
||
gpu_groups[gpu].append(record)
|
||
framework_groups[f"{fw}"].append(record)
|
||
combo_groups[(gpu, fw, tt)].append(record)
|
||
profile = record.get("modelProfile") or {}
|
||
model_type = str(profile.get("modelType") or "").strip()
|
||
if model_type:
|
||
quantization = str(profile.get("quantizationMethod") or "none").strip()
|
||
profile_groups[(gpu, fw, tt, model_type, quantization)].append(record)
|
||
|
||
gpu_summaries = {gpu: _summarize(records) for gpu, records in gpu_groups.items()}
|
||
framework_summaries = {fw: _summarize(records) for fw, records in framework_groups.items()}
|
||
combination_stats = {
|
||
f"{gpu}|{fw}|{tt}": {"targetGpu": gpu, "framework": fw, "taskType": tt, **_summarize(records)}
|
||
for (gpu, fw, tt), records in combo_groups.items()
|
||
}
|
||
recent_combination_stats: dict[str, dict[str, Any]] = {}
|
||
for (gpu, fw, tt), records in combo_groups.items():
|
||
recent = sorted(records, key=_outcome_record_timestamp, reverse=True)[:20]
|
||
consecutive_failures = _consecutive_attributable_failures(recent)
|
||
consecutive_platform_failures = _consecutive_platform_failures(recent)
|
||
last_terminal_at = None
|
||
if recent:
|
||
last_terminal_at = (
|
||
parse_datetime(recent[0].get("lastSyncTime"))
|
||
or parse_datetime(recent[0].get("submitTime"))
|
||
)
|
||
recent_combination_stats[f"{gpu}|{fw}|{tt}"] = {
|
||
"targetGpu": gpu,
|
||
"framework": fw,
|
||
"taskType": tt,
|
||
**_summarize(recent),
|
||
"consecutiveFailures": consecutive_failures,
|
||
"consecutivePlatformFailures": consecutive_platform_failures,
|
||
"lastPlatformFailureAt": _latest_platform_failure_at(recent),
|
||
"lastTerminalAt": last_terminal_at.isoformat() if last_terminal_at else None,
|
||
}
|
||
|
||
profile_combination_stats: dict[str, dict[str, Any]] = {}
|
||
recent_profile_combination_stats: dict[str, dict[str, Any]] = {}
|
||
for (gpu, fw, tt, model_type, quantization), records in profile_groups.items():
|
||
key = f"{gpu}|{fw}|{tt}|{model_type}|{quantization}"
|
||
profile_combination_stats[key] = {
|
||
"targetGpu": gpu,
|
||
"framework": fw,
|
||
"taskType": tt,
|
||
"modelType": model_type,
|
||
"quantizationMethod": quantization,
|
||
**_summarize(records),
|
||
}
|
||
recent = sorted(records, key=_outcome_record_timestamp, reverse=True)[:20]
|
||
consecutive_failures = _consecutive_attributable_failures(recent)
|
||
last_terminal_at = None
|
||
if recent:
|
||
last_terminal_at = (
|
||
parse_datetime(recent[0].get("lastSyncTime"))
|
||
or parse_datetime(recent[0].get("submitTime"))
|
||
)
|
||
recent_profile_combination_stats[key] = {
|
||
**profile_combination_stats[key],
|
||
**_summarize(recent),
|
||
"consecutiveFailures": consecutive_failures,
|
||
"lastTerminalAt": last_terminal_at.isoformat() if last_terminal_at else None,
|
||
}
|
||
|
||
warnings: list[str] = []
|
||
for gpu, summary in gpu_summaries.items():
|
||
if summary["decisionTotal"] >= 4 and summary["decisionFailureRate"] >= 0.5:
|
||
warnings.append(f"GPU {gpu} 本地统计失败率偏高（≥50%），建议重点关注。")
|
||
for key, stat in combination_stats.items():
|
||
if stat["decisionTotal"] >= 3 and stat["decisionFailureRate"] >= 0.6:
|
||
warnings.append(f"组合 {key} 近期失败集中，建议降低该 GPU+框架的提交优先级。")
|
||
|
||
pending_count = sum(1 for r in self._records if r.get("outcome") == "pending")
|
||
policy_cancelled_count = sum(
|
||
1 for record in self._records if record.get("outcome") == "policy_cancelled"
|
||
)
|
||
observed_gpu_memory: dict[str, float] = {}
|
||
for record in self._records:
|
||
gpu = str(record.get("targetGpu") or "")
|
||
try:
|
||
memory_gib = float(record.get("failureObservedGpuMemoryGiB"))
|
||
except (TypeError, ValueError):
|
||
continue
|
||
if gpu and 0 < memory_gib <= 1024:
|
||
previous = observed_gpu_memory.get(gpu)
|
||
observed_gpu_memory[gpu] = min(previous, memory_gib) if previous else memory_gib
|
||
|
||
architecture_block_ttl_days = _architecture_block_ttl_days()
|
||
architecture_blocks = _build_architecture_compatibility_blocks(
|
||
terminal,
|
||
now=now_datetime,
|
||
ttl_days=architecture_block_ttl_days,
|
||
)
|
||
architecture_blocks_by_gpu_framework: dict[str, int] = defaultdict(int)
|
||
for block in architecture_blocks.values():
|
||
combination = f"{block['targetGpu']}|{block['framework']}"
|
||
architecture_blocks_by_gpu_framework[combination] += 1
|
||
|
||
return {
|
||
"generatedAt": now,
|
||
"totalRecords": len(self._records),
|
||
"pendingRecords": pending_count,
|
||
"policyCancelledRecords": policy_cancelled_count,
|
||
"terminalRecords": len(terminal),
|
||
"gpuSummaries": gpu_summaries,
|
||
"frameworkSummaries": framework_summaries,
|
||
"combinationStats": combination_stats,
|
||
"recentCombinationStats": recent_combination_stats,
|
||
"profileCombinationStats": profile_combination_stats,
|
||
"recentProfileCombinationStats": recent_profile_combination_stats,
|
||
"architectureCompatibilityBlocks": architecture_blocks,
|
||
"architectureCompatibilitySummary": {
|
||
"activeBlockCount": len(architecture_blocks),
|
||
"ttlDays": architecture_block_ttl_days,
|
||
"byGpuFramework": dict(architecture_blocks_by_gpu_framework),
|
||
},
|
||
"observedGpuMemoryGiB": observed_gpu_memory,
|
||
"totals": _summarize(terminal),
|
||
"warnings": warnings,
|
||
}
|
||
|
||
def save(self) -> None:
|
||
local_records = list(self._records)
|
||
|
||
def merge(existing: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
merged = list(existing)
|
||
index_by_key = {_outcome_record_key(record): index for index, record in enumerate(merged)}
|
||
for record in local_records:
|
||
key = _outcome_record_key(record)
|
||
existing_index = index_by_key.get(key)
|
||
if existing_index is None:
|
||
index_by_key[key] = len(merged)
|
||
merged.append(record)
|
||
continue
|
||
merged[existing_index] = _prefer_newer_outcome(merged[existing_index], record)
|
||
return merged
|
||
|
||
self._records = update_jsonl(self.path, merge)
|
||
self._rebuild_indexes()
|
||
|
||
def _rebuild_failed_index(self) -> None:
|
||
self._failed_model_gpus.clear()
|
||
latest_by_combo: dict[tuple[str, str], tuple[datetime, dict[str, Any]]] = {}
|
||
for record in self._records:
|
||
# Infrastructure failures and our own policy cancellations neither
|
||
# clear nor create a model/GPU cooldown. Look through them to the
|
||
# latest attributable outcome.
|
||
if _is_platform_failure(record) or _is_policy_cancelled(record):
|
||
continue
|
||
model_id = record.get("modelId") or ""
|
||
target_gpu = record.get("targetGpu") or ""
|
||
event_time = parse_datetime(record.get("submitTime")) or parse_datetime(record.get("lastSyncTime"))
|
||
if not model_id or not target_gpu or event_time is None:
|
||
continue
|
||
key = (model_id, target_gpu)
|
||
current = latest_by_combo.get(key)
|
||
if current is None or event_time >= current[0]:
|
||
latest_by_combo[key] = (event_time, record)
|
||
|
||
for key, (event_time, record) in latest_by_combo.items():
|
||
if record.get("outcome") == "failed":
|
||
self._failed_model_gpus[key] = event_time
|
||
|
||
@staticmethod
|
||
def _update_record_from_task(record: dict[str, Any], task: dict[str, Any]) -> None:
|
||
record["status"] = task.get("status")
|
||
record["verifyResult"] = task.get("verifyResult")
|
||
record["lastSyncTime"] = _now_iso()
|
||
if task.get("logCosUrl"):
|
||
record["logCosUrl"] = task.get("logCosUrl")
|
||
status = str(task.get("status") or "").strip().lower()
|
||
if is_success(task):
|
||
record["outcome"] = "success"
|
||
record["failReason"] = None
|
||
if record.get("policyCancelled"):
|
||
record["policyCancellationResolvedAsSuccess"] = True
|
||
record["policyCancelled"] = False
|
||
elif record.get("policyCancelled"):
|
||
# A successful stop may be observed as waiting/running briefly before
|
||
# the platform publishes its terminal cancellation state. None of
|
||
# those intermediate states should become failure evidence.
|
||
record["outcome"] = "policy_cancelled"
|
||
record["failReason"] = None
|
||
elif is_failure(task):
|
||
record["outcome"] = "failed"
|
||
record["failReason"] = classify_failure(task)
|
||
else:
|
||
record["outcome"] = "pending"
|
||
|
||
@staticmethod
|
||
def _create_record_from_task(task: dict[str, Any]) -> dict[str, Any]:
|
||
create_time = (
|
||
parse_datetime(task.get("submitTime"))
|
||
or parse_datetime(task.get("createTime"))
|
||
or parse_datetime(task.get("updateTime"))
|
||
)
|
||
model_profile = task.get("modelProfile")
|
||
record: dict[str, Any] = {
|
||
"modelId": task.get("modelId") or task.get("model_id") or "",
|
||
"targetGpu": task.get("gpuType") or task.get("targetGpu") or "",
|
||
"framework": task.get("framework") or "",
|
||
"taskType": task.get("taskType") or task_type_from_history_task(task) or "",
|
||
"taskId": str(task.get("taskId")) if task.get("taskId") is not None else None,
|
||
"submitTime": create_time.isoformat() if create_time else _now_iso(),
|
||
"lastSyncTime": _now_iso(),
|
||
"status": task.get("status"),
|
||
"verifyResult": task.get("verifyResult"),
|
||
"outcome": "pending",
|
||
"failReason": None,
|
||
"logCosUrl": task.get("logCosUrl"),
|
||
"modelProfile": dict(model_profile) if isinstance(model_profile, dict) else {},
|
||
}
|
||
if is_success(task):
|
||
record["outcome"] = "success"
|
||
elif is_failure(task):
|
||
record["outcome"] = "failed"
|
||
record["failReason"] = classify_failure(task)
|
||
return record
|
||
|
||
|
||
def _summarize(records: list[dict[str, Any]]) -> dict[str, Any]:
|
||
total = len(records)
|
||
success_count = sum(1 for r in records if r.get("outcome") == "success")
|
||
failure_count = sum(1 for r in records if r.get("outcome") == "failed")
|
||
pending_count = sum(1 for r in records if r.get("outcome") == "pending")
|
||
attributable_failure_count = sum(
|
||
1 for record in records
|
||
if record.get("outcome") == "failed" and not _is_platform_failure(record)
|
||
)
|
||
platform_failure_count = failure_count - attributable_failure_count
|
||
decision_total = success_count + attributable_failure_count
|
||
|
||
failure_breakdown: dict[str, int] = defaultdict(int)
|
||
for r in records:
|
||
reason = r.get("failureCategory") or r.get("failReason")
|
||
if reason:
|
||
failure_breakdown[reason] += 1
|
||
|
||
return {
|
||
"total": total,
|
||
"successCount": success_count,
|
||
"failureCount": failure_count,
|
||
"attributableFailureCount": attributable_failure_count,
|
||
"platformFailureCount": platform_failure_count,
|
||
"decisionTotal": decision_total,
|
||
"pendingCount": pending_count,
|
||
"successRate": round(success_count / total, 4) if total > 0 else 0.0,
|
||
"failureRate": round(failure_count / total, 4) if total > 0 else 0.0,
|
||
"decisionSuccessRate": round(success_count / decision_total, 4) if decision_total > 0 else 0.0,
|
||
"decisionFailureRate": round(attributable_failure_count / decision_total, 4) if decision_total > 0 else 0.0,
|
||
"pendingRate": round(pending_count / total, 4) if total > 0 else 0.0,
|
||
"failureBreakdown": dict(failure_breakdown),
|
||
}
|
||
|
||
|
||
def _architecture_block_ttl_days() -> int:
|
||
raw = os.getenv("MODELHUB_ARCHITECTURE_BLOCK_TTL_DAYS")
|
||
if raw is None:
|
||
return DEFAULT_ARCHITECTURE_BLOCK_TTL_DAYS
|
||
try:
|
||
value = int(raw)
|
||
except ValueError:
|
||
return DEFAULT_ARCHITECTURE_BLOCK_TTL_DAYS
|
||
return min(365, max(1, value))
|
||
|
||
|
||
def _build_architecture_compatibility_blocks(
|
||
records: list[dict[str, Any]],
|
||
*,
|
||
now: datetime,
|
||
ttl_days: int,
|
||
) -> dict[str, dict[str, Any]]:
|
||
successes: dict[str, list[tuple[datetime, dict[str, Any]]]] = defaultdict(list)
|
||
failures: dict[str, list[tuple[datetime, dict[str, Any], dict[str, Any]]]] = defaultdict(list)
|
||
cutoff = now - timedelta(days=max(1, int(ttl_days)))
|
||
|
||
for record in records:
|
||
target_gpu = str(record.get("targetGpu") or "").strip()
|
||
framework = str(record.get("framework") or "").strip()
|
||
task_type = str(record.get("taskType") or "").strip()
|
||
event_time = (
|
||
parse_datetime(record.get("submitTime"))
|
||
or parse_datetime(record.get("lastSyncTime"))
|
||
)
|
||
if not target_gpu or not framework or not task_type or event_time is None:
|
||
continue
|
||
if record.get("outcome") == "success":
|
||
for profile in _stored_architecture_profiles(record, include_model_type=True):
|
||
key = architecture_compatibility_key(
|
||
target_gpu,
|
||
framework,
|
||
task_type,
|
||
profile["signature"],
|
||
)
|
||
if key is not None:
|
||
successes[key].append((event_time, record))
|
||
continue
|
||
if not _is_explicit_architecture_failure(record) or event_time < cutoff:
|
||
continue
|
||
for profile in _failure_architecture_profiles(record):
|
||
key = architecture_compatibility_key(
|
||
target_gpu,
|
||
framework,
|
||
task_type,
|
||
profile["signature"],
|
||
)
|
||
if key is not None:
|
||
failures[key].append((event_time, record, profile))
|
||
|
||
blocks: dict[str, dict[str, Any]] = {}
|
||
for key, failure_events in failures.items():
|
||
failure_events.sort(key=lambda item: item[0])
|
||
latest_failure_at, latest_failure, latest_profile = failure_events[-1]
|
||
success_events = successes.get(key) or []
|
||
latest_success_at = max((item[0] for item in success_events), default=None)
|
||
if latest_success_at is not None and latest_success_at >= latest_failure_at:
|
||
continue
|
||
effective_failures = [
|
||
item
|
||
for item in failure_events
|
||
if latest_success_at is None or item[0] > latest_success_at
|
||
]
|
||
expires_at = latest_failure_at + timedelta(days=max(1, int(ttl_days)))
|
||
if expires_at <= now:
|
||
continue
|
||
blocks[key] = {
|
||
"targetGpu": str(latest_failure.get("targetGpu") or ""),
|
||
"framework": str(latest_failure.get("framework") or ""),
|
||
"taskType": str(latest_failure.get("taskType") or ""),
|
||
"matchType": latest_profile["matchType"],
|
||
"architectureSignature": latest_profile["signature"],
|
||
"architectures": latest_profile["architectures"],
|
||
"modelType": latest_profile["modelType"],
|
||
"evidenceCount": len(effective_failures),
|
||
"latestFailureAt": latest_failure_at.isoformat(),
|
||
"latestSuccessfulAt": latest_success_at.isoformat() if latest_success_at else None,
|
||
"expiresAt": expires_at.isoformat(),
|
||
"sourceTaskIds": _bounded_unique(
|
||
record.get("taskId") for _, record, _ in reversed(effective_failures)
|
||
),
|
||
"sourceModelIds": _bounded_unique(
|
||
record.get("modelId") for _, record, _ in reversed(effective_failures)
|
||
),
|
||
}
|
||
return blocks
|
||
|
||
|
||
def _stored_architecture_profiles(
|
||
record: dict[str, Any],
|
||
*,
|
||
include_model_type: bool,
|
||
) -> list[dict[str, Any]]:
|
||
profile_data = record.get("modelProfile")
|
||
if not isinstance(profile_data, dict):
|
||
return []
|
||
if include_model_type:
|
||
return architecture_profiles(
|
||
profile_data.get("modelType"),
|
||
profile_data.get("architectures"),
|
||
)
|
||
profile = architecture_profile(
|
||
profile_data.get("modelType"),
|
||
profile_data.get("architectures"),
|
||
)
|
||
return [profile] if profile is not None else []
|
||
|
||
|
||
def _failure_architecture_profiles(record: dict[str, Any]) -> list[dict[str, Any]]:
|
||
profiles: list[dict[str, Any]] = []
|
||
unsupported_architectures = record.get("failureUnsupportedArchitectures")
|
||
if isinstance(unsupported_architectures, list) and unsupported_architectures:
|
||
profile = architecture_profile(None, unsupported_architectures)
|
||
if profile is not None:
|
||
profiles.append(profile)
|
||
unsupported_model_types = record.get("failureUnsupportedModelTypes")
|
||
if isinstance(unsupported_model_types, list):
|
||
for model_type in unsupported_model_types:
|
||
profile = architecture_profile(model_type, None)
|
||
if profile is not None:
|
||
profiles.append(profile)
|
||
if not profiles:
|
||
profiles.extend(_stored_architecture_profiles(record, include_model_type=False))
|
||
deduped: dict[str, dict[str, Any]] = {}
|
||
for profile in profiles:
|
||
deduped[profile["signature"]] = profile
|
||
return list(deduped.values())
|
||
|
||
|
||
def _is_explicit_architecture_failure(record: dict[str, Any]) -> bool:
|
||
return bool(
|
||
record.get("outcome") == "failed"
|
||
and record.get("failureCategory") == EXPLICIT_ARCHITECTURE_FAILURE_CATEGORY
|
||
and record.get("failureDeterministic") is True
|
||
and record.get("failureClassificationReason") == EXPLICIT_ARCHITECTURE_FAILURE_REASON
|
||
)
|
||
|
||
|
||
def _bounded_unique(values: Any, limit: int = 5) -> list[str]:
|
||
result: list[str] = []
|
||
for value in values:
|
||
rendered = str(value or "").strip()
|
||
if not rendered or rendered in result:
|
||
continue
|
||
result.append(rendered)
|
||
if len(result) >= limit:
|
||
break
|
||
return result
|
||
|
||
|
||
def _is_platform_failure(record: dict[str, Any]) -> bool:
|
||
if record.get("outcome") != "failed":
|
||
return False
|
||
category = str(record.get("failureCategory") or "").lower()
|
||
scope = str(record.get("failureScope") or "").lower()
|
||
return scope == "platform" or category.startswith("platform_")
|
||
|
||
|
||
def _is_policy_cancelled(record: dict[str, Any]) -> bool:
|
||
return bool(record.get("policyCancelled")) or record.get("outcome") == "policy_cancelled"
|
||
|
||
|
||
def _consecutive_attributable_failures(records: list[dict[str, Any]]) -> int:
|
||
count = 0
|
||
for record in records:
|
||
if record.get("outcome") == "success":
|
||
break
|
||
if record.get("outcome") == "failed" and not _is_platform_failure(record):
|
||
count += 1
|
||
return count
|
||
|
||
|
||
def _consecutive_platform_failures(records: list[dict[str, Any]]) -> int:
|
||
count = 0
|
||
for record in records:
|
||
if not _is_platform_failure(record):
|
||
break
|
||
count += 1
|
||
return count
|
||
|
||
|
||
def _latest_platform_failure_at(records: list[dict[str, Any]]) -> str | None:
|
||
for record in records:
|
||
if not _is_platform_failure(record):
|
||
continue
|
||
timestamp = parse_datetime(record.get("submitTime")) or parse_datetime(record.get("lastSyncTime"))
|
||
return timestamp.isoformat() if timestamp else None
|
||
return None
|
||
|
||
|
||
def _outcome_record_timestamp(record: dict[str, Any]) -> float:
|
||
timestamp = parse_datetime(record.get("submitTime")) or parse_datetime(record.get("lastSyncTime"))
|
||
return timestamp.timestamp() if timestamp else 0.0
|
||
|
||
|
||
def _outcome_record_key(record: dict[str, Any]) -> str:
|
||
task_id = record.get("taskId")
|
||
if task_id is not None:
|
||
return f"task:{task_id}"
|
||
return "fallback:{model}|{gpu}|{time}".format(
|
||
model=record.get("modelId") or "",
|
||
gpu=record.get("targetGpu") or "",
|
||
time=record.get("submitTime") or "",
|
||
)
|
||
|
||
|
||
def _outcome_version(record: dict[str, Any]) -> tuple[int, float, float]:
|
||
last_sync = parse_datetime(record.get("lastSyncTime"))
|
||
submit_time = parse_datetime(record.get("submitTime"))
|
||
outcome_rank = 1 if record.get("outcome") in {"success", "failed", "policy_cancelled"} else 0
|
||
return (
|
||
outcome_rank,
|
||
last_sync.timestamp() if last_sync else 0.0,
|
||
submit_time.timestamp() if submit_time else 0.0,
|
||
)
|
||
|
||
|
||
def _prefer_newer_outcome(existing: dict[str, Any], candidate: dict[str, Any]) -> dict[str, Any]:
|
||
if _outcome_version(candidate) >= _outcome_version(existing):
|
||
return candidate
|
||
return existing
|