501 lines
21 KiB
Python
501 lines
21 KiB
Python
from __future__ import annotations
|
||
|
||
from collections import defaultdict
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
from datetime import datetime, timedelta
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from common import append_jsonl, parse_datetime, read_jsonl, update_jsonl, utc_now
|
||
from failure_log_inspector import fetch_and_classify_failure_log
|
||
from history_stats import classify_failure, is_failure, is_success
|
||
from llm_classifier import LLMAssistedClassifier
|
||
from modelhub_client import ModelHubClient, ModelHubClientPool
|
||
|
||
|
||
DEFAULT_OUTCOMES_PATH = Path("outcomes/submissions.jsonl")
|
||
FAILURE_ENRICHMENT_LIMIT = 40
|
||
FAILURE_ENRICHMENT_WORKERS = 4
|
||
FAILURE_ENRICHMENT_MAX_ATTEMPTS = 3
|
||
|
||
|
||
def _now_iso() -> str:
|
||
return utc_now().isoformat()
|
||
|
||
|
||
class OutcomeTracker:
|
||
def __init__(self, path: Path | str) -> None:
|
||
self.path = Path(path)
|
||
self._records: list[dict[str, Any]] = []
|
||
self._by_task_id: dict[str, dict[str, Any]] = {}
|
||
self._by_model_gpu: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
|
||
self._failed_model_gpus: dict[tuple[str, str], datetime] = {}
|
||
self._failure_llm_classifier: LLMAssistedClassifier | None = None
|
||
|
||
self._records = read_jsonl(self.path)
|
||
self._rebuild_indexes()
|
||
|
||
last_sync_times = [
|
||
parse_datetime(record.get("lastSyncTime"))
|
||
for record in self._records
|
||
if record.get("lastSyncTime")
|
||
]
|
||
self._last_sync_time: datetime = max(last_sync_times) if last_sync_times else utc_now() - timedelta(days=7)
|
||
|
||
def set_failure_llm_classifier(self, classifier: LLMAssistedClassifier | None) -> None:
|
||
self._failure_llm_classifier = classifier
|
||
|
||
def _rebuild_indexes(self) -> None:
|
||
self._by_task_id.clear()
|
||
self._by_model_gpu.clear()
|
||
for record in self._records:
|
||
task_id = record.get("taskId")
|
||
if task_id:
|
||
self._by_task_id[str(task_id)] = record
|
||
model_id = record.get("modelId") or ""
|
||
target_gpu = record.get("targetGpu") or ""
|
||
self._by_model_gpu[(model_id, target_gpu)].append(record)
|
||
|
||
self._rebuild_failed_index()
|
||
|
||
def record_submission(
|
||
self,
|
||
model_id: str,
|
||
target_gpu: str,
|
||
framework: str,
|
||
task_type: str,
|
||
task_id: str | None,
|
||
submit_time: str,
|
||
model_profile: dict[str, Any] | None = None,
|
||
) -> None:
|
||
record: dict[str, Any] = {
|
||
"modelId": model_id,
|
||
"targetGpu": target_gpu,
|
||
"framework": framework,
|
||
"taskType": task_type,
|
||
"taskId": task_id,
|
||
"submitTime": submit_time,
|
||
"lastSyncTime": None,
|
||
"status": "pending",
|
||
"verifyResult": None,
|
||
"outcome": "pending",
|
||
"failReason": None,
|
||
"modelProfile": dict(model_profile or {}),
|
||
}
|
||
self._records.append(record)
|
||
if task_id:
|
||
self._by_task_id[task_id] = record
|
||
self._by_model_gpu[(model_id, target_gpu)].append(record)
|
||
append_jsonl(self.path, record)
|
||
|
||
def sync_from_api(self, client: ModelHubClient | ModelHubClientPool) -> int:
|
||
try:
|
||
begin = self._last_sync_time
|
||
end = utc_now()
|
||
# Use fanout for pools to sync outcomes across all accounts
|
||
list_kwargs: dict[str, Any] = {"begin_time": begin, "end_time": end, "page_size": 100, "only_mine": True}
|
||
if isinstance(client, ModelHubClientPool):
|
||
list_kwargs["_fanout_all"] = True
|
||
tasks = client.list_tasks(**list_kwargs)
|
||
except Exception:
|
||
return 0
|
||
|
||
updated_count = 0
|
||
enrichment_candidates: list[dict[str, Any]] = []
|
||
for task in tasks:
|
||
task_id = str(task.get("taskId")) if task.get("taskId") is not None else None
|
||
if not task_id:
|
||
continue
|
||
|
||
existing = self._by_task_id.get(task_id)
|
||
if existing is not None:
|
||
if existing.get("outcome") == "pending":
|
||
self._update_record_from_task(existing, task)
|
||
if existing.get("outcome") == "failed" and existing.get("logCosUrl"):
|
||
enrichment_candidates.append(existing)
|
||
updated_count += 1
|
||
else:
|
||
status = str(task.get("status") or "").lower()
|
||
if status in {"success", "failed", "error", "cancelled", "completed"}:
|
||
record = self._create_record_from_task(task)
|
||
self._records.append(record)
|
||
self._by_task_id[task_id] = record
|
||
model_id = record["modelId"]
|
||
target_gpu = record["targetGpu"]
|
||
self._by_model_gpu[(model_id, target_gpu)].append(record)
|
||
updated_count += 1
|
||
|
||
# Retry a small bounded set of our own failed submissions. Historical
|
||
# tasks without the locally recorded framework/profile are intentionally
|
||
# excluded to avoid downloading thousands of old log archives at once.
|
||
candidate_ids = {id(record) for record in enrichment_candidates}
|
||
for record in self._records:
|
||
if len(enrichment_candidates) >= FAILURE_ENRICHMENT_LIMIT:
|
||
break
|
||
if id(record) in candidate_ids:
|
||
continue
|
||
if (
|
||
record.get("outcome") == "failed"
|
||
and record.get("logCosUrl")
|
||
and record.get("framework")
|
||
and record.get("modelProfile")
|
||
and not record.get("failureCategory")
|
||
and int(record.get("failureEnrichmentAttempts") or 0) < FAILURE_ENRICHMENT_MAX_ATTEMPTS
|
||
):
|
||
enrichment_candidates.append(record)
|
||
candidate_ids.add(id(record))
|
||
|
||
enrichment_attempts = self._enrich_failure_records(
|
||
enrichment_candidates[:FAILURE_ENRICHMENT_LIMIT]
|
||
)
|
||
|
||
if updated_count or enrichment_attempts:
|
||
self._last_sync_time = end
|
||
self._rebuild_failed_index()
|
||
self.save()
|
||
|
||
return updated_count
|
||
|
||
def _enrich_failure_records(self, records: list[dict[str, Any]]) -> int:
|
||
if not records:
|
||
return 0
|
||
|
||
def inspect(record: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any] | None, str | None]:
|
||
try:
|
||
result = fetch_and_classify_failure_log(
|
||
str(record["logCosUrl"]),
|
||
task_context=record,
|
||
llm_classifier=self._failure_llm_classifier,
|
||
)
|
||
return record, result, None
|
||
except Exception as exc:
|
||
return record, None, f"{type(exc).__name__}: {exc}"
|
||
|
||
attempted = 0
|
||
with ThreadPoolExecutor(max_workers=min(FAILURE_ENRICHMENT_WORKERS, len(records))) as executor:
|
||
futures = [executor.submit(inspect, record) for record in records]
|
||
for future in as_completed(futures):
|
||
record, result, error = future.result()
|
||
attempted += 1
|
||
record["failureEnrichmentAttempts"] = int(record.get("failureEnrichmentAttempts") or 0) + 1
|
||
if result is None:
|
||
record["failureEnrichmentError"] = error
|
||
continue
|
||
record.update(result)
|
||
record["failReason"] = result.get("failureCategory") or record.get("failReason")
|
||
record["failureEnrichmentError"] = None
|
||
record.pop("logCosUrl", None)
|
||
return attempted
|
||
|
||
def is_model_gpu_failed(
|
||
self,
|
||
model_id: str,
|
||
target_gpu: str,
|
||
*,
|
||
cooldown_hours: int = 24,
|
||
now: datetime | None = None,
|
||
) -> bool:
|
||
failed_at = self._failed_model_gpus.get((model_id, target_gpu))
|
||
if failed_at is None:
|
||
return False
|
||
now = now or utc_now()
|
||
return failed_at >= now - timedelta(hours=max(0, int(cooldown_hours)))
|
||
|
||
def get_stats_report(self) -> dict[str, Any]:
|
||
now = _now_iso()
|
||
terminal = [r for r in self._records if r.get("outcome") in {"success", "failed"}]
|
||
|
||
gpu_groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||
framework_groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||
combo_groups: dict[tuple[str, str, str], list[dict[str, Any]]] = defaultdict(list)
|
||
profile_groups: dict[tuple[str, str, str, str, str], list[dict[str, Any]]] = defaultdict(list)
|
||
|
||
for record in terminal:
|
||
gpu = record.get("targetGpu") or "unknown"
|
||
fw = record.get("framework") or "unknown"
|
||
tt = record.get("taskType") or "unknown"
|
||
gpu_groups[gpu].append(record)
|
||
framework_groups[f"{fw}"].append(record)
|
||
combo_groups[(gpu, fw, tt)].append(record)
|
||
profile = record.get("modelProfile") or {}
|
||
model_type = str(profile.get("modelType") or "").strip()
|
||
if model_type:
|
||
quantization = str(profile.get("quantizationMethod") or "none").strip()
|
||
profile_groups[(gpu, fw, tt, model_type, quantization)].append(record)
|
||
|
||
gpu_summaries = {gpu: _summarize(records) for gpu, records in gpu_groups.items()}
|
||
framework_summaries = {fw: _summarize(records) for fw, records in framework_groups.items()}
|
||
combination_stats = {
|
||
f"{gpu}|{fw}|{tt}": {"targetGpu": gpu, "framework": fw, "taskType": tt, **_summarize(records)}
|
||
for (gpu, fw, tt), records in combo_groups.items()
|
||
}
|
||
recent_combination_stats: dict[str, dict[str, Any]] = {}
|
||
for (gpu, fw, tt), records in combo_groups.items():
|
||
recent = sorted(records, key=_outcome_record_timestamp, reverse=True)[:20]
|
||
consecutive_failures = _consecutive_attributable_failures(recent)
|
||
consecutive_platform_failures = _consecutive_platform_failures(recent)
|
||
last_terminal_at = None
|
||
if recent:
|
||
last_terminal_at = (
|
||
parse_datetime(recent[0].get("lastSyncTime"))
|
||
or parse_datetime(recent[0].get("submitTime"))
|
||
)
|
||
recent_combination_stats[f"{gpu}|{fw}|{tt}"] = {
|
||
"targetGpu": gpu,
|
||
"framework": fw,
|
||
"taskType": tt,
|
||
**_summarize(recent),
|
||
"consecutiveFailures": consecutive_failures,
|
||
"consecutivePlatformFailures": consecutive_platform_failures,
|
||
"lastPlatformFailureAt": _latest_platform_failure_at(recent),
|
||
"lastTerminalAt": last_terminal_at.isoformat() if last_terminal_at else None,
|
||
}
|
||
|
||
profile_combination_stats: dict[str, dict[str, Any]] = {}
|
||
recent_profile_combination_stats: dict[str, dict[str, Any]] = {}
|
||
for (gpu, fw, tt, model_type, quantization), records in profile_groups.items():
|
||
key = f"{gpu}|{fw}|{tt}|{model_type}|{quantization}"
|
||
profile_combination_stats[key] = {
|
||
"targetGpu": gpu,
|
||
"framework": fw,
|
||
"taskType": tt,
|
||
"modelType": model_type,
|
||
"quantizationMethod": quantization,
|
||
**_summarize(records),
|
||
}
|
||
recent = sorted(records, key=_outcome_record_timestamp, reverse=True)[:20]
|
||
consecutive_failures = _consecutive_attributable_failures(recent)
|
||
last_terminal_at = None
|
||
if recent:
|
||
last_terminal_at = (
|
||
parse_datetime(recent[0].get("lastSyncTime"))
|
||
or parse_datetime(recent[0].get("submitTime"))
|
||
)
|
||
recent_profile_combination_stats[key] = {
|
||
**profile_combination_stats[key],
|
||
**_summarize(recent),
|
||
"consecutiveFailures": consecutive_failures,
|
||
"lastTerminalAt": last_terminal_at.isoformat() if last_terminal_at else None,
|
||
}
|
||
|
||
warnings: list[str] = []
|
||
for gpu, summary in gpu_summaries.items():
|
||
if summary["decisionTotal"] >= 4 and summary["decisionFailureRate"] >= 0.5:
|
||
warnings.append(f"GPU {gpu} 本地统计失败率偏高（≥50%），建议重点关注。")
|
||
for key, stat in combination_stats.items():
|
||
if stat["decisionTotal"] >= 3 and stat["decisionFailureRate"] >= 0.6:
|
||
warnings.append(f"组合 {key} 近期失败集中，建议降低该 GPU+框架的提交优先级。")
|
||
|
||
pending_count = sum(1 for r in self._records if r.get("outcome") == "pending")
|
||
observed_gpu_memory: dict[str, float] = {}
|
||
for record in self._records:
|
||
gpu = str(record.get("targetGpu") or "")
|
||
try:
|
||
memory_gib = float(record.get("failureObservedGpuMemoryGiB"))
|
||
except (TypeError, ValueError):
|
||
continue
|
||
if gpu and 0 < memory_gib <= 1024:
|
||
previous = observed_gpu_memory.get(gpu)
|
||
observed_gpu_memory[gpu] = min(previous, memory_gib) if previous else memory_gib
|
||
|
||
return {
|
||
"generatedAt": now,
|
||
"totalRecords": len(self._records),
|
||
"pendingRecords": pending_count,
|
||
"terminalRecords": len(terminal),
|
||
"gpuSummaries": gpu_summaries,
|
||
"frameworkSummaries": framework_summaries,
|
||
"combinationStats": combination_stats,
|
||
"recentCombinationStats": recent_combination_stats,
|
||
"profileCombinationStats": profile_combination_stats,
|
||
"recentProfileCombinationStats": recent_profile_combination_stats,
|
||
"observedGpuMemoryGiB": observed_gpu_memory,
|
||
"totals": _summarize(terminal),
|
||
"warnings": warnings,
|
||
}
|
||
|
||
def save(self) -> None:
|
||
local_records = list(self._records)
|
||
|
||
def merge(existing: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
merged = list(existing)
|
||
index_by_key = {_outcome_record_key(record): index for index, record in enumerate(merged)}
|
||
for record in local_records:
|
||
key = _outcome_record_key(record)
|
||
existing_index = index_by_key.get(key)
|
||
if existing_index is None:
|
||
index_by_key[key] = len(merged)
|
||
merged.append(record)
|
||
continue
|
||
merged[existing_index] = _prefer_newer_outcome(merged[existing_index], record)
|
||
return merged
|
||
|
||
self._records = update_jsonl(self.path, merge)
|
||
self._rebuild_indexes()
|
||
|
||
def _rebuild_failed_index(self) -> None:
|
||
self._failed_model_gpus.clear()
|
||
latest_by_combo: dict[tuple[str, str], tuple[datetime, dict[str, Any]]] = {}
|
||
for record in self._records:
|
||
# Infrastructure failures neither clear nor create a model/GPU
|
||
# cooldown. Look through them to the latest attributable outcome.
|
||
if _is_platform_failure(record):
|
||
continue
|
||
model_id = record.get("modelId") or ""
|
||
target_gpu = record.get("targetGpu") or ""
|
||
event_time = parse_datetime(record.get("lastSyncTime")) or parse_datetime(record.get("submitTime"))
|
||
if not model_id or not target_gpu or event_time is None:
|
||
continue
|
||
key = (model_id, target_gpu)
|
||
current = latest_by_combo.get(key)
|
||
if current is None or event_time >= current[0]:
|
||
latest_by_combo[key] = (event_time, record)
|
||
|
||
for key, (event_time, record) in latest_by_combo.items():
|
||
if record.get("outcome") == "failed":
|
||
self._failed_model_gpus[key] = event_time
|
||
|
||
@staticmethod
|
||
def _update_record_from_task(record: dict[str, Any], task: dict[str, Any]) -> None:
|
||
record["status"] = task.get("status")
|
||
record["verifyResult"] = task.get("verifyResult")
|
||
record["lastSyncTime"] = _now_iso()
|
||
if task.get("logCosUrl"):
|
||
record["logCosUrl"] = task.get("logCosUrl")
|
||
if is_success(task):
|
||
record["outcome"] = "success"
|
||
record["failReason"] = None
|
||
elif is_failure(task):
|
||
record["outcome"] = "failed"
|
||
record["failReason"] = classify_failure(task)
|
||
else:
|
||
record["outcome"] = "pending"
|
||
|
||
@staticmethod
|
||
def _create_record_from_task(task: dict[str, Any]) -> dict[str, Any]:
|
||
create_time = parse_datetime(task.get("createTime"))
|
||
record: dict[str, Any] = {
|
||
"modelId": task.get("modelId") or task.get("model_id") or "",
|
||
"targetGpu": task.get("gpuType") or task.get("targetGpu") or "",
|
||
"framework": task.get("framework") or "",
|
||
"taskType": task.get("taskType") or "",
|
||
"taskId": str(task.get("taskId")) if task.get("taskId") is not None else None,
|
||
"submitTime": create_time.isoformat() if create_time else _now_iso(),
|
||
"lastSyncTime": _now_iso(),
|
||
"status": task.get("status"),
|
||
"verifyResult": task.get("verifyResult"),
|
||
"outcome": "pending",
|
||
"failReason": None,
|
||
"logCosUrl": task.get("logCosUrl"),
|
||
}
|
||
if is_success(task):
|
||
record["outcome"] = "success"
|
||
elif is_failure(task):
|
||
record["outcome"] = "failed"
|
||
record["failReason"] = classify_failure(task)
|
||
return record
|
||
|
||
|
||
def _summarize(records: list[dict[str, Any]]) -> dict[str, Any]:
|
||
total = len(records)
|
||
success_count = sum(1 for r in records if r.get("outcome") == "success")
|
||
failure_count = sum(1 for r in records if r.get("outcome") == "failed")
|
||
pending_count = sum(1 for r in records if r.get("outcome") == "pending")
|
||
attributable_failure_count = sum(
|
||
1 for record in records
|
||
if record.get("outcome") == "failed" and not _is_platform_failure(record)
|
||
)
|
||
platform_failure_count = failure_count - attributable_failure_count
|
||
decision_total = success_count + attributable_failure_count
|
||
|
||
failure_breakdown: dict[str, int] = defaultdict(int)
|
||
for r in records:
|
||
reason = r.get("failureCategory") or r.get("failReason")
|
||
if reason:
|
||
failure_breakdown[reason] += 1
|
||
|
||
return {
|
||
"total": total,
|
||
"successCount": success_count,
|
||
"failureCount": failure_count,
|
||
"attributableFailureCount": attributable_failure_count,
|
||
"platformFailureCount": platform_failure_count,
|
||
"decisionTotal": decision_total,
|
||
"pendingCount": pending_count,
|
||
"successRate": round(success_count / total, 4) if total > 0 else 0.0,
|
||
"failureRate": round(failure_count / total, 4) if total > 0 else 0.0,
|
||
"decisionSuccessRate": round(success_count / decision_total, 4) if decision_total > 0 else 0.0,
|
||
"decisionFailureRate": round(attributable_failure_count / decision_total, 4) if decision_total > 0 else 0.0,
|
||
"pendingRate": round(pending_count / total, 4) if total > 0 else 0.0,
|
||
"failureBreakdown": dict(failure_breakdown),
|
||
}
|
||
|
||
|
||
def _is_platform_failure(record: dict[str, Any]) -> bool:
|
||
if record.get("outcome") != "failed":
|
||
return False
|
||
category = str(record.get("failureCategory") or "").lower()
|
||
scope = str(record.get("failureScope") or "").lower()
|
||
return scope == "platform" or category.startswith("platform_")
|
||
|
||
|
||
def _consecutive_attributable_failures(records: list[dict[str, Any]]) -> int:
|
||
count = 0
|
||
for record in records:
|
||
if record.get("outcome") == "success":
|
||
break
|
||
if record.get("outcome") == "failed" and not _is_platform_failure(record):
|
||
count += 1
|
||
return count
|
||
|
||
|
||
def _consecutive_platform_failures(records: list[dict[str, Any]]) -> int:
|
||
count = 0
|
||
for record in records:
|
||
if not _is_platform_failure(record):
|
||
break
|
||
count += 1
|
||
return count
|
||
|
||
|
||
def _latest_platform_failure_at(records: list[dict[str, Any]]) -> str | None:
|
||
for record in records:
|
||
if not _is_platform_failure(record):
|
||
continue
|
||
timestamp = parse_datetime(record.get("lastSyncTime")) or parse_datetime(record.get("submitTime"))
|
||
return timestamp.isoformat() if timestamp else None
|
||
return None
|
||
|
||
|
||
def _outcome_record_timestamp(record: dict[str, Any]) -> float:
|
||
timestamp = parse_datetime(record.get("lastSyncTime")) or parse_datetime(record.get("submitTime"))
|
||
return timestamp.timestamp() if timestamp else 0.0
|
||
|
||
|
||
def _outcome_record_key(record: dict[str, Any]) -> str:
|
||
task_id = record.get("taskId")
|
||
if task_id is not None:
|
||
return f"task:{task_id}"
|
||
return "fallback:{model}|{gpu}|{time}".format(
|
||
model=record.get("modelId") or "",
|
||
gpu=record.get("targetGpu") or "",
|
||
time=record.get("submitTime") or "",
|
||
)
|
||
|
||
|
||
def _outcome_version(record: dict[str, Any]) -> tuple[int, float, float]:
|
||
last_sync = parse_datetime(record.get("lastSyncTime"))
|
||
submit_time = parse_datetime(record.get("submitTime"))
|
||
outcome_rank = 1 if record.get("outcome") in {"success", "failed"} else 0
|
||
return (
|
||
outcome_rank,
|
||
last_sync.timestamp() if last_sync else 0.0,
|
||
submit_time.timestamp() if submit_time else 0.0,
|
||
)
|
||
|
||
|
||
def _prefer_newer_outcome(existing: dict[str, Any], candidate: dict[str, Any]) -> dict[str, Any]:
|
||
if _outcome_version(candidate) >= _outcome_version(existing):
|
||
return candidate
|
||
return existing
|