CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: add failure-aware preflight and Qwen review

Browse Source
This commit is contained in:
CoolBoy
2026-08-10 21:44:42 +08:00
parent 5e47d9e695
commit 3d15f60284
21 changed files with 2672 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

187
modelhub_submmit_api/outcome_tracker.py
View File

@@ -1,16 +1,22 @@
from __future__ import annotations
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any
from common import append_jsonl, parse_datetime, read_jsonl, update_jsonl, utc_now
from failure_log_inspector import fetch_and_classify_failure_log
from history_stats import classify_failure, is_failure, is_success
from llm_classifier import LLMAssistedClassifier
from modelhub_client import ModelHubClient, ModelHubClientPool
DEFAULT_OUTCOMES_PATH = Path("outcomes/submissions.jsonl")
FAILURE_ENRICHMENT_LIMIT = 40
FAILURE_ENRICHMENT_WORKERS = 4
FAILURE_ENRICHMENT_MAX_ATTEMPTS = 3
def _now_iso() -> str:
@@ -24,6 +30,7 @@ class OutcomeTracker:
self._by_task_id: dict[str, dict[str, Any]] = {}
self._by_model_gpu: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
self._failed_model_gpus: dict[tuple[str, str], datetime] = {}
self._failure_llm_classifier: LLMAssistedClassifier | None = None
self._records = read_jsonl(self.path)
self._rebuild_indexes()
@@ -35,6 +42,9 @@ class OutcomeTracker:
]
self._last_sync_time: datetime = max(last_sync_times) if last_sync_times else utc_now() - timedelta(days=7)
def set_failure_llm_classifier(self, classifier: LLMAssistedClassifier | None) -> None:
self._failure_llm_classifier = classifier
def _rebuild_indexes(self) -> None:
self._by_task_id.clear()
self._by_model_gpu.clear()
@@ -56,6 +66,7 @@ class OutcomeTracker:
task_type: str,
task_id: str | None,
submit_time: str,
model_profile: dict[str, Any] | None = None,
) -> None:
record: dict[str, Any] = {
"modelId": model_id,
@@ -69,6 +80,7 @@ class OutcomeTracker:
"verifyResult": None,
"outcome": "pending",
"failReason": None,
"modelProfile": dict(model_profile or {}),
}
self._records.append(record)
if task_id:
@@ -89,6 +101,7 @@ class OutcomeTracker:
return 0
updated_count = 0
enrichment_candidates: list[dict[str, Any]] = []
for task in tasks:
task_id = str(task.get("taskId")) if task.get("taskId") is not None else None
if not task_id:
@@ -98,6 +111,8 @@ class OutcomeTracker:
if existing is not None:
if existing.get("outcome") == "pending":
self._update_record_from_task(existing, task)
if existing.get("outcome") == "failed" and existing.get("logCosUrl"):
enrichment_candidates.append(existing)
updated_count += 1
else:
status = str(task.get("status") or "").lower()
@@ -110,13 +125,68 @@ class OutcomeTracker:
self._by_model_gpu[(model_id, target_gpu)].append(record)
updated_count += 1
if updated_count:
# Retry a small bounded set of our own failed submissions. Historical
# tasks without the locally recorded framework/profile are intentionally
# excluded to avoid downloading thousands of old log archives at once.
candidate_ids = {id(record) for record in enrichment_candidates}
for record in self._records:
if len(enrichment_candidates) >= FAILURE_ENRICHMENT_LIMIT:
break
if id(record) in candidate_ids:
continue
if (
record.get("outcome") == "failed"
and record.get("logCosUrl")
and record.get("framework")
and record.get("modelProfile")
and not record.get("failureCategory")
and int(record.get("failureEnrichmentAttempts") or 0) < FAILURE_ENRICHMENT_MAX_ATTEMPTS
):
enrichment_candidates.append(record)
candidate_ids.add(id(record))
enrichment_attempts = self._enrich_failure_records(
enrichment_candidates[:FAILURE_ENRICHMENT_LIMIT]
)
if updated_count or enrichment_attempts:
self._last_sync_time = end
self._rebuild_failed_index()
self.save()
return updated_count
def _enrich_failure_records(self, records: list[dict[str, Any]]) -> int:
if not records:
return 0
def inspect(record: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any] | None, str | None]:
try:
result = fetch_and_classify_failure_log(
str(record["logCosUrl"]),
task_context=record,
llm_classifier=self._failure_llm_classifier,
)
return record, result, None
except Exception as exc:
return record, None, f"{type(exc).__name__}: {exc}"
attempted = 0
with ThreadPoolExecutor(max_workers=min(FAILURE_ENRICHMENT_WORKERS, len(records))) as executor:
futures = [executor.submit(inspect, record) for record in records]
for future in as_completed(futures):
record, result, error = future.result()
attempted += 1
record["failureEnrichmentAttempts"] = int(record.get("failureEnrichmentAttempts") or 0) + 1
if result is None:
record["failureEnrichmentError"] = error
continue
record.update(result)
record["failReason"] = result.get("failureCategory") or record.get("failReason")
record["failureEnrichmentError"] = None
record.pop("logCosUrl", None)
return attempted
def is_model_gpu_failed(
self,
model_id: str,
@@ -138,6 +208,7 @@ class OutcomeTracker:
gpu_groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
framework_groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
combo_groups: dict[tuple[str, str, str], list[dict[str, Any]]] = defaultdict(list)
profile_groups: dict[tuple[str, str, str, str, str], list[dict[str, Any]]] = defaultdict(list)
for record in terminal:
gpu = record.get("targetGpu") or "unknown"
@@ -146,6 +217,11 @@ class OutcomeTracker:
gpu_groups[gpu].append(record)
framework_groups[f"{fw}"].append(record)
combo_groups[(gpu, fw, tt)].append(record)
profile = record.get("modelProfile") or {}
model_type = str(profile.get("modelType") or "").strip()
if model_type:
quantization = str(profile.get("quantizationMethod") or "none").strip()
profile_groups[(gpu, fw, tt, model_type, quantization)].append(record)
gpu_summaries = {gpu: _summarize(records) for gpu, records in gpu_groups.items()}
framework_summaries = {fw: _summarize(records) for fw, records in framework_groups.items()}
@@ -156,11 +232,8 @@ class OutcomeTracker:
recent_combination_stats: dict[str, dict[str, Any]] = {}
for (gpu, fw, tt), records in combo_groups.items():
recent = sorted(records, key=_outcome_record_timestamp, reverse=True)[:20]
consecutive_failures = 0
for record in recent:
if record.get("outcome") != "failed":
break
consecutive_failures += 1
consecutive_failures = _consecutive_attributable_failures(recent)
consecutive_platform_failures = _consecutive_platform_failures(recent)
last_terminal_at = None
if recent:
last_terminal_at = (
@@ -173,18 +246,57 @@ class OutcomeTracker:
"taskType": tt,
**_summarize(recent),
"consecutiveFailures": consecutive_failures,
"consecutivePlatformFailures": consecutive_platform_failures,
"lastPlatformFailureAt": _latest_platform_failure_at(recent),
"lastTerminalAt": last_terminal_at.isoformat() if last_terminal_at else None,
}
profile_combination_stats: dict[str, dict[str, Any]] = {}
recent_profile_combination_stats: dict[str, dict[str, Any]] = {}
for (gpu, fw, tt, model_type, quantization), records in profile_groups.items():
key = f"{gpu}|{fw}|{tt}|{model_type}|{quantization}"
profile_combination_stats[key] = {
"targetGpu": gpu,
"framework": fw,
"taskType": tt,
"modelType": model_type,
"quantizationMethod": quantization,
**_summarize(records),
}
recent = sorted(records, key=_outcome_record_timestamp, reverse=True)[:20]
consecutive_failures = _consecutive_attributable_failures(recent)
last_terminal_at = None
if recent:
last_terminal_at = (
parse_datetime(recent[0].get("lastSyncTime"))
or parse_datetime(recent[0].get("submitTime"))
)
recent_profile_combination_stats[key] = {
**profile_combination_stats[key],
**_summarize(recent),
"consecutiveFailures": consecutive_failures,
"lastTerminalAt": last_terminal_at.isoformat() if last_terminal_at else None,
}
warnings: list[str] = []
for gpu, summary in gpu_summaries.items():
if summary["total"] >= 4 and summary["failureRate"] >= 0.5:
if summary["decisionTotal"] >= 4 and summary["decisionFailureRate"] >= 0.5:
warnings.append(f"GPU {gpu} 本地统计失败率偏高（≥50%），建议重点关注。")
for key, stat in combination_stats.items():
if stat["total"] >= 3 and stat["failureRate"] >= 0.6:
if stat["decisionTotal"] >= 3 and stat["decisionFailureRate"] >= 0.6:
warnings.append(f"组合 {key} 近期失败集中，建议降低该 GPU+框架的提交优先级。")
pending_count = sum(1 for r in self._records if r.get("outcome") == "pending")
observed_gpu_memory: dict[str, float] = {}
for record in self._records:
gpu = str(record.get("targetGpu") or "")
try:
memory_gib = float(record.get("failureObservedGpuMemoryGiB"))
except (TypeError, ValueError):
continue
if gpu and 0 < memory_gib <= 1024:
previous = observed_gpu_memory.get(gpu)
observed_gpu_memory[gpu] = min(previous, memory_gib) if previous else memory_gib
return {
"generatedAt": now,
@@ -195,6 +307,9 @@ class OutcomeTracker:
"frameworkSummaries": framework_summaries,
"combinationStats": combination_stats,
"recentCombinationStats": recent_combination_stats,
"profileCombinationStats": profile_combination_stats,
"recentProfileCombinationStats": recent_profile_combination_stats,
"observedGpuMemoryGiB": observed_gpu_memory,
"totals": _summarize(terminal),
"warnings": warnings,
}
@@ -222,6 +337,10 @@ class OutcomeTracker:
self._failed_model_gpus.clear()
latest_by_combo: dict[tuple[str, str], tuple[datetime, dict[str, Any]]] = {}
for record in self._records:
# Infrastructure failures neither clear nor create a model/GPU
# cooldown. Look through them to the latest attributable outcome.
if _is_platform_failure(record):
continue
model_id = record.get("modelId") or ""
target_gpu = record.get("targetGpu") or ""
event_time = parse_datetime(record.get("lastSyncTime")) or parse_datetime(record.get("submitTime"))
@@ -241,6 +360,8 @@ class OutcomeTracker:
record["status"] = task.get("status")
record["verifyResult"] = task.get("verifyResult")
record["lastSyncTime"] = _now_iso()
if task.get("logCosUrl"):
record["logCosUrl"] = task.get("logCosUrl")
if is_success(task):
record["outcome"] = "success"
record["failReason"] = None
@@ -265,6 +386,7 @@ class OutcomeTracker:
"verifyResult": task.get("verifyResult"),
"outcome": "pending",
"failReason": None,
"logCosUrl": task.get("logCosUrl"),
}
if is_success(task):
record["outcome"] = "success"
@@ -279,10 +401,16 @@ def _summarize(records: list[dict[str, Any]]) -> dict[str, Any]:
success_count = sum(1 for r in records if r.get("outcome") == "success")
failure_count = sum(1 for r in records if r.get("outcome") == "failed")
pending_count = sum(1 for r in records if r.get("outcome") == "pending")
attributable_failure_count = sum(
1 for record in records
if record.get("outcome") == "failed" and not _is_platform_failure(record)
)
platform_failure_count = failure_count - attributable_failure_count
decision_total = success_count + attributable_failure_count
failure_breakdown: dict[str, int] = defaultdict(int)
for r in records:
reason = r.get("failReason")
reason = r.get("failureCategory") or r.get("failReason")
if reason:
failure_breakdown[reason] += 1
@@ -290,14 +418,55 @@ def _summarize(records: list[dict[str, Any]]) -> dict[str, Any]:
"total": total,
"successCount": success_count,
"failureCount": failure_count,
"attributableFailureCount": attributable_failure_count,
"platformFailureCount": platform_failure_count,
"decisionTotal": decision_total,
"pendingCount": pending_count,
"successRate": round(success_count / total, 4) if total > 0 else 0.0,
"failureRate": round(failure_count / total, 4) if total > 0 else 0.0,
"decisionSuccessRate": round(success_count / decision_total, 4) if decision_total > 0 else 0.0,
"decisionFailureRate": round(attributable_failure_count / decision_total, 4) if decision_total > 0 else 0.0,
"pendingRate": round(pending_count / total, 4) if total > 0 else 0.0,
"failureBreakdown": dict(failure_breakdown),
}
def _is_platform_failure(record: dict[str, Any]) -> bool:
if record.get("outcome") != "failed":
return False
category = str(record.get("failureCategory") or "").lower()
scope = str(record.get("failureScope") or "").lower()
return scope == "platform" or category.startswith("platform_")
def _consecutive_attributable_failures(records: list[dict[str, Any]]) -> int:
count = 0
for record in records:
if record.get("outcome") == "success":
break
if record.get("outcome") == "failed" and not _is_platform_failure(record):
count += 1
return count
def _consecutive_platform_failures(records: list[dict[str, Any]]) -> int:
count = 0
for record in records:
if not _is_platform_failure(record):
break
count += 1
return count
def _latest_platform_failure_at(records: list[dict[str, Any]]) -> str | None:
for record in records:
if not _is_platform_failure(record):
continue
timestamp = parse_datetime(record.get("lastSyncTime")) or parse_datetime(record.get("submitTime"))
return timestamp.isoformat() if timestamp else None
return None
def _outcome_record_timestamp(record: dict[str, Any]) -> float:
timestamp = parse_datetime(record.get("lastSyncTime")) or parse_datetime(record.get("submitTime"))
return timestamp.timestamp() if timestamp else 0.0