CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Bootstrap architecture rules from task history

Browse Source
This commit is contained in:
CoolBoy
2026-08-12 08:41:07 +08:00
parent 7ec875563e
commit d8ed7fa1c2
12 changed files with 653 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
modelhub_submmit_api/outcome_tracker.py
View File

@@ -20,12 +20,14 @@ from failure_log_inspector import fetch_and_classify_failure_log
from history_stats import classify_failure, is_failure, is_success
from llm_classifier import LLMAssistedClassifier
from modelhub_client import ModelHubClient, ModelHubClientPool
from task_registry import task_type_from_history_task
DEFAULT_OUTCOMES_PATH = Path("outcomes/submissions.jsonl")
FAILURE_ENRICHMENT_LIMIT = 40
FAILURE_ENRICHMENT_WORKERS = 4
FAILURE_ENRICHMENT_MAX_ATTEMPTS = 3
TERMINAL_TASK_STATUSES = {"success", "failed", "error", "cancelled", "completed"}
def _now_iso() -> str:
@@ -188,7 +190,7 @@ class OutcomeTracker:
updated_count += 1
else:
status = str(task.get("status") or "").lower()
if status in {"success", "failed", "error", "cancelled", "completed"}:
if status in TERMINAL_TASK_STATUSES:
record = self._create_record_from_task(task)
self._records.append(record)
self._by_task_id[task_id] = record
@@ -197,10 +199,9 @@ class OutcomeTracker:
self._by_model_gpu[(model_id, target_gpu)].append(record)
updated_count += 1
# Retry a small bounded set of our own failed submissions. A stored
# framework is sufficient: fixed MODEL_NOT_SUPPORTED log text can yield
# an exact architecture/model_type even for older records that predate
# local modelProfile capture.
# Retry a small bounded set of our own failed submissions. Newer logs
# contain the target image, so enrichment can recover the framework even
# when the history API omits it.
candidate_ids = {id(record) for record in enrichment_candidates}
for record in self._records:
if len(enrichment_candidates) >= FAILURE_ENRICHMENT_LIMIT:
@@ -210,7 +211,6 @@ class OutcomeTracker:
if (
record.get("outcome") == "failed"
and record.get("logCosUrl")
and record.get("framework")
and not record.get("failureCategory")
and int(record.get("failureEnrichmentAttempts") or 0) < FAILURE_ENRICHMENT_MAX_ATTEMPTS
):
@@ -228,7 +228,185 @@ class OutcomeTracker:
return updated_count
def _enrich_failure_records(self, records: list[dict[str, Any]]) -> int:
def bootstrap_from_history_tasks(
self,
tasks: list[dict[str, Any]],
*,
task_contexts: dict[str, dict[str, Any]] | None = None,
enrichment_limit: int = 0,
enrichment_workers: int = 8,
log: Any = None,
) -> dict[str, int]:
"""Import terminal history and classify every usable historical failure."""
contexts = task_contexts if isinstance(task_contexts, dict) else {}
imported = 0
refreshed = 0
terminal_seen = 0
seen_task_ids: set[str] = set()
for raw_task in tasks:
if not isinstance(raw_task, dict):
continue
task_id_value = raw_task.get("taskId")
if task_id_value is None:
continue
task_id = str(task_id_value)
if not task_id or task_id in seen_task_ids:
continue
seen_task_ids.add(task_id)
status = str(raw_task.get("status") or "").strip().lower()
if status not in TERMINAL_TASK_STATUSES:
continue
terminal_seen += 1
task = self._enrich_history_task(raw_task, contexts.get(task_id))
existing = self._by_task_id.get(task_id)
if existing is None:
record = self._create_record_from_task(task)
self._records.append(record)
self._by_task_id[task_id] = record
self._by_model_gpu[(record["modelId"], record["targetGpu"])].append(record)
imported += 1
continue
self._merge_record_metadata(existing, task)
if existing.get("outcome") in {"pending", "policy_cancelled"}:
self._update_record_from_task(existing, task)
refreshed += 1
elif task.get("logCosUrl") and not existing.get("failureCategory"):
existing["logCosUrl"] = task.get("logCosUrl")
candidates = [
record
for record in self._records
if (
record.get("outcome") == "failed"
and record.get("logCosUrl")
and not record.get("failureCategory")
and int(record.get("failureEnrichmentAttempts") or 0)
< FAILURE_ENRICHMENT_MAX_ATTEMPTS
)
]
candidates.sort(key=_outcome_record_timestamp, reverse=True)
if enrichment_limit > 0:
candidates = candidates[:enrichment_limit]
self._last_sync_time = utc_now()
self._rebuild_failed_index()
if imported or refreshed:
self.save()
candidate_task_ids = [
str(record["taskId"])
for record in candidates
if record.get("taskId") is not None
]
enrichment_attempts = 0
classified_total = 0
errors_total = 0
batch_size = 200
total_candidates = len(candidate_task_ids)
for offset in range(0, total_candidates, batch_size):
batch_ids = candidate_task_ids[offset : offset + batch_size]
batch_records = [
self._by_task_id[task_id]
for task_id in batch_ids
if task_id in self._by_task_id
]
def progress(
completed: int,
_total: int,
classified: int,
errors: int,
) -> None:
if log is None:
return
global_completed = offset + completed
if global_completed == total_candidates or global_completed % 50 == 0:
log(
f"[architecture-bootstrap] failure_logs={global_completed}/{total_candidates} "
f"classified={classified_total + classified} "
f"errors={errors_total + errors}"
)
enrichment_attempts += self._enrich_failure_records(
batch_records,
workers=max(1, int(enrichment_workers)),
progress=progress,
)
classified_total += sum(
1 for record in batch_records if record.get("failureCategory")
)
errors_total += sum(
1 for record in batch_records if record.get("failureEnrichmentError")
)
self._rebuild_failed_index()
self.save()
explicit_architecture_failures = sum(
1 for record in self._records if _is_explicit_architecture_failure(record)
)
recovered_frameworks = sum(
1
for record in self._records
if record.get("failureDetectedFramework") and record.get("framework")
)
return {
"recordsScanned": len(seen_task_ids),
"terminalRecords": terminal_seen,
"importedRecords": imported,
"refreshedRecords": refreshed,
"eligibleFailureLogs": total_candidates,
"enrichmentAttempts": enrichment_attempts,
"explicitArchitectureFailures": explicit_architecture_failures,
"recoveredFrameworks": recovered_frameworks,
}
@staticmethod
def _enrich_history_task(
task: dict[str, Any],
context: dict[str, Any] | None,
) -> dict[str, Any]:
enriched = dict(task)
context = context if isinstance(context, dict) else {}
if not enriched.get("modelId") and context.get("modelId"):
enriched["modelId"] = context.get("modelId")
if not (enriched.get("gpuType") or enriched.get("targetGpu")) and context.get("targetGpu"):
enriched["targetGpu"] = context.get("targetGpu")
if not enriched.get("framework") and context.get("framework"):
enriched["framework"] = context.get("framework")
if not enriched.get("taskType"):
enriched["taskType"] = context.get("taskType") or task_type_from_history_task(enriched)
if context.get("modelProfile") and not enriched.get("modelProfile"):
enriched["modelProfile"] = context.get("modelProfile")
if context.get("submitTime") and not (
enriched.get("submitTime") or enriched.get("createTime")
):
enriched["submitTime"] = context.get("submitTime")
return enriched
@staticmethod
def _merge_record_metadata(record: dict[str, Any], task: dict[str, Any]) -> None:
field_sources = {
"modelId": task.get("modelId") or task.get("model_id"),
"targetGpu": task.get("gpuType") or task.get("targetGpu"),
"framework": task.get("framework"),
"taskType": task.get("taskType") or task_type_from_history_task(task),
"modelProfile": task.get("modelProfile"),
}
for field, value in field_sources.items():
if not record.get(field) and value not in (None, "", "unknown"):
record[field] = dict(value) if field == "modelProfile" and isinstance(value, dict) else value
if task.get("logCosUrl") and not record.get("failureCategory"):
record["logCosUrl"] = task.get("logCosUrl")
def _enrich_failure_records(
self,
records: list[dict[str, Any]],
*,
workers: int = FAILURE_ENRICHMENT_WORKERS,
progress: Any = None,
) -> int:
if not records:
return 0
@@ -244,19 +422,29 @@ class OutcomeTracker:
return record, None, f"{type(exc).__name__}: {exc}"
attempted = 0
with ThreadPoolExecutor(max_workers=min(FAILURE_ENRICHMENT_WORKERS, len(records))) as executor:
classified = 0
errors = 0
with ThreadPoolExecutor(max_workers=min(max(1, int(workers)), len(records))) as executor:
futures = [executor.submit(inspect, record) for record in records]
for future in as_completed(futures):
record, result, error = future.result()
attempted += 1
record["failureEnrichmentAttempts"] = int(record.get("failureEnrichmentAttempts") or 0) + 1
if result is None:
errors += 1
record["failureEnrichmentError"] = error
if progress is not None:
progress(attempted, len(records), classified, errors)
continue
record.update(result)
if not record.get("framework") and result.get("failureDetectedFramework"):
record["framework"] = result.get("failureDetectedFramework")
record["failReason"] = result.get("failureCategory") or record.get("failReason")
record["failureEnrichmentError"] = None
record.pop("logCosUrl", None)
classified += 1
if progress is not None:
progress(attempted, len(records), classified, errors)
return attempted
def is_model_gpu_failed(
@@ -438,7 +626,7 @@ class OutcomeTracker:
continue
model_id = record.get("modelId") or ""
target_gpu = record.get("targetGpu") or ""
event_time = parse_datetime(record.get("lastSyncTime")) or parse_datetime(record.get("submitTime"))
event_time = parse_datetime(record.get("submitTime")) or parse_datetime(record.get("lastSyncTime"))
if not model_id or not target_gpu or event_time is None:
continue
key = (model_id, target_gpu)
@@ -478,12 +666,17 @@ class OutcomeTracker:
@staticmethod
def _create_record_from_task(task: dict[str, Any]) -> dict[str, Any]:
create_time = parse_datetime(task.get("createTime"))
create_time = (
parse_datetime(task.get("submitTime"))
or parse_datetime(task.get("createTime"))
or parse_datetime(task.get("updateTime"))
)
model_profile = task.get("modelProfile")
record: dict[str, Any] = {
"modelId": task.get("modelId") or task.get("model_id") or "",
"targetGpu": task.get("gpuType") or task.get("targetGpu") or "",
"framework": task.get("framework") or "",
"taskType": task.get("taskType") or "",
"taskType": task.get("taskType") or task_type_from_history_task(task) or "",
"taskId": str(task.get("taskId")) if task.get("taskId") is not None else None,
"submitTime": create_time.isoformat() if create_time else _now_iso(),
"lastSyncTime": _now_iso(),
@@ -492,6 +685,7 @@ class OutcomeTracker:
"outcome": "pending",
"failReason": None,
"logCosUrl": task.get("logCosUrl"),
"modelProfile": dict(model_profile) if isinstance(model_profile, dict) else {},
}
if is_success(task):
record["outcome"] = "success"
@@ -725,13 +919,13 @@ def _latest_platform_failure_at(records: list[dict[str, Any]]) -> str | None:
for record in records:
if not _is_platform_failure(record):
continue
timestamp = parse_datetime(record.get("lastSyncTime")) or parse_datetime(record.get("submitTime"))
timestamp = parse_datetime(record.get("submitTime")) or parse_datetime(record.get("lastSyncTime"))
return timestamp.isoformat() if timestamp else None
return None
def _outcome_record_timestamp(record: dict[str, Any]) -> float:
timestamp = parse_datetime(record.get("lastSyncTime")) or parse_datetime(record.get("submitTime"))
timestamp = parse_datetime(record.get("submitTime")) or parse_datetime(record.get("lastSyncTime"))
return timestamp.timestamp() if timestamp else 0.0