CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: dynamically clean incompatible architectures

Browse Source
This commit is contained in:
CoolBoy
2026-08-12 08:19:53 +08:00
parent 615bcad124
commit 7ec875563e
12 changed files with 979 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
modelhub_submmit_api/outcome_tracker.py
View File

@@ -13,6 +13,7 @@ from architecture_compatibility import (
EXPLICIT_ARCHITECTURE_FAILURE_REASON,
architecture_compatibility_key,
architecture_profile,
architecture_profiles,
)
from common import append_jsonl, parse_datetime, read_jsonl, update_jsonl, utc_now
from failure_log_inspector import fetch_and_classify_failure_log
@@ -53,6 +54,26 @@ class OutcomeTracker:
def set_failure_llm_classifier(self, classifier: LLMAssistedClassifier | None) -> None:
self._failure_llm_classifier = classifier
def get_task_compatibility_contexts(self) -> dict[str, dict[str, Any]]:
"""Return locally known submit metadata needed for account queue cleanup."""
contexts: dict[str, dict[str, Any]] = {}
for task_id, record in self._by_task_id.items():
framework = str(record.get("framework") or "").strip()
task_type = str(record.get("taskType") or "").strip()
if not framework or not task_type:
continue
profile = record.get("modelProfile")
contexts[task_id] = {
"taskId": task_id,
"modelId": str(record.get("modelId") or ""),
"targetGpu": str(record.get("targetGpu") or ""),
"framework": framework,
"taskType": task_type,
"modelProfile": dict(profile) if isinstance(profile, dict) else {},
"submitTime": record.get("submitTime"),
}
return contexts
def _rebuild_indexes(self) -> None:
self._by_task_id.clear()
self._by_model_gpu.clear()
@@ -176,9 +197,10 @@ class OutcomeTracker:
self._by_model_gpu[(model_id, target_gpu)].append(record)
updated_count += 1
# Retry a small bounded set of our own failed submissions. Historical
# tasks without the locally recorded framework/profile are intentionally
# excluded to avoid downloading thousands of old log archives at once.
# Retry a small bounded set of our own failed submissions. A stored
# framework is sufficient: fixed MODEL_NOT_SUPPORTED log text can yield
# an exact architecture/model_type even for older records that predate
# local modelProfile capture.
candidate_ids = {id(record) for record in enrichment_candidates}
for record in self._records:
if len(enrichment_candidates) >= FAILURE_ENRICHMENT_LIMIT:
@@ -189,7 +211,6 @@ class OutcomeTracker:
record.get("outcome") == "failed"
and record.get("logCosUrl")
and record.get("framework")
and record.get("modelProfile")
and not record.get("failureCategory")
and int(record.get("failureEnrichmentAttempts") or 0) < FAILURE_ENRICHMENT_MAX_ATTEMPTS
):
@@ -537,36 +558,37 @@ def _build_architecture_compatibility_blocks(
cutoff = now - timedelta(days=max(1, int(ttl_days)))
for record in records:
profile_data = record.get("modelProfile")
if not isinstance(profile_data, dict):
continue
profile = architecture_profile(
profile_data.get("modelType"),
profile_data.get("architectures"),
)
if profile is None:
continue
target_gpu = str(record.get("targetGpu") or "").strip()
framework = str(record.get("framework") or "").strip()
task_type = str(record.get("taskType") or "").strip()
key = architecture_compatibility_key(
target_gpu,
framework,
task_type,
profile["signature"],
)
event_time = (
parse_datetime(record.get("submitTime"))
or parse_datetime(record.get("lastSyncTime"))
)
if key is None or event_time is None:
if not target_gpu or not framework or not task_type or event_time is None:
continue
if record.get("outcome") == "success":
successes[key].append((event_time, record))
for profile in _stored_architecture_profiles(record, include_model_type=True):
key = architecture_compatibility_key(
target_gpu,
framework,
task_type,
profile["signature"],
)
if key is not None:
successes[key].append((event_time, record))
continue
if not _is_explicit_architecture_failure(record) or event_time < cutoff:
continue
failures[key].append((event_time, record, profile))
for profile in _failure_architecture_profiles(record):
key = architecture_compatibility_key(
target_gpu,
framework,
task_type,
profile["signature"],
)
if key is not None:
failures[key].append((event_time, record, profile))
blocks: dict[str, dict[str, Any]] = {}
for key, failure_events in failures.items():
@@ -606,6 +628,47 @@ def _build_architecture_compatibility_blocks(
return blocks
def _stored_architecture_profiles(
record: dict[str, Any],
*,
include_model_type: bool,
) -> list[dict[str, Any]]:
profile_data = record.get("modelProfile")
if not isinstance(profile_data, dict):
return []
if include_model_type:
return architecture_profiles(
profile_data.get("modelType"),
profile_data.get("architectures"),
)
profile = architecture_profile(
profile_data.get("modelType"),
profile_data.get("architectures"),
)
return [profile] if profile is not None else []
def _failure_architecture_profiles(record: dict[str, Any]) -> list[dict[str, Any]]:
profiles: list[dict[str, Any]] = []
unsupported_architectures = record.get("failureUnsupportedArchitectures")
if isinstance(unsupported_architectures, list) and unsupported_architectures:
profile = architecture_profile(None, unsupported_architectures)
if profile is not None:
profiles.append(profile)
unsupported_model_types = record.get("failureUnsupportedModelTypes")
if isinstance(unsupported_model_types, list):
for model_type in unsupported_model_types:
profile = architecture_profile(model_type, None)
if profile is not None:
profiles.append(profile)
if not profiles:
profiles.extend(_stored_architecture_profiles(record, include_model_type=False))
deduped: dict[str, dict[str, Any]] = {}
for profile in profiles:
deduped[profile["signature"]] = profile
return list(deduped.values())
def _is_explicit_architecture_failure(record: dict[str, Any]) -> bool:
return bool(
record.get("outcome") == "failed"