CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

fix: bound unproven profile submissions and support single-worker takeover

Browse Source
This commit is contained in:
CoolBoy
2026-10-26 14:56:55 +08:00
parent 484a10607c
commit e38d5cf823
13 changed files with 376 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
modelhub_submmit_api/outcome_tracker.py
View File

@@ -20,7 +20,7 @@ from common import append_jsonl, parse_datetime, read_json, read_jsonl, update_j
from failure_log_inspector import fetch_and_classify_failure_log
from history_stats import classify_failure, is_failure, is_success
from llm_classifier import LLMAssistedClassifier
from modelhub_client import ModelHubClient, ModelHubClientPool
from modelhub_client import ModelHubClient, ModelHubClientPool, is_active_task
from task_registry import task_type_from_history_task
from common import runtime_instance_id
from version import AGENT_VERSION
@@ -361,6 +361,31 @@ class OutcomeTracker:
pending_gpus.add(target_gpu)
return len(pending_gpus)
def pending_records(self) -> list[dict[str, Any]]:
return [row for row in self._records if row.get("outcome") == "pending"]
def adopt_active_tasks(self, tasks, contexts=None) -> int:
"""Adopt by task ID without inventing ownership or resubmitting work."""
adopted = 0
for task in tasks:
task_id = str(task.get("taskId") or "")
if not task_id or task_id in self._by_task_id or not is_active_task(task):
continue
enriched = self._enrich_history_task(task, (contexts or {}).get(task_id))
record = {"taskId": task_id, "modelId": enriched.get("modelId"),
"targetGpu": enriched.get("gpuType"), "framework": enriched.get("framework"),
"taskType": enriched.get("taskType") or task_type_from_history_task(enriched),
"modelProfile": enriched.get("modelProfile") or {},
"status": enriched.get("status"), "outcome": "pending",
"submitTime": enriched.get("createTime") or enriched.get("submitTime"),
"lastSyncTime": _now_iso()}
self._records.append(record)
self._by_task_id[task_id] = record
self._by_model_gpu[(record["modelId"], record["targetGpu"])].append(record)
append_jsonl(self.path, record)
adopted += 1
return adopted
def mark_policy_cancellations(self, decisions: list[dict[str, Any]]) -> int:
"""Persist intentional task stops so they never become failure evidence."""
marked = 0
@@ -760,11 +785,14 @@ class OutcomeTracker:
framework_groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
combo_groups: dict[tuple[str, str, str], list[dict[str, Any]]] = defaultdict(list)
profile_groups: dict[tuple[str, str, str, str, str], list[dict[str, Any]]] = defaultdict(list)
encoding_groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
agent_groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
sized_profile_groups: dict[tuple[str, str, str, str, str, int], list[dict[str, Any]]] = defaultdict(list)
recent_combo_groups: dict[tuple[str, str, str], list[dict[str, Any]]] = defaultdict(list)
recent_profile_groups: dict[tuple[str, str, str, str, str], list[dict[str, Any]]] = defaultdict(list)
for record in terminal:
agent_groups[str(record.get("agentVersion") or "unknown")].append(record)
gpu = record.get("targetGpu") or "unknown"
fw = record.get("framework") or "unknown"
tt = record.get("taskType") or "unknown"
@@ -776,6 +804,8 @@ class OutcomeTracker:
if model_type:
quantization = str(profile.get("quantizationMethod") or "none").strip()
profile_groups[(gpu, fw, tt, model_type, quantization)].append(record)
if profile.get("quantizationSignature"):
encoding_groups[f"{gpu}|{fw}|{tt}|{model_type}|{profile['quantizationSignature']}"].append(record)
try:
load_bytes = max(1, int(profile.get("estimatedLoadBytes") or 0))
except (TypeError, ValueError):
@@ -914,6 +944,8 @@ class OutcomeTracker:
"combinationStats": combination_stats,
"recentCombinationStats": recent_combination_stats,
"profileCombinationStats": profile_combination_stats,
"encodingCombinationStats": {key: _summarize(rows) for key, rows in encoding_groups.items()},
"agentSummaries": {key: _summarize(rows) for key, rows in agent_groups.items()},
"recentProfileCombinationStats": recent_profile_combination_stats,
"sizedProfileCombinationStats": sized_profile_combination_stats,
"architectureCompatibilityBlocks": architecture_blocks,
@@ -944,6 +976,8 @@ class OutcomeTracker:
"frameworkSummaries",
"combinationStats",
"profileCombinationStats",
"encodingCombinationStats",
"agentSummaries",
"sizedProfileCombinationStats",
):
base_items = baseline.get(field) if isinstance(baseline.get(field), dict) else {}