CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: add tiered durable state and memory bounds

Browse Source
This commit is contained in:
CoolBoy
2026-08-22 14:15:25 +08:00
parent 6eb7ded984
commit 5b1ec4d3eb
11 changed files with 722 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

322
modelhub_submmit_api/outcome_tracker.py
View File

@@ -3,9 +3,12 @@ from __future__ import annotations
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timedelta
import gzip
import json
import os
from pathlib import Path
from typing import Any
import uuid
from architecture_compatibility import (
DEFAULT_ARCHITECTURE_BLOCK_TTL_DAYS,
@@ -15,7 +18,7 @@ from architecture_compatibility import (
architecture_profile,
architecture_profiles,
)
from common import append_jsonl, parse_datetime, read_jsonl, update_jsonl, utc_now
from common import append_jsonl, parse_datetime, read_json, read_jsonl, update_jsonl, utc_now, write_json, write_jsonl
from failure_log_inspector import fetch_and_classify_failure_log
from history_stats import classify_failure, is_failure, is_success
from llm_classifier import LLMAssistedClassifier
@@ -24,6 +27,12 @@ from task_registry import task_type_from_history_task
DEFAULT_OUTCOMES_PATH = Path("outcomes/submissions.jsonl")
DEFAULT_OUTCOME_CHECKPOINT_PATH = Path(".modelhub_state/outcome_checkpoint.json")
DEFAULT_RECENT_OUTCOMES_PATH = Path(".modelhub_state/recent_outcomes.jsonl")
DEFAULT_ARCHIVE_PENDING_DIR = Path(".modelhub_state/archive_pending/outcomes")
OUTCOME_CHECKPOINT_VERSION = 1
DEFAULT_OUTCOME_COMPACT_THRESHOLD = 2000
DEFAULT_RECENT_OUTCOME_LIMIT = 1000
FAILURE_ENRICHMENT_LIMIT = 40
FAILURE_ENRICHMENT_WORKERS = 4
FAILURE_ENRICHMENT_MAX_ATTEMPTS = 3
@@ -35,9 +44,21 @@ def _now_iso() -> str:
class OutcomeTracker:
def __init__(self, path: Path | str) -> None:
def __init__(
self,
path: Path | str,
*,
checkpoint_path: Path | str = DEFAULT_OUTCOME_CHECKPOINT_PATH,
recent_path: Path | str = DEFAULT_RECENT_OUTCOMES_PATH,
archive_pending_dir: Path | str = DEFAULT_ARCHIVE_PENDING_DIR,
) -> None:
self.path = Path(path)
self.checkpoint_path = Path(checkpoint_path)
self.recent_path = Path(recent_path)
self.archive_pending_dir = Path(archive_pending_dir)
self._records: list[dict[str, Any]] = []
self._recent_records: list[dict[str, Any]] = read_jsonl(self.recent_path)
self._checkpoint: dict[str, Any] = self._load_checkpoint()
self._by_task_id: dict[str, dict[str, Any]] = {}
self._by_model_gpu: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
self._failed_model_gpus: dict[tuple[str, str], datetime] = {}
@@ -46,13 +67,131 @@ class OutcomeTracker:
self._records = read_jsonl(self.path)
self._rebuild_indexes()
self._compact_if_needed(force=not bool(self._checkpoint) and len(self._records) > self._compact_threshold())
last_sync_times = [
parse_datetime(record.get("lastSyncTime"))
for record in self._records
for record in [*self._records, *self._recent_records]
if record.get("lastSyncTime")
]
checkpoint_sync = parse_datetime(self._checkpoint.get("lastSyncTime"))
if checkpoint_sync is not None:
last_sync_times.append(checkpoint_sync)
self._last_sync_time: datetime = max(last_sync_times) if last_sync_times else utc_now() - timedelta(days=7)
def _compact_threshold(self) -> int:
raw = os.getenv("MODELHUB_AGENT_OUTCOME_COMPACT_THRESHOLD", str(DEFAULT_OUTCOME_COMPACT_THRESHOLD))
try:
return max(500, int(raw))
except ValueError:
return DEFAULT_OUTCOME_COMPACT_THRESHOLD
def _recent_limit(self) -> int:
raw = os.getenv("MODELHUB_AGENT_RECENT_OUTCOME_LIMIT", str(DEFAULT_RECENT_OUTCOME_LIMIT))
try:
return max(100, int(raw))
except ValueError:
return DEFAULT_RECENT_OUTCOME_LIMIT
def _load_checkpoint(self) -> dict[str, Any]:
try:
payload = read_json(self.checkpoint_path)
except (FileNotFoundError, ValueError, TypeError):
return {}
if not isinstance(payload, dict) or int(payload.get("version") or 0) != OUTCOME_CHECKPOINT_VERSION:
return {}
return payload
@property
def has_durable_checkpoint(self) -> bool:
return bool(self._checkpoint and isinstance(self._checkpoint.get("report"), dict))
def _combined_recent_terminal(self) -> list[dict[str, Any]]:
by_key: dict[str, dict[str, Any]] = {}
for record in [*self._recent_records, *self._records]:
if record.get("outcome") not in {"success", "failed"}:
continue
key = _outcome_record_key(record)
existing = by_key.get(key)
by_key[key] = record if existing is None else _prefer_newer_outcome(existing, record)
return sorted(by_key.values(), key=_outcome_record_timestamp, reverse=True)[: self._recent_limit()]
@staticmethod
def _archive_record(record: dict[str, Any]) -> dict[str, Any]:
return {
key: value
for key, value in record.items()
if not any(
marker in key.casefold()
for marker in ("url", "token", "cookie", "authorization", "configparams", "response")
)
}
def _write_archive_shard(self, records: list[dict[str, Any]]) -> str | None:
if not records:
return None
now = utc_now()
month = now.strftime("%Y-%m")
name = f"{now.strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:10]}.jsonl.gz"
path = self.archive_pending_dir / month / name
path.parent.mkdir(parents=True, exist_ok=True)
with gzip.open(path, "wt", encoding="utf-8", compresslevel=6) as handle:
for record in records:
handle.write(json.dumps(self._archive_record(record), ensure_ascii=False, sort_keys=True) + "\n")
return f"{month}/{name}"
def _compact_if_needed(self, *, force: bool = False) -> bool:
if not force and len(self._records) <= self._compact_threshold():
return False
removed = [
record
for record in self._records
if record.get("outcome") in {"success", "failed", "policy_cancelled"}
]
if not removed:
return False
retained = [
record
for record in self._records
if record.get("outcome") not in {"success", "failed", "policy_cancelled"}
]
full_report = self.get_stats_report()
recent_records = self._combined_recent_terminal()
shard = self._write_archive_shard(removed)
archived_total = max(0, int(full_report.get("totalRecords") or 0) - len(retained))
checkpoint_report = dict(full_report)
checkpoint_report.update(
{
"totalRecords": archived_total,
"pendingRecords": 0,
}
)
previous_shards = list(self._checkpoint.get("archiveShards") or [])
if shard:
previous_shards.append(shard)
sync_times = [
parse_datetime(record.get("lastSyncTime"))
for record in [*removed, *retained]
if record.get("lastSyncTime")
]
last_sync = max((value for value in sync_times if value is not None), default=None)
self._checkpoint = {
"version": OUTCOME_CHECKPOINT_VERSION,
"generatedAt": utc_now().isoformat(),
"lastSyncTime": last_sync.isoformat() if last_sync else self._checkpoint.get("lastSyncTime"),
"archivedRecords": archived_total,
"archiveShards": previous_shards[-200:],
"recentLimit": self._recent_limit(),
"report": checkpoint_report,
}
self._records = retained
self._recent_records = recent_records
write_json(self.checkpoint_path, self._checkpoint)
write_jsonl(self.recent_path, self._recent_records)
write_jsonl(self.path, self._records)
self._rebuild_indexes()
return True
def set_failure_llm_classifier(self, classifier: LLMAssistedClassifier | None) -> None:
self._failure_llm_classifier = classifier
@@ -79,7 +218,9 @@ class OutcomeTracker:
def get_strategy_history_records(self) -> list[dict[str, Any]]:
"""Expose only successes and evidence-attributable failures for GPU ranking."""
records: list[dict[str, Any]] = []
for record in self._records:
recent_by_key = {_outcome_record_key(record): record for record in self._recent_records}
recent_by_key.update({_outcome_record_key(record): record for record in self._records})
for record in recent_by_key.values():
outcome = record.get("outcome")
if outcome == "success":
verify_result = 1
@@ -101,7 +242,13 @@ class OutcomeTracker:
def _rebuild_indexes(self) -> None:
self._by_task_id.clear()
self._by_model_gpu.clear()
for record in self._records:
# Keep the bounded recent window in the identity index as well. The
# platform history API uses an inclusive time cursor, so the first page
# after a restart can contain terminal tasks already represented by the
# checkpoint. Remembering their task IDs prevents double-counting
# without loading the cold archive.
indexed_records = [*self._recent_records, *self._records]
for record in indexed_records:
task_id = record.get("taskId")
if task_id:
self._by_task_id[str(task_id)] = record
@@ -487,12 +634,15 @@ class OutcomeTracker:
now_datetime = utc_now()
now = now_datetime.isoformat()
terminal = [r for r in self._records if r.get("outcome") in {"success", "failed"}]
recent_terminal = self._combined_recent_terminal()
gpu_groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
framework_groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
combo_groups: dict[tuple[str, str, str], list[dict[str, Any]]] = defaultdict(list)
profile_groups: dict[tuple[str, str, str, str, str], list[dict[str, Any]]] = defaultdict(list)
sized_profile_groups: dict[tuple[str, str, str, str, str, int], list[dict[str, Any]]] = defaultdict(list)
recent_combo_groups: dict[tuple[str, str, str], list[dict[str, Any]]] = defaultdict(list)
recent_profile_groups: dict[tuple[str, str, str, str, str], list[dict[str, Any]]] = defaultdict(list)
for record in terminal:
gpu = record.get("targetGpu") or "unknown"
@@ -514,6 +664,17 @@ class OutcomeTracker:
size_bucket = int(load_bytes).bit_length() - 1
sized_profile_groups[(gpu, fw, tt, model_type, quantization, size_bucket)].append(record)
for record in recent_terminal:
gpu = record.get("targetGpu") or "unknown"
fw = record.get("framework") or "unknown"
tt = record.get("taskType") or "unknown"
recent_combo_groups[(gpu, fw, tt)].append(record)
profile = record.get("modelProfile") or {}
model_type = str(profile.get("modelType") or "").strip()
if model_type:
quantization = str(profile.get("quantizationMethod") or "none").strip()
recent_profile_groups[(gpu, fw, tt, model_type, quantization)].append(record)
gpu_summaries = {gpu: _summarize(records) for gpu, records in gpu_groups.items()}
framework_summaries = {fw: _summarize(records) for fw, records in framework_groups.items()}
combination_stats = {
@@ -521,7 +682,7 @@ class OutcomeTracker:
for (gpu, fw, tt), records in combo_groups.items()
}
recent_combination_stats: dict[str, dict[str, Any]] = {}
for (gpu, fw, tt), records in combo_groups.items():
for (gpu, fw, tt), records in recent_combo_groups.items():
recent = sorted(records, key=_outcome_record_timestamp, reverse=True)[:20]
consecutive_failures = _consecutive_attributable_failures(recent)
consecutive_platform_failures = _consecutive_platform_failures(recent)
@@ -554,6 +715,8 @@ class OutcomeTracker:
"quantizationMethod": quantization,
**_summarize(records),
}
for (gpu, fw, tt, model_type, quantization), records in recent_profile_groups.items():
key = f"{gpu}|{fw}|{tt}|{model_type}|{quantization}"
recent = sorted(records, key=_outcome_record_timestamp, reverse=True)[:20]
consecutive_failures = _consecutive_attributable_failures(recent)
last_terminal_at = None
@@ -563,7 +726,11 @@ class OutcomeTracker:
or parse_datetime(recent[0].get("submitTime"))
)
recent_profile_combination_stats[key] = {
**profile_combination_stats[key],
"targetGpu": gpu,
"framework": fw,
"taskType": tt,
"modelType": model_type,
"quantizationMethod": quantization,
**_summarize(recent),
"consecutiveFailures": consecutive_failures,
"lastTerminalAt": last_terminal_at.isoformat() if last_terminal_at else None,
@@ -607,7 +774,7 @@ class OutcomeTracker:
architecture_block_ttl_days = _architecture_block_ttl_days()
architecture_blocks = _build_architecture_compatibility_blocks(
terminal,
recent_terminal,
now=now_datetime,
ttl_days=architecture_block_ttl_days,
)
@@ -616,7 +783,7 @@ class OutcomeTracker:
combination = f"{block['targetGpu']}|{block['framework']}"
architecture_blocks_by_gpu_framework[combination] += 1
return {
report = {
"generatedAt": now,
"totalRecords": len(self._records),
"pendingRecords": pending_count,
@@ -639,6 +806,102 @@ class OutcomeTracker:
"totals": _summarize(terminal),
"warnings": warnings,
}
return self._merge_checkpoint_report(report, recent_terminal=recent_terminal, now=now_datetime)
def _merge_checkpoint_report(
self,
report: dict[str, Any],
*,
recent_terminal: list[dict[str, Any]],
now: datetime,
) -> dict[str, Any]:
baseline = self._checkpoint.get("report")
if not isinstance(baseline, dict):
return report
merged = dict(report)
for field in (
"gpuSummaries",
"frameworkSummaries",
"combinationStats",
"profileCombinationStats",
"sizedProfileCombinationStats",
):
base_items = baseline.get(field) if isinstance(baseline.get(field), dict) else {}
delta_items = report.get(field) if isinstance(report.get(field), dict) else {}
combined: dict[str, dict[str, Any]] = {}
for key in set(base_items) | set(delta_items):
combined[str(key)] = _merge_stat_items(base_items.get(key), delta_items.get(key))
merged[field] = combined
merged["totals"] = _merge_stat_items(baseline.get("totals"), report.get("totals"))
merged["totalRecords"] = int(self._checkpoint.get("archivedRecords") or 0) + len(self._records)
merged["terminalRecords"] = int((merged.get("totals") or {}).get("total") or 0)
merged["pendingRecords"] = sum(1 for record in self._records if record.get("outcome") == "pending")
merged["policyCancelledRecords"] = int(baseline.get("policyCancelledRecords") or 0) + sum(
1 for record in self._records if record.get("outcome") == "policy_cancelled"
)
observed: dict[str, float] = {}
for source in (baseline.get("observedGpuMemoryGiB") or {}, report.get("observedGpuMemoryGiB") or {}):
if not isinstance(source, dict):
continue
for gpu, raw in source.items():
try:
value = float(raw)
except (TypeError, ValueError):
continue
previous = observed.get(str(gpu))
observed[str(gpu)] = min(previous, value) if previous else value
merged["observedGpuMemoryGiB"] = observed
blocks = {
str(key): dict(value)
for key, value in (baseline.get("architectureCompatibilityBlocks") or {}).items()
if isinstance(value, dict)
and (parse_datetime(value.get("expiresAt")) or now + timedelta(seconds=1)) > now
}
for record in recent_terminal:
if record.get("outcome") != "success":
continue
event_time = parse_datetime(record.get("lastSyncTime")) or parse_datetime(record.get("submitTime"))
if event_time is None:
continue
for profile in _stored_architecture_profiles(record, include_model_type=True):
key = architecture_compatibility_key(
str(record.get("targetGpu") or ""),
str(record.get("framework") or ""),
str(record.get("taskType") or ""),
profile["signature"],
)
block = blocks.get(str(key)) if key is not None else None
if block is not None and event_time >= (parse_datetime(block.get("latestFailureAt")) or event_time):
blocks.pop(str(key), None)
blocks.update(
{
str(key): dict(value)
for key, value in (report.get("architectureCompatibilityBlocks") or {}).items()
if isinstance(value, dict)
}
)
by_gpu_framework: dict[str, int] = defaultdict(int)
for block in blocks.values():
by_gpu_framework[f"{block.get('targetGpu')}|{block.get('framework')}"] += 1
merged["architectureCompatibilityBlocks"] = blocks
merged["architectureCompatibilitySummary"] = {
"activeBlockCount": len(blocks),
"ttlDays": _architecture_block_ttl_days(),
"byGpuFramework": dict(by_gpu_framework),
}
warnings: list[str] = []
for gpu, summary in (merged.get("gpuSummaries") or {}).items():
if int(summary.get("decisionTotal") or 0) >= 4 and float(summary.get("decisionFailureRate") or 0) >= 0.5:
warnings.append(f"GPU {gpu} 本地统计失败率偏高（≥50%），建议重点关注。")
for key, stat in (merged.get("combinationStats") or {}).items():
if int(stat.get("decisionTotal") or 0) >= 3 and float(stat.get("decisionFailureRate") or 0) >= 0.6:
warnings.append(f"组合 {key} 近期失败集中，建议降低该 GPU+框架的提交优先级。")
merged["warnings"] = warnings
return merged
def save(self) -> None:
local_records = list(self._records)
@@ -658,11 +921,14 @@ class OutcomeTracker:
self._records = update_jsonl(self.path, merge)
self._rebuild_indexes()
self._compact_if_needed()
def _rebuild_failed_index(self) -> None:
self._failed_model_gpus.clear()
latest_by_combo: dict[tuple[str, str], tuple[datetime, dict[str, Any]]] = {}
for record in self._records:
recent_by_key = {_outcome_record_key(record): record for record in self._recent_records}
recent_by_key.update({_outcome_record_key(record): record for record in self._records})
for record in recent_by_key.values():
# Platform, unresolved, and policy outcomes neither clear nor
# create a model/GPU cooldown. The full-history audit showed that
# more than half of failures lack enough evidence for attribution.
@@ -781,6 +1047,42 @@ def _summarize(records: list[dict[str, Any]]) -> dict[str, Any]:
}
def _merge_stat_items(base: Any, delta: Any) -> dict[str, Any]:
base = base if isinstance(base, dict) else {}
delta = delta if isinstance(delta, dict) else {}
merged = {**base, **delta}
count_fields = (
"total",
"successCount",
"failureCount",
"attributableFailureCount",
"platformFailureCount",
"unresolvedFailureCount",
"decisionTotal",
"pendingCount",
)
for field in count_fields:
merged[field] = int(base.get(field) or 0) + int(delta.get(field) or 0)
breakdown: dict[str, int] = defaultdict(int)
for source in (base.get("failureBreakdown") or {}, delta.get("failureBreakdown") or {}):
if isinstance(source, dict):
for key, value in source.items():
breakdown[str(key)] += int(value or 0)
merged["failureBreakdown"] = dict(breakdown)
total = merged["total"]
decisions = merged["decisionTotal"]
merged.update(
{
"successRate": round(merged["successCount"] / total, 4) if total else 0.0,
"failureRate": round(merged["failureCount"] / total, 4) if total else 0.0,
"decisionSuccessRate": round(merged["successCount"] / decisions, 4) if decisions else 0.0,
"decisionFailureRate": round(merged["attributableFailureCount"] / decisions, 4) if decisions else 0.0,
"pendingRate": round(merged["pendingCount"] / total, 4) if total else 0.0,
}
)
return merged
def _architecture_block_ttl_days() -> int:
raw = os.getenv("MODELHUB_ARCHITECTURE_BLOCK_TTL_DAYS")
if raw is None: