CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

refactor: retain decision state instead of full logs

Browse Source
This commit is contained in:
CoolBoy
2026-09-04 10:49:04 +08:00
parent ff73768537
commit d60551e130
9 changed files with 206 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
modelhub_submmit_api/outcome_tracker.py
View File

@@ -3,12 +3,10 @@ from __future__ import annotations
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timedelta
import gzip
import json
import os
from pathlib import Path
from typing import Any
import uuid
from architecture_compatibility import (
DEFAULT_ARCHITECTURE_BLOCK_TTL_DAYS,
@@ -29,15 +27,47 @@ from task_registry import task_type_from_history_task
DEFAULT_OUTCOMES_PATH = Path("outcomes/submissions.jsonl")
DEFAULT_OUTCOME_CHECKPOINT_PATH = Path(".modelhub_state/outcome_checkpoint.json")
DEFAULT_RECENT_OUTCOMES_PATH = Path(".modelhub_state/recent_outcomes.jsonl")
DEFAULT_ARCHIVE_PENDING_DIR = Path(".modelhub_state/archive_pending/outcomes")
OUTCOME_CHECKPOINT_VERSION = 1
DEFAULT_OUTCOME_COMPACT_THRESHOLD = 1000
DEFAULT_RECENT_OUTCOME_LIMIT = 1000
DEFAULT_OUTCOME_COMPACT_THRESHOLD = 500
DEFAULT_RECENT_OUTCOME_LIMIT = 300
FAILURE_ENRICHMENT_LIMIT = 40
FAILURE_ENRICHMENT_WORKERS = 4
FAILURE_ENRICHMENT_MAX_ATTEMPTS = 3
TERMINAL_TASK_STATUSES = {"success", "failed", "error", "cancelled", "completed"}
DECISION_RECORD_FIELDS = {
"taskId",
"modelId",
"targetGpu",
"framework",
"taskType",
"submitTime",
"lastSyncTime",
"status",
"verifyResult",
"outcome",
"failReason",
"modelProfile",
"failureCode",
"failureCategory",
"failureScope",
"failureAction",
"failureDeterministic",
"failureNeedsLlm",
"failureClassificationReason",
"failureObservedGpuMemoryGiB",
"failureUnsupportedArchitectures",
"failureUnsupportedModelTypes",
"failureDetectedFramework",
"failureEnrichmentAttempts",
"failureEnrichmentError",
"platformFailure",
"policyCancelled",
"policyCancellationReasons",
"policyCancelledAt",
"policyCancellationResolvedAsSuccess",
}
def _now_iso() -> str:
return utc_now().isoformat()
@@ -50,12 +80,10 @@ class OutcomeTracker:
*,
checkpoint_path: Path | str = DEFAULT_OUTCOME_CHECKPOINT_PATH,
recent_path: Path | str = DEFAULT_RECENT_OUTCOMES_PATH,
archive_pending_dir: Path | str = DEFAULT_ARCHIVE_PENDING_DIR,
) -> None:
self.path = Path(path)
self.checkpoint_path = Path(checkpoint_path)
self.recent_path = Path(recent_path)
self.archive_pending_dir = Path(archive_pending_dir)
self._records: list[dict[str, Any]] = []
self._recent_records: list[dict[str, Any]] = read_jsonl(self.recent_path)
self._checkpoint: dict[str, Any] = self._load_checkpoint()
@@ -65,6 +93,14 @@ class OutcomeTracker:
self._failure_llm_classifier: LLMAssistedClassifier | None = None
self._records = read_jsonl(self.path)
compact_recent = sorted(
(self._decision_record(record) for record in self._recent_records),
key=_outcome_record_timestamp,
reverse=True,
)[: self._recent_limit()]
if compact_recent != self._recent_records:
self._recent_records = compact_recent
write_jsonl(self.recent_path, self._recent_records)
self._rebuild_indexes()
self._compact_if_needed(force=not bool(self._checkpoint) and len(self._records) > self._compact_threshold())
@@ -100,6 +136,21 @@ class OutcomeTracker:
return {}
if not isinstance(payload, dict) or int(payload.get("version") or 0) != OUTCOME_CHECKPOINT_VERSION:
return {}
changed = False
if "archiveShards" in payload:
payload.pop("archiveShards", None)
changed = True
if "archivedRecords" in payload:
payload["summarizedRecords"] = max(
int(payload.get("summarizedRecords") or 0),
int(payload.pop("archivedRecords") or 0),
)
changed = True
if payload.get("storageMode") != "decision_state_only":
payload["storageMode"] = "decision_state_only"
changed = True
if changed:
write_json(self.checkpoint_path, payload)
return payload
@property
@@ -117,28 +168,18 @@ class OutcomeTracker:
return sorted(by_key.values(), key=_outcome_record_timestamp, reverse=True)[: self._recent_limit()]
@staticmethod
def _archive_record(record: dict[str, Any]) -> dict[str, Any]:
return {
key: value
for key, value in record.items()
if not any(
marker in key.casefold()
for marker in ("url", "token", "cookie", "authorization", "configparams", "response")
)
}
def _write_archive_shard(self, records: list[dict[str, Any]]) -> str | None:
if not records:
return None
now = utc_now()
month = now.strftime("%Y-%m")
name = f"{now.strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:10]}.jsonl.gz"
path = self.archive_pending_dir / month / name
path.parent.mkdir(parents=True, exist_ok=True)
with gzip.open(path, "wt", encoding="utf-8", compresslevel=6) as handle:
for record in records:
handle.write(json.dumps(self._archive_record(record), ensure_ascii=False, sort_keys=True) + "\n")
return f"{month}/{name}"
def _decision_record(record: dict[str, Any]) -> dict[str, Any]:
compact = {key: value for key, value in record.items() if key in DECISION_RECORD_FIELDS}
if (
record.get("outcome") == "failed"
and not record.get("failureCategory")
and int(record.get("failureEnrichmentAttempts") or 0) < FAILURE_ENRICHMENT_MAX_ATTEMPTS
and record.get("logCosUrl")
):
# Retain a temporary signed log location only until bounded
# classification retries finish; never copy it to recent history.
compact["logCosUrl"] = record.get("logCosUrl")
return compact
def _compact_if_needed(self, *, force: bool = False) -> bool:
if not force and len(self._records) <= self._compact_threshold():
@@ -157,18 +198,14 @@ class OutcomeTracker:
]
full_report = self.get_stats_report()
recent_records = self._combined_recent_terminal()
shard = self._write_archive_shard(removed)
archived_total = max(0, int(full_report.get("totalRecords") or 0) - len(retained))
summarized_total = max(0, int(full_report.get("totalRecords") or 0) - len(retained))
checkpoint_report = dict(full_report)
checkpoint_report.update(
{
"totalRecords": archived_total,
"totalRecords": summarized_total,
"pendingRecords": 0,
}
)
previous_shards = list(self._checkpoint.get("archiveShards") or [])
if shard:
previous_shards.append(shard)
sync_times = [
parse_datetime(record.get("lastSyncTime"))
for record in [*removed, *retained]
@@ -177,15 +214,15 @@ class OutcomeTracker:
last_sync = max((value for value in sync_times if value is not None), default=None)
self._checkpoint = {
"version": OUTCOME_CHECKPOINT_VERSION,
"storageMode": "decision_state_only",
"generatedAt": utc_now().isoformat(),
"lastSyncTime": last_sync.isoformat() if last_sync else self._checkpoint.get("lastSyncTime"),
"archivedRecords": archived_total,
"archiveShards": previous_shards[-200:],
"summarizedRecords": summarized_total,
"recentLimit": self._recent_limit(),
"report": checkpoint_report,
}
self._records = retained
self._recent_records = recent_records
self._recent_records = [self._decision_record(record) for record in recent_records]
write_json(self.checkpoint_path, self._checkpoint)
write_jsonl(self.recent_path, self._recent_records)
write_jsonl(self.path, self._records)
@@ -265,7 +302,7 @@ class OutcomeTracker:
# platform history API uses an inclusive time cursor, so the first page
# after a restart can contain terminal tasks already represented by the
# checkpoint. Remembering their task IDs prevents double-counting
# without loading the cold archive.
# without retaining full historical rows.
indexed_records = [*self._recent_records, *self._records]
for record in indexed_records:
task_id = record.get("taskId")
@@ -890,7 +927,11 @@ class OutcomeTracker:
merged[field] = combined
merged["totals"] = _merge_stat_items(baseline.get("totals"), report.get("totals"))
merged["totalRecords"] = int(self._checkpoint.get("archivedRecords") or 0) + len(self._records)
merged["totalRecords"] = int(
self._checkpoint.get("summarizedRecords")
or self._checkpoint.get("archivedRecords")
or 0
) + len(self._records)
merged["terminalRecords"] = int((merged.get("totals") or {}).get("total") or 0)
merged["pendingRecords"] = sum(1 for record in self._records if record.get("outcome") == "pending")
merged["policyCancelledRecords"] = int(baseline.get("policyCancelledRecords") or 0) + sum(
@@ -960,7 +1001,12 @@ class OutcomeTracker:
return merged
def save(self) -> None:
local_records = list(self._records)
local_records = [
self._decision_record(record)
if record.get("outcome") in {"success", "failed", "policy_cancelled"}
else record
for record in self._records
]
def merge(existing: list[dict[str, Any]]) -> list[dict[str, Any]]:
merged = list(existing)
@@ -973,7 +1019,12 @@ class OutcomeTracker:
merged.append(record)
continue
merged[existing_index] = _prefer_newer_outcome(merged[existing_index], record)
return merged
return [
self._decision_record(record)
if record.get("outcome") in {"success", "failed", "policy_cancelled"}
else record
for record in merged
]
self._records = update_jsonl(self.path, merge)
self._rebuild_indexes()