CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

refactor: retain decision state instead of full logs

Browse Source
This commit is contained in:
CoolBoy
2026-09-04 10:49:04 +08:00
parent ff73768537
commit d60551e130
9 changed files with 206 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files

254
modelhub_submmit_api/state_sync.py
View File

@@ -25,11 +25,24 @@ from version import AGENT_VERSION
STATE_SCHEMA_VERSION = 1
DEFAULT_REMOTE = "https://dev.modelhub.org.cn/CoolBoy/submmit.git"
DEFAULT_BRANCH = "agent-state"
DEFAULT_ARCHIVE_BRANCH_PREFIX = "agent-archive"
DEFAULT_BATCH_SIZE = 100
DEFAULT_RETENTION_DAYS = 30
DEFAULT_HISTORY_DEPTH = 200
DEFAULT_RECENT_TERMINAL_INTENTS = 200
DEFAULT_RECENT_TERMINAL_INTENTS = 300
DEFAULT_LEDGER_RECORDS = 500
DEFAULT_CRASH_RECORDS = 50
STATE_OUTCOME_FIELDS = {
"taskId", "modelId", "targetGpu", "framework", "taskType", "submitTime",
"lastSyncTime", "status", "verifyResult", "outcome", "failReason",
"modelProfile", "failureCode", "failureCategory", "failureScope",
"failureAction", "failureDeterministic", "failureNeedsLlm",
"failureClassificationReason", "failureObservedGpuMemoryGiB",
"failureUnsupportedArchitectures", "failureUnsupportedModelTypes",
"failureDetectedFramework", "failureEnrichmentAttempts",
"failureEnrichmentError", "platformFailure", "policyCancelled",
"policyCancellationReasons", "policyCancelledAt",
"policyCancellationResolvedAsSuccess",
}
# Only these runtime files may cross the trust boundary into the state branch.
# Credentials, raw stdout, downloaded archives and run directories are excluded.
@@ -134,7 +147,6 @@ class StateGitSync:
remote: str = DEFAULT_REMOTE,
branch: str = DEFAULT_BRANCH,
batch_size: int = DEFAULT_BATCH_SIZE,
retention_days: int = DEFAULT_RETENTION_DAYS,
history_depth: int = DEFAULT_HISTORY_DEPTH,
log_fn=None,
) -> None:
@@ -143,7 +155,6 @@ class StateGitSync:
self.remote = remote
self.branch = branch
self.batch_size = max(1, min(100, int(batch_size)))
self.retention_days = max(1, int(retention_days))
self.history_depth = max(2, int(history_depth))
self.log = log_fn or (lambda message: print(message, flush=True))
self.writer_id = uuid.uuid4().hex
@@ -219,97 +230,6 @@ class StateGitSync:
oid = result.refs.get(f"refs/heads/{branch}".encode("utf-8"))
return oid.decode("ascii") if oid else None
def _sync_archive_pending(self) -> None:
pending_root = self.state_dir / "archive_pending"
files = sorted(path for path in pending_root.glob("*/*/*.jsonl.gz") if path.is_file())
if not files:
return
by_month: dict[str, list[Path]] = {}
for path in files:
by_month.setdefault(path.parent.name, []).append(path)
for month, month_files in sorted(by_month.items()):
branch = f"{DEFAULT_ARCHIVE_BRANCH_PREFIX}-{month}"
remote_oid = self._remote_oid_for(branch)
parent = Path(tempfile.mkdtemp(prefix="modelhub-agent-archive-"))
workspace = parent / "archive"
try:
if remote_oid:
porcelain.clone(
self.remote,
workspace,
branch=branch,
depth=1,
checkout=True,
errstream=io.BytesIO(),
**self._auth_kwargs(),
)
else:
workspace.mkdir(parents=True)
repo = porcelain.init(workspace)
repo.refs.set_symbolic_ref(b"HEAD", f"refs/heads/{branch}".encode("utf-8"))
repo = Repo(str(workspace))
manifest_path = workspace / "manifest.json"
try:
manifest = read_json(manifest_path)
except (FileNotFoundError, ValueError, TypeError):
manifest = {}
archived = manifest.get("files") if isinstance(manifest.get("files"), dict) else {}
for source in month_files:
archive_kind = source.parent.parent.name
destination = workspace / archive_kind / source.name
destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(source, destination)
archived[f"{archive_kind}/{source.name}"] = {
"sha256": _sha256_file(destination),
"bytes": destination.stat().st_size,
}
write_json(
manifest_path,
{
"schemaVersion": 1,
"month": month,
"updatedAt": _utc_now().isoformat(),
"files": archived,
},
)
manifest_path.with_name(f".{manifest_path.name}.lock").unlink(missing_ok=True)
porcelain.add(repo)
status = porcelain.status(repo)
if any(status.staged.get(kind) for kind in ("add", "delete", "modify")):
porcelain.commit(
repo,
message=f"archive: durable records {month}".encode("utf-8"),
author=self._author,
committer=self._author,
)
if self._remote_oid_for(branch) != remote_oid:
raise StateSyncError(f"archive branch {branch} changed remotely")
porcelain.push(
repo,
self.remote,
refspecs=f"HEAD:refs/heads/{branch}",
force=True,
outstream=io.BytesIO(),
errstream=io.BytesIO(),
**self._auth_kwargs(),
)
if self._remote_oid_for(branch) != repo.head().decode("ascii"):
raise StateSyncError(f"archive branch {branch} verification failed")
for source in month_files:
source.unlink(missing_ok=True)
self.log(
f"[archive-sync] branch={branch} shards={len(month_files)} "
f"files_total={len(archived)} status=ok"
)
finally:
shutil.rmtree(parent, ignore_errors=True)
def _sync_archive_pending_safely(self) -> None:
try:
self._sync_archive_pending()
except Exception as exc:
self.log(f"[archive-sync] status=deferred reason={_safe_text(exc)}")
def _create_workspace(self, remote_oid: str | None) -> None:
parent = Path(tempfile.mkdtemp(prefix="modelhub-agent-state-"))
workspace = parent / "state"
@@ -383,7 +303,7 @@ class StateGitSync:
for row in [*read_jsonl(source), *read_jsonl(destination)]:
key = json.dumps(row, ensure_ascii=False, sort_keys=True)
merged[key] = row
rows = sorted(merged.values(), key=lambda row: str(row.get("at") or ""))[-200:]
rows = sorted(merged.values(), key=lambda row: str(row.get("at") or ""))[-DEFAULT_CRASH_RECORDS:]
write_jsonl(temporary, rows)
else:
shutil.copy2(source, temporary)
@@ -409,30 +329,8 @@ class StateGitSync:
self._expected_remote_oid = None
return self.restore()
def _archive_intents(self, records: list[dict[str, Any]]) -> str | None:
if not records:
return None
now = _utc_now()
month = now.strftime("%Y-%m")
name = f"{now.strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:10]}.jsonl.gz"
path = self.state_dir / "archive_pending" / "attempts" / month / name
path.parent.mkdir(parents=True, exist_ok=True)
import gzip
with gzip.open(path, "wt", encoding="utf-8", compresslevel=6) as handle:
for record in records:
safe = {
key: value
for key, value in record.items()
if not any(marker in key.casefold() for marker in ("token", "password", "authorization"))
}
handle.write(json.dumps(safe, ensure_ascii=False, sort_keys=True) + "\n")
return f"{month}/{name}"
def _compact_intents(self) -> int:
records = read_jsonl(self.intents_path)
if len(records) <= DEFAULT_RECENT_TERMINAL_INTENTS:
return 0
active_statuses = {"pending", "submitted", "recovered_active"}
active = [row for row in records if str(row.get("status") or "") in active_statuses]
terminal = [row for row in records if str(row.get("status") or "") not in active_statuses]
@@ -441,45 +339,32 @@ class StateGitSync:
reverse=True,
)
retained_terminal = terminal[:DEFAULT_RECENT_TERMINAL_INTENTS]
archived = terminal[DEFAULT_RECENT_TERMINAL_INTENTS:]
if not archived:
discarded = terminal[DEFAULT_RECENT_TERMINAL_INTENTS:]
compact_fields = {
"intentId", "batchId", "status", "createdAt", "completedAt",
"repoId", "targetGpu", "taskType", "framework", "configSource",
"configFingerprint", "safeConfigVector", "taskId",
}
compacted = [
*active,
*(
{key: value for key, value in row.items() if key in compact_fields}
for row in retained_terminal
),
]
if compacted == records:
return 0
self._archive_intents(archived)
write_jsonl(self.intents_path, [*active, *retained_terminal])
write_jsonl(self.intents_path, compacted)
self.log(
f"[state-compact] intents_archived={len(archived)} "
f"[state-compact] mode=decision_state_only intents_discarded={len(discarded)} "
f"active={len(active)} recent_terminal={len(retained_terminal)}"
)
return len(archived)
return len(discarded)
def _event_files(self) -> list[Path]:
event_dir = self.state_dir / "events"
if not event_dir.exists():
return []
cutoff = (_utc_now() - timedelta(days=self.retention_days)).date()
result: list[Path] = []
for path in sorted(event_dir.glob("*.jsonl")):
try:
event_day = datetime.strptime(path.stem, "%Y-%m-%d").date()
except ValueError:
continue
if event_day >= cutoff:
result.append(path)
else:
path.unlink(missing_ok=True)
return result
def _append_event(self, event: dict[str, Any]) -> None:
now = _utc_now()
path = self.state_dir / "events" / f"{now.date().isoformat()}.jsonl"
existing = read_jsonl(path)
sanitized = {key: value for key, value in event.items() if key not in {"configParams", "token", "password"}}
sanitized["at"] = sanitized.get("at") or now.isoformat()
sanitized["eventId"] = sanitized.get("eventId") or uuid.uuid4().hex
if "reason" in sanitized:
sanitized["reason"] = _safe_text(sanitized["reason"])
existing.append(sanitized)
write_jsonl(path, existing)
# The recovery intent WAL already records both transitions. Persisting
# a second event stream doubled state without improving recovery.
return []
@staticmethod
def _intent(candidate: dict[str, Any], batch_id: str) -> dict[str, Any]:
@@ -508,8 +393,6 @@ class StateGitSync:
intents = [self._intent(candidate, batch_id) for candidate in candidates]
records.extend(intents)
write_jsonl(self.intents_path, records)
for intent in intents:
self._append_event({**intent, "event": "submission_intent"})
if not self.sync("intent"):
return None
return batch_id
@@ -539,7 +422,6 @@ class StateGitSync:
intent["completedAt"] = _utc_now().isoformat()
intent["taskId"] = result.get("taskId")
intent["reason"] = _safe_text(result.get("reason")) if result.get("reason") else None
self._append_event({**intent, "event": "submission_result"})
write_jsonl(self.intents_path, records)
return self.sync("result")
@@ -634,27 +516,9 @@ class StateGitSync:
intent["completedAt"] = now.isoformat()
elif intent.get("status") == "pending":
unresolved += 1
retention_cutoff = now - timedelta(days=self.retention_days)
retained: list[dict[str, Any]] = []
expired: list[dict[str, Any]] = []
for intent in intents:
completed_text = intent.get("completedAt")
if not completed_text:
retained.append(intent)
continue
try:
completed_at = datetime.fromisoformat(str(completed_text).replace("Z", "+00:00"))
except ValueError:
retained.append(intent)
continue
if completed_at.tzinfo is None:
completed_at = completed_at.replace(tzinfo=timezone.utc)
if completed_at >= retention_cutoff:
retained.append(intent)
else:
expired.append(intent)
self._archive_intents(expired)
write_jsonl(self.intents_path, retained)
# Terminal intents are compacted by count and reduced to decision
# fields during snapshot creation. No full historical rows are kept.
write_jsonl(self.intents_path, intents)
self.record_active_tasks(enriched)
return {"active": len(enriched), "reconciled": reconciled, "unresolved": unresolved}
@@ -694,13 +558,13 @@ class StateGitSync:
elif relative == ".modelhub_state/official_capabilities.json":
payload = read_json(source)
cache = payload.get("modelGpuTaskTypes") if isinstance(payload, dict) else None
if isinstance(cache, dict) and len(cache) > 1500:
if isinstance(cache, dict) and len(cache) > 750:
ordered = sorted(
cache.items(),
key=lambda pair: str((pair[1] or {}).get("updatedAt") or ""),
reverse=True,
)
payload["modelGpuTaskTypes"] = dict(ordered[:1500])
payload["modelGpuTaskTypes"] = dict(ordered[:750])
write_json(destination, payload)
elif relative in {"outcomes/submissions.jsonl", ".modelhub_state/recent_outcomes.jsonl"}:
sanitized_outcomes: list[dict[str, Any]] = []
@@ -709,13 +573,33 @@ class StateGitSync:
{
key: value
for key, value in row.items()
if not any(
marker in key.casefold()
for marker in ("url", "token", "cookie", "authorization", "configparams")
)
if key in STATE_OUTCOME_FIELDS
}
)
if relative == ".modelhub_state/recent_outcomes.jsonl":
sanitized_outcomes = sanitized_outcomes[:300]
write_jsonl(destination, sanitized_outcomes)
elif relative == "ledger/submissions.jsonl":
ledger_rows = read_jsonl(source)
active_ids = {
str(row.get("taskId"))
for row in read_jsonl(self.active_tasks_path)
if row.get("taskId") is not None
}
selected_by_task: dict[str, dict[str, Any]] = {}
anonymous: list[dict[str, Any]] = []
for row in [
*(item for item in ledger_rows if str(item.get("taskId") or "") in active_ids),
*ledger_rows[-DEFAULT_LEDGER_RECORDS:],
]:
task_id = str(row.get("taskId") or "")
if task_id:
selected_by_task[task_id] = row
else:
anonymous.append(row)
write_jsonl(destination, [*selected_by_task.values(), *anonymous[-20:]])
elif relative == ".modelhub_state/worker_crashes.jsonl":
write_jsonl(destination, read_jsonl(source)[-DEFAULT_CRASH_RECORDS:])
else:
shutil.copy2(source, destination)
checksums[relative] = _sha256_file(destination)
@@ -766,7 +650,6 @@ class StateGitSync:
self.last_error = None
self.healthy = True
self.log(f"[state-sync] generation={self.generation} phase={phase} status=ok")
self._sync_archive_pending_safely()
return True
def sync(self, phase: str) -> bool:
@@ -790,7 +673,6 @@ class StateGitSync:
):
self.healthy = True
self.last_error = None
self._sync_archive_pending_safely()
return True
next_generation = self.generation + 1
manifest = {
@@ -809,10 +691,6 @@ class StateGitSync:
staged = any(status.staged.get(kind) for kind in ("add", "delete", "modify"))
if not staged:
self.healthy = True
# A previous archive push may have been deferred while the
# hot snapshot was already current. Retry cold shards even
# when this cycle has no hot-state commit to publish.
self._sync_archive_pending_safely()
return True
porcelain.commit(
repo,