CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: compact durable logs and refine framework routing

Browse Source
This commit is contained in:
CoolBoy
2026-09-04 10:30:21 +08:00
parent 5b1ec4d3eb
commit ff73768537
13 changed files with 621 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

167
modelhub_submmit_api/state_sync.py
View File

@@ -26,9 +26,10 @@ STATE_SCHEMA_VERSION = 1
DEFAULT_REMOTE = "https://dev.modelhub.org.cn/CoolBoy/submmit.git"
DEFAULT_BRANCH = "agent-state"
DEFAULT_ARCHIVE_BRANCH_PREFIX = "agent-archive"
DEFAULT_BATCH_SIZE = 20
DEFAULT_BATCH_SIZE = 100
DEFAULT_RETENTION_DAYS = 30
DEFAULT_HISTORY_DEPTH = 20
DEFAULT_HISTORY_DEPTH = 200
DEFAULT_RECENT_TERMINAL_INTENTS = 200
# Only these runtime files may cross the trust boundary into the state branch.
# Credentials, raw stdout, downloaded archives and run directories are excluded.
@@ -154,6 +155,9 @@ class StateGitSync:
self._mutex = threading.Lock()
self._workspace: Path | None = None
self._expected_remote_oid: str | None = None
self._pending_push_oid: str | None = None
self._pending_generation: int | None = None
self._pending_manifest: dict[str, Any] | None = None
@property
def state_dir(self) -> Path:
@@ -216,8 +220,8 @@ class StateGitSync:
return oid.decode("ascii") if oid else None
def _sync_archive_pending(self) -> None:
pending_root = self.state_dir / "archive_pending" / "outcomes"
files = sorted(path for path in pending_root.glob("*/*.jsonl.gz") if path.is_file())
pending_root = self.state_dir / "archive_pending"
files = sorted(path for path in pending_root.glob("*/*/*.jsonl.gz") if path.is_file())
if not files:
return
by_month: dict[str, list[Path]] = {}
@@ -251,10 +255,11 @@ class StateGitSync:
manifest = {}
archived = manifest.get("files") if isinstance(manifest.get("files"), dict) else {}
for source in month_files:
destination = workspace / "outcomes" / source.name
archive_kind = source.parent.parent.name
destination = workspace / archive_kind / source.name
destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(source, destination)
archived[f"outcomes/{source.name}"] = {
archived[f"{archive_kind}/{source.name}"] = {
"sha256": _sha256_file(destination),
"bytes": destination.stat().st_size,
}
@@ -273,7 +278,7 @@ class StateGitSync:
if any(status.staged.get(kind) for kind in ("add", "delete", "modify")):
porcelain.commit(
repo,
message=f"archive: outcomes {month}".encode("utf-8"),
message=f"archive: durable records {month}".encode("utf-8"),
author=self._author,
committer=self._author,
)
@@ -404,6 +409,49 @@ class StateGitSync:
self._expected_remote_oid = None
return self.restore()
def _archive_intents(self, records: list[dict[str, Any]]) -> str | None:
if not records:
return None
now = _utc_now()
month = now.strftime("%Y-%m")
name = f"{now.strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:10]}.jsonl.gz"
path = self.state_dir / "archive_pending" / "attempts" / month / name
path.parent.mkdir(parents=True, exist_ok=True)
import gzip
with gzip.open(path, "wt", encoding="utf-8", compresslevel=6) as handle:
for record in records:
safe = {
key: value
for key, value in record.items()
if not any(marker in key.casefold() for marker in ("token", "password", "authorization"))
}
handle.write(json.dumps(safe, ensure_ascii=False, sort_keys=True) + "\n")
return f"{month}/{name}"
def _compact_intents(self) -> int:
records = read_jsonl(self.intents_path)
if len(records) <= DEFAULT_RECENT_TERMINAL_INTENTS:
return 0
active_statuses = {"pending", "submitted", "recovered_active"}
active = [row for row in records if str(row.get("status") or "") in active_statuses]
terminal = [row for row in records if str(row.get("status") or "") not in active_statuses]
terminal.sort(
key=lambda row: str(row.get("completedAt") or row.get("createdAt") or ""),
reverse=True,
)
retained_terminal = terminal[:DEFAULT_RECENT_TERMINAL_INTENTS]
archived = terminal[DEFAULT_RECENT_TERMINAL_INTENTS:]
if not archived:
return 0
self._archive_intents(archived)
write_jsonl(self.intents_path, [*active, *retained_terminal])
self.log(
f"[state-compact] intents_archived={len(archived)} "
f"active={len(active)} recent_terminal={len(retained_terminal)}"
)
return len(archived)
def _event_files(self) -> list[Path]:
event_dir = self.state_dir / "events"
if not event_dir.exists():
@@ -588,6 +636,7 @@ class StateGitSync:
unresolved += 1
retention_cutoff = now - timedelta(days=self.retention_days)
retained: list[dict[str, Any]] = []
expired: list[dict[str, Any]] = []
for intent in intents:
completed_text = intent.get("completedAt")
if not completed_text:
@@ -602,12 +651,16 @@ class StateGitSync:
completed_at = completed_at.replace(tzinfo=timezone.utc)
if completed_at >= retention_cutoff:
retained.append(intent)
else:
expired.append(intent)
self._archive_intents(expired)
write_jsonl(self.intents_path, retained)
self.record_active_tasks(enriched)
return {"active": len(enriched), "reconciled": reconciled, "unresolved": unresolved}
def _copy_snapshot(self) -> dict[str, str]:
assert self._workspace is not None
self._compact_intents()
checksums: dict[str, str] = {}
for relative in STATE_ALLOWLIST:
source = self.project_root / relative
@@ -634,7 +687,21 @@ class StateGitSync:
if isinstance(sanitized_market, dict):
# A restored snapshot must refresh official configs before routing.
sanitized_market["frameworkUpdatedAt"] = None
# Raw community rows are a disposable refresh cache. The
# aggregated framework/GPU statistics are the durable input.
sanitized_market["communitySample"] = {}
write_json(destination, sanitized_market)
elif relative == ".modelhub_state/official_capabilities.json":
payload = read_json(source)
cache = payload.get("modelGpuTaskTypes") if isinstance(payload, dict) else None
if isinstance(cache, dict) and len(cache) > 1500:
ordered = sorted(
cache.items(),
key=lambda pair: str((pair[1] or {}).get("updatedAt") or ""),
reverse=True,
)
payload["modelGpuTaskTypes"] = dict(ordered[:1500])
write_json(destination, payload)
elif relative in {"outcomes/submissions.jsonl", ".modelhub_state/recent_outcomes.jsonl"}:
sanitized_outcomes: list[dict[str, Any]] = []
for row in read_jsonl(source):
@@ -666,16 +733,69 @@ class StateGitSync:
path.unlink(missing_ok=True)
return checksums
def _push_pending(self, *, phase: str) -> bool:
if self._pending_push_oid is None:
return True
assert self._workspace is not None
repo = Repo(str(self._workspace))
remote_oid = self._remote_oid()
if remote_oid == self._pending_push_oid:
pushed = True
elif remote_oid != self._expected_remote_oid:
raise StateSyncError("state branch changed remotely; refusing to overwrite a newer snapshot")
else:
porcelain.push(
repo,
self.remote,
refspecs=f"HEAD:refs/heads/{self.branch}",
force=True,
outstream=io.BytesIO(),
errstream=io.BytesIO(),
**self._auth_kwargs(),
)
pushed = self._remote_oid() == self._pending_push_oid
if not pushed:
raise StateSyncError("state branch verification failed after push")
self._expected_remote_oid = self._pending_push_oid
self.generation = int(self._pending_generation or self.generation)
manifest = self._pending_manifest or {}
self.last_sync_at = manifest.get("updatedAt")
self._pending_push_oid = None
self._pending_generation = None
self._pending_manifest = None
self.last_error = None
self.healthy = True
self.log(f"[state-sync] generation={self.generation} phase={phase} status=ok")
self._sync_archive_pending_safely()
return True
def sync(self, phase: str) -> bool:
with self._mutex:
try:
if self._workspace is None:
raise StateSyncError("state workspace is not initialized")
# A failed network push is retried byte-for-byte. Do not create
# another commit or generation for the same state transition.
if self._pending_push_oid is not None:
return self._push_pending(phase=phase)
previous_manifest: dict[str, Any] = {}
try:
previous_manifest = read_json(self._workspace / "manifest.json")
except (FileNotFoundError, ValueError, TypeError):
pass
checksums = self._copy_snapshot()
self.generation += 1
if (
previous_manifest.get("checksums") == checksums
and previous_manifest.get("agentVersion") == AGENT_VERSION
):
self.healthy = True
self.last_error = None
self._sync_archive_pending_safely()
return True
next_generation = self.generation + 1
manifest = {
"schemaVersion": STATE_SCHEMA_VERSION,
"generation": self.generation,
"generation": next_generation,
"updatedAt": _utc_now().isoformat(),
"agentVersion": AGENT_VERSION,
"writerId": self.writer_id,
@@ -696,7 +816,7 @@ class StateGitSync:
return True
porcelain.commit(
repo,
message=f"state: generation {self.generation} ({phase})".encode("utf-8"),
message=f"state: generation {next_generation} ({phase})".encode("utf-8"),
author=self._author,
committer=self._author,
)
@@ -707,32 +827,15 @@ class StateGitSync:
porcelain.add(repo)
porcelain.commit(
repo,
message=f"state: compacted generation {self.generation}".encode("utf-8"),
message=f"state: compacted generation {next_generation}".encode("utf-8"),
author=self._author,
committer=self._author,
)
current_remote_oid = self._remote_oid()
if current_remote_oid != self._expected_remote_oid:
raise StateSyncError("state branch changed remotely; refusing to overwrite a newer snapshot")
porcelain.push(
repo,
self.remote,
refspecs=f"HEAD:refs/heads/{self.branch}",
force=True,
outstream=io.BytesIO(),
errstream=io.BytesIO(),
**self._auth_kwargs(),
)
local_oid = repo.head().decode("ascii")
if self._remote_oid() != local_oid:
raise StateSyncError("state branch verification failed after push")
self._expected_remote_oid = local_oid
self.last_sync_at = manifest["updatedAt"]
self.last_error = None
self.healthy = True
self.log(f"[state-sync] generation={self.generation} phase={phase} status=ok")
self._sync_archive_pending_safely()
return True
self._pending_push_oid = local_oid
self._pending_generation = next_generation
self._pending_manifest = manifest
return self._push_pending(phase=phase)
except Exception as exc:
self.healthy = False
self.last_error = _safe_text(exc)