feat: add tiered durable state and memory bounds

modelhub_submmit_api/state_sync.py

@@ -25,6 +25,7 @@ from version import AGENT_VERSION
 STATE_SCHEMA_VERSION = 1
 DEFAULT_REMOTE = "https://dev.modelhub.org.cn/CoolBoy/submmit.git"
 DEFAULT_BRANCH = "agent-state"
+DEFAULT_ARCHIVE_BRANCH_PREFIX = "agent-archive"
 DEFAULT_BATCH_SIZE = 20
 DEFAULT_RETENTION_DAYS = 30
 DEFAULT_HISTORY_DEPTH = 20
@@ -37,7 +38,9 @@ STATE_ALLOWLIST = (
     ".modelhub_state/gpu_strategy.json",
     ".modelhub_state/market_intelligence.json",
     ".modelhub_state/official_capabilities.json",
+    ".modelhub_state/outcome_checkpoint.json",
     ".modelhub_state/queue_cleanup_latest.json",
+    ".modelhub_state/recent_outcomes.jsonl",
     ".modelhub_state/recovery_active_tasks.jsonl",
     ".modelhub_state/recovery_intents.jsonl",
     ".modelhub_state/routing_intelligence.json",
@@ -205,10 +208,103 @@ class StateGitSync:
         return f"{name} <{email}>".encode("utf-8")
 
     def _remote_oid(self) -> str | None:
+        return self._remote_oid_for(self.branch)
+
+    def _remote_oid_for(self, branch: str) -> str | None:
         result = porcelain.ls_remote(self.remote, quiet=True, **self._auth_kwargs())
-        oid = result.refs.get(f"refs/heads/{self.branch}".encode("utf-8"))
+        oid = result.refs.get(f"refs/heads/{branch}".encode("utf-8"))
         return oid.decode("ascii") if oid else None
 
+    def _sync_archive_pending(self) -> None:
+        pending_root = self.state_dir / "archive_pending" / "outcomes"
+        files = sorted(path for path in pending_root.glob("*/*.jsonl.gz") if path.is_file())
+        if not files:
+            return
+        by_month: dict[str, list[Path]] = {}
+        for path in files:
+            by_month.setdefault(path.parent.name, []).append(path)
+        for month, month_files in sorted(by_month.items()):
+            branch = f"{DEFAULT_ARCHIVE_BRANCH_PREFIX}-{month}"
+            remote_oid = self._remote_oid_for(branch)
+            parent = Path(tempfile.mkdtemp(prefix="modelhub-agent-archive-"))
+            workspace = parent / "archive"
+            try:
+                if remote_oid:
+                    porcelain.clone(
+                        self.remote,
+                        workspace,
+                        branch=branch,
+                        depth=1,
+                        checkout=True,
+                        errstream=io.BytesIO(),
+                        **self._auth_kwargs(),
+                    )
+                else:
+                    workspace.mkdir(parents=True)
+                    repo = porcelain.init(workspace)
+                    repo.refs.set_symbolic_ref(b"HEAD", f"refs/heads/{branch}".encode("utf-8"))
+                repo = Repo(str(workspace))
+                manifest_path = workspace / "manifest.json"
+                try:
+                    manifest = read_json(manifest_path)
+                except (FileNotFoundError, ValueError, TypeError):
+                    manifest = {}
+                archived = manifest.get("files") if isinstance(manifest.get("files"), dict) else {}
+                for source in month_files:
+                    destination = workspace / "outcomes" / source.name
+                    destination.parent.mkdir(parents=True, exist_ok=True)
+                    shutil.copy2(source, destination)
+                    archived[f"outcomes/{source.name}"] = {
+                        "sha256": _sha256_file(destination),
+                        "bytes": destination.stat().st_size,
+                    }
+                write_json(
+                    manifest_path,
+                    {
+                        "schemaVersion": 1,
+                        "month": month,
+                        "updatedAt": _utc_now().isoformat(),
+                        "files": archived,
+                    },
+                )
+                manifest_path.with_name(f".{manifest_path.name}.lock").unlink(missing_ok=True)
+                porcelain.add(repo)
+                status = porcelain.status(repo)
+                if any(status.staged.get(kind) for kind in ("add", "delete", "modify")):
+                    porcelain.commit(
+                        repo,
+                        message=f"archive: outcomes {month}".encode("utf-8"),
+                        author=self._author,
+                        committer=self._author,
+                    )
+                    if self._remote_oid_for(branch) != remote_oid:
+                        raise StateSyncError(f"archive branch {branch} changed remotely")
+                    porcelain.push(
+                        repo,
+                        self.remote,
+                        refspecs=f"HEAD:refs/heads/{branch}",
+                        force=True,
+                        outstream=io.BytesIO(),
+                        errstream=io.BytesIO(),
+                        **self._auth_kwargs(),
+                    )
+                    if self._remote_oid_for(branch) != repo.head().decode("ascii"):
+                        raise StateSyncError(f"archive branch {branch} verification failed")
+                for source in month_files:
+                    source.unlink(missing_ok=True)
+                self.log(
+                    f"[archive-sync] branch={branch} shards={len(month_files)} "
+                    f"files_total={len(archived)} status=ok"
+                )
+            finally:
+                shutil.rmtree(parent, ignore_errors=True)
+
+    def _sync_archive_pending_safely(self) -> None:
+        try:
+            self._sync_archive_pending()
+        except Exception as exc:
+            self.log(f"[archive-sync] status=deferred reason={_safe_text(exc)}")
+
     def _create_workspace(self, remote_oid: str | None) -> None:
         parent = Path(tempfile.mkdtemp(prefix="modelhub-agent-state-"))
         workspace = parent / "state"
@@ -539,7 +635,7 @@ class StateGitSync:
                     # A restored snapshot must refresh official configs before routing.
                     sanitized_market["frameworkUpdatedAt"] = None
                 write_json(destination, sanitized_market)
-            elif relative == "outcomes/submissions.jsonl":
+            elif relative in {"outcomes/submissions.jsonl", ".modelhub_state/recent_outcomes.jsonl"}:
                 sanitized_outcomes: list[dict[str, Any]] = []
                 for row in read_jsonl(source):
                     sanitized_outcomes.append(
@@ -593,6 +689,10 @@ class StateGitSync:
                 staged = any(status.staged.get(kind) for kind in ("add", "delete", "modify"))
                 if not staged:
                     self.healthy = True
+                    # A previous archive push may have been deferred while the
+                    # hot snapshot was already current.  Retry cold shards even
+                    # when this cycle has no hot-state commit to publish.
+                    self._sync_archive_pending_safely()
                     return True
                 porcelain.commit(
                     repo,
@@ -631,6 +731,7 @@ class StateGitSync:
                 self.last_error = None
                 self.healthy = True
                 self.log(f"[state-sync] generation={self.generation} phase={phase} status=ok")
+                self._sync_archive_pending_safely()
                 return True
             except Exception as exc:
                 self.healthy = False