fix: keep agent alive across worker crashes

modelhub_submmit_api/poll_runner.py

@@ -5,6 +5,7 @@ import json
 import os
 import sys
 import time
+from collections import deque
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import timedelta
 from pathlib import Path
@@ -600,8 +601,10 @@ def run_poll_loop(
     else:
         architecture_bootstrap_summary["reason"] = "outcome_sync_disabled"
 
-    cycle_summaries: list[dict[str, Any]] = []
-    queue_cleanup_runs: list[dict[str, Any]] = []
+    retained_cycles = max(10, int(os.getenv("MODELHUB_AGENT_RETAINED_CYCLE_SUMMARIES", "50")))
+    retained_cleanups = max(5, int(os.getenv("MODELHUB_AGENT_RETAINED_CLEANUP_SUMMARIES", "20")))
+    cycle_summaries: deque[dict[str, Any]] = deque(maxlen=retained_cycles)
+    queue_cleanup_runs: deque[dict[str, Any]] = deque(maxlen=retained_cleanups)
     submitted_total = 0
     cycles = 0
     stopped_reason = "max_cycles_reached"
@@ -920,8 +923,12 @@ def run_poll_loop(
         "submittedTotal": submitted_total,
         "stoppedReason": stopped_reason,
         "pollRunDir": str(poll_run_dir),
-        "cycleSummaries": cycle_summaries,
-        "queueCleanupRuns": queue_cleanup_runs,
+        "cycleSummaries": list(cycle_summaries),
+        "cycleSummariesRetained": len(cycle_summaries),
+        "cycleSummariesRetentionLimit": retained_cycles,
+        "queueCleanupRuns": list(queue_cleanup_runs),
+        "queueCleanupRunsRetained": len(queue_cleanup_runs),
+        "queueCleanupRunsRetentionLimit": retained_cleanups,
         "architectureBootstrap": architecture_bootstrap_summary,
         "outcomeStats": stats_report,
     }

modelhub_submmit_api/state_sync.py

@@ -42,6 +42,7 @@ STATE_ALLOWLIST = (
     ".modelhub_state/recovery_intents.jsonl",
     ".modelhub_state/routing_intelligence.json",
     ".modelhub_state/submission_exclusions.jsonl",
+    ".modelhub_state/worker_crashes.jsonl",
     "ledger/submissions.jsonl",
     "outcomes/submissions.jsonl",
 )
@@ -276,7 +277,15 @@ class StateGitSync:
                         destination = self.project_root / relative
                         destination.parent.mkdir(parents=True, exist_ok=True)
                         temporary = destination.with_name(f".{destination.name}.restore-{uuid.uuid4().hex}")
-                        shutil.copy2(source, temporary)
+                        if relative == ".modelhub_state/worker_crashes.jsonl":
+                            merged: dict[str, dict[str, Any]] = {}
+                            for row in [*read_jsonl(source), *read_jsonl(destination)]:
+                                key = json.dumps(row, ensure_ascii=False, sort_keys=True)
+                                merged[key] = row
+                            rows = sorted(merged.values(), key=lambda row: str(row.get("at") or ""))[-200:]
+                            write_jsonl(temporary, rows)
+                        else:
+                            shutil.copy2(source, temporary)
                         os.replace(temporary, destination)
                     self.log(
                         f"[state-recovery] generation={self.generation} source=remote branch={self.branch} status=ok"

modelhub_submmit_api/version.py

@@ -1 +1 @@
-AGENT_VERSION = "2026.08.15.3"
+AGENT_VERSION = "2026.08.21.1"