fix: bound unproven profile submissions and support single-worker takeover

main.py

@@ -187,6 +187,33 @@ def _write_supervisor_readiness(reason: str, *, exit_code: int | None = None) ->
     os.replace(temporary, READINESS_PATH)
 
 
+_worker_memory_sample: dict[str, object] = {}
+_worker_memory_sample_at = 0.0
+
+
+def _sample_worker_memory(now: float) -> None:
+    global _worker_memory_sample, _worker_memory_sample_at
+    if now - _worker_memory_sample_at < 60:
+        return
+    _worker_memory_sample_at = now
+    sample: dict[str, object] = {"sampledAt": datetime.now(timezone.utc).isoformat()}
+    try:
+        pid = int(getattr(worker, "pid", 0))
+        for line in Path(f"/proc/{pid}/status").read_text().splitlines():
+            if line.startswith(("VmRSS:", "VmHWM:")):
+                name, value, *_ = line.split()
+                sample[name.rstrip(":") + "KiB"] = int(value)
+    except (OSError, ValueError, TypeError):
+        pass
+    for name in ("memory.current", "memory.peak", "memory.max", "memory.events"):
+        try:
+            sample[name] = (Path("/sys/fs/cgroup") / name).read_text().strip()[:500]
+        except OSError:
+            pass
+    _worker_memory_sample = sample
+    print("[worker-memory] " + json.dumps(sample, separators=(",", ":")), flush=True)
+
+
 def _record_worker_crash(*, exit_code: int | None, uptime_seconds: float, reason: str) -> None:
     WORKER_CRASH_PATH.parent.mkdir(parents=True, exist_ok=True)
     event = {
@@ -195,13 +222,21 @@ def _record_worker_crash(*, exit_code: int | None, uptime_seconds: float, reason
         "uptimeSeconds": round(max(0.0, uptime_seconds), 3),
         "restartCount": worker_restart_count,
         "reason": reason[:500],
+        "lastMemorySample": dict(_worker_memory_sample),
     }
+    try:
+        event["cgroupMemoryEventsAtExit"] = Path("/sys/fs/cgroup/memory.events").read_text().strip()[:500]
+    except OSError:
+        pass
     with WORKER_CRASH_PATH.open("a", encoding="utf-8") as handle:
         handle.write(json.dumps(event, ensure_ascii=False, separators=(",", ":")) + "\n")
 
 
 def _launch_worker(*, now: float | None = None) -> bool:
     global worker, worker_started_at, worker_last_exit_code
+    global _worker_memory_sample, _worker_memory_sample_at
+    _worker_memory_sample = {}
+    _worker_memory_sample_at = 0.0
     now = time.monotonic() if now is None else now
     cmd = _worker_command()
     print("starting submission worker: " + " ".join(cmd), flush=True)
@@ -227,6 +262,7 @@ def _supervise_worker(*, now: float | None = None) -> None:
     if worker is not None:
         return_code = worker.poll()
         if return_code is None:
+            _sample_worker_memory(now)
             if worker_started_at is not None and now - worker_started_at >= WORKER_STABLE_SECONDS:
                 worker_restart_count = 0
             return