CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

fix: keep agent alive across worker crashes

Browse Source
This commit is contained in:
CoolBoy
2026-08-21 03:38:06 +08:00
parent 85f6cb5157
commit 6eb7ded984
7 changed files with 183 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
tests/test_agent_entrypoint.py
View File

@@ -46,6 +46,31 @@ class HostedAgentEntrypointTests(unittest.TestCase):
self.assertNotIn("apt-get", dockerfile)
self.assertIn("dulwich", requirements.casefold())
def test_exited_worker_is_backed_off_without_exiting_supervisor(self) -> None:
class ExitedWorker:
def poll(self) -> int:
return 137
with tempfile.TemporaryDirectory() as temporary_dir:
root = Path(temporary_dir)
with (
patch.object(ENTRYPOINT, "worker", ExitedWorker()),
patch.object(ENTRYPOINT, "worker_start_enabled", True),
patch.object(ENTRYPOINT, "shutdown_requested", False),
patch.object(ENTRYPOINT, "config_error", None),
patch.object(ENTRYPOINT, "worker_started_at", 90.0),
patch.object(ENTRYPOINT, "worker_restart_count", 0),
patch.object(ENTRYPOINT, "worker_next_restart_at", 0.0),
patch.object(ENTRYPOINT, "WORKER_CRASH_PATH", root / "worker_crashes.jsonl"),
patch.object(ENTRYPOINT, "READINESS_PATH", root / "readiness.json"),
):
ENTRYPOINT._supervise_worker(now=100.0)
self.assertIsNone(ENTRYPOINT.worker)
self.assertEqual(1, ENTRYPOINT.worker_restart_count)
self.assertEqual(105.0, ENTRYPOINT.worker_next_restart_at)
self.assertEqual("worker_restarting", ENTRYPOINT._readiness()["reason"])
self.assertIn('"exitCode":137', (root / "worker_crashes.jsonl").read_text(encoding="utf-8"))
if __name__ == "__main__":
unittest.main()

10
tests/test_super_agent.py
View File

@@ -141,6 +141,14 @@ class SuperAgentTests(unittest.TestCase):
restored_project.mkdir()
porcelain.init(remote, bare=True)
write_json(project / ".modelhub_state" / "account_capacity.json", {"version": 1})
write_jsonl(
project / ".modelhub_state" / "worker_crashes.jsonl",
[{"at": "2026-08-21T00:00:00+00:00", "exitCode": 137}],
)
write_jsonl(
restored_project / ".modelhub_state" / "worker_crashes.jsonl",
[{"at": "2026-08-21T01:00:00+00:00", "exitCode": 1}],
)
credentials = {"username": "tester", "email": "tester@example.com", "password": "secret-value"}
manager = StateGitSync(
project_root=project,
@@ -175,6 +183,8 @@ class SuperAgentTests(unittest.TestCase):
self.assertTrue(restored.restore())
intents = read_jsonl(restored_project / ".modelhub_state" / "recovery_intents.jsonl")
self.assertEqual("owner/model", intents[0]["repoId"])
crashes = read_jsonl(restored_project / ".modelhub_state" / "worker_crashes.jsonl")
self.assertEqual([137, 1], [row["exitCode"] for row in crashes])
state_text = "\n".join(
path.read_text(encoding="utf-8")
for path in restored._workspace.rglob("*")