feat: clean deterministic OOM tasks on startup

modelhub_submmit_api/poll_runner.py

@@ -23,6 +23,7 @@ from market_intelligence import (
 )
 from modelhub_client import DEFAULT_CAPACITY_STATE_PATH, ModelHubClient, ModelHubClientPool
 from outcome_tracker import DEFAULT_OUTCOMES_PATH, OutcomeTracker
+from queue_cleanup import cleanup_certain_oom_tasks
 from runner_common import DEFAULT_KEY_PATH, ensure_tokens
 from submission_claims import DEFAULT_CLAIMS_PATH
 from template_selector import TemplateSelector
@@ -167,6 +168,24 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument("--post-cycle-cooldown-seconds", type=int, default=2, help="Short sleep after a successful cycle")
     parser.add_argument("--max-cycles", type=int, default=0, help="Optional hard stop after N cycles; 0 means run until quota is reached")
     parser.add_argument("--print-stats", action="store_true", help="Load outcomes, sync, print stats report, and exit")
+    parser.add_argument("--disable-queue-cleanup", action="store_true", help=argparse.SUPPRESS)
+    parser.add_argument(
+        "--queue-cleanup-interval-cycles",
+        type=int,
+        default=int(os.getenv("MODELHUB_QUEUE_CLEANUP_INTERVAL_CYCLES", "120")),
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--queue-cleanup-read-concurrency",
+        type=int,
+        default=int(os.getenv("MODELHUB_QUEUE_CLEANUP_READ_CONCURRENCY", "6")),
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--queue-cleanup-report-path",
+        default=os.getenv("MODELHUB_QUEUE_CLEANUP_REPORT_PATH", ".modelhub_state/queue_cleanup_latest.json"),
+        help=argparse.SUPPRESS,
+    )
     return parser
 
 
@@ -220,6 +239,7 @@ def run_poll_loop(
     )
 
     cycle_summaries: list[dict[str, Any]] = []
+    queue_cleanup_runs: list[dict[str, Any]] = []
     submitted_total = 0
     cycles = 0
     stopped_reason = "max_cycles_reached"
@@ -232,6 +252,47 @@ def run_poll_loop(
         cycles += 1
         if hasattr(modelhub_client, "configure_capacity_probe"):
             modelhub_client.configure_capacity_probe(cycles)
+
+        cleanup_interval = max(0, int(getattr(base_args, "queue_cleanup_interval_cycles", 120) or 0))
+        should_cleanup_queue = (
+            not bool(getattr(base_args, "disable_queue_cleanup", False))
+            and isinstance(modelhub_client, ModelHubClientPool)
+            and (cycles == 1 or (cleanup_interval > 0 and cycles % cleanup_interval == 0))
+        )
+        if should_cleanup_queue:
+            try:
+                cleanup_feedback = outcome_tracker.get_stats_report()
+                cleanup_gpu_memory = cleanup_feedback.get("observedGpuMemoryGiB") or {}
+                cleanup_summary = cleanup_certain_oom_tasks(
+                    modelhub_client,
+                    hf_discovery,
+                    dry_run=bool(base_args.dry_run),
+                    read_concurrency=max(1, int(getattr(base_args, "queue_cleanup_read_concurrency", 6) or 6)),
+                    gpu_memory_gib=cleanup_gpu_memory if isinstance(cleanup_gpu_memory, dict) else None,
+                    log=log,
+                )
+                write_json(
+                    Path(
+                        getattr(
+                            base_args,
+                            "queue_cleanup_report_path",
+                            ".modelhub_state/queue_cleanup_latest.json",
+                        )
+                    ),
+                    cleanup_summary,
+                )
+                queue_cleanup_runs.append(
+                    {
+                        "cycle": cycles,
+                        "activeScanned": cleanup_summary["activeScanned"],
+                        "certainOomCount": cleanup_summary["certainOomCount"],
+                        "cancelledCount": cleanup_summary["cancelledCount"],
+                        "stopErrorCount": len(cleanup_summary["stopErrors"]),
+                    }
+                )
+            except Exception as exc:
+                log(f"[queue-cleanup] error={type(exc).__name__}: {exc} continue_polling=true")
+
         active_counts = modelhub_client.active_task_counts() if hasattr(modelhub_client, "active_task_counts") else []
         capacity_limits = modelhub_client.account_capacity_limits() if hasattr(modelhub_client, "account_capacity_limits") else []
         capacity_probe = modelhub_client.capacity_probe_enabled() if hasattr(modelhub_client, "capacity_probe_enabled") else False
@@ -335,6 +396,7 @@ def run_poll_loop(
         "stoppedReason": stopped_reason,
         "pollRunDir": str(poll_run_dir),
         "cycleSummaries": cycle_summaries,
+        "queueCleanupRuns": queue_cleanup_runs,
         "outcomeStats": stats_report,
     }
     write_json(poll_run_dir / "summary.json", summary)