feat: make model age admission-only

tests/test_queue_cleanup.py

@@ -19,7 +19,6 @@ from queue_cleanup import (  # noqa: E402
     cleanup_certain_oom_tasks,
     find_architecture_incompatible_tasks,
     find_certain_oom_tasks,
-    find_old_overflow_tasks,
 )
 
 
@@ -394,98 +393,34 @@ class QueueCleanupTests(unittest.TestCase):
         self.assertEqual("task_started_running", summary["policyNoLongerAppliesTasks"][0]["policyChangeReason"])
         self.assertEqual([], client.stopped)
 
-    def test_old_models_use_each_accounts_own_capacity_minus_ten_threshold(self) -> None:
-        now = datetime(2026, 8, 11, tzinfo=timezone.utc)
-        tasks = [
-            OwnedTask(0, index, "owner/old", "Iluvatar_bi-100", "waiting")
-            for index in range(1, 92)
-        ]
-        tasks.extend(
-            OwnedTask(1, 1000 + index, "owner/old", "Iluvatar_bi-100", "waiting")
-            for index in range(1, 192)
-        )
-        tasks.append(OwnedTask(0, 92, "owner/recent", "Iluvatar_bi-100", "waiting"))
-        tasks.append(OwnedTask(0, 93, "owner/old", "Iluvatar_bi-100", "running"))
-        tasks.append(OwnedTask(1, 1192, "owner/recent", "Iluvatar_bi-100", "waiting"))
-
-        selected, skipped = find_old_overflow_tasks(
-            tasks,
-            model_last_modified={
-                "owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc),
-                "owner/recent": datetime(2026, 8, 10, tzinfo=timezone.utc),
-            },
-            queue_threshold={0: 90, 1: 190},
-            recent_model_days=7,
-            reference_time=now,
-        )
-
-        self.assertEqual([91, 1191], [item["taskId"] for item in selected])
-        self.assertEqual([91, 191], [item["queuePosition"] for item in selected])
-        self.assertEqual(2, skipped["recentOverflowTasks"])
-        self.assertEqual(1, skipped["runningOverflowProtected"])
-
-    def test_age_cleanup_never_stops_running_overflow_task(self) -> None:
+    def test_initial_and_later_cleanup_never_stop_old_waiting_or_running_tasks(self) -> None:
         records = [
             {
                 "taskId": index,
                 "modelId": "owner/old",
                 "gpuType": "Iluvatar_bi-100",
-                "status": "running" if index == 91 else "waiting",
+                "status": "running" if index == 100 else "waiting",
             }
-            for index in range(1, 92)
+            for index in range(1, 101)
         ]
         client = FakeQueueClient(records)
         pool = ModelHubClientPool(
             [client],  # type: ignore[list-item]
             active_task_cap=100,
-            recent_model_reserve_slots=10,
+            recent_model_reserve_slots=5,
         )
-        summary = cleanup_certain_oom_tasks(
-            pool,
-            FakeDiscovery(
-                {"owner/old": 1 * GIB},
-                {"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
-            ),  # type: ignore[arg-type]
-            reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
-            log=lambda _message: None,
-        )
-
-        self.assertEqual(0, summary["oldOverflowCount"])
-        self.assertEqual(1, summary["agePolicySkipped"]["runningOverflowProtected"])
-        self.assertEqual([], client.stopped)
-
-    def test_age_cleanup_recheck_releases_task_that_started_running(self) -> None:
-        records = [
-            {
-                "taskId": index,
-                "modelId": "owner/old",
-                "gpuType": "Iluvatar_bi-100",
-                "status": "waiting",
-            }
-            for index in range(1, 92)
+        summaries = [
+            cleanup_certain_oom_tasks(
+                pool,
+                FakeDiscovery({"owner/old": 1 * GIB}),  # type: ignore[arg-type]
+                log=lambda _message: None,
+            )
+            for _ in range(2)
         ]
-        # Read 1 is discovery, read 2 is the account-wide mutation recheck,
-        # and read 3 is the final age-only recheck after the OOM phase.
-        client = FakeQueueClient(records, promote_on_waiting_read=3)
-        pool = ModelHubClientPool(
-            [client],  # type: ignore[list-item]
-            active_task_cap=100,
-            recent_model_reserve_slots=10,
-        )
-        summary = cleanup_certain_oom_tasks(
-            pool,
-            FakeDiscovery(
-                {"owner/old": 1 * GIB},
-                {"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
-            ),  # type: ignore[arg-type]
-            read_concurrency=1,
-            reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
-            log=lambda _message: None,
-        )
 
-        self.assertEqual(1, summary["oldOverflowCount"])
-        self.assertEqual(0, summary["cancelledCount"])
-        self.assertEqual("task_started_running", summary["policyNoLongerAppliesTasks"][0]["policyChangeReason"])
+        self.assertTrue(all(item["ageCleanupMode"] == "admission_only" for item in summaries))
+        self.assertTrue(all(item["oldOverflowCount"] == 0 for item in summaries))
+        self.assertTrue(all(item["cancelledCount"] == 0 for item in summaries))
         self.assertEqual([], client.stopped)
 
     def test_certain_oom_cleanup_can_still_stop_running_task(self) -> None:
@@ -510,38 +445,7 @@ class QueueCleanupTests(unittest.TestCase):
         self.assertEqual(1, summary["cancelledCount"])
         self.assertEqual([[1]], client.stopped)
 
-    def test_old_overflow_task_is_not_stopped_if_it_moves_inside_dynamic_threshold(self) -> None:
-        records = [
-            {
-                "taskId": index,
-                "modelId": "owner/old",
-                "gpuType": "Iluvatar_bi-100",
-                "status": "waiting",
-            }
-            for index in range(1, 92)
-        ]
-        client = FakeQueueClient(records, drop_first_on_recheck=True)
-        pool = ModelHubClientPool(
-            [client],  # type: ignore[list-item]
-            active_task_cap=100,
-            recent_model_reserve_slots=10,
-        )
-        summary = cleanup_certain_oom_tasks(
-            pool,
-            FakeDiscovery(
-                {"owner/old": 1 * GIB},
-                {"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
-            ),  # type: ignore[arg-type]
-            reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
-            log=lambda _message: None,
-        )
-
-        self.assertEqual(1, summary["oldOverflowCount"])
-        self.assertEqual(0, summary["cancelledCount"])
-        self.assertEqual(1, summary["policyNoLongerAppliesCount"])
-        self.assertEqual([], client.stopped)
-
-    def test_cleanup_promotes_capacity_from_complete_active_listing(self) -> None:
+    def test_capacity_decline_pauses_submissions_and_never_deletes_existing_tasks(self) -> None:
         records = [
             {
                 "taskId": index,
@@ -555,114 +459,20 @@ class QueueCleanupTests(unittest.TestCase):
         pool = ModelHubClientPool(
             [client],  # type: ignore[list-item]
             active_task_cap=100,
-            recent_model_reserve_slots=10,
+            recent_model_reserve_slots=5,
             capacity_state_path=None,
         )
         summary = cleanup_certain_oom_tasks(
             pool,
-            FakeDiscovery(
-                {"owner/old": 1 * GIB},
-                {"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
-            ),  # type: ignore[arg-type]
-            reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
+            FakeDiscovery({"owner/old": 1 * GIB}),  # type: ignore[arg-type]
             log=lambda _message: None,
         )
 
         self.assertEqual([150], summary["accountCapacityLimits"])
-        self.assertEqual([140], summary["oldModelQueueThresholds"])
-        self.assertEqual(list(range(141, 151)), [item["taskId"] for item in summary["oldOverflowTasks"]])
-
-    def test_initial_cleanup_stops_old_task_beyond_capacity_minus_ten(self) -> None:
-        records = [
-            {
-                "taskId": index,
-                "modelId": "owner/old",
-                "gpuType": "Iluvatar_bi-100",
-                "status": "waiting",
-            }
-            for index in range(1, 92)
-        ]
-        client = FakeQueueClient(records)
-        pool = ModelHubClientPool(
-            [client],  # type: ignore[list-item]
-            active_task_cap=100,
-            recent_model_reserve_slots=10,
-        )
-        summary = cleanup_certain_oom_tasks(
-            pool,
-            FakeDiscovery(
-                {"owner/old": 1 * GIB},
-                {"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
-            ),  # type: ignore[arg-type]
-            reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
-            log=lambda _message: None,
-        )
-
-        self.assertEqual(1, summary["oldOverflowCount"])
-        self.assertEqual(1, summary["cancelledCount"])
-        self.assertEqual([90], summary["oldModelQueueThresholds"])
-        self.assertEqual([[91]], client.stopped)
-
-    def test_scheduled_cleanup_uses_capacity_minus_five(self) -> None:
-        records = [
-            {
-                "taskId": index,
-                "modelId": "owner/old",
-                "gpuType": "Iluvatar_bi-100",
-                "status": "waiting",
-            }
-            for index in range(1, 97)
-        ]
-        client = FakeQueueClient(records)
-        pool = ModelHubClientPool(
-            [client],  # type: ignore[list-item]
-            active_task_cap=100,
-            recent_model_reserve_slots=10,
-        )
-        summary = cleanup_certain_oom_tasks(
-            pool,
-            FakeDiscovery(
-                {"owner/old": 1 * GIB},
-                {"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
-            ),  # type: ignore[arg-type]
-            age_reserved_slots=5,
-            reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
-            log=lambda _message: None,
-        )
-
-        self.assertEqual([95], summary["oldModelQueueThresholds"])
-        self.assertEqual([96], [item["taskId"] for item in summary["oldOverflowTasks"]])
-        self.assertEqual([[96]], client.stopped)
-
-    def test_oom_is_removed_before_recalculating_old_overflow_positions(self) -> None:
-        records = [
-            {
-                "taskId": index,
-                "modelId": "owner/large" if index == 1 else "owner/old",
-                "gpuType": "Iluvatar_bi-100",
-                "status": "waiting",
-            }
-            for index in range(1, 93)
-        ]
-        client = FakeQueueClient(records)
-        pool = ModelHubClientPool(
-            [client],  # type: ignore[list-item]
-            active_task_cap=100,
-            recent_model_reserve_slots=10,
-        )
-        summary = cleanup_certain_oom_tasks(
-            pool,
-            FakeDiscovery(
-                {"owner/large": 40 * GIB, "owner/old": 1 * GIB},
-                {"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
-            ),  # type: ignore[arg-type]
-            reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
-            log=lambda _message: None,
-        )
-
-        self.assertEqual(1, summary["certainOomCount"])
-        self.assertEqual([92], [item["taskId"] for item in summary["oldOverflowTasks"]])
-        self.assertEqual([[1], [92]], client.stopped)
+        self.assertEqual([145], summary["oldModelQueueThresholds"])
+        self.assertEqual(0, summary["oldOverflowCount"])
+        self.assertEqual(0, pool.available_submit_slots())
+        self.assertEqual([], client.stopped)
 
     def test_stop_tasks_uses_documented_put_endpoint_and_integer_ids(self) -> None:
         http = RecordingHttpClient()