from __future__ import annotations import unittest import sys from datetime import datetime, timezone from pathlib import Path from typing import Any PACKAGE_DIR = Path(__file__).resolve().parents[1] / "modelhub_submmit_api" if str(PACKAGE_DIR) in sys.path: sys.path.remove(str(PACKAGE_DIR)) sys.path.insert(0, str(PACKAGE_DIR)) from modelhub_client import ModelHubClient, ModelHubClientPool # noqa: E402 from queue_cleanup import ( # noqa: E402 OwnedTask, cleanup_certain_oom_tasks, find_certain_oom_tasks, find_old_overflow_tasks, ) GIB = 1024**3 class FakeQueueClient: def __init__( self, records: list[dict[str, Any]], *, disappear_on_recheck: bool = False, drop_first_on_recheck: bool = False, ) -> None: self.records = list(records) self.disappear_on_recheck = disappear_on_recheck self.drop_first_on_recheck = drop_first_on_recheck self.waiting_reads = 0 self.stopped: list[list[int]] = [] def list_tasks_page(self, *, status: str, **_kwargs: Any) -> dict[str, Any]: if status == "waiting": self.waiting_reads += 1 if self.disappear_on_recheck and self.waiting_reads >= 2: records: list[dict[str, Any]] = [] else: records = [record for record in self.records if record["status"] == "waiting"] if self.drop_first_on_recheck and self.waiting_reads >= 2: records = sorted(records, key=lambda item: int(item["taskId"]))[1:] else: records = [record for record in self.records if record["status"] == status] return {"code": 0, "data": {"records": records, "pages": 1}} def stop_tasks(self, task_ids: list[int]) -> dict[str, Any]: self.stopped.append(list(task_ids)) ids = set(task_ids) self.records = [record for record in self.records if int(record["taskId"]) not in ids] return {"code": 0, "data": None} def count_active_tasks(self, **_kwargs: Any) -> int: return len(self.records) class FakeDiscovery: def __init__( self, sizes: dict[str, int | None], last_modified: dict[str, datetime | None] | None = None, ) -> None: self.sizes = sizes self.last_modified = last_modified or {} def list_repo_tree(self, repo_id: str) -> list[dict[str, Any]]: size = self.sizes[repo_id] if size is None: return [{"Path": "model.safetensors"}] return [{"Path": "model.safetensors", "Size": size}] def get_model_last_modified(self, repo_id: str) -> datetime | None: return self.last_modified.get(repo_id) class RecordingHttpClient: def __init__(self) -> None: self.calls: list[tuple[str, str, dict[str, Any] | None, dict[str, Any] | None]] = [] def request_json( self, method: str, path: str, *, query: dict[str, Any] | None = None, data: dict[str, Any] | None = None, ) -> dict[str, Any]: self.calls.append((method, path, query, data)) return {"code": 0, "data": None} class QueueCleanupTests(unittest.TestCase): def test_old_models_are_selected_only_after_each_accounts_first_eighty_tasks(self) -> None: now = datetime(2026, 8, 11, tzinfo=timezone.utc) tasks = [ OwnedTask(0, index, "owner/old", "Iluvatar_bi-100", "waiting") for index in range(1, 82) ] tasks.append(OwnedTask(0, 82, "owner/recent", "Iluvatar_bi-100", "waiting")) selected, skipped = find_old_overflow_tasks( tasks, model_last_modified={ "owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc), "owner/recent": datetime(2026, 8, 10, tzinfo=timezone.utc), }, queue_threshold=80, recent_model_days=7, reference_time=now, ) self.assertEqual([81], [item["taskId"] for item in selected]) self.assertEqual(81, selected[0]["queuePosition"]) self.assertEqual(1, skipped["recentOverflowTasks"]) def test_old_overflow_task_is_not_stopped_if_it_moves_into_first_eighty(self) -> None: records = [ { "taskId": index, "modelId": "owner/old", "gpuType": "Iluvatar_bi-100", "status": "waiting", } for index in range(1, 82) ] client = FakeQueueClient(records, drop_first_on_recheck=True) pool = ModelHubClientPool( [client], # type: ignore[list-item] active_task_cap=100, old_model_queue_threshold=80, ) summary = cleanup_certain_oom_tasks( pool, FakeDiscovery( {"owner/old": 1 * GIB}, {"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)}, ), # type: ignore[arg-type] reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc), log=lambda _message: None, ) self.assertEqual(1, summary["oldOverflowCount"]) self.assertEqual(0, summary["cancelledCount"]) self.assertEqual(1, summary["policyNoLongerAppliesCount"]) self.assertEqual([], client.stopped) def test_cleanup_stops_old_task_beyond_eightieth_position(self) -> None: records = [ { "taskId": index, "modelId": "owner/old", "gpuType": "Iluvatar_bi-100", "status": "waiting", } for index in range(1, 82) ] client = FakeQueueClient(records) pool = ModelHubClientPool( [client], # type: ignore[list-item] active_task_cap=100, old_model_queue_threshold=80, ) summary = cleanup_certain_oom_tasks( pool, FakeDiscovery( {"owner/old": 1 * GIB}, {"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)}, ), # type: ignore[arg-type] reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc), log=lambda _message: None, ) self.assertEqual(1, summary["oldOverflowCount"]) self.assertEqual(1, summary["cancelledCount"]) self.assertEqual([[81]], client.stopped) def test_dynamic_cleanup_keeps_positions_through_ninety_five(self) -> None: records = [ { "taskId": index, "modelId": "owner/old", "gpuType": "Iluvatar_bi-100", "status": "waiting", } for index in range(1, 97) ] client = FakeQueueClient(records) pool = ModelHubClientPool( [client], # type: ignore[list-item] active_task_cap=100, old_model_queue_threshold=80, ) summary = cleanup_certain_oom_tasks( pool, FakeDiscovery( {"owner/old": 1 * GIB}, {"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)}, ), # type: ignore[arg-type] age_queue_threshold=95, reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc), log=lambda _message: None, ) self.assertEqual(95, summary["oldModelQueueThreshold"]) self.assertEqual([96], [item["taskId"] for item in summary["oldOverflowTasks"]]) self.assertEqual([[96]], client.stopped) def test_oom_is_removed_before_recalculating_old_overflow_positions(self) -> None: records = [ { "taskId": index, "modelId": "owner/large" if index == 1 else "owner/old", "gpuType": "Iluvatar_bi-100", "status": "waiting", } for index in range(1, 83) ] client = FakeQueueClient(records) pool = ModelHubClientPool( [client], # type: ignore[list-item] active_task_cap=100, old_model_queue_threshold=80, ) summary = cleanup_certain_oom_tasks( pool, FakeDiscovery( {"owner/large": 40 * GIB, "owner/old": 1 * GIB}, {"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)}, ), # type: ignore[arg-type] reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc), log=lambda _message: None, ) self.assertEqual(1, summary["certainOomCount"]) self.assertEqual([82], [item["taskId"] for item in summary["oldOverflowTasks"]]) self.assertEqual([[1], [82]], client.stopped) def test_stop_tasks_uses_documented_put_endpoint_and_integer_ids(self) -> None: http = RecordingHttpClient() client = ModelHubClient(http_client=http) # type: ignore[arg-type] client.stop_tasks(["12", 12, 13]) self.assertEqual( [ ( "PUT", "/api/async/task/stop-create-contest-task", None, {"taskIds": [12, 13]}, ) ], http.calls, ) def test_only_exact_size_capacity_failures_are_selected(self) -> None: tasks = [ OwnedTask(0, 1, "owner/too-large", "Iluvatar_bi-100", "waiting"), OwnedTask(0, 2, "owner/fits", "Iluvatar_bi-100", "waiting"), OwnedTask(0, 3, "owner/unknown-size", "Iluvatar_bi-100", "waiting"), OwnedTask(0, 4, "owner/unknown-gpu", "new-gpu", "waiting"), ] selected, skipped = find_certain_oom_tasks( tasks, repository_sizes={ "owner/too-large": 30 * GIB, "owner/fits": 20 * GIB, "owner/unknown-gpu": 30 * GIB, }, ) self.assertEqual([1], [item["taskId"] for item in selected]) self.assertEqual(1, skipped["repositorySizeUnknown"]) self.assertEqual(1, skipped["gpuCapacityUnknown"]) self.assertEqual(1, skipped["fitsKnownCapacity"]) def test_cleanup_stops_only_certain_oom_tasks_on_the_owning_account(self) -> None: first = FakeQueueClient( [ {"taskId": 1, "modelId": "owner/large", "gpuType": "Iluvatar_bi-100", "status": "waiting"}, {"taskId": 2, "modelId": "owner/small", "gpuType": "Iluvatar_bi-100", "status": "waiting"}, ] ) second = FakeQueueClient( [{"taskId": 3, "modelId": "owner/large", "gpuType": "MetaX_c-500", "status": "waiting"}] ) pool = ModelHubClientPool([first, second], active_task_cap=100) # type: ignore[list-item] summary = cleanup_certain_oom_tasks( pool, FakeDiscovery({"owner/large": 40 * GIB, "owner/small": 20 * GIB}), # type: ignore[arg-type] log=lambda _message: None, ) self.assertEqual(1, summary["certainOomCount"]) self.assertEqual(1, summary["cancelledCount"]) self.assertEqual([[1]], first.stopped) self.assertEqual([], second.stopped) def test_task_that_disappears_during_scan_is_not_stopped(self) -> None: client = FakeQueueClient( [{"taskId": 1, "modelId": "owner/large", "gpuType": "Iluvatar_bi-100", "status": "waiting"}], disappear_on_recheck=True, ) pool = ModelHubClientPool([client], active_task_cap=100) # type: ignore[list-item] summary = cleanup_certain_oom_tasks( pool, FakeDiscovery({"owner/large": 40 * GIB}), # type: ignore[arg-type] log=lambda _message: None, ) self.assertEqual(1, summary["certainOomCount"]) self.assertEqual(0, summary["cancelledCount"]) self.assertEqual(1, summary["noLongerActiveCount"]) self.assertEqual([], client.stopped) if __name__ == "__main__": unittest.main()