CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: derive recent queue reserve from account capacity

Browse Source
This commit is contained in:
CoolBoy
2026-08-11 01:41:26 +08:00
parent f12d96b138
commit 2065ad6abc
11 changed files with 266 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
tests/test_queue_cleanup.py
View File

@@ -97,13 +97,18 @@ class RecordingHttpClient:
class QueueCleanupTests(unittest.TestCase):
def test_old_models_are_selected_only_after_each_accounts_first_eighty_tasks(self) -> None:
def test_old_models_use_each_accounts_own_capacity_minus_ten_threshold(self) -> None:
now = datetime(2026, 8, 11, tzinfo=timezone.utc)
tasks = [
OwnedTask(0, index, "owner/old", "Iluvatar_bi-100", "waiting")
for index in range(1, 82)
for index in range(1, 92)
]
tasks.append(OwnedTask(0, 82, "owner/recent", "Iluvatar_bi-100", "waiting"))
tasks.extend(
OwnedTask(1, 1000 + index, "owner/old", "Iluvatar_bi-100", "waiting")
for index in range(1, 192)
)
tasks.append(OwnedTask(0, 92, "owner/recent", "Iluvatar_bi-100", "waiting"))
tasks.append(OwnedTask(1, 1192, "owner/recent", "Iluvatar_bi-100", "waiting"))
selected, skipped = find_old_overflow_tasks(
tasks,
@@ -111,16 +116,16 @@ class QueueCleanupTests(unittest.TestCase):
"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc),
"owner/recent": datetime(2026, 8, 10, tzinfo=timezone.utc),
},
queue_threshold=80,
queue_threshold={0: 90, 1: 190},
recent_model_days=7,
reference_time=now,
)
self.assertEqual([81], [item["taskId"] for item in selected])
self.assertEqual(81, selected[0]["queuePosition"])
self.assertEqual(1, skipped["recentOverflowTasks"])
self.assertEqual([91, 1191], [item["taskId"] for item in selected])
self.assertEqual([91, 191], [item["queuePosition"] for item in selected])
self.assertEqual(2, skipped["recentOverflowTasks"])
def test_old_overflow_task_is_not_stopped_if_it_moves_into_first_eighty(self) -> None:
def test_old_overflow_task_is_not_stopped_if_it_moves_inside_dynamic_threshold(self) -> None:
records = [
{
"taskId": index,
@@ -128,13 +133,13 @@ class QueueCleanupTests(unittest.TestCase):
"gpuType": "Iluvatar_bi-100",
"status": "waiting",
}
for index in range(1, 82)
for index in range(1, 92)
]
client = FakeQueueClient(records, drop_first_on_recheck=True)
pool = ModelHubClientPool(
[client], # type: ignore[list-item]
active_task_cap=100,
old_model_queue_threshold=80,
recent_model_reserve_slots=10,
)
summary = cleanup_certain_oom_tasks(
pool,
@@ -151,7 +156,7 @@ class QueueCleanupTests(unittest.TestCase):
self.assertEqual(1, summary["policyNoLongerAppliesCount"])
self.assertEqual([], client.stopped)
def test_cleanup_stops_old_task_beyond_eightieth_position(self) -> None:
def test_cleanup_promotes_capacity_from_complete_active_listing(self) -> None:
records = [
{
"taskId": index,
@@ -159,13 +164,44 @@ class QueueCleanupTests(unittest.TestCase):
"gpuType": "Iluvatar_bi-100",
"status": "waiting",
}
for index in range(1, 82)
for index in range(1, 151)
]
client = FakeQueueClient(records)
pool = ModelHubClientPool(
[client], # type: ignore[list-item]
active_task_cap=100,
old_model_queue_threshold=80,
recent_model_reserve_slots=10,
capacity_state_path=None,
)
summary = cleanup_certain_oom_tasks(
pool,
FakeDiscovery(
{"owner/old": 1 * GIB},
{"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
), # type: ignore[arg-type]
reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
log=lambda _message: None,
)
self.assertEqual([150], summary["accountCapacityLimits"])
self.assertEqual([140], summary["oldModelQueueThresholds"])
self.assertEqual(list(range(141, 151)), [item["taskId"] for item in summary["oldOverflowTasks"]])
def test_initial_cleanup_stops_old_task_beyond_capacity_minus_ten(self) -> None:
records = [
{
"taskId": index,
"modelId": "owner/old",
"gpuType": "Iluvatar_bi-100",
"status": "waiting",
}
for index in range(1, 92)
]
client = FakeQueueClient(records)
pool = ModelHubClientPool(
[client], # type: ignore[list-item]
active_task_cap=100,
recent_model_reserve_slots=10,
)
summary = cleanup_certain_oom_tasks(
pool,
@@ -179,9 +215,10 @@ class QueueCleanupTests(unittest.TestCase):
self.assertEqual(1, summary["oldOverflowCount"])
self.assertEqual(1, summary["cancelledCount"])
self.assertEqual([[81]], client.stopped)
self.assertEqual([90], summary["oldModelQueueThresholds"])
self.assertEqual([[91]], client.stopped)
def test_dynamic_cleanup_keeps_positions_through_ninety_five(self) -> None:
def test_scheduled_cleanup_uses_capacity_minus_five(self) -> None:
records = [
{
"taskId": index,
@@ -195,7 +232,7 @@ class QueueCleanupTests(unittest.TestCase):
pool = ModelHubClientPool(
[client], # type: ignore[list-item]
active_task_cap=100,
old_model_queue_threshold=80,
recent_model_reserve_slots=10,
)
summary = cleanup_certain_oom_tasks(
pool,
@@ -203,12 +240,12 @@ class QueueCleanupTests(unittest.TestCase):
{"owner/old": 1 * GIB},
{"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
), # type: ignore[arg-type]
age_queue_threshold=95,
age_reserved_slots=5,
reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
log=lambda _message: None,
)
self.assertEqual(95, summary["oldModelQueueThreshold"])
self.assertEqual([95], summary["oldModelQueueThresholds"])
self.assertEqual([96], [item["taskId"] for item in summary["oldOverflowTasks"]])
self.assertEqual([[96]], client.stopped)
@@ -220,13 +257,13 @@ class QueueCleanupTests(unittest.TestCase):
"gpuType": "Iluvatar_bi-100",
"status": "waiting",
}
for index in range(1, 83)
for index in range(1, 93)
]
client = FakeQueueClient(records)
pool = ModelHubClientPool(
[client], # type: ignore[list-item]
active_task_cap=100,
old_model_queue_threshold=80,
recent_model_reserve_slots=10,
)
summary = cleanup_certain_oom_tasks(
pool,
@@ -239,8 +276,8 @@ class QueueCleanupTests(unittest.TestCase):
)
self.assertEqual(1, summary["certainOomCount"])
self.assertEqual([82], [item["taskId"] for item in summary["oldOverflowTasks"]])
self.assertEqual([[1], [82]], client.stopped)
self.assertEqual([92], [item["taskId"] for item in summary["oldOverflowTasks"]])
self.assertEqual([[1], [92]], client.stopped)
def test_stop_tasks_uses_documented_put_endpoint_and_integer_ids(self) -> None:
http = RecordingHttpClient()