fix: enrich cleanup from active queue context

This commit is contained in:
CoolBoy
2026-08-12 09:00:51 +08:00
parent d8ed7fa1c2
commit 86088ce577
4 changed files with 297 additions and 33 deletions

View File

@@ -65,6 +65,13 @@ class FakeQueueClient:
self.records = [record for record in self.records if int(record["taskId"]) not in ids]
return {"code": 0, "data": None}
def list_framework_stats(self, task_type: str, target_gpu: str) -> list[dict[str, Any]]:
del task_type, target_gpu
return [
{"framework": "vllm", "modelCount": 100, "successCount": 60},
{"framework": "transformers", "modelCount": 100, "successCount": 50},
]
def count_active_tasks(self, **_kwargs: Any) -> int:
return len(self.records)
@@ -173,6 +180,107 @@ class QueueCleanupTests(unittest.TestCase):
self.assertEqual(1, skipped["runningMatchedProtected"])
self.assertEqual(1, skipped["noMatchingBlock"])
def test_architecture_cleanup_uses_current_queue_task_type_when_every_framework_is_blocked(self) -> None:
vllm_key, vllm_block = self.architecture_block(framework="vllm")
transformers_key, transformers_block = self.architecture_block(framework="transformers")
tasks = [
OwnedTask(
0,
1,
"owner/model",
"Iluvatar_bi-100",
"waiting",
task_type="text-generation",
)
]
selected, skipped = find_architecture_incompatible_tasks(
tasks,
architecture_blocks={
vllm_key: vllm_block,
transformers_key: transformers_block,
},
task_contexts={},
model_configs={
"owner/model": {"architectures": ["Qwen2ForCausalLM"]},
},
framework_catalog={
("iluvatar_bi-100", "text-generation"): {"vllm", "transformers"},
},
)
self.assertEqual([1], [item["taskId"] for item in selected])
self.assertEqual("all_live_frameworks", selected[0]["architectureMatchScope"])
self.assertEqual(["transformers", "vllm"], selected[0]["evaluatedFrameworks"])
self.assertEqual(1, skipped["frameworkContextUnknown"])
def test_architecture_cleanup_does_not_guess_when_only_some_frameworks_are_blocked(self) -> None:
key, block = self.architecture_block(framework="vllm")
tasks = [
OwnedTask(
0,
1,
"owner/model",
"Iluvatar_bi-100",
"waiting",
task_type="text-generation",
)
]
selected, skipped = find_architecture_incompatible_tasks(
tasks,
architecture_blocks={key: block},
task_contexts={},
model_configs={
"owner/model": {"architectures": ["Qwen2ForCausalLM"]},
},
framework_catalog={
("iluvatar_bi-100", "text-generation"): {"vllm", "transformers"},
},
)
self.assertEqual([], selected)
self.assertEqual(1, skipped["partiallyBlockedFrameworkSet"])
def test_queue_cleanup_enriches_active_api_record_and_stops_only_all_framework_block(self) -> None:
vllm_key, vllm_block = self.architecture_block(framework="vllm")
transformers_key, transformers_block = self.architecture_block(framework="transformers")
client = FakeQueueClient(
[
{
"taskId": 1,
"modelId": "owner/model",
"gpuType": "Iluvatar_bi-100",
"status": "waiting",
"modelTaskLevelId": 23,
"modelTaskLevel": "文本生成",
}
]
)
pool = ModelHubClientPool([client], active_task_cap=100) # type: ignore[list-item]
summary = cleanup_certain_oom_tasks(
pool,
FakeDiscovery(
{"owner/model": 1 * GIB},
configs={"owner/model": {"architectures": ["Qwen2ForCausalLM"]}},
), # type: ignore[arg-type]
architecture_compatibility_blocks={
vllm_key: vllm_block,
transformers_key: transformers_block,
},
task_compatibility_contexts={},
log=lambda _message: None,
)
self.assertEqual(1, summary["architectureIncompatibleCount"])
self.assertEqual(1, summary["cancelledCount"])
self.assertEqual(
["transformers", "vllm"],
summary["architectureFrameworkCatalog"]["iluvatar_bi-100|text-generation"],
)
self.assertEqual([[1]], client.stopped)
def test_queue_cleanup_fetches_config_and_stops_known_incompatible_waiting_task(self) -> None:
key, block = self.architecture_block()
client = FakeQueueClient(