fix: enrich cleanup from active queue context
This commit is contained in:
@@ -65,6 +65,13 @@ class FakeQueueClient:
|
||||
self.records = [record for record in self.records if int(record["taskId"]) not in ids]
|
||||
return {"code": 0, "data": None}
|
||||
|
||||
def list_framework_stats(self, task_type: str, target_gpu: str) -> list[dict[str, Any]]:
|
||||
del task_type, target_gpu
|
||||
return [
|
||||
{"framework": "vllm", "modelCount": 100, "successCount": 60},
|
||||
{"framework": "transformers", "modelCount": 100, "successCount": 50},
|
||||
]
|
||||
|
||||
def count_active_tasks(self, **_kwargs: Any) -> int:
|
||||
return len(self.records)
|
||||
|
||||
@@ -173,6 +180,107 @@ class QueueCleanupTests(unittest.TestCase):
|
||||
self.assertEqual(1, skipped["runningMatchedProtected"])
|
||||
self.assertEqual(1, skipped["noMatchingBlock"])
|
||||
|
||||
def test_architecture_cleanup_uses_current_queue_task_type_when_every_framework_is_blocked(self) -> None:
|
||||
vllm_key, vllm_block = self.architecture_block(framework="vllm")
|
||||
transformers_key, transformers_block = self.architecture_block(framework="transformers")
|
||||
tasks = [
|
||||
OwnedTask(
|
||||
0,
|
||||
1,
|
||||
"owner/model",
|
||||
"Iluvatar_bi-100",
|
||||
"waiting",
|
||||
task_type="text-generation",
|
||||
)
|
||||
]
|
||||
|
||||
selected, skipped = find_architecture_incompatible_tasks(
|
||||
tasks,
|
||||
architecture_blocks={
|
||||
vllm_key: vllm_block,
|
||||
transformers_key: transformers_block,
|
||||
},
|
||||
task_contexts={},
|
||||
model_configs={
|
||||
"owner/model": {"architectures": ["Qwen2ForCausalLM"]},
|
||||
},
|
||||
framework_catalog={
|
||||
("iluvatar_bi-100", "text-generation"): {"vllm", "transformers"},
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual([1], [item["taskId"] for item in selected])
|
||||
self.assertEqual("all_live_frameworks", selected[0]["architectureMatchScope"])
|
||||
self.assertEqual(["transformers", "vllm"], selected[0]["evaluatedFrameworks"])
|
||||
self.assertEqual(1, skipped["frameworkContextUnknown"])
|
||||
|
||||
def test_architecture_cleanup_does_not_guess_when_only_some_frameworks_are_blocked(self) -> None:
|
||||
key, block = self.architecture_block(framework="vllm")
|
||||
tasks = [
|
||||
OwnedTask(
|
||||
0,
|
||||
1,
|
||||
"owner/model",
|
||||
"Iluvatar_bi-100",
|
||||
"waiting",
|
||||
task_type="text-generation",
|
||||
)
|
||||
]
|
||||
|
||||
selected, skipped = find_architecture_incompatible_tasks(
|
||||
tasks,
|
||||
architecture_blocks={key: block},
|
||||
task_contexts={},
|
||||
model_configs={
|
||||
"owner/model": {"architectures": ["Qwen2ForCausalLM"]},
|
||||
},
|
||||
framework_catalog={
|
||||
("iluvatar_bi-100", "text-generation"): {"vllm", "transformers"},
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual([], selected)
|
||||
self.assertEqual(1, skipped["partiallyBlockedFrameworkSet"])
|
||||
|
||||
def test_queue_cleanup_enriches_active_api_record_and_stops_only_all_framework_block(self) -> None:
|
||||
vllm_key, vllm_block = self.architecture_block(framework="vllm")
|
||||
transformers_key, transformers_block = self.architecture_block(framework="transformers")
|
||||
client = FakeQueueClient(
|
||||
[
|
||||
{
|
||||
"taskId": 1,
|
||||
"modelId": "owner/model",
|
||||
"gpuType": "Iluvatar_bi-100",
|
||||
"status": "waiting",
|
||||
"modelTaskLevelId": 23,
|
||||
"modelTaskLevel": "文本生成",
|
||||
}
|
||||
]
|
||||
)
|
||||
pool = ModelHubClientPool([client], active_task_cap=100) # type: ignore[list-item]
|
||||
|
||||
summary = cleanup_certain_oom_tasks(
|
||||
pool,
|
||||
FakeDiscovery(
|
||||
{"owner/model": 1 * GIB},
|
||||
configs={"owner/model": {"architectures": ["Qwen2ForCausalLM"]}},
|
||||
), # type: ignore[arg-type]
|
||||
architecture_compatibility_blocks={
|
||||
vllm_key: vllm_block,
|
||||
transformers_key: transformers_block,
|
||||
},
|
||||
task_compatibility_contexts={},
|
||||
log=lambda _message: None,
|
||||
)
|
||||
|
||||
self.assertEqual(1, summary["architectureIncompatibleCount"])
|
||||
self.assertEqual(1, summary["cancelledCount"])
|
||||
self.assertEqual(
|
||||
["transformers", "vllm"],
|
||||
summary["architectureFrameworkCatalog"]["iluvatar_bi-100|text-generation"],
|
||||
)
|
||||
self.assertEqual([[1]], client.stopped)
|
||||
|
||||
def test_queue_cleanup_fetches_config_and_stops_known_incompatible_waiting_task(self) -> None:
|
||||
key, block = self.architecture_block()
|
||||
client = FakeQueueClient(
|
||||
|
||||
Reference in New Issue
Block a user