CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

fix: recover eligible model GPU routes

Browse Source
This commit is contained in:
CoolBoy
2026-09-20 10:47:22 +08:00
parent 3077b169c5
commit 75c034d588
13 changed files with 436 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

182
tests/test_submission_safety.py
View File

@@ -13,6 +13,7 @@ if str(PACKAGE_DIR) in sys.path:
sys.path.insert(0, str(PACKAGE_DIR))
from main import build_parser, one_candidate_per_model, process_model_for_candidates, run_submission, submit_candidate # noqa: E402
from candidate_preflight import CandidatePreflightAdvisor # noqa: E402
from modelhub_client import ( # noqa: E402
ModelHubAPIError,
ModelHubClientPool,
@@ -68,6 +69,23 @@ class ExactGpuProcessedClient:
return {"code": 0, "data": {"id": "should-not-submit"}}
class FailedFrameworkClient:
def __init__(self) -> None:
self.add_calls = 0
@staticmethod
def model_submission_precheck(_model_id: str, *, force_refresh: bool = False) -> dict:
del force_refresh
return {
"processedGpus": set(),
"failedFrameworksByGpu": {"Cambricon_mlu-370-x4": {"vllm"}},
}
def add_task(self, _payload: dict) -> dict:
self.add_calls += 1
return {"code": 0, "data": {"id": str(self.add_calls)}}
class AllGpuOfficialRegistry:
@staticmethod
def task_types_for(_client, *, model_address: str, model_last_modified: str | None, gpu: str) -> list[str]:
@@ -100,7 +118,11 @@ class AllGpuVettedMarket:
@staticmethod
def framework_metadata(_task_type: str, _target_gpu: str, _framework: str) -> dict:
return {"frameworkCombinedScore": 1.0, "frameworkOfficialConfigValid": True}
return {
"frameworkCombinedScore": 1.0,
"frameworkOfficialConfigValid": True,
"frameworkEvidenceQualified": True,
}
@staticmethod
def gpu_metadata(_target_gpu: str) -> dict:
@@ -113,6 +135,15 @@ class SeventhGpuFallbackMarket(AllGpuVettedMarket):
return target_gpu == "gpu-7"
class TwoFrameworkMarket(AllGpuVettedMarket):
@staticmethod
def selectable_frameworks(
*, task_type: str, target_gpu: str, incumbent_frameworks: list[str], inspection
) -> list[str]:
del task_type, target_gpu, incumbent_frameworks, inspection
return ["vllm", "vllm-mlu"]
class UniquenessRejectingClient:
@staticmethod
def add_task(_payload: dict) -> dict:
@@ -230,6 +261,155 @@ class SubmissionSafetyTests(unittest.TestCase):
self.assertEqual("already_processed_for_gpu", skipped[0]["reason"])
self.assertEqual([], failed)
def test_precheck_distinguishes_unverified_failed_and_active_gpus(self) -> None:
payload = {
"data": {
"isInDB": True,
"verifyResult": {
"free": {"result": "未验证", "records": []},
"failed": {
"result": "验证失败",
"records": [{"framework": "vllm", "verifyResult": -1}],
},
"running": {"result": "验证中", "records": []},
"success": {
"result": "已验证",
"records": [{"framework": "vllm", "verifyResult": 1}],
},
"unknown": {"result": "未知状态", "records": []},
"failed_without_framework": {
"result": "验证失败",
"records": [{"verifyResult": -1}],
},
},
},
}
precheck = parse_model_submission_precheck(payload)
self.assertEqual(
{"running", "success", "unknown", "failed_without_framework"},
precheck["processedGpus"],
)
self.assertEqual({"failed": {"vllm"}}, precheck["failedFrameworksByGpu"])
def test_unverified_gpu_becomes_candidate_but_validating_gpu_does_not(self) -> None:
class StatusClient:
def __init__(self, status: str) -> None:
self.status = status
def model_submission_precheck(self, _model_id: str) -> dict:
return parse_model_submission_precheck({
"data": {
"verifyResult": {
"Vastai_va16": {"result": self.status, "records": []}
}
}
})
def collect(status: str):
return process_model_for_candidates(
model=SafeDiscovery().model,
hf_discovery=SafeDiscovery(), # type: ignore[arg-type]
modelhub_client=StatusClient(status), # type: ignore[arg-type]
template_selector=TemplateSelector(),
target_gpus=["Vastai_va16"],
allowed_task_types=["text-generation"],
)
open_candidates, _, _ = collect("未验证")
active_candidates, active_skips, _ = collect("验证中")
self.assertEqual(["Vastai_va16"], [item["targetGpu"] for item in open_candidates])
self.assertEqual([], active_candidates)
self.assertEqual("already_processed_for_gpu", active_skips[0]["reason"])
def test_failed_gpu_requires_an_alternative_framework(self) -> None:
model = SafeDiscovery().model
client = FailedFrameworkClient()
candidates, skipped, failed = process_model_for_candidates(
model=model,
hf_discovery=SafeDiscovery(), # type: ignore[arg-type]
modelhub_client=client, # type: ignore[arg-type]
template_selector=TemplateSelector(),
target_gpus=["Cambricon_mlu-370-x4"],
allowed_task_types=["text-generation"],
market_intelligence=TwoFrameworkMarket(), # type: ignore[arg-type]
official_registry=AllGpuOfficialRegistry(), # type: ignore[arg-type]
)
self.assertEqual([], skipped)
self.assertEqual([], failed)
self.assertEqual(["vllm-mlu"], [item["framework"] for item in candidates])
self.assertEqual(["vllm"], candidates[0]["previousFailedFrameworks"])
blocked = submit_candidate({**candidates[0], "framework": "vllm"}, client) # type: ignore[arg-type]
accepted = submit_candidate(candidates[0], client) # type: ignore[arg-type]
self.assertEqual("duplicate", blocked["outcome"])
self.assertEqual("prior_failed_framework_for_gpu", blocked["reason"])
self.assertEqual("submitted", accepted["outcome"])
self.assertEqual(1, client.add_calls)
unvetted = submit_candidate(
{**candidates[0], "frameworkEvidenceQualified": False}, client # type: ignore[arg-type]
)
self.assertEqual("framework_evidence_deferred", unvetted["outcome"])
self.assertEqual(1, client.add_calls)
no_routes, no_route_skips, _ = process_model_for_candidates(
model=model,
hf_discovery=SafeDiscovery(), # type: ignore[arg-type]
modelhub_client=client, # type: ignore[arg-type]
template_selector=TemplateSelector(),
target_gpus=["Cambricon_mlu-370-x4"],
allowed_task_types=["text-generation"],
market_intelligence=AllGpuVettedMarket(), # type: ignore[arg-type]
official_registry=AllGpuOfficialRegistry(), # type: ignore[arg-type]
)
self.assertEqual([], no_routes)
self.assertEqual("prior_failed_framework_no_alternative", no_route_skips[-1]["reason"])
no_market_routes, no_market_skips, _ = process_model_for_candidates(
model=model,
hf_discovery=SafeDiscovery(), # type: ignore[arg-type]
modelhub_client=client, # type: ignore[arg-type]
template_selector=TemplateSelector(),
target_gpus=["Cambricon_mlu-370-x4"],
allowed_task_types=["text-generation"],
)
self.assertEqual([], no_market_routes)
self.assertEqual(
"prior_failed_gpu_requires_public_framework_evidence",
no_market_skips[-1]["reason"],
)
def test_published_size_avoids_official_calls_for_certain_oom(self) -> None:
class CountingRegistry(AllGpuOfficialRegistry):
calls = 0
def task_types_for(self, *_args, **_kwargs) -> list[str]:
self.calls += 1
return ["text-generation"]
registry = CountingRegistry()
model = HFModelSummary(
repo_id="owner/oversized",
downloads=100,
last_modified=datetime(2026, 1, 1, tzinfo=timezone.utc),
pipeline_tag="text-generation",
file_size=100 * 1024**3,
)
candidates, skipped, failed = process_model_for_candidates(
model=model,
hf_discovery=SafeDiscovery(), # type: ignore[arg-type]
modelhub_client=OtherGpuProcessedClient(), # type: ignore[arg-type]
template_selector=TemplateSelector(),
target_gpus=["MetaX_c-500"],
allowed_task_types=["text-generation"],
preflight_advisor=CandidatePreflightAdvisor(),
official_registry=registry, # type: ignore[arg-type]
)
self.assertEqual([], candidates)
self.assertEqual("preflight_predicted_oom", skipped[0]["reason"])
self.assertEqual([], failed)
self.assertEqual(0, registry.calls)
def test_lookup_failure_is_fail_closed(self) -> None:
pool = ModelHubClientPool([FailingLookupClient()]) # type: ignore[arg-type]
with self.assertRaises(ModelHubAPIError):