CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

fix: recover eligible model GPU routes

Browse Source
This commit is contained in:
CoolBoy
2026-09-20 10:47:22 +08:00
parent 3077b169c5
commit 75c034d588
13 changed files with 436 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
tests/test_concurrency.py
View File

@@ -544,7 +544,8 @@ class ClientPoolConcurrencyTests(unittest.TestCase):
self.assertEqual(2, summary["submittedCount"])
self.assertEqual(len(supported_gpus), summary["skipReasonCounts"]["already_processed_for_gpu"])
self.assertEqual("all_history", summary["scanStages"][-1]["name"])
self.assertEqual("popular_history", summary["scanStages"][-1]["name"])
self.assertEqual("downloads", summary["scanStages"][-1]["sort"])
self.assertIsNone(discovery.calls[-1])
def test_adaptive_strategy_counts_only_platform_accepted_tasks(self) -> None:

8
tests/test_market_intelligence.py
View File

@@ -115,6 +115,14 @@ class MarketIntelligenceTests(unittest.TestCase):
self.assertTrue(
state["frameworkStats"]["text-generation"]["fast"]["transformers"]["officialConfigValid"]
)
self.assertTrue(
manager.has_vetted_alternative_framework("text-generation", "fast", {"vllm"})
)
self.assertFalse(
manager.has_vetted_alternative_framework(
"text-generation", "fast", {"vllm", "transformers"}
)
)
def test_snapshot_cache_prevents_repeated_public_api_scans(self) -> None:
with tempfile.TemporaryDirectory() as temporary_dir:

25
tests/test_modelscope_discovery.py
View File

@@ -93,6 +93,31 @@ class ModelScopeDiscoveryTests(unittest.TestCase):
self.assertEqual([], models)
self.assertEqual(50, http_client.queries[0]["page_size"])
def test_popular_sort_uses_separate_page_cache(self) -> None:
http_client = RecordingHttpClient()
discovery = HuggingFaceDiscovery(http_client=http_client) # type: ignore[arg-type]
discovery.list_recent_models(pipeline_tags=["text-generation"], limit=50, min_downloads=0)
discovery.list_recent_models(
pipeline_tags=["text-generation"], limit=50, min_downloads=0, sort="downloads"
)
self.assertEqual(["last_modified", "downloads"], [item["sort"] for item in http_client.queries])
def test_popular_scan_rotates_bounded_page_windows(self) -> None:
http_client = RecordingHttpClient()
discovery = HuggingFaceDiscovery(http_client=http_client) # type: ignore[arg-type]
for _ in range(2):
discovery.list_recent_models(
pipeline_tags=["text-generation"],
limit=200,
min_downloads=50,
sort="downloads",
rotate_pages=True,
)
self.assertEqual([1, 5], [item["page_number"] for item in http_client.queries])
def test_rate_limited_page_keeps_previous_pages_and_resumes_from_cache(self) -> None:
http_client = RateLimitedPageClient()
discovery = HuggingFaceDiscovery(

4
tests/test_poll_policy.py
View File

@@ -27,10 +27,12 @@ class PollPolicyTests(unittest.TestCase):
def test_architecture_history_backfill_resumes_and_keeps_only_decision_state(self) -> None:
class PagedHistoryClient:
calls: list[int] = []
end_times: list[datetime] = []
def list_tasks_page(self, **kwargs): # noqa: ANN003, ANN201
current = int(kwargs["current"])
self.calls.append(current)
self.end_times.append(kwargs["end_time"])
records = {
1: [
{
@@ -101,6 +103,8 @@ class PollPolicyTests(unittest.TestCase):
self.assertTrue(second["complete"])
self.assertNotIn("seenTaskIds", second)
self.assertEqual([1, 2], client.calls)
self.assertTrue(all(isinstance(value, datetime) for value in client.end_times))
self.assertEqual(client.end_times[0], client.end_times[1])
self.assertEqual(3, restored.get_stats_report()["terminalRecords"])
self.assertEqual([], json.loads(outcomes.read_text(encoding="utf-8") or "[]"))
persisted_progress = progress_path.read_text(encoding="utf-8")

182
tests/test_submission_safety.py
View File

@@ -13,6 +13,7 @@ if str(PACKAGE_DIR) in sys.path:
sys.path.insert(0, str(PACKAGE_DIR))
from main import build_parser, one_candidate_per_model, process_model_for_candidates, run_submission, submit_candidate # noqa: E402
from candidate_preflight import CandidatePreflightAdvisor # noqa: E402
from modelhub_client import ( # noqa: E402
ModelHubAPIError,
ModelHubClientPool,
@@ -68,6 +69,23 @@ class ExactGpuProcessedClient:
return {"code": 0, "data": {"id": "should-not-submit"}}
class FailedFrameworkClient:
def __init__(self) -> None:
self.add_calls = 0
@staticmethod
def model_submission_precheck(_model_id: str, *, force_refresh: bool = False) -> dict:
del force_refresh
return {
"processedGpus": set(),
"failedFrameworksByGpu": {"Cambricon_mlu-370-x4": {"vllm"}},
}
def add_task(self, _payload: dict) -> dict:
self.add_calls += 1
return {"code": 0, "data": {"id": str(self.add_calls)}}
class AllGpuOfficialRegistry:
@staticmethod
def task_types_for(_client, *, model_address: str, model_last_modified: str | None, gpu: str) -> list[str]:
@@ -100,7 +118,11 @@ class AllGpuVettedMarket:
@staticmethod
def framework_metadata(_task_type: str, _target_gpu: str, _framework: str) -> dict:
return {"frameworkCombinedScore": 1.0, "frameworkOfficialConfigValid": True}
return {
"frameworkCombinedScore": 1.0,
"frameworkOfficialConfigValid": True,
"frameworkEvidenceQualified": True,
}
@staticmethod
def gpu_metadata(_target_gpu: str) -> dict:
@@ -113,6 +135,15 @@ class SeventhGpuFallbackMarket(AllGpuVettedMarket):
return target_gpu == "gpu-7"
class TwoFrameworkMarket(AllGpuVettedMarket):
@staticmethod
def selectable_frameworks(
*, task_type: str, target_gpu: str, incumbent_frameworks: list[str], inspection
) -> list[str]:
del task_type, target_gpu, incumbent_frameworks, inspection
return ["vllm", "vllm-mlu"]
class UniquenessRejectingClient:
@staticmethod
def add_task(_payload: dict) -> dict:
@@ -230,6 +261,155 @@ class SubmissionSafetyTests(unittest.TestCase):
self.assertEqual("already_processed_for_gpu", skipped[0]["reason"])
self.assertEqual([], failed)
def test_precheck_distinguishes_unverified_failed_and_active_gpus(self) -> None:
payload = {
"data": {
"isInDB": True,
"verifyResult": {
"free": {"result": "未验证", "records": []},
"failed": {
"result": "验证失败",
"records": [{"framework": "vllm", "verifyResult": -1}],
},
"running": {"result": "验证中", "records": []},
"success": {
"result": "已验证",
"records": [{"framework": "vllm", "verifyResult": 1}],
},
"unknown": {"result": "未知状态", "records": []},
"failed_without_framework": {
"result": "验证失败",
"records": [{"verifyResult": -1}],
},
},
},
}
precheck = parse_model_submission_precheck(payload)
self.assertEqual(
{"running", "success", "unknown", "failed_without_framework"},
precheck["processedGpus"],
)
self.assertEqual({"failed": {"vllm"}}, precheck["failedFrameworksByGpu"])
def test_unverified_gpu_becomes_candidate_but_validating_gpu_does_not(self) -> None:
class StatusClient:
def __init__(self, status: str) -> None:
self.status = status
def model_submission_precheck(self, _model_id: str) -> dict:
return parse_model_submission_precheck({
"data": {
"verifyResult": {
"Vastai_va16": {"result": self.status, "records": []}
}
}
})
def collect(status: str):
return process_model_for_candidates(
model=SafeDiscovery().model,
hf_discovery=SafeDiscovery(), # type: ignore[arg-type]
modelhub_client=StatusClient(status), # type: ignore[arg-type]
template_selector=TemplateSelector(),
target_gpus=["Vastai_va16"],
allowed_task_types=["text-generation"],
)
open_candidates, _, _ = collect("未验证")
active_candidates, active_skips, _ = collect("验证中")
self.assertEqual(["Vastai_va16"], [item["targetGpu"] for item in open_candidates])
self.assertEqual([], active_candidates)
self.assertEqual("already_processed_for_gpu", active_skips[0]["reason"])
def test_failed_gpu_requires_an_alternative_framework(self) -> None:
model = SafeDiscovery().model
client = FailedFrameworkClient()
candidates, skipped, failed = process_model_for_candidates(
model=model,
hf_discovery=SafeDiscovery(), # type: ignore[arg-type]
modelhub_client=client, # type: ignore[arg-type]
template_selector=TemplateSelector(),
target_gpus=["Cambricon_mlu-370-x4"],
allowed_task_types=["text-generation"],
market_intelligence=TwoFrameworkMarket(), # type: ignore[arg-type]
official_registry=AllGpuOfficialRegistry(), # type: ignore[arg-type]
)
self.assertEqual([], skipped)
self.assertEqual([], failed)
self.assertEqual(["vllm-mlu"], [item["framework"] for item in candidates])
self.assertEqual(["vllm"], candidates[0]["previousFailedFrameworks"])
blocked = submit_candidate({**candidates[0], "framework": "vllm"}, client) # type: ignore[arg-type]
accepted = submit_candidate(candidates[0], client) # type: ignore[arg-type]
self.assertEqual("duplicate", blocked["outcome"])
self.assertEqual("prior_failed_framework_for_gpu", blocked["reason"])
self.assertEqual("submitted", accepted["outcome"])
self.assertEqual(1, client.add_calls)
unvetted = submit_candidate(
{**candidates[0], "frameworkEvidenceQualified": False}, client # type: ignore[arg-type]
)
self.assertEqual("framework_evidence_deferred", unvetted["outcome"])
self.assertEqual(1, client.add_calls)
no_routes, no_route_skips, _ = process_model_for_candidates(
model=model,
hf_discovery=SafeDiscovery(), # type: ignore[arg-type]
modelhub_client=client, # type: ignore[arg-type]
template_selector=TemplateSelector(),
target_gpus=["Cambricon_mlu-370-x4"],
allowed_task_types=["text-generation"],
market_intelligence=AllGpuVettedMarket(), # type: ignore[arg-type]
official_registry=AllGpuOfficialRegistry(), # type: ignore[arg-type]
)
self.assertEqual([], no_routes)
self.assertEqual("prior_failed_framework_no_alternative", no_route_skips[-1]["reason"])
no_market_routes, no_market_skips, _ = process_model_for_candidates(
model=model,
hf_discovery=SafeDiscovery(), # type: ignore[arg-type]
modelhub_client=client, # type: ignore[arg-type]
template_selector=TemplateSelector(),
target_gpus=["Cambricon_mlu-370-x4"],
allowed_task_types=["text-generation"],
)
self.assertEqual([], no_market_routes)
self.assertEqual(
"prior_failed_gpu_requires_public_framework_evidence",
no_market_skips[-1]["reason"],
)
def test_published_size_avoids_official_calls_for_certain_oom(self) -> None:
class CountingRegistry(AllGpuOfficialRegistry):
calls = 0
def task_types_for(self, *_args, **_kwargs) -> list[str]:
self.calls += 1
return ["text-generation"]
registry = CountingRegistry()
model = HFModelSummary(
repo_id="owner/oversized",
downloads=100,
last_modified=datetime(2026, 1, 1, tzinfo=timezone.utc),
pipeline_tag="text-generation",
file_size=100 * 1024**3,
)
candidates, skipped, failed = process_model_for_candidates(
model=model,
hf_discovery=SafeDiscovery(), # type: ignore[arg-type]
modelhub_client=OtherGpuProcessedClient(), # type: ignore[arg-type]
template_selector=TemplateSelector(),
target_gpus=["MetaX_c-500"],
allowed_task_types=["text-generation"],
preflight_advisor=CandidatePreflightAdvisor(),
official_registry=registry, # type: ignore[arg-type]
)
self.assertEqual([], candidates)
self.assertEqual("preflight_predicted_oom", skipped[0]["reason"])
self.assertEqual([], failed)
self.assertEqual(0, registry.calls)
def test_lookup_failure_is_fail_closed(self) -> None:
pool = ModelHubClientPool([FailingLookupClient()]) # type: ignore[arg-type]
with self.assertRaises(ModelHubAPIError):