CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

fix: restore accuracy-first route admission

Browse Source
This commit is contained in:
CoolBoy
2026-09-22 15:26:42 +08:00
parent 4afce879dc
commit 6154dc738f
11 changed files with 343 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
tests/test_concurrency.py
View File

@@ -733,9 +733,9 @@ class ClientPoolConcurrencyTests(unittest.TestCase):
)
self.assertEqual(2, summary["submittedCount"])
self.assertEqual(len(supported_gpus), summary["skipReasonCounts"]["already_processed_for_gpu"])
self.assertEqual("popular_history", summary["scanStages"][-1]["name"])
self.assertEqual("downloads", summary["scanStages"][-1]["sort"])
self.assertEqual(1, summary["skipReasonCounts"]["model_route_limit_reached"])
self.assertEqual("all_history", summary["scanStages"][-1]["name"])
self.assertEqual("last_modified", summary["scanStages"][-1]["sort"])
self.assertIsNone(discovery.calls[-1])
def test_adaptive_strategy_counts_only_platform_accepted_tasks(self) -> None:

124
tests/test_market_intelligence.py
View File

@@ -219,7 +219,7 @@ class MarketIntelligenceTests(unittest.TestCase):
"frameworkStats": {
"text-generation": {
"gpu": {
"vllm": {"modelCount": 1000, "wilsonLowerBound": 0.10},
"vllm": {"modelCount": 1000, "wilsonLowerBound": 0.30},
"vllm-mlu": {"modelCount": 800, "wilsonLowerBound": 0.35},
"vllm-customized": {"modelCount": 2, "wilsonLowerBound": 0.90},
}
@@ -357,7 +357,7 @@ class MarketIntelligenceTests(unittest.TestCase):
"frameworkStats": {
"text-generation": {
"Cambricon_mlu-370-x4": {
"vllm": {"modelCount": 1000, "wilsonLowerBound": 0.10},
"vllm": {"modelCount": 1000, "wilsonLowerBound": 0.30},
"future-engine": {
"modelCount": 1000,
"successCount": 500,
@@ -396,7 +396,7 @@ class MarketIntelligenceTests(unittest.TestCase):
"frameworkStats": {
"text-generation": {
"Cambricon_mlu-370-x4": {
"vllm": {"modelCount": 1000, "wilsonLowerBound": 0.10},
"vllm": {"modelCount": 1000, "wilsonLowerBound": 0.30},
"future-engine": {
"modelCount": 2,
"wilsonLowerBound": 0.90,
@@ -454,6 +454,124 @@ class MarketIntelligenceTests(unittest.TestCase):
)
self.assertIsNone(candidate)
def test_low_confidence_public_framework_is_rejected_even_with_many_samples(self) -> None:
manager = MarketIntelligenceManager("unused.json", framework_min_samples=100)
manager.state = {
"frameworkStats": {
"text-generation": {
"Cambricon_mlu-370-x4": {
"vllm": {"modelCount": 10_000, "wilsonLowerBound": 0.12},
}
}
}
}
candidate = choose_candidate_for_gpu(
model=HFModelSummary(
repo_id="owner/model",
downloads=100,
last_modified=None,
pipeline_tag="text-generation",
),
inspection=ModelInspection(repo_id="owner/model", weight_files=["model.safetensors"]),
template_selector=TemplateSelector(),
task_types=["text-generation"],
target_gpu="Cambricon_mlu-370-x4",
market_intelligence=manager,
)
self.assertIsNone(candidate)
def test_repeated_ambiguous_failures_open_operational_circuit(self) -> None:
manager = MarketIntelligenceManager("unused.json", framework_min_samples=100)
manager.state = {
"frameworkStats": {
"text-generation": {
"Biren_166m": {
"vllm_fix_tokenizer": {"modelCount": 10_000, "wilsonLowerBound": 0.35},
}
}
}
}
manager.set_local_outcome_stats(
{
"recentCombinationStats": {
"Biren_166m|vllm_fix_tokenizer|text-generation": {
"successCount": 0,
"failureCount": 5,
"attributableFailureCount": 0,
"consecutiveFailures": 0,
"lastTerminalAt": datetime.now(timezone.utc).isoformat(),
}
}
}
)
metadata = manager.framework_metadata(
"text-generation", "Biren_166m", "vllm_fix_tokenizer"
)
self.assertTrue(metadata["frameworkCircuitOpen"])
self.assertEqual(
"recent_terminal_success_below_10_percent",
metadata["frameworkCircuitReason"],
)
def test_low_sample_gpu_with_recent_success_and_proven_framework_remains_eligible(self) -> None:
manager = MarketIntelligenceManager("unused.json", framework_min_samples=100)
manager.state = {
"gpuStats": {
"Sunrise_pt-200-x1": {
"canVerify": True,
"submissionEligible": False,
"recentTerminal": 10,
"recentSuccess": 5,
"recentWilsonLowerBound": 0.24,
"healthFactor": 1.0,
"selectionWeight": 0.5,
}
},
"frameworkStats": {
"text-generation": {
"Sunrise_pt-200-x1": {
"vllm_fix_tokenizer": {
"modelCount": 6_000,
"wilsonLowerBound": 0.41,
"officialConfigValid": True,
}
}
}
},
}
self.assertEqual(
["Sunrise_pt-200-x1"],
manager.eligible_gpus(["Sunrise_pt-200-x1"]),
)
def test_low_sample_gpu_does_not_bypass_absolute_quality_floor(self) -> None:
manager = MarketIntelligenceManager("unused.json", framework_min_samples=100)
manager.state = {
"gpuStats": {
"gpu-low-quality": {
"canVerify": True,
"submissionEligible": False,
"recentTerminal": 10,
"recentSuccess": 1,
"recentWilsonLowerBound": 0.02,
"healthFactor": 1.0,
"selectionWeight": 0.5,
}
},
"frameworkStats": {
"text-generation": {
"gpu-low-quality": {
"vllm": {
"modelCount": 6_000,
"wilsonLowerBound": 0.41,
"officialConfigValid": True,
}
}
}
},
}
self.assertEqual([], manager.eligible_gpus(["gpu-low-quality"]))
def test_five_consecutive_local_failures_circuit_break_framework(self) -> None:
manager = MarketIntelligenceManager("unused.json", framework_min_samples=100)
manager.state = {

84
tests/test_submission_safety.py
View File

@@ -21,6 +21,7 @@ from modelhub_client import ( # noqa: E402
parse_model_submission_precheck,
)
from models import HFModelSummary, ModelInspection # noqa: E402
from outcome_tracker import OutcomeTracker # noqa: E402
from submission_exclusions import SubmissionExclusionStore # noqa: E402
from template_selector import TemplateSelector # noqa: E402
@@ -286,9 +287,10 @@ class SubmissionSafetyTests(unittest.TestCase):
}
precheck = parse_model_submission_precheck(payload)
self.assertEqual(
{"running", "success", "unknown", "failed_without_framework"},
{"failed", "running", "success", "unknown", "failed_without_framework"},
precheck["processedGpus"],
)
self.assertEqual({"success"}, precheck["successfulGpus"])
self.assertEqual({"failed": {"vllm"}}, precheck["failedFrameworksByGpu"])
def test_unverified_gpu_becomes_candidate_but_validating_gpu_does_not(self) -> None:
@@ -321,7 +323,7 @@ class SubmissionSafetyTests(unittest.TestCase):
self.assertEqual([], active_candidates)
self.assertEqual("already_processed_for_gpu", active_skips[0]["reason"])
def test_failed_gpu_requires_an_alternative_framework(self) -> None:
def test_failed_gpu_remains_closed_even_when_an_alternative_framework_exists(self) -> None:
model = SafeDiscovery().model
client = FailedFrameworkClient()
candidates, skipped, failed = process_model_for_candidates(
@@ -334,23 +336,14 @@ class SubmissionSafetyTests(unittest.TestCase):
market_intelligence=TwoFrameworkMarket(), # type: ignore[arg-type]
official_registry=AllGpuOfficialRegistry(), # type: ignore[arg-type]
)
self.assertEqual([], skipped)
self.assertEqual("prior_failed_gpu_closed_accuracy_first", skipped[-1]["reason"])
self.assertEqual([], failed)
self.assertEqual(["vllm-mlu"], [item["framework"] for item in candidates])
self.assertEqual(["vllm"], candidates[0]["previousFailedFrameworks"])
self.assertEqual([], candidates)
blocked = submit_candidate({**candidates[0], "framework": "vllm"}, client) # type: ignore[arg-type]
accepted = submit_candidate(candidates[0], client) # type: ignore[arg-type]
blocked = submit_candidate(candidate("Cambricon_mlu-370-x4"), client) # type: ignore[arg-type]
self.assertEqual("duplicate", blocked["outcome"])
self.assertEqual("prior_failed_framework_for_gpu", blocked["reason"])
self.assertEqual("submitted", accepted["outcome"])
self.assertEqual(1, client.add_calls)
unvetted = submit_candidate(
{**candidates[0], "frameworkEvidenceQualified": False}, client # type: ignore[arg-type]
)
self.assertEqual("framework_evidence_deferred", unvetted["outcome"])
self.assertEqual(1, client.add_calls)
self.assertEqual("prior_failed_gpu_closed_accuracy_first", blocked["reason"])
self.assertEqual(0, client.add_calls)
no_routes, no_route_skips, _ = process_model_for_candidates(
model=model,
@@ -363,7 +356,7 @@ class SubmissionSafetyTests(unittest.TestCase):
official_registry=AllGpuOfficialRegistry(), # type: ignore[arg-type]
)
self.assertEqual([], no_routes)
self.assertEqual("prior_failed_framework_no_alternative", no_route_skips[-1]["reason"])
self.assertEqual("prior_failed_gpu_closed_accuracy_first", no_route_skips[-1]["reason"])
no_market_routes, no_market_skips, _ = process_model_for_candidates(
model=model,
@@ -375,10 +368,65 @@ class SubmissionSafetyTests(unittest.TestCase):
)
self.assertEqual([], no_market_routes)
self.assertEqual(
"prior_failed_gpu_requires_public_framework_evidence",
"prior_failed_gpu_closed_accuracy_first",
no_market_skips[-1]["reason"],
)
def test_model_route_limit_is_global_across_poll_cycles(self) -> None:
class SixRouteClient:
@staticmethod
def model_submission_precheck(_model_id: str) -> dict:
return {
"processedGpus": {f"gpu-{index}" for index in range(1, 7)},
"successfulGpus": {"gpu-1"},
"failedFrameworksByGpu": {},
}
candidates, skipped, failed = process_model_for_candidates(
model=SafeDiscovery().model,
hf_discovery=SafeDiscovery(), # type: ignore[arg-type]
modelhub_client=SixRouteClient(), # type: ignore[arg-type]
template_selector=TemplateSelector(),
target_gpus=["gpu-7"],
allowed_task_types=["text-generation"],
)
self.assertEqual([], candidates)
self.assertEqual([], failed)
self.assertEqual("model_route_limit_reached", skipped[0]["reason"])
self.assertEqual(6, skipped[0]["processedGpuCount"])
def test_model_waits_for_feedback_after_two_pending_routes(self) -> None:
with tempfile.TemporaryDirectory() as temporary_dir:
root = Path(temporary_dir)
tracker = OutcomeTracker(
root / "outcomes.jsonl",
checkpoint_path=root / "checkpoint.json",
recent_path=root / "recent.jsonl",
)
tracker.record_submission(
"owner/model", "gpu-1", "vllm", "text-generation", "task-1",
"2026-09-22T00:00:00+00:00",
)
tracker.record_submission(
"owner/model", "gpu-2", "vllm", "text-generation", "task-2",
"2026-09-22T00:01:00+00:00",
)
candidates, skipped, failed = process_model_for_candidates(
model=SafeDiscovery().model,
hf_discovery=SafeDiscovery(), # type: ignore[arg-type]
modelhub_client=OtherGpuProcessedClient(), # type: ignore[arg-type]
template_selector=TemplateSelector(),
target_gpus=["gpu-3"],
allowed_task_types=["text-generation"],
outcome_tracker=tracker,
)
self.assertEqual([], candidates)
self.assertEqual([], failed)
self.assertEqual("model_pending_feedback_limit", skipped[0]["reason"])
self.assertEqual(2, skipped[0]["pendingRouteCount"])
def test_published_size_avoids_official_calls_for_certain_oom(self) -> None:
class CountingRegistry(AllGpuOfficialRegistry):
calls = 0