CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

fix: restore accuracy-first route admission

Browse Source
This commit is contained in:
CoolBoy
2026-09-22 15:26:42 +08:00
parent 4afce879dc
commit 6154dc738f
11 changed files with 343 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

124
tests/test_market_intelligence.py
View File

@@ -219,7 +219,7 @@ class MarketIntelligenceTests(unittest.TestCase):
"frameworkStats": {
"text-generation": {
"gpu": {
"vllm": {"modelCount": 1000, "wilsonLowerBound": 0.10},
"vllm": {"modelCount": 1000, "wilsonLowerBound": 0.30},
"vllm-mlu": {"modelCount": 800, "wilsonLowerBound": 0.35},
"vllm-customized": {"modelCount": 2, "wilsonLowerBound": 0.90},
}
@@ -357,7 +357,7 @@ class MarketIntelligenceTests(unittest.TestCase):
"frameworkStats": {
"text-generation": {
"Cambricon_mlu-370-x4": {
"vllm": {"modelCount": 1000, "wilsonLowerBound": 0.10},
"vllm": {"modelCount": 1000, "wilsonLowerBound": 0.30},
"future-engine": {
"modelCount": 1000,
"successCount": 500,
@@ -396,7 +396,7 @@ class MarketIntelligenceTests(unittest.TestCase):
"frameworkStats": {
"text-generation": {
"Cambricon_mlu-370-x4": {
"vllm": {"modelCount": 1000, "wilsonLowerBound": 0.10},
"vllm": {"modelCount": 1000, "wilsonLowerBound": 0.30},
"future-engine": {
"modelCount": 2,
"wilsonLowerBound": 0.90,
@@ -454,6 +454,124 @@ class MarketIntelligenceTests(unittest.TestCase):
)
self.assertIsNone(candidate)
def test_low_confidence_public_framework_is_rejected_even_with_many_samples(self) -> None:
manager = MarketIntelligenceManager("unused.json", framework_min_samples=100)
manager.state = {
"frameworkStats": {
"text-generation": {
"Cambricon_mlu-370-x4": {
"vllm": {"modelCount": 10_000, "wilsonLowerBound": 0.12},
}
}
}
}
candidate = choose_candidate_for_gpu(
model=HFModelSummary(
repo_id="owner/model",
downloads=100,
last_modified=None,
pipeline_tag="text-generation",
),
inspection=ModelInspection(repo_id="owner/model", weight_files=["model.safetensors"]),
template_selector=TemplateSelector(),
task_types=["text-generation"],
target_gpu="Cambricon_mlu-370-x4",
market_intelligence=manager,
)
self.assertIsNone(candidate)
def test_repeated_ambiguous_failures_open_operational_circuit(self) -> None:
manager = MarketIntelligenceManager("unused.json", framework_min_samples=100)
manager.state = {
"frameworkStats": {
"text-generation": {
"Biren_166m": {
"vllm_fix_tokenizer": {"modelCount": 10_000, "wilsonLowerBound": 0.35},
}
}
}
}
manager.set_local_outcome_stats(
{
"recentCombinationStats": {
"Biren_166m|vllm_fix_tokenizer|text-generation": {
"successCount": 0,
"failureCount": 5,
"attributableFailureCount": 0,
"consecutiveFailures": 0,
"lastTerminalAt": datetime.now(timezone.utc).isoformat(),
}
}
}
)
metadata = manager.framework_metadata(
"text-generation", "Biren_166m", "vllm_fix_tokenizer"
)
self.assertTrue(metadata["frameworkCircuitOpen"])
self.assertEqual(
"recent_terminal_success_below_10_percent",
metadata["frameworkCircuitReason"],
)
def test_low_sample_gpu_with_recent_success_and_proven_framework_remains_eligible(self) -> None:
manager = MarketIntelligenceManager("unused.json", framework_min_samples=100)
manager.state = {
"gpuStats": {
"Sunrise_pt-200-x1": {
"canVerify": True,
"submissionEligible": False,
"recentTerminal": 10,
"recentSuccess": 5,
"recentWilsonLowerBound": 0.24,
"healthFactor": 1.0,
"selectionWeight": 0.5,
}
},
"frameworkStats": {
"text-generation": {
"Sunrise_pt-200-x1": {
"vllm_fix_tokenizer": {
"modelCount": 6_000,
"wilsonLowerBound": 0.41,
"officialConfigValid": True,
}
}
}
},
}
self.assertEqual(
["Sunrise_pt-200-x1"],
manager.eligible_gpus(["Sunrise_pt-200-x1"]),
)
def test_low_sample_gpu_does_not_bypass_absolute_quality_floor(self) -> None:
manager = MarketIntelligenceManager("unused.json", framework_min_samples=100)
manager.state = {
"gpuStats": {
"gpu-low-quality": {
"canVerify": True,
"submissionEligible": False,
"recentTerminal": 10,
"recentSuccess": 1,
"recentWilsonLowerBound": 0.02,
"healthFactor": 1.0,
"selectionWeight": 0.5,
}
},
"frameworkStats": {
"text-generation": {
"gpu-low-quality": {
"vllm": {
"modelCount": 6_000,
"wilsonLowerBound": 0.41,
"officialConfigValid": True,
}
}
}
},
}
self.assertEqual([], manager.eligible_gpus(["gpu-low-quality"]))
def test_five_consecutive_local_failures_circuit_break_framework(self) -> None:
manager = MarketIntelligenceManager("unused.json", framework_min_samples=100)
manager.state = {