CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: prioritize proven GPU framework combinations

Browse Source
This commit is contained in:
CoolBoy
2026-08-05 18:21:59 +08:00
parent 882479e43e
commit 5e47d9e695
11 changed files with 431 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
tests/test_market_intelligence.py
View File

@@ -13,6 +13,7 @@ if str(PACKAGE_DIR) in sys.path:
sys.path.insert(0, str(PACKAGE_DIR))
from main import choose_candidate_for_gpu # noqa: E402
from common import write_json # noqa: E402
from market_intelligence import MarketIntelligenceManager # noqa: E402
from models import HFModelSummary, ModelInspection # noqa: E402
from template_selector import TemplateSelector # noqa: E402
@@ -158,6 +159,52 @@ class MarketIntelligenceTests(unittest.TestCase):
)
self.assertEqual(first_call_count, client.calls)
def test_version_two_snapshot_is_migrated_without_losing_public_evidence(self) -> None:
with tempfile.TemporaryDirectory() as temporary_dir:
path = Path(temporary_dir) / "market.json"
now = datetime.now(timezone.utc)
write_json(
path,
{
"version": 2,
"supportedGpus": ["fast"],
"taskTypes": ["text-generation"],
"queueUpdatedAt": now.isoformat(),
"frameworkUpdatedAt": now.isoformat(),
"queueError": None,
"frameworkError": None,
"gpuStats": {
"fast": {
"gpu": "fast",
"canVerify": True,
"recentTerminal": 100,
"recentSuccess": 80,
"recentWilsonLowerBound": 0.70,
"backlogHours": 10.0,
"healthFactor": 1.0,
}
},
"frameworkStats": {
"text-generation": {
"fast": {
"vllm": {"modelCount": 1000, "wilsonLowerBound": 0.20}
}
}
},
},
)
client = FailingMarketClient()
manager = MarketIntelligenceManager(path, log_fn=lambda _message: None)
state = manager.prepare(
client,
supported_gpus=["fast"],
task_types=["text-generation"],
now=now + timedelta(seconds=30),
)
self.assertEqual(3, state["version"])
self.assertTrue(state["gpuStats"]["fast"]["submissionEligible"])
self.assertEqual(0, client.calls)
def test_framework_ranking_uses_confidence_bound_and_ignores_tiny_samples(self) -> None:
manager = MarketIntelligenceManager("unused.json", framework_min_samples=100)
manager.state = {
@@ -332,6 +379,76 @@ class MarketIntelligenceTests(unittest.TestCase):
assert candidate is not None
self.assertEqual("vllm", candidate.framework)
def test_unproven_public_framework_receives_no_submission(self) -> None:
manager = MarketIntelligenceManager("unused.json")
manager.state = {
"frameworkStats": {
"text-generation": {
"Cambricon_mlu-370-x4": {
"vllm": {"modelCount": 299, "wilsonLowerBound": 0.50},
}
}
}
}
candidate = choose_candidate_for_gpu(
model=HFModelSummary(
repo_id="owner/model",
downloads=100,
last_modified=None,
pipeline_tag="text-generation",
),
inspection=ModelInspection(repo_id="owner/model", weight_files=["model.safetensors"]),
template_selector=TemplateSelector(),
task_types=["text-generation"],
target_gpu="Cambricon_mlu-370-x4",
market_intelligence=manager,
)
self.assertIsNone(candidate)
def test_five_consecutive_local_failures_circuit_break_framework(self) -> None:
manager = MarketIntelligenceManager("unused.json", framework_min_samples=100)
manager.state = {
"frameworkStats": {
"text-generation": {
"Iluvatar_bi-100": {
"vllm": {"modelCount": 1000, "wilsonLowerBound": 0.40},
"transformers": {"modelCount": 1000, "wilsonLowerBound": 0.20},
}
}
}
}
manager.set_local_outcome_stats(
{
"combinationStats": {},
"recentCombinationStats": {
"Iluvatar_bi-100|vllm|text-generation": {
"successCount": 0,
"failureCount": 5,
"consecutiveFailures": 5,
"lastTerminalAt": datetime.now(timezone.utc).isoformat(),
}
},
}
)
candidate = choose_candidate_for_gpu(
model=HFModelSummary(
repo_id="owner/model",
downloads=100,
last_modified=None,
pipeline_tag="text-generation",
),
inspection=ModelInspection(repo_id="owner/model", weight_files=["model.safetensors"]),
template_selector=TemplateSelector(),
task_types=["text-generation"],
target_gpu="Iluvatar_bi-100",
market_intelligence=manager,
)
self.assertIsNotNone(candidate)
assert candidate is not None
self.assertEqual("transformers", candidate.framework)
metadata = manager.framework_metadata("text-generation", "Iluvatar_bi-100", "vllm")
self.assertTrue(metadata["frameworkCircuitOpen"])
if __name__ == "__main__":
unittest.main()