from __future__ import annotations import sys import tempfile import unittest from collections import Counter from datetime import datetime, timedelta, timezone from pathlib import Path PACKAGE_DIR = Path(__file__).resolve().parents[1] / "modelhub_submmit_api" if str(PACKAGE_DIR) in sys.path: sys.path.remove(str(PACKAGE_DIR)) sys.path.insert(0, str(PACKAGE_DIR)) from gpu_strategy import ( # noqa: E402 LONG_TERM, RECENT, GPUStrategyManager, build_strategy_snapshot, choose_next_category, ) def make_history(gpu: str, success: int, failure: int, *, start: datetime) -> list[dict]: records: list[dict] = [] for index in range(success + failure): passed = index < success records.append( { "taskId": f"{gpu}-{index}", "gpuType": gpu, "status": "success", "verifyResult": 1 if passed else -1, "updateTime": (start + timedelta(seconds=index)).isoformat(), } ) return records class HistoryClient: def __init__(self, tasks: list[dict]) -> None: self.tasks = tasks self.list_calls = 0 def list_tasks(self, **_kwargs) -> list[dict]: # noqa: ANN003 self.list_calls += 1 return list(self.tasks) class WeightedMarket: def gpu_weight(self, gpu: str, *, category: str) -> float: del category return {"gpu-a": 3.0, "gpu-b": 0.1, "gpu-c": 0.1, "gpu-d": 0.1}[gpu] def gpu_metadata(self, gpu: str) -> dict: return {"marketWeight": self.gpu_weight(gpu, category=LONG_TERM)} def eligible_gpus(self, supported_gpus: list[str]) -> list[str]: return list(supported_gpus) class VettedMarket(WeightedMarket): def eligible_gpus(self, supported_gpus: list[str]) -> list[str]: return [gpu for gpu in supported_gpus if gpu in {"gpu-b", "gpu-c"}] class GPUStrategyTests(unittest.TestCase): def test_long_term_ranking_rejects_tiny_high_rate_sample(self) -> None: start = datetime(2026, 1, 1, tzinfo=timezone.utc) supported = ["tiny", "best", "second", "third", "bad"] tasks = [ *make_history("tiny", 8, 0, start=start), *make_history("best", 40, 160, start=start), *make_history("second", 30, 270, start=start), *make_history("third", 45, 455, start=start), *make_history("bad", 5, 495, start=start), ] snapshot = build_strategy_snapshot( tasks, supported_gpus=supported, long_term_min_samples=100, ) self.assertEqual(["best", "second", "third"], snapshot["longTermGpus"]) self.assertNotIn("tiny", snapshot["longTermGpus"]) def test_weighted_category_planner_is_exact_over_200_accepts(self) -> None: counts = {LONG_TERM: 0, RECENT: 0} for _ in range(200): category = choose_next_category(counts) counts[category] += 1 self.assertEqual({LONG_TERM: 140, RECENT: 60}, counts) def test_candidate_order_uses_70_30_without_exploration(self) -> None: supported = ["gpu-a", "gpu-b", "gpu-c", "gpu-d"] snapshot = build_strategy_snapshot([], supported_gpus=supported) manager = GPUStrategyManager("unused.json") manager.state = snapshot candidates = [ {"repoId": f"owner/model-{model}", "targetGpu": gpu} for model in range(250) for gpu in supported ] ordered = manager.order_candidates(candidates) categories = Counter(candidate["strategyCategory"] for candidate in ordered[:200]) self.assertEqual(140, categories[LONG_TERM]) self.assertEqual(60, categories[RECENT]) self.assertNotIn("all_supported", categories) def test_refresh_happens_only_after_200_accepted_submissions(self) -> None: with tempfile.TemporaryDirectory() as temporary_dir: path = Path(temporary_dir) / "strategy.json" client = HistoryClient([]) manager = GPUStrategyManager(path, refresh_submissions=200) manager.prepare(client, supported_gpus=["gpu-a", "gpu-b", "gpu-c"]) self.assertEqual(1, client.list_calls) manager.record_accepted([{"strategyCategory": LONG_TERM}] * 199) self.assertEqual(1, manager.submissions_until_refresh) manager.prepare(client, supported_gpus=["gpu-a", "gpu-b", "gpu-c"]) self.assertEqual(1, client.list_calls) manager.record_accepted([{"strategyCategory": RECENT}]) self.assertEqual(0, manager.submissions_until_refresh) manager.prepare(client, supported_gpus=["gpu-a", "gpu-b", "gpu-c"]) self.assertEqual(2, client.list_calls) self.assertEqual(1, manager.state["generation"]) self.assertEqual(200, manager.state["acceptedTotal"]) self.assertEqual(0, manager.state["acceptedSinceRefresh"]) def test_market_weights_change_gpu_mix_without_changing_70_30_split(self) -> None: supported = ["gpu-a", "gpu-b", "gpu-c", "gpu-d"] manager = GPUStrategyManager("unused.json", market_intelligence=WeightedMarket()) manager.state = build_strategy_snapshot([], supported_gpus=supported) candidates = [ {"repoId": f"owner/model-{model}", "targetGpu": gpu} for model in range(300) for gpu in supported ] ordered = manager.order_candidates(candidates)[:200] category_counts = Counter(candidate["strategyCategory"] for candidate in ordered) gpu_counts = Counter(candidate["targetGpu"] for candidate in ordered) self.assertEqual({LONG_TERM: 140, RECENT: 60}, category_counts) self.assertGreater(gpu_counts["gpu-a"], gpu_counts["gpu-b"]) self.assertNotIn("all_supported", category_counts) def test_unvetted_gpus_receive_zero_submissions(self) -> None: supported = ["gpu-a", "gpu-b", "gpu-c", "gpu-d"] manager = GPUStrategyManager("unused.json", market_intelligence=VettedMarket()) manager.state = build_strategy_snapshot([], supported_gpus=supported) candidates = [ {"repoId": f"owner/model-{model}", "targetGpu": gpu} for model in range(50) for gpu in supported ] ordered = manager.order_candidates(candidates) self.assertTrue(ordered) self.assertEqual({"gpu-b", "gpu-c"}, {candidate["targetGpu"] for candidate in ordered}) if __name__ == "__main__": unittest.main()