CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: make model age admission-only

Browse Source
This commit is contained in:
CoolBoy
2026-08-12 10:18:09 +08:00
parent 86088ce577
commit 91d1d3d87d
15 changed files with 438 additions and 653 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
tests/test_concurrency.py
View File

@@ -215,21 +215,21 @@ def make_candidate(index: int) -> dict:
class ClientPoolConcurrencyTests(unittest.TestCase):
def test_old_models_reserve_each_accounts_last_ten_queue_positions(self) -> None:
below_threshold = FakeClient(active_count=89)
at_threshold = FakeClient(active_count=90)
def test_old_models_use_another_account_and_reserve_last_five_positions(self) -> None:
below_threshold = FakeClient(active_count=94)
at_threshold = FakeClient(active_count=95)
pool = ModelHubClientPool(
[below_threshold, at_threshold], # type: ignore[list-item]
active_task_cap=100,
active_counts_ttl=60,
recent_model_reserve_slots=10,
recent_model_reserve_slots=5,
recent_model_days=7,
instance_id="old-model-threshold-test",
)
submitted_at = datetime(2026, 8, 11, tzinfo=timezone.utc)
pool.add_task_for_model(
{"model": "old-allowed-as-position-90"},
{"model": "old-allowed-as-position-95"},
model_last_modified=datetime(2026, 7, 1, tzinfo=timezone.utc),
submitted_at=submitted_at,
)
@@ -251,13 +251,13 @@ class ClientPoolConcurrencyTests(unittest.TestCase):
)
self.assertEqual(2, len(below_threshold.submitted) + len(at_threshold.submitted))
def test_concurrent_old_model_submissions_cannot_enter_reserved_ten_slots(self) -> None:
client = FakeClient(active_count=88)
def test_concurrent_old_model_submissions_cannot_enter_reserved_five_slots(self) -> None:
client = FakeClient(active_count=93)
pool = ModelHubClientPool(
[client], # type: ignore[list-item]
active_task_cap=100,
active_counts_ttl=60,
recent_model_reserve_slots=10,
recent_model_reserve_slots=5,
recent_model_days=7,
)
submitted_at = datetime(2026, 8, 11, tzinfo=timezone.utc)
@@ -295,11 +295,11 @@ class ClientPoolConcurrencyTests(unittest.TestCase):
self.assertTrue(all(stage["updatedAfter"] >= now - timedelta(days=7) for stage in stages))
def test_submit_candidate_reports_old_model_policy_skip_at_dynamic_threshold(self) -> None:
client = FakeClient(active_count=90)
client = FakeClient(active_count=95)
pool = ModelHubClientPool(
[client], # type: ignore[list-item]
active_task_cap=100,
recent_model_reserve_slots=10,
recent_model_reserve_slots=5,
recent_model_days=7,
)
result = submit_candidate(
@@ -315,25 +315,118 @@ class ClientPoolConcurrencyTests(unittest.TestCase):
pool, # type: ignore[arg-type]
)
self.assertEqual("old_model_policy_skipped", result["outcome"])
self.assertEqual("age_policy_deferred", result["outcome"])
self.assertEqual("age_policy_skipped", result["reason"])
self.assertEqual([], client.submitted)
def test_old_model_threshold_tracks_a_discovered_capacity_increase(self) -> None:
client = DynamicCapacityClient(active=100, limit=101)
def test_age_policy_deferred_candidate_is_skipped_not_failed(self) -> None:
class AdmissionRaceClient:
@staticmethod
def begin_cycle() -> None:
return None
@staticmethod
def old_model_submit_slots() -> int:
return 1
@staticmethod
def old_model_queue_thresholds() -> list[int]:
return [95]
@staticmethod
def available_submit_slots() -> int:
return 1
@staticmethod
def processed_gpus_for_model(_model_id: str) -> set[str]:
return set()
@staticmethod
def list_tasks_page(**_kwargs) -> dict: # noqa: ANN003
return {"code": 0, "data": {"records": [], "pages": 0}}
@staticmethod
def add_task_for_model(*_args, **_kwargs) -> dict: # noqa: ANN002, ANN003
raise OldModelQueuePolicyError("old-model capacity filled during submission")
with tempfile.TemporaryDirectory() as temporary_dir:
root = Path(temporary_dir)
args = build_parser().parse_args(
[
"--gpus",
"Iluvatar_bi-150",
"--task-types",
"text-generation",
"--limit",
"1",
"--max-scan-models",
"1",
"--skip-outcome-sync",
"--skip-history-archive",
"--disable-candidate-preflight",
"--disable-gpu-strategy",
"--disable-market-intelligence",
]
)
args.runs_dir = str(root / "runs")
args.ledger_path = str(root / "ledger.jsonl")
args.outcomes_path = str(root / "outcomes.jsonl")
args.claims_path = str(root / "claims.jsonl")
args.history_archive_path = str(root / "history.jsonl")
args.submission_exclusions_path = str(root / "exclusions.jsonl")
client = AdmissionRaceClient()
summary = run_submission(
args,
now=datetime(2026, 2, 1, tzinfo=timezone.utc),
hf_discovery=FakeDiscovery(1), # type: ignore[arg-type]
modelhub_client=client, # type: ignore[arg-type]
template_selector=TemplateSelector(),
)
self.assertEqual(0, summary["submittedCount"])
self.assertEqual(0, summary["failedCount"])
self.assertEqual(1, summary["skipReasonCounts"]["age_policy_skipped"])
def test_old_model_threshold_tracks_capacity_increase_from_100_to_200(self) -> None:
client = DynamicCapacityClient(active=100, limit=200)
pool = ModelHubClientPool(
[client], # type: ignore[list-item]
active_task_cap=100,
recent_model_reserve_slots=10,
recent_model_reserve_slots=5,
capacity_probe_interval_cycles=1,
capacity_state_path=None,
)
self.assertEqual([90], pool.old_model_queue_thresholds())
pool.configure_capacity_probe(1)
pool.add_task({"model": "recent-capacity-probe"})
self.assertEqual([100], pool.active_task_counts())
self.assertEqual([95], pool.old_model_queue_thresholds())
pool.observe_capacity_lower_bounds([200])
self.assertEqual([101], pool.account_capacity_limits())
self.assertEqual([91], pool.old_model_queue_thresholds())
self.assertEqual([200], pool.account_capacity_limits())
self.assertEqual([195], pool.old_model_queue_thresholds())
def test_old_model_admission_fails_closed_when_active_count_is_unknown(self) -> None:
class UnreadableCountClient(FakeClient):
def count_active_tasks(self, **_kwargs) -> int: # noqa: ANN003
raise TimeoutError("count unavailable")
client = UnreadableCountClient(active_count=0)
pool = ModelHubClientPool(
[client], # type: ignore[list-item]
active_task_cap=100,
recent_model_reserve_slots=5,
capacity_state_path=None,
)
with self.assertRaises(OldModelQueuePolicyError):
pool.add_task_for_model(
{"model": "old-deferred"},
model_last_modified=datetime(2026, 7, 1, tzinfo=timezone.utc),
submitted_at=datetime(2026, 8, 11, tzinfo=timezone.utc),
)
self.assertEqual([None], pool.old_model_queue_thresholds())
self.assertEqual([], client.submitted)
def test_online_submission_does_not_construct_llm_even_when_key_is_present(self) -> None:
with tempfile.TemporaryDirectory() as temporary_dir:
@@ -617,6 +710,8 @@ class ProcessCoordinationTests(unittest.TestCase):
"modelId": "owner/old-failure",
"targetGpu": "gpu-a",
"outcome": "failed",
"failureCategory": "model_load",
"failureScope": "model_gpu_framework",
"submitTime": (now - timedelta(hours=26)).isoformat(),
"lastSyncTime": (now - timedelta(hours=25)).isoformat(),
},
@@ -624,6 +719,8 @@ class ProcessCoordinationTests(unittest.TestCase):
"modelId": "owner/recent-failure",
"targetGpu": "gpu-a",
"outcome": "failed",
"failureCategory": "model_load",
"failureScope": "model_gpu_framework",
"submitTime": (now - timedelta(hours=2)).isoformat(),
"lastSyncTime": (now - timedelta(hours=1)).isoformat(),
},