feat: make model age admission-only
This commit is contained in:
@@ -385,6 +385,8 @@ class CandidatePreflightTests(unittest.TestCase):
|
||||
model_profile={"modelType": "qwen3", "quantizationMethod": "awq"},
|
||||
)
|
||||
tracker._records[0]["outcome"] = "failed" # noqa: SLF001
|
||||
tracker._records[0]["failureCategory"] = "model_load" # noqa: SLF001
|
||||
tracker._records[0]["failureScope"] = "model_gpu_framework" # noqa: SLF001
|
||||
report = tracker.get_stats_report()
|
||||
|
||||
key = "gpu|vllm|text-generation|qwen3|awq"
|
||||
@@ -830,8 +832,50 @@ class CandidatePreflightTests(unittest.TestCase):
|
||||
self.assertEqual(1, combo["platformFailureCount"])
|
||||
self.assertEqual(0, profile["consecutiveFailures"])
|
||||
self.assertFalse(tracker.is_model_gpu_failed("owner/model", "gpu"))
|
||||
self.assertEqual([], tracker.get_strategy_history_records())
|
||||
self.assertNotIn("logCosUrl", read_jsonl(path)[0])
|
||||
|
||||
def test_ambiguous_failure_is_unresolved_and_does_not_poison_strategy_feedback(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_dir:
|
||||
path = Path(temporary_dir) / "outcomes.jsonl"
|
||||
tracker = OutcomeTracker(path)
|
||||
tracker.record_submission(
|
||||
"owner/model",
|
||||
"gpu",
|
||||
"vllm",
|
||||
"text-generation",
|
||||
"task-ambiguous",
|
||||
datetime.now(timezone.utc).isoformat(),
|
||||
)
|
||||
task = {
|
||||
"taskId": "task-ambiguous",
|
||||
"status": "failed",
|
||||
"verifyResult": -1,
|
||||
"logCosUrl": "https://logs.invalid/task-ambiguous.zip",
|
||||
}
|
||||
classification = {
|
||||
"failureCategory": "ambiguous_runtime",
|
||||
"failureScope": "unknown",
|
||||
"failureAction": "offline_review",
|
||||
"failureNeedsLlm": False,
|
||||
}
|
||||
with patch(
|
||||
"outcome_tracker.fetch_and_classify_failure_log",
|
||||
return_value=classification,
|
||||
):
|
||||
tracker.sync_from_api(TaskClient([task])) # type: ignore[arg-type]
|
||||
|
||||
combo = tracker.get_stats_report()["combinationStats"][
|
||||
"gpu|vllm|text-generation"
|
||||
]
|
||||
|
||||
self.assertEqual(1, combo["failureCount"])
|
||||
self.assertEqual(0, combo["attributableFailureCount"])
|
||||
self.assertEqual(0, combo["platformFailureCount"])
|
||||
self.assertEqual(1, combo["unresolvedFailureCount"])
|
||||
self.assertFalse(tracker.is_model_gpu_failed("owner/model", "gpu"))
|
||||
self.assertEqual([], tracker.get_strategy_history_records())
|
||||
|
||||
def test_failed_log_enrichment_attempt_is_persisted_and_bounded(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_dir:
|
||||
path = Path(temporary_dir) / "outcomes.jsonl"
|
||||
|
||||
@@ -215,21 +215,21 @@ def make_candidate(index: int) -> dict:
|
||||
|
||||
|
||||
class ClientPoolConcurrencyTests(unittest.TestCase):
|
||||
def test_old_models_reserve_each_accounts_last_ten_queue_positions(self) -> None:
|
||||
below_threshold = FakeClient(active_count=89)
|
||||
at_threshold = FakeClient(active_count=90)
|
||||
def test_old_models_use_another_account_and_reserve_last_five_positions(self) -> None:
|
||||
below_threshold = FakeClient(active_count=94)
|
||||
at_threshold = FakeClient(active_count=95)
|
||||
pool = ModelHubClientPool(
|
||||
[below_threshold, at_threshold], # type: ignore[list-item]
|
||||
active_task_cap=100,
|
||||
active_counts_ttl=60,
|
||||
recent_model_reserve_slots=10,
|
||||
recent_model_reserve_slots=5,
|
||||
recent_model_days=7,
|
||||
instance_id="old-model-threshold-test",
|
||||
)
|
||||
submitted_at = datetime(2026, 8, 11, tzinfo=timezone.utc)
|
||||
|
||||
pool.add_task_for_model(
|
||||
{"model": "old-allowed-as-position-90"},
|
||||
{"model": "old-allowed-as-position-95"},
|
||||
model_last_modified=datetime(2026, 7, 1, tzinfo=timezone.utc),
|
||||
submitted_at=submitted_at,
|
||||
)
|
||||
@@ -251,13 +251,13 @@ class ClientPoolConcurrencyTests(unittest.TestCase):
|
||||
)
|
||||
self.assertEqual(2, len(below_threshold.submitted) + len(at_threshold.submitted))
|
||||
|
||||
def test_concurrent_old_model_submissions_cannot_enter_reserved_ten_slots(self) -> None:
|
||||
client = FakeClient(active_count=88)
|
||||
def test_concurrent_old_model_submissions_cannot_enter_reserved_five_slots(self) -> None:
|
||||
client = FakeClient(active_count=93)
|
||||
pool = ModelHubClientPool(
|
||||
[client], # type: ignore[list-item]
|
||||
active_task_cap=100,
|
||||
active_counts_ttl=60,
|
||||
recent_model_reserve_slots=10,
|
||||
recent_model_reserve_slots=5,
|
||||
recent_model_days=7,
|
||||
)
|
||||
submitted_at = datetime(2026, 8, 11, tzinfo=timezone.utc)
|
||||
@@ -295,11 +295,11 @@ class ClientPoolConcurrencyTests(unittest.TestCase):
|
||||
self.assertTrue(all(stage["updatedAfter"] >= now - timedelta(days=7) for stage in stages))
|
||||
|
||||
def test_submit_candidate_reports_old_model_policy_skip_at_dynamic_threshold(self) -> None:
|
||||
client = FakeClient(active_count=90)
|
||||
client = FakeClient(active_count=95)
|
||||
pool = ModelHubClientPool(
|
||||
[client], # type: ignore[list-item]
|
||||
active_task_cap=100,
|
||||
recent_model_reserve_slots=10,
|
||||
recent_model_reserve_slots=5,
|
||||
recent_model_days=7,
|
||||
)
|
||||
result = submit_candidate(
|
||||
@@ -315,25 +315,118 @@ class ClientPoolConcurrencyTests(unittest.TestCase):
|
||||
pool, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
self.assertEqual("old_model_policy_skipped", result["outcome"])
|
||||
self.assertEqual("age_policy_deferred", result["outcome"])
|
||||
self.assertEqual("age_policy_skipped", result["reason"])
|
||||
self.assertEqual([], client.submitted)
|
||||
|
||||
def test_old_model_threshold_tracks_a_discovered_capacity_increase(self) -> None:
|
||||
client = DynamicCapacityClient(active=100, limit=101)
|
||||
def test_age_policy_deferred_candidate_is_skipped_not_failed(self) -> None:
|
||||
class AdmissionRaceClient:
|
||||
@staticmethod
|
||||
def begin_cycle() -> None:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def old_model_submit_slots() -> int:
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def old_model_queue_thresholds() -> list[int]:
|
||||
return [95]
|
||||
|
||||
@staticmethod
|
||||
def available_submit_slots() -> int:
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def processed_gpus_for_model(_model_id: str) -> set[str]:
|
||||
return set()
|
||||
|
||||
@staticmethod
|
||||
def list_tasks_page(**_kwargs) -> dict: # noqa: ANN003
|
||||
return {"code": 0, "data": {"records": [], "pages": 0}}
|
||||
|
||||
@staticmethod
|
||||
def add_task_for_model(*_args, **_kwargs) -> dict: # noqa: ANN002, ANN003
|
||||
raise OldModelQueuePolicyError("old-model capacity filled during submission")
|
||||
|
||||
with tempfile.TemporaryDirectory() as temporary_dir:
|
||||
root = Path(temporary_dir)
|
||||
args = build_parser().parse_args(
|
||||
[
|
||||
"--gpus",
|
||||
"Iluvatar_bi-150",
|
||||
"--task-types",
|
||||
"text-generation",
|
||||
"--limit",
|
||||
"1",
|
||||
"--max-scan-models",
|
||||
"1",
|
||||
"--skip-outcome-sync",
|
||||
"--skip-history-archive",
|
||||
"--disable-candidate-preflight",
|
||||
"--disable-gpu-strategy",
|
||||
"--disable-market-intelligence",
|
||||
]
|
||||
)
|
||||
args.runs_dir = str(root / "runs")
|
||||
args.ledger_path = str(root / "ledger.jsonl")
|
||||
args.outcomes_path = str(root / "outcomes.jsonl")
|
||||
args.claims_path = str(root / "claims.jsonl")
|
||||
args.history_archive_path = str(root / "history.jsonl")
|
||||
args.submission_exclusions_path = str(root / "exclusions.jsonl")
|
||||
client = AdmissionRaceClient()
|
||||
|
||||
summary = run_submission(
|
||||
args,
|
||||
now=datetime(2026, 2, 1, tzinfo=timezone.utc),
|
||||
hf_discovery=FakeDiscovery(1), # type: ignore[arg-type]
|
||||
modelhub_client=client, # type: ignore[arg-type]
|
||||
template_selector=TemplateSelector(),
|
||||
)
|
||||
|
||||
self.assertEqual(0, summary["submittedCount"])
|
||||
self.assertEqual(0, summary["failedCount"])
|
||||
self.assertEqual(1, summary["skipReasonCounts"]["age_policy_skipped"])
|
||||
|
||||
def test_old_model_threshold_tracks_capacity_increase_from_100_to_200(self) -> None:
|
||||
client = DynamicCapacityClient(active=100, limit=200)
|
||||
pool = ModelHubClientPool(
|
||||
[client], # type: ignore[list-item]
|
||||
active_task_cap=100,
|
||||
recent_model_reserve_slots=10,
|
||||
recent_model_reserve_slots=5,
|
||||
capacity_probe_interval_cycles=1,
|
||||
capacity_state_path=None,
|
||||
)
|
||||
|
||||
self.assertEqual([90], pool.old_model_queue_thresholds())
|
||||
pool.configure_capacity_probe(1)
|
||||
pool.add_task({"model": "recent-capacity-probe"})
|
||||
self.assertEqual([100], pool.active_task_counts())
|
||||
self.assertEqual([95], pool.old_model_queue_thresholds())
|
||||
pool.observe_capacity_lower_bounds([200])
|
||||
|
||||
self.assertEqual([101], pool.account_capacity_limits())
|
||||
self.assertEqual([91], pool.old_model_queue_thresholds())
|
||||
self.assertEqual([200], pool.account_capacity_limits())
|
||||
self.assertEqual([195], pool.old_model_queue_thresholds())
|
||||
|
||||
def test_old_model_admission_fails_closed_when_active_count_is_unknown(self) -> None:
|
||||
class UnreadableCountClient(FakeClient):
|
||||
def count_active_tasks(self, **_kwargs) -> int: # noqa: ANN003
|
||||
raise TimeoutError("count unavailable")
|
||||
|
||||
client = UnreadableCountClient(active_count=0)
|
||||
pool = ModelHubClientPool(
|
||||
[client], # type: ignore[list-item]
|
||||
active_task_cap=100,
|
||||
recent_model_reserve_slots=5,
|
||||
capacity_state_path=None,
|
||||
)
|
||||
|
||||
with self.assertRaises(OldModelQueuePolicyError):
|
||||
pool.add_task_for_model(
|
||||
{"model": "old-deferred"},
|
||||
model_last_modified=datetime(2026, 7, 1, tzinfo=timezone.utc),
|
||||
submitted_at=datetime(2026, 8, 11, tzinfo=timezone.utc),
|
||||
)
|
||||
|
||||
self.assertEqual([None], pool.old_model_queue_thresholds())
|
||||
self.assertEqual([], client.submitted)
|
||||
|
||||
def test_online_submission_does_not_construct_llm_even_when_key_is_present(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_dir:
|
||||
@@ -617,6 +710,8 @@ class ProcessCoordinationTests(unittest.TestCase):
|
||||
"modelId": "owner/old-failure",
|
||||
"targetGpu": "gpu-a",
|
||||
"outcome": "failed",
|
||||
"failureCategory": "model_load",
|
||||
"failureScope": "model_gpu_framework",
|
||||
"submitTime": (now - timedelta(hours=26)).isoformat(),
|
||||
"lastSyncTime": (now - timedelta(hours=25)).isoformat(),
|
||||
},
|
||||
@@ -624,6 +719,8 @@ class ProcessCoordinationTests(unittest.TestCase):
|
||||
"modelId": "owner/recent-failure",
|
||||
"targetGpu": "gpu-a",
|
||||
"outcome": "failed",
|
||||
"failureCategory": "model_load",
|
||||
"failureScope": "model_gpu_framework",
|
||||
"submitTime": (now - timedelta(hours=2)).isoformat(),
|
||||
"lastSyncTime": (now - timedelta(hours=1)).isoformat(),
|
||||
},
|
||||
|
||||
@@ -133,6 +133,34 @@ class GPUStrategyTests(unittest.TestCase):
|
||||
self.assertEqual(200, manager.state["acceptedTotal"])
|
||||
self.assertEqual(0, manager.state["acceptedSinceRefresh"])
|
||||
|
||||
def test_refresh_prefers_classified_attributable_outcomes(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_dir:
|
||||
path = Path(temporary_dir) / "strategy.json"
|
||||
raw_history = make_history(
|
||||
"gpu-a",
|
||||
0,
|
||||
200,
|
||||
start=datetime(2026, 1, 1, tzinfo=timezone.utc),
|
||||
)
|
||||
classified_history = make_history(
|
||||
"gpu-b",
|
||||
200,
|
||||
0,
|
||||
start=datetime(2026, 1, 1, tzinfo=timezone.utc),
|
||||
)
|
||||
client = HistoryClient(raw_history)
|
||||
manager = GPUStrategyManager(path, long_term_min_samples=100)
|
||||
|
||||
state = manager.prepare(
|
||||
client,
|
||||
supported_gpus=["gpu-a", "gpu-b", "gpu-c"],
|
||||
history_records=classified_history,
|
||||
)
|
||||
|
||||
self.assertEqual(0, client.list_calls)
|
||||
self.assertEqual("classified_attributable_outcomes", state["historySource"])
|
||||
self.assertEqual("gpu-b", state["longTermGpus"][0])
|
||||
|
||||
def test_market_weights_change_gpu_mix_without_changing_70_30_split(self) -> None:
|
||||
supported = ["gpu-a", "gpu-b", "gpu-c", "gpu-d"]
|
||||
manager = GPUStrategyManager("unused.json", market_intelligence=WeightedMarket())
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
@@ -18,7 +17,7 @@ from outcome_tracker import OutcomeTracker # noqa: E402
|
||||
from poll_runner import ( # noqa: E402
|
||||
_bootstrap_architecture_history,
|
||||
_load_task_compatibility_contexts,
|
||||
resolve_age_cleanup_policy,
|
||||
build_parser,
|
||||
)
|
||||
|
||||
|
||||
@@ -107,31 +106,11 @@ class PollPolicyTests(unittest.TestCase):
|
||||
self.assertEqual("mindie", contexts["task-old"]["framework"])
|
||||
self.assertEqual({}, contexts["task-old"]["modelProfile"])
|
||||
|
||||
def test_age_cleanup_uses_minus_ten_once_then_minus_five(self) -> None:
|
||||
args = argparse.Namespace(
|
||||
recent_model_reserve_slots=10,
|
||||
dynamic_old_model_cleanup_reserve_slots=5,
|
||||
)
|
||||
def test_age_policy_defaults_to_admission_only_reserve_five(self) -> None:
|
||||
args = build_parser().parse_args([])
|
||||
|
||||
self.assertEqual(
|
||||
("initial", 10),
|
||||
resolve_age_cleanup_policy(args, initial_cleanup_pending=True),
|
||||
)
|
||||
self.assertEqual(
|
||||
("dynamic", 5),
|
||||
resolve_age_cleanup_policy(args, initial_cleanup_pending=False),
|
||||
)
|
||||
|
||||
def test_dynamic_cleanup_cannot_be_stricter_than_initial_cleanup(self) -> None:
|
||||
args = argparse.Namespace(
|
||||
recent_model_reserve_slots=10,
|
||||
dynamic_old_model_cleanup_reserve_slots=20,
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
("dynamic", 10),
|
||||
resolve_age_cleanup_policy(args, initial_cleanup_pending=False),
|
||||
)
|
||||
self.assertEqual(5, args.recent_model_reserve_slots)
|
||||
self.assertFalse(hasattr(args, "dynamic_old_model_cleanup_reserve_slots"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -19,7 +19,6 @@ from queue_cleanup import ( # noqa: E402
|
||||
cleanup_certain_oom_tasks,
|
||||
find_architecture_incompatible_tasks,
|
||||
find_certain_oom_tasks,
|
||||
find_old_overflow_tasks,
|
||||
)
|
||||
|
||||
|
||||
@@ -394,98 +393,34 @@ class QueueCleanupTests(unittest.TestCase):
|
||||
self.assertEqual("task_started_running", summary["policyNoLongerAppliesTasks"][0]["policyChangeReason"])
|
||||
self.assertEqual([], client.stopped)
|
||||
|
||||
def test_old_models_use_each_accounts_own_capacity_minus_ten_threshold(self) -> None:
|
||||
now = datetime(2026, 8, 11, tzinfo=timezone.utc)
|
||||
tasks = [
|
||||
OwnedTask(0, index, "owner/old", "Iluvatar_bi-100", "waiting")
|
||||
for index in range(1, 92)
|
||||
]
|
||||
tasks.extend(
|
||||
OwnedTask(1, 1000 + index, "owner/old", "Iluvatar_bi-100", "waiting")
|
||||
for index in range(1, 192)
|
||||
)
|
||||
tasks.append(OwnedTask(0, 92, "owner/recent", "Iluvatar_bi-100", "waiting"))
|
||||
tasks.append(OwnedTask(0, 93, "owner/old", "Iluvatar_bi-100", "running"))
|
||||
tasks.append(OwnedTask(1, 1192, "owner/recent", "Iluvatar_bi-100", "waiting"))
|
||||
|
||||
selected, skipped = find_old_overflow_tasks(
|
||||
tasks,
|
||||
model_last_modified={
|
||||
"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc),
|
||||
"owner/recent": datetime(2026, 8, 10, tzinfo=timezone.utc),
|
||||
},
|
||||
queue_threshold={0: 90, 1: 190},
|
||||
recent_model_days=7,
|
||||
reference_time=now,
|
||||
)
|
||||
|
||||
self.assertEqual([91, 1191], [item["taskId"] for item in selected])
|
||||
self.assertEqual([91, 191], [item["queuePosition"] for item in selected])
|
||||
self.assertEqual(2, skipped["recentOverflowTasks"])
|
||||
self.assertEqual(1, skipped["runningOverflowProtected"])
|
||||
|
||||
def test_age_cleanup_never_stops_running_overflow_task(self) -> None:
|
||||
def test_initial_and_later_cleanup_never_stop_old_waiting_or_running_tasks(self) -> None:
|
||||
records = [
|
||||
{
|
||||
"taskId": index,
|
||||
"modelId": "owner/old",
|
||||
"gpuType": "Iluvatar_bi-100",
|
||||
"status": "running" if index == 91 else "waiting",
|
||||
"status": "running" if index == 100 else "waiting",
|
||||
}
|
||||
for index in range(1, 92)
|
||||
for index in range(1, 101)
|
||||
]
|
||||
client = FakeQueueClient(records)
|
||||
pool = ModelHubClientPool(
|
||||
[client], # type: ignore[list-item]
|
||||
active_task_cap=100,
|
||||
recent_model_reserve_slots=10,
|
||||
recent_model_reserve_slots=5,
|
||||
)
|
||||
summary = cleanup_certain_oom_tasks(
|
||||
pool,
|
||||
FakeDiscovery(
|
||||
{"owner/old": 1 * GIB},
|
||||
{"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
|
||||
), # type: ignore[arg-type]
|
||||
reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
|
||||
log=lambda _message: None,
|
||||
)
|
||||
|
||||
self.assertEqual(0, summary["oldOverflowCount"])
|
||||
self.assertEqual(1, summary["agePolicySkipped"]["runningOverflowProtected"])
|
||||
self.assertEqual([], client.stopped)
|
||||
|
||||
def test_age_cleanup_recheck_releases_task_that_started_running(self) -> None:
|
||||
records = [
|
||||
{
|
||||
"taskId": index,
|
||||
"modelId": "owner/old",
|
||||
"gpuType": "Iluvatar_bi-100",
|
||||
"status": "waiting",
|
||||
}
|
||||
for index in range(1, 92)
|
||||
summaries = [
|
||||
cleanup_certain_oom_tasks(
|
||||
pool,
|
||||
FakeDiscovery({"owner/old": 1 * GIB}), # type: ignore[arg-type]
|
||||
log=lambda _message: None,
|
||||
)
|
||||
for _ in range(2)
|
||||
]
|
||||
# Read 1 is discovery, read 2 is the account-wide mutation recheck,
|
||||
# and read 3 is the final age-only recheck after the OOM phase.
|
||||
client = FakeQueueClient(records, promote_on_waiting_read=3)
|
||||
pool = ModelHubClientPool(
|
||||
[client], # type: ignore[list-item]
|
||||
active_task_cap=100,
|
||||
recent_model_reserve_slots=10,
|
||||
)
|
||||
summary = cleanup_certain_oom_tasks(
|
||||
pool,
|
||||
FakeDiscovery(
|
||||
{"owner/old": 1 * GIB},
|
||||
{"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
|
||||
), # type: ignore[arg-type]
|
||||
read_concurrency=1,
|
||||
reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
|
||||
log=lambda _message: None,
|
||||
)
|
||||
|
||||
self.assertEqual(1, summary["oldOverflowCount"])
|
||||
self.assertEqual(0, summary["cancelledCount"])
|
||||
self.assertEqual("task_started_running", summary["policyNoLongerAppliesTasks"][0]["policyChangeReason"])
|
||||
self.assertTrue(all(item["ageCleanupMode"] == "admission_only" for item in summaries))
|
||||
self.assertTrue(all(item["oldOverflowCount"] == 0 for item in summaries))
|
||||
self.assertTrue(all(item["cancelledCount"] == 0 for item in summaries))
|
||||
self.assertEqual([], client.stopped)
|
||||
|
||||
def test_certain_oom_cleanup_can_still_stop_running_task(self) -> None:
|
||||
@@ -510,38 +445,7 @@ class QueueCleanupTests(unittest.TestCase):
|
||||
self.assertEqual(1, summary["cancelledCount"])
|
||||
self.assertEqual([[1]], client.stopped)
|
||||
|
||||
def test_old_overflow_task_is_not_stopped_if_it_moves_inside_dynamic_threshold(self) -> None:
|
||||
records = [
|
||||
{
|
||||
"taskId": index,
|
||||
"modelId": "owner/old",
|
||||
"gpuType": "Iluvatar_bi-100",
|
||||
"status": "waiting",
|
||||
}
|
||||
for index in range(1, 92)
|
||||
]
|
||||
client = FakeQueueClient(records, drop_first_on_recheck=True)
|
||||
pool = ModelHubClientPool(
|
||||
[client], # type: ignore[list-item]
|
||||
active_task_cap=100,
|
||||
recent_model_reserve_slots=10,
|
||||
)
|
||||
summary = cleanup_certain_oom_tasks(
|
||||
pool,
|
||||
FakeDiscovery(
|
||||
{"owner/old": 1 * GIB},
|
||||
{"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
|
||||
), # type: ignore[arg-type]
|
||||
reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
|
||||
log=lambda _message: None,
|
||||
)
|
||||
|
||||
self.assertEqual(1, summary["oldOverflowCount"])
|
||||
self.assertEqual(0, summary["cancelledCount"])
|
||||
self.assertEqual(1, summary["policyNoLongerAppliesCount"])
|
||||
self.assertEqual([], client.stopped)
|
||||
|
||||
def test_cleanup_promotes_capacity_from_complete_active_listing(self) -> None:
|
||||
def test_capacity_decline_pauses_submissions_and_never_deletes_existing_tasks(self) -> None:
|
||||
records = [
|
||||
{
|
||||
"taskId": index,
|
||||
@@ -555,114 +459,20 @@ class QueueCleanupTests(unittest.TestCase):
|
||||
pool = ModelHubClientPool(
|
||||
[client], # type: ignore[list-item]
|
||||
active_task_cap=100,
|
||||
recent_model_reserve_slots=10,
|
||||
recent_model_reserve_slots=5,
|
||||
capacity_state_path=None,
|
||||
)
|
||||
summary = cleanup_certain_oom_tasks(
|
||||
pool,
|
||||
FakeDiscovery(
|
||||
{"owner/old": 1 * GIB},
|
||||
{"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
|
||||
), # type: ignore[arg-type]
|
||||
reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
|
||||
FakeDiscovery({"owner/old": 1 * GIB}), # type: ignore[arg-type]
|
||||
log=lambda _message: None,
|
||||
)
|
||||
|
||||
self.assertEqual([150], summary["accountCapacityLimits"])
|
||||
self.assertEqual([140], summary["oldModelQueueThresholds"])
|
||||
self.assertEqual(list(range(141, 151)), [item["taskId"] for item in summary["oldOverflowTasks"]])
|
||||
|
||||
def test_initial_cleanup_stops_old_task_beyond_capacity_minus_ten(self) -> None:
|
||||
records = [
|
||||
{
|
||||
"taskId": index,
|
||||
"modelId": "owner/old",
|
||||
"gpuType": "Iluvatar_bi-100",
|
||||
"status": "waiting",
|
||||
}
|
||||
for index in range(1, 92)
|
||||
]
|
||||
client = FakeQueueClient(records)
|
||||
pool = ModelHubClientPool(
|
||||
[client], # type: ignore[list-item]
|
||||
active_task_cap=100,
|
||||
recent_model_reserve_slots=10,
|
||||
)
|
||||
summary = cleanup_certain_oom_tasks(
|
||||
pool,
|
||||
FakeDiscovery(
|
||||
{"owner/old": 1 * GIB},
|
||||
{"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
|
||||
), # type: ignore[arg-type]
|
||||
reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
|
||||
log=lambda _message: None,
|
||||
)
|
||||
|
||||
self.assertEqual(1, summary["oldOverflowCount"])
|
||||
self.assertEqual(1, summary["cancelledCount"])
|
||||
self.assertEqual([90], summary["oldModelQueueThresholds"])
|
||||
self.assertEqual([[91]], client.stopped)
|
||||
|
||||
def test_scheduled_cleanup_uses_capacity_minus_five(self) -> None:
|
||||
records = [
|
||||
{
|
||||
"taskId": index,
|
||||
"modelId": "owner/old",
|
||||
"gpuType": "Iluvatar_bi-100",
|
||||
"status": "waiting",
|
||||
}
|
||||
for index in range(1, 97)
|
||||
]
|
||||
client = FakeQueueClient(records)
|
||||
pool = ModelHubClientPool(
|
||||
[client], # type: ignore[list-item]
|
||||
active_task_cap=100,
|
||||
recent_model_reserve_slots=10,
|
||||
)
|
||||
summary = cleanup_certain_oom_tasks(
|
||||
pool,
|
||||
FakeDiscovery(
|
||||
{"owner/old": 1 * GIB},
|
||||
{"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
|
||||
), # type: ignore[arg-type]
|
||||
age_reserved_slots=5,
|
||||
reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
|
||||
log=lambda _message: None,
|
||||
)
|
||||
|
||||
self.assertEqual([95], summary["oldModelQueueThresholds"])
|
||||
self.assertEqual([96], [item["taskId"] for item in summary["oldOverflowTasks"]])
|
||||
self.assertEqual([[96]], client.stopped)
|
||||
|
||||
def test_oom_is_removed_before_recalculating_old_overflow_positions(self) -> None:
|
||||
records = [
|
||||
{
|
||||
"taskId": index,
|
||||
"modelId": "owner/large" if index == 1 else "owner/old",
|
||||
"gpuType": "Iluvatar_bi-100",
|
||||
"status": "waiting",
|
||||
}
|
||||
for index in range(1, 93)
|
||||
]
|
||||
client = FakeQueueClient(records)
|
||||
pool = ModelHubClientPool(
|
||||
[client], # type: ignore[list-item]
|
||||
active_task_cap=100,
|
||||
recent_model_reserve_slots=10,
|
||||
)
|
||||
summary = cleanup_certain_oom_tasks(
|
||||
pool,
|
||||
FakeDiscovery(
|
||||
{"owner/large": 40 * GIB, "owner/old": 1 * GIB},
|
||||
{"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
|
||||
), # type: ignore[arg-type]
|
||||
reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
|
||||
log=lambda _message: None,
|
||||
)
|
||||
|
||||
self.assertEqual(1, summary["certainOomCount"])
|
||||
self.assertEqual([92], [item["taskId"] for item in summary["oldOverflowTasks"]])
|
||||
self.assertEqual([[1], [92]], client.stopped)
|
||||
self.assertEqual([145], summary["oldModelQueueThresholds"])
|
||||
self.assertEqual(0, summary["oldOverflowCount"])
|
||||
self.assertEqual(0, pool.available_submit_slots())
|
||||
self.assertEqual([], client.stopped)
|
||||
|
||||
def test_stop_tasks_uses_documented_put_endpoint_and_integer_ids(self) -> None:
|
||||
http = RecordingHttpClient()
|
||||
|
||||
Reference in New Issue
Block a user