feat: make model age admission-only

This commit is contained in:
CoolBoy
2026-08-12 10:18:09 +08:00
parent 86088ce577
commit 91d1d3d87d
15 changed files with 438 additions and 653 deletions

View File

@@ -385,6 +385,8 @@ class CandidatePreflightTests(unittest.TestCase):
model_profile={"modelType": "qwen3", "quantizationMethod": "awq"},
)
tracker._records[0]["outcome"] = "failed" # noqa: SLF001
tracker._records[0]["failureCategory"] = "model_load" # noqa: SLF001
tracker._records[0]["failureScope"] = "model_gpu_framework" # noqa: SLF001
report = tracker.get_stats_report()
key = "gpu|vllm|text-generation|qwen3|awq"
@@ -830,8 +832,50 @@ class CandidatePreflightTests(unittest.TestCase):
self.assertEqual(1, combo["platformFailureCount"])
self.assertEqual(0, profile["consecutiveFailures"])
self.assertFalse(tracker.is_model_gpu_failed("owner/model", "gpu"))
self.assertEqual([], tracker.get_strategy_history_records())
self.assertNotIn("logCosUrl", read_jsonl(path)[0])
def test_ambiguous_failure_is_unresolved_and_does_not_poison_strategy_feedback(self) -> None:
with tempfile.TemporaryDirectory() as temporary_dir:
path = Path(temporary_dir) / "outcomes.jsonl"
tracker = OutcomeTracker(path)
tracker.record_submission(
"owner/model",
"gpu",
"vllm",
"text-generation",
"task-ambiguous",
datetime.now(timezone.utc).isoformat(),
)
task = {
"taskId": "task-ambiguous",
"status": "failed",
"verifyResult": -1,
"logCosUrl": "https://logs.invalid/task-ambiguous.zip",
}
classification = {
"failureCategory": "ambiguous_runtime",
"failureScope": "unknown",
"failureAction": "offline_review",
"failureNeedsLlm": False,
}
with patch(
"outcome_tracker.fetch_and_classify_failure_log",
return_value=classification,
):
tracker.sync_from_api(TaskClient([task])) # type: ignore[arg-type]
combo = tracker.get_stats_report()["combinationStats"][
"gpu|vllm|text-generation"
]
self.assertEqual(1, combo["failureCount"])
self.assertEqual(0, combo["attributableFailureCount"])
self.assertEqual(0, combo["platformFailureCount"])
self.assertEqual(1, combo["unresolvedFailureCount"])
self.assertFalse(tracker.is_model_gpu_failed("owner/model", "gpu"))
self.assertEqual([], tracker.get_strategy_history_records())
def test_failed_log_enrichment_attempt_is_persisted_and_bounded(self) -> None:
with tempfile.TemporaryDirectory() as temporary_dir:
path = Path(temporary_dir) / "outcomes.jsonl"

View File

@@ -215,21 +215,21 @@ def make_candidate(index: int) -> dict:
class ClientPoolConcurrencyTests(unittest.TestCase):
def test_old_models_reserve_each_accounts_last_ten_queue_positions(self) -> None:
below_threshold = FakeClient(active_count=89)
at_threshold = FakeClient(active_count=90)
def test_old_models_use_another_account_and_reserve_last_five_positions(self) -> None:
below_threshold = FakeClient(active_count=94)
at_threshold = FakeClient(active_count=95)
pool = ModelHubClientPool(
[below_threshold, at_threshold], # type: ignore[list-item]
active_task_cap=100,
active_counts_ttl=60,
recent_model_reserve_slots=10,
recent_model_reserve_slots=5,
recent_model_days=7,
instance_id="old-model-threshold-test",
)
submitted_at = datetime(2026, 8, 11, tzinfo=timezone.utc)
pool.add_task_for_model(
{"model": "old-allowed-as-position-90"},
{"model": "old-allowed-as-position-95"},
model_last_modified=datetime(2026, 7, 1, tzinfo=timezone.utc),
submitted_at=submitted_at,
)
@@ -251,13 +251,13 @@ class ClientPoolConcurrencyTests(unittest.TestCase):
)
self.assertEqual(2, len(below_threshold.submitted) + len(at_threshold.submitted))
def test_concurrent_old_model_submissions_cannot_enter_reserved_ten_slots(self) -> None:
client = FakeClient(active_count=88)
def test_concurrent_old_model_submissions_cannot_enter_reserved_five_slots(self) -> None:
client = FakeClient(active_count=93)
pool = ModelHubClientPool(
[client], # type: ignore[list-item]
active_task_cap=100,
active_counts_ttl=60,
recent_model_reserve_slots=10,
recent_model_reserve_slots=5,
recent_model_days=7,
)
submitted_at = datetime(2026, 8, 11, tzinfo=timezone.utc)
@@ -295,11 +295,11 @@ class ClientPoolConcurrencyTests(unittest.TestCase):
self.assertTrue(all(stage["updatedAfter"] >= now - timedelta(days=7) for stage in stages))
def test_submit_candidate_reports_old_model_policy_skip_at_dynamic_threshold(self) -> None:
client = FakeClient(active_count=90)
client = FakeClient(active_count=95)
pool = ModelHubClientPool(
[client], # type: ignore[list-item]
active_task_cap=100,
recent_model_reserve_slots=10,
recent_model_reserve_slots=5,
recent_model_days=7,
)
result = submit_candidate(
@@ -315,25 +315,118 @@ class ClientPoolConcurrencyTests(unittest.TestCase):
pool, # type: ignore[arg-type]
)
self.assertEqual("old_model_policy_skipped", result["outcome"])
self.assertEqual("age_policy_deferred", result["outcome"])
self.assertEqual("age_policy_skipped", result["reason"])
self.assertEqual([], client.submitted)
def test_old_model_threshold_tracks_a_discovered_capacity_increase(self) -> None:
client = DynamicCapacityClient(active=100, limit=101)
def test_age_policy_deferred_candidate_is_skipped_not_failed(self) -> None:
class AdmissionRaceClient:
@staticmethod
def begin_cycle() -> None:
return None
@staticmethod
def old_model_submit_slots() -> int:
return 1
@staticmethod
def old_model_queue_thresholds() -> list[int]:
return [95]
@staticmethod
def available_submit_slots() -> int:
return 1
@staticmethod
def processed_gpus_for_model(_model_id: str) -> set[str]:
return set()
@staticmethod
def list_tasks_page(**_kwargs) -> dict: # noqa: ANN003
return {"code": 0, "data": {"records": [], "pages": 0}}
@staticmethod
def add_task_for_model(*_args, **_kwargs) -> dict: # noqa: ANN002, ANN003
raise OldModelQueuePolicyError("old-model capacity filled during submission")
with tempfile.TemporaryDirectory() as temporary_dir:
root = Path(temporary_dir)
args = build_parser().parse_args(
[
"--gpus",
"Iluvatar_bi-150",
"--task-types",
"text-generation",
"--limit",
"1",
"--max-scan-models",
"1",
"--skip-outcome-sync",
"--skip-history-archive",
"--disable-candidate-preflight",
"--disable-gpu-strategy",
"--disable-market-intelligence",
]
)
args.runs_dir = str(root / "runs")
args.ledger_path = str(root / "ledger.jsonl")
args.outcomes_path = str(root / "outcomes.jsonl")
args.claims_path = str(root / "claims.jsonl")
args.history_archive_path = str(root / "history.jsonl")
args.submission_exclusions_path = str(root / "exclusions.jsonl")
client = AdmissionRaceClient()
summary = run_submission(
args,
now=datetime(2026, 2, 1, tzinfo=timezone.utc),
hf_discovery=FakeDiscovery(1), # type: ignore[arg-type]
modelhub_client=client, # type: ignore[arg-type]
template_selector=TemplateSelector(),
)
self.assertEqual(0, summary["submittedCount"])
self.assertEqual(0, summary["failedCount"])
self.assertEqual(1, summary["skipReasonCounts"]["age_policy_skipped"])
def test_old_model_threshold_tracks_capacity_increase_from_100_to_200(self) -> None:
client = DynamicCapacityClient(active=100, limit=200)
pool = ModelHubClientPool(
[client], # type: ignore[list-item]
active_task_cap=100,
recent_model_reserve_slots=10,
recent_model_reserve_slots=5,
capacity_probe_interval_cycles=1,
capacity_state_path=None,
)
self.assertEqual([90], pool.old_model_queue_thresholds())
pool.configure_capacity_probe(1)
pool.add_task({"model": "recent-capacity-probe"})
self.assertEqual([100], pool.active_task_counts())
self.assertEqual([95], pool.old_model_queue_thresholds())
pool.observe_capacity_lower_bounds([200])
self.assertEqual([101], pool.account_capacity_limits())
self.assertEqual([91], pool.old_model_queue_thresholds())
self.assertEqual([200], pool.account_capacity_limits())
self.assertEqual([195], pool.old_model_queue_thresholds())
def test_old_model_admission_fails_closed_when_active_count_is_unknown(self) -> None:
class UnreadableCountClient(FakeClient):
def count_active_tasks(self, **_kwargs) -> int: # noqa: ANN003
raise TimeoutError("count unavailable")
client = UnreadableCountClient(active_count=0)
pool = ModelHubClientPool(
[client], # type: ignore[list-item]
active_task_cap=100,
recent_model_reserve_slots=5,
capacity_state_path=None,
)
with self.assertRaises(OldModelQueuePolicyError):
pool.add_task_for_model(
{"model": "old-deferred"},
model_last_modified=datetime(2026, 7, 1, tzinfo=timezone.utc),
submitted_at=datetime(2026, 8, 11, tzinfo=timezone.utc),
)
self.assertEqual([None], pool.old_model_queue_thresholds())
self.assertEqual([], client.submitted)
def test_online_submission_does_not_construct_llm_even_when_key_is_present(self) -> None:
with tempfile.TemporaryDirectory() as temporary_dir:
@@ -617,6 +710,8 @@ class ProcessCoordinationTests(unittest.TestCase):
"modelId": "owner/old-failure",
"targetGpu": "gpu-a",
"outcome": "failed",
"failureCategory": "model_load",
"failureScope": "model_gpu_framework",
"submitTime": (now - timedelta(hours=26)).isoformat(),
"lastSyncTime": (now - timedelta(hours=25)).isoformat(),
},
@@ -624,6 +719,8 @@ class ProcessCoordinationTests(unittest.TestCase):
"modelId": "owner/recent-failure",
"targetGpu": "gpu-a",
"outcome": "failed",
"failureCategory": "model_load",
"failureScope": "model_gpu_framework",
"submitTime": (now - timedelta(hours=2)).isoformat(),
"lastSyncTime": (now - timedelta(hours=1)).isoformat(),
},

View File

@@ -133,6 +133,34 @@ class GPUStrategyTests(unittest.TestCase):
self.assertEqual(200, manager.state["acceptedTotal"])
self.assertEqual(0, manager.state["acceptedSinceRefresh"])
def test_refresh_prefers_classified_attributable_outcomes(self) -> None:
with tempfile.TemporaryDirectory() as temporary_dir:
path = Path(temporary_dir) / "strategy.json"
raw_history = make_history(
"gpu-a",
0,
200,
start=datetime(2026, 1, 1, tzinfo=timezone.utc),
)
classified_history = make_history(
"gpu-b",
200,
0,
start=datetime(2026, 1, 1, tzinfo=timezone.utc),
)
client = HistoryClient(raw_history)
manager = GPUStrategyManager(path, long_term_min_samples=100)
state = manager.prepare(
client,
supported_gpus=["gpu-a", "gpu-b", "gpu-c"],
history_records=classified_history,
)
self.assertEqual(0, client.list_calls)
self.assertEqual("classified_attributable_outcomes", state["historySource"])
self.assertEqual("gpu-b", state["longTermGpus"][0])
def test_market_weights_change_gpu_mix_without_changing_70_30_split(self) -> None:
supported = ["gpu-a", "gpu-b", "gpu-c", "gpu-d"]
manager = GPUStrategyManager("unused.json", market_intelligence=WeightedMarket())

View File

@@ -1,6 +1,5 @@
from __future__ import annotations
import argparse
import json
import sys
import tempfile
@@ -18,7 +17,7 @@ from outcome_tracker import OutcomeTracker # noqa: E402
from poll_runner import ( # noqa: E402
_bootstrap_architecture_history,
_load_task_compatibility_contexts,
resolve_age_cleanup_policy,
build_parser,
)
@@ -107,31 +106,11 @@ class PollPolicyTests(unittest.TestCase):
self.assertEqual("mindie", contexts["task-old"]["framework"])
self.assertEqual({}, contexts["task-old"]["modelProfile"])
def test_age_cleanup_uses_minus_ten_once_then_minus_five(self) -> None:
args = argparse.Namespace(
recent_model_reserve_slots=10,
dynamic_old_model_cleanup_reserve_slots=5,
)
def test_age_policy_defaults_to_admission_only_reserve_five(self) -> None:
args = build_parser().parse_args([])
self.assertEqual(
("initial", 10),
resolve_age_cleanup_policy(args, initial_cleanup_pending=True),
)
self.assertEqual(
("dynamic", 5),
resolve_age_cleanup_policy(args, initial_cleanup_pending=False),
)
def test_dynamic_cleanup_cannot_be_stricter_than_initial_cleanup(self) -> None:
args = argparse.Namespace(
recent_model_reserve_slots=10,
dynamic_old_model_cleanup_reserve_slots=20,
)
self.assertEqual(
("dynamic", 10),
resolve_age_cleanup_policy(args, initial_cleanup_pending=False),
)
self.assertEqual(5, args.recent_model_reserve_slots)
self.assertFalse(hasattr(args, "dynamic_old_model_cleanup_reserve_slots"))
if __name__ == "__main__":

View File

@@ -19,7 +19,6 @@ from queue_cleanup import ( # noqa: E402
cleanup_certain_oom_tasks,
find_architecture_incompatible_tasks,
find_certain_oom_tasks,
find_old_overflow_tasks,
)
@@ -394,98 +393,34 @@ class QueueCleanupTests(unittest.TestCase):
self.assertEqual("task_started_running", summary["policyNoLongerAppliesTasks"][0]["policyChangeReason"])
self.assertEqual([], client.stopped)
def test_old_models_use_each_accounts_own_capacity_minus_ten_threshold(self) -> None:
now = datetime(2026, 8, 11, tzinfo=timezone.utc)
tasks = [
OwnedTask(0, index, "owner/old", "Iluvatar_bi-100", "waiting")
for index in range(1, 92)
]
tasks.extend(
OwnedTask(1, 1000 + index, "owner/old", "Iluvatar_bi-100", "waiting")
for index in range(1, 192)
)
tasks.append(OwnedTask(0, 92, "owner/recent", "Iluvatar_bi-100", "waiting"))
tasks.append(OwnedTask(0, 93, "owner/old", "Iluvatar_bi-100", "running"))
tasks.append(OwnedTask(1, 1192, "owner/recent", "Iluvatar_bi-100", "waiting"))
selected, skipped = find_old_overflow_tasks(
tasks,
model_last_modified={
"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc),
"owner/recent": datetime(2026, 8, 10, tzinfo=timezone.utc),
},
queue_threshold={0: 90, 1: 190},
recent_model_days=7,
reference_time=now,
)
self.assertEqual([91, 1191], [item["taskId"] for item in selected])
self.assertEqual([91, 191], [item["queuePosition"] for item in selected])
self.assertEqual(2, skipped["recentOverflowTasks"])
self.assertEqual(1, skipped["runningOverflowProtected"])
def test_age_cleanup_never_stops_running_overflow_task(self) -> None:
def test_initial_and_later_cleanup_never_stop_old_waiting_or_running_tasks(self) -> None:
records = [
{
"taskId": index,
"modelId": "owner/old",
"gpuType": "Iluvatar_bi-100",
"status": "running" if index == 91 else "waiting",
"status": "running" if index == 100 else "waiting",
}
for index in range(1, 92)
for index in range(1, 101)
]
client = FakeQueueClient(records)
pool = ModelHubClientPool(
[client], # type: ignore[list-item]
active_task_cap=100,
recent_model_reserve_slots=10,
recent_model_reserve_slots=5,
)
summary = cleanup_certain_oom_tasks(
pool,
FakeDiscovery(
{"owner/old": 1 * GIB},
{"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
), # type: ignore[arg-type]
reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
log=lambda _message: None,
)
self.assertEqual(0, summary["oldOverflowCount"])
self.assertEqual(1, summary["agePolicySkipped"]["runningOverflowProtected"])
self.assertEqual([], client.stopped)
def test_age_cleanup_recheck_releases_task_that_started_running(self) -> None:
records = [
{
"taskId": index,
"modelId": "owner/old",
"gpuType": "Iluvatar_bi-100",
"status": "waiting",
}
for index in range(1, 92)
summaries = [
cleanup_certain_oom_tasks(
pool,
FakeDiscovery({"owner/old": 1 * GIB}), # type: ignore[arg-type]
log=lambda _message: None,
)
for _ in range(2)
]
# Read 1 is discovery, read 2 is the account-wide mutation recheck,
# and read 3 is the final age-only recheck after the OOM phase.
client = FakeQueueClient(records, promote_on_waiting_read=3)
pool = ModelHubClientPool(
[client], # type: ignore[list-item]
active_task_cap=100,
recent_model_reserve_slots=10,
)
summary = cleanup_certain_oom_tasks(
pool,
FakeDiscovery(
{"owner/old": 1 * GIB},
{"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
), # type: ignore[arg-type]
read_concurrency=1,
reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
log=lambda _message: None,
)
self.assertEqual(1, summary["oldOverflowCount"])
self.assertEqual(0, summary["cancelledCount"])
self.assertEqual("task_started_running", summary["policyNoLongerAppliesTasks"][0]["policyChangeReason"])
self.assertTrue(all(item["ageCleanupMode"] == "admission_only" for item in summaries))
self.assertTrue(all(item["oldOverflowCount"] == 0 for item in summaries))
self.assertTrue(all(item["cancelledCount"] == 0 for item in summaries))
self.assertEqual([], client.stopped)
def test_certain_oom_cleanup_can_still_stop_running_task(self) -> None:
@@ -510,38 +445,7 @@ class QueueCleanupTests(unittest.TestCase):
self.assertEqual(1, summary["cancelledCount"])
self.assertEqual([[1]], client.stopped)
def test_old_overflow_task_is_not_stopped_if_it_moves_inside_dynamic_threshold(self) -> None:
records = [
{
"taskId": index,
"modelId": "owner/old",
"gpuType": "Iluvatar_bi-100",
"status": "waiting",
}
for index in range(1, 92)
]
client = FakeQueueClient(records, drop_first_on_recheck=True)
pool = ModelHubClientPool(
[client], # type: ignore[list-item]
active_task_cap=100,
recent_model_reserve_slots=10,
)
summary = cleanup_certain_oom_tasks(
pool,
FakeDiscovery(
{"owner/old": 1 * GIB},
{"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
), # type: ignore[arg-type]
reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
log=lambda _message: None,
)
self.assertEqual(1, summary["oldOverflowCount"])
self.assertEqual(0, summary["cancelledCount"])
self.assertEqual(1, summary["policyNoLongerAppliesCount"])
self.assertEqual([], client.stopped)
def test_cleanup_promotes_capacity_from_complete_active_listing(self) -> None:
def test_capacity_decline_pauses_submissions_and_never_deletes_existing_tasks(self) -> None:
records = [
{
"taskId": index,
@@ -555,114 +459,20 @@ class QueueCleanupTests(unittest.TestCase):
pool = ModelHubClientPool(
[client], # type: ignore[list-item]
active_task_cap=100,
recent_model_reserve_slots=10,
recent_model_reserve_slots=5,
capacity_state_path=None,
)
summary = cleanup_certain_oom_tasks(
pool,
FakeDiscovery(
{"owner/old": 1 * GIB},
{"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
), # type: ignore[arg-type]
reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
FakeDiscovery({"owner/old": 1 * GIB}), # type: ignore[arg-type]
log=lambda _message: None,
)
self.assertEqual([150], summary["accountCapacityLimits"])
self.assertEqual([140], summary["oldModelQueueThresholds"])
self.assertEqual(list(range(141, 151)), [item["taskId"] for item in summary["oldOverflowTasks"]])
def test_initial_cleanup_stops_old_task_beyond_capacity_minus_ten(self) -> None:
records = [
{
"taskId": index,
"modelId": "owner/old",
"gpuType": "Iluvatar_bi-100",
"status": "waiting",
}
for index in range(1, 92)
]
client = FakeQueueClient(records)
pool = ModelHubClientPool(
[client], # type: ignore[list-item]
active_task_cap=100,
recent_model_reserve_slots=10,
)
summary = cleanup_certain_oom_tasks(
pool,
FakeDiscovery(
{"owner/old": 1 * GIB},
{"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
), # type: ignore[arg-type]
reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
log=lambda _message: None,
)
self.assertEqual(1, summary["oldOverflowCount"])
self.assertEqual(1, summary["cancelledCount"])
self.assertEqual([90], summary["oldModelQueueThresholds"])
self.assertEqual([[91]], client.stopped)
def test_scheduled_cleanup_uses_capacity_minus_five(self) -> None:
records = [
{
"taskId": index,
"modelId": "owner/old",
"gpuType": "Iluvatar_bi-100",
"status": "waiting",
}
for index in range(1, 97)
]
client = FakeQueueClient(records)
pool = ModelHubClientPool(
[client], # type: ignore[list-item]
active_task_cap=100,
recent_model_reserve_slots=10,
)
summary = cleanup_certain_oom_tasks(
pool,
FakeDiscovery(
{"owner/old": 1 * GIB},
{"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
), # type: ignore[arg-type]
age_reserved_slots=5,
reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
log=lambda _message: None,
)
self.assertEqual([95], summary["oldModelQueueThresholds"])
self.assertEqual([96], [item["taskId"] for item in summary["oldOverflowTasks"]])
self.assertEqual([[96]], client.stopped)
def test_oom_is_removed_before_recalculating_old_overflow_positions(self) -> None:
records = [
{
"taskId": index,
"modelId": "owner/large" if index == 1 else "owner/old",
"gpuType": "Iluvatar_bi-100",
"status": "waiting",
}
for index in range(1, 93)
]
client = FakeQueueClient(records)
pool = ModelHubClientPool(
[client], # type: ignore[list-item]
active_task_cap=100,
recent_model_reserve_slots=10,
)
summary = cleanup_certain_oom_tasks(
pool,
FakeDiscovery(
{"owner/large": 40 * GIB, "owner/old": 1 * GIB},
{"owner/old": datetime(2026, 7, 1, tzinfo=timezone.utc)},
), # type: ignore[arg-type]
reference_time=datetime(2026, 8, 11, tzinfo=timezone.utc),
log=lambda _message: None,
)
self.assertEqual(1, summary["certainOomCount"])
self.assertEqual([92], [item["taskId"] for item in summary["oldOverflowTasks"]])
self.assertEqual([[1], [92]], client.stopped)
self.assertEqual([145], summary["oldModelQueueThresholds"])
self.assertEqual(0, summary["oldOverflowCount"])
self.assertEqual(0, pool.available_submit_slots())
self.assertEqual([], client.stopped)
def test_stop_tasks_uses_documented_put_endpoint_and_integer_ids(self) -> None:
http = RecordingHttpClient()