feat: reserve queue capacity for recent models
This commit is contained in:
@@ -17,8 +17,13 @@ if str(PACKAGE_DIR) in sys.path:
|
||||
sys.path.insert(0, str(PACKAGE_DIR))
|
||||
|
||||
from common import read_json, read_jsonl # noqa: E402
|
||||
from main import build_parser, make_run_dir, run_submission, submit_candidate # noqa: E402
|
||||
from modelhub_client import ModelHubAPIError, ModelHubClientPool, is_duplicate_submission_error # noqa: E402
|
||||
from main import build_adaptive_scan_stages, build_parser, make_run_dir, run_submission, submit_candidate # noqa: E402
|
||||
from modelhub_client import ( # noqa: E402
|
||||
ModelHubAPIError,
|
||||
ModelHubClientPool,
|
||||
OldModelQueuePolicyError,
|
||||
is_duplicate_submission_error,
|
||||
)
|
||||
from models import HFModelSummary, ModelInspection # noqa: E402
|
||||
from outcome_tracker import OutcomeTracker # noqa: E402
|
||||
from submission_claims import SubmissionClaimStore, candidate_key # noqa: E402
|
||||
@@ -210,6 +215,109 @@ def make_candidate(index: int) -> dict:
|
||||
|
||||
|
||||
class ClientPoolConcurrencyTests(unittest.TestCase):
|
||||
def test_old_models_use_only_each_accounts_first_eighty_queue_positions(self) -> None:
|
||||
below_threshold = FakeClient(active_count=79)
|
||||
at_threshold = FakeClient(active_count=80)
|
||||
pool = ModelHubClientPool(
|
||||
[below_threshold, at_threshold], # type: ignore[list-item]
|
||||
active_task_cap=100,
|
||||
active_counts_ttl=60,
|
||||
old_model_queue_threshold=80,
|
||||
recent_model_days=7,
|
||||
instance_id="old-model-threshold-test",
|
||||
)
|
||||
submitted_at = datetime(2026, 8, 11, tzinfo=timezone.utc)
|
||||
|
||||
pool.add_task_for_model(
|
||||
{"model": "old-allowed-as-position-80"},
|
||||
model_last_modified=datetime(2026, 7, 1, tzinfo=timezone.utc),
|
||||
submitted_at=submitted_at,
|
||||
)
|
||||
|
||||
self.assertEqual(1, len(below_threshold.submitted))
|
||||
self.assertEqual(0, len(at_threshold.submitted))
|
||||
self.assertEqual(0, pool.old_model_submit_slots())
|
||||
with self.assertRaises(OldModelQueuePolicyError):
|
||||
pool.add_task_for_model(
|
||||
{"model": "old-rejected"},
|
||||
model_last_modified=datetime(2026, 7, 1, tzinfo=timezone.utc),
|
||||
submitted_at=submitted_at,
|
||||
)
|
||||
|
||||
pool.add_task_for_model(
|
||||
{"model": "recent-allowed"},
|
||||
model_last_modified=datetime(2026, 8, 10, tzinfo=timezone.utc),
|
||||
submitted_at=submitted_at,
|
||||
)
|
||||
self.assertEqual(2, len(below_threshold.submitted) + len(at_threshold.submitted))
|
||||
|
||||
def test_concurrent_old_model_submissions_cannot_cross_eighty(self) -> None:
|
||||
client = FakeClient(active_count=78)
|
||||
pool = ModelHubClientPool(
|
||||
[client], # type: ignore[list-item]
|
||||
active_task_cap=100,
|
||||
active_counts_ttl=60,
|
||||
old_model_queue_threshold=80,
|
||||
recent_model_days=7,
|
||||
)
|
||||
submitted_at = datetime(2026, 8, 11, tzinfo=timezone.utc)
|
||||
|
||||
def submit(index: int) -> str:
|
||||
try:
|
||||
pool.add_task_for_model(
|
||||
{"model": f"old-{index}"},
|
||||
model_last_modified=datetime(2026, 7, 1, tzinfo=timezone.utc),
|
||||
submitted_at=submitted_at,
|
||||
)
|
||||
return "submitted"
|
||||
except OldModelQueuePolicyError:
|
||||
return "policy_skipped"
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as executor:
|
||||
outcomes = list(executor.map(submit, range(8)))
|
||||
|
||||
self.assertEqual(2, outcomes.count("submitted"))
|
||||
self.assertEqual(6, outcomes.count("policy_skipped"))
|
||||
self.assertEqual(2, len(client.submitted))
|
||||
self.assertEqual(0, pool.old_model_submit_slots())
|
||||
|
||||
def test_scan_does_not_expand_beyond_recent_window_when_old_slots_are_full(self) -> None:
|
||||
now = datetime(2026, 8, 11, tzinfo=timezone.utc)
|
||||
stages = build_adaptive_scan_stages(
|
||||
now=now,
|
||||
initial_updated_after=now - timedelta(hours=48),
|
||||
initial_limit=100,
|
||||
allow_older_than_recent_window=False,
|
||||
recent_model_days=7,
|
||||
)
|
||||
|
||||
self.assertEqual(["configured_window", "last_7_days"], [stage["name"] for stage in stages])
|
||||
self.assertTrue(all(stage["updatedAfter"] >= now - timedelta(days=7) for stage in stages))
|
||||
|
||||
def test_submit_candidate_reports_old_model_policy_skip_at_eighty(self) -> None:
|
||||
client = FakeClient(active_count=80)
|
||||
pool = ModelHubClientPool(
|
||||
[client], # type: ignore[list-item]
|
||||
active_task_cap=100,
|
||||
old_model_queue_threshold=80,
|
||||
recent_model_days=7,
|
||||
)
|
||||
result = submit_candidate(
|
||||
{
|
||||
"repoId": "owner/old",
|
||||
"modelAddress": "https://modelscope.cn/models/owner/old",
|
||||
"targetGpu": "Iluvatar_bi-150",
|
||||
"taskType": "text-generation",
|
||||
"framework": "vllm",
|
||||
"configParams": "framework: vllm",
|
||||
"lastModified": "2025-01-01T00:00:00+00:00",
|
||||
},
|
||||
pool, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
self.assertEqual("old_model_policy_skipped", result["outcome"])
|
||||
self.assertEqual([], client.submitted)
|
||||
|
||||
def test_online_submission_does_not_construct_llm_even_when_key_is_present(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_dir:
|
||||
root = Path(temporary_dir)
|
||||
|
||||
Reference in New Issue
Block a user