feat: make model age admission-only
This commit is contained in:
CoolBoy
@@ -111,7 +111,7 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
parser.add_argument(
|
||||
"--recent-model-reserve-slots",
|
||||
type=int,
|
||||
default=int(os.getenv("MODELHUB_RECENT_MODEL_RESERVE_SLOTS", "10")),
|
||||
default=int(os.getenv("MODELHUB_RECENT_MODEL_RESERVE_SLOTS", "5")),
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
parser.add_argument(
|
||||
@@ -749,10 +749,15 @@ def submit_candidate(
|
||||
}
|
||||
except ModelHubAPIError as exc:
|
||||
if isinstance(exc, OldModelQueuePolicyError):
|
||||
print(
|
||||
f"[submit] deferred repo={candidate['repoId']} gpu={candidate['targetGpu']} "
|
||||
f"framework={candidate['framework']} reason=age_policy_skipped",
|
||||
flush=True,
|
||||
)
|
||||
return {
|
||||
"outcome": "old_model_policy_skipped",
|
||||
"outcome": "age_policy_deferred",
|
||||
"candidate": candidate,
|
||||
"reason": "all_accounts_at_old_model_queue_threshold",
|
||||
"reason": "age_policy_skipped",
|
||||
}
|
||||
if is_model_uniqueness_error(exc):
|
||||
print(
|
||||
@@ -839,7 +844,7 @@ def run_submission(
|
||||
clients,
|
||||
capacity_probe_interval_cycles=max(0, int(getattr(args, "capacity_probe_interval_cycles", 3) or 0)),
|
||||
capacity_state_path=Path(getattr(args, "capacity_state_path", DEFAULT_CAPACITY_STATE_PATH)),
|
||||
recent_model_reserve_slots=max(0, int(getattr(args, "recent_model_reserve_slots", 10) or 0)),
|
||||
recent_model_reserve_slots=max(0, int(getattr(args, "recent_model_reserve_slots", 5) or 0)),
|
||||
recent_model_days=max(1, int(getattr(args, "recent_model_days", 7) or 7)),
|
||||
)
|
||||
|
||||
@@ -1054,7 +1059,13 @@ def run_submission(
|
||||
long_term_min_samples=max(1, int(getattr(args, "gpu_strategy_min_long_samples", 100) or 100)),
|
||||
market_intelligence=market_intelligence,
|
||||
)
|
||||
strategy_manager.prepare(modelhub_client, supported_gpus=target_gpus, now=now)
|
||||
strategy_history_records = outcome_tracker.get_strategy_history_records()
|
||||
strategy_manager.prepare(
|
||||
modelhub_client,
|
||||
supported_gpus=target_gpus,
|
||||
now=now,
|
||||
history_records=strategy_history_records,
|
||||
)
|
||||
strategy_summary = strategy_manager.summary()
|
||||
|
||||
if getattr(args, "skip_history_archive", False):
|
||||
@@ -1108,17 +1119,18 @@ def run_submission(
|
||||
pipeline_tags = pipeline_tags_for_task_types(selected_task_types)
|
||||
explicit_scan_cap = max(0, int(getattr(args, "max_scan_models", 0) or 0))
|
||||
recent_model_days = max(1, int(getattr(args, "recent_model_days", 7) or 7))
|
||||
recent_model_reserve_slots = max(0, int(getattr(args, "recent_model_reserve_slots", 10) or 0))
|
||||
recent_model_reserve_slots = max(0, int(getattr(args, "recent_model_reserve_slots", 5) or 0))
|
||||
old_model_slots_before_scan: int | None = None
|
||||
if hasattr(modelhub_client, "old_model_submit_slots"):
|
||||
old_model_slots_before_scan = int(modelhub_client.old_model_submit_slots())
|
||||
old_model_queue_thresholds: list[int] | None = None
|
||||
old_model_queue_thresholds: list[int | None] | None = None
|
||||
if hasattr(modelhub_client, "old_model_queue_thresholds"):
|
||||
old_model_queue_thresholds = list(modelhub_client.old_model_queue_thresholds())
|
||||
allow_older_models_for_scan = old_model_slots_before_scan is None or old_model_slots_before_scan > 0
|
||||
print(
|
||||
f"[age-policy] recent_days={recent_model_days} reserve_recent_slots={recent_model_reserve_slots} "
|
||||
f"account_thresholds={','.join(str(value) for value in old_model_queue_thresholds) if old_model_queue_thresholds is not None else 'n/a'} "
|
||||
f"[age-policy] mode=admission_only recent_days={recent_model_days} "
|
||||
f"reserve_recent_slots={recent_model_reserve_slots} "
|
||||
f"account_thresholds={','.join('n/a' if value is None else str(value) for value in old_model_queue_thresholds) if old_model_queue_thresholds is not None else 'n/a'} "
|
||||
f"old_model_slots={old_model_slots_before_scan if old_model_slots_before_scan is not None else 'n/a'} "
|
||||
f"scan_older={'on' if allow_older_models_for_scan else 'off'}",
|
||||
flush=True,
|
||||
@@ -1133,6 +1145,7 @@ def run_submission(
|
||||
):
|
||||
if candidate_goal <= 0 or len(candidates) >= candidate_goal:
|
||||
break
|
||||
skipped_before_stage = len(skipped)
|
||||
stage_updated_after = stage["updatedAfter"]
|
||||
stage_limit = int(stage["limit"])
|
||||
query_kwargs = {
|
||||
@@ -1149,15 +1162,26 @@ def run_submission(
|
||||
except TypeError:
|
||||
models = hf_discovery.list_recent_models(**query_kwargs)
|
||||
if not allow_older_models_for_scan:
|
||||
models = [
|
||||
model
|
||||
for model in models
|
||||
recent_models: list[HFModelSummary] = []
|
||||
for model in models:
|
||||
if is_recent_model(
|
||||
model.last_modified,
|
||||
reference_time=now,
|
||||
recent_model_days=recent_model_days,
|
||||
):
|
||||
recent_models.append(model)
|
||||
continue
|
||||
if model.repo_id in seen_model_ids:
|
||||
continue
|
||||
seen_model_ids.add(model.repo_id)
|
||||
skipped.append(
|
||||
{
|
||||
"repoId": model.repo_id,
|
||||
"reason": "age_policy_skipped",
|
||||
"deferred": True,
|
||||
}
|
||||
)
|
||||
]
|
||||
models = recent_models
|
||||
|
||||
stage_candidates, stage_skipped, stage_failed, processed_count = collect_candidates_from_models(
|
||||
models=models,
|
||||
@@ -1186,14 +1210,14 @@ def run_submission(
|
||||
"newModelsProcessed": processed_count,
|
||||
"candidatesAdded": len(stage_candidates),
|
||||
"candidateCountAfterStage": len(candidates),
|
||||
"skippedAdded": len(stage_skipped),
|
||||
"skippedAdded": len(skipped) - skipped_before_stage,
|
||||
"failedAdded": len(stage_failed),
|
||||
}
|
||||
scan_stages.append(stage_summary)
|
||||
print(
|
||||
f"[scan] stage={stage['name']} discovered={len(models)} new_processed={processed_count} "
|
||||
f"candidates_added={len(stage_candidates)} candidates_total={len(candidates)}/{candidate_goal} "
|
||||
f"skipped_added={len(stage_skipped)}",
|
||||
f"skipped_added={len(skipped) - skipped_before_stage}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
@@ -1235,25 +1259,11 @@ def run_submission(
|
||||
# already-scanned pool until the desired number of real submissions is
|
||||
# reached or account capacity is genuinely exhausted.
|
||||
while len(submitted) < target_submit_count and len(attempted_candidates) < max_submit_attempts:
|
||||
allow_older_models_now = (
|
||||
bool(modelhub_client.can_submit_old_models())
|
||||
if hasattr(modelhub_client, "can_submit_old_models")
|
||||
else True
|
||||
)
|
||||
submission_reference_time = utc_now()
|
||||
remaining_candidates = [
|
||||
candidate
|
||||
for candidate in diversified_candidates
|
||||
if candidate_key(candidate) not in attempted_keys
|
||||
and not submission_exclusion_store.is_blocked(candidate["repoId"], candidate["targetGpu"])
|
||||
and (
|
||||
allow_older_models_now
|
||||
or is_recent_model(
|
||||
candidate.get("lastModified"),
|
||||
reference_time=submission_reference_time,
|
||||
recent_model_days=recent_model_days,
|
||||
)
|
||||
)
|
||||
]
|
||||
remaining_candidates = one_candidate_per_model(remaining_candidates)
|
||||
desired_count = min(
|
||||
@@ -1328,13 +1338,13 @@ def run_submission(
|
||||
}
|
||||
)
|
||||
continue
|
||||
if result["outcome"] == "old_model_policy_skipped":
|
||||
if result["outcome"] == "age_policy_deferred":
|
||||
batch_policy_skipped_candidates.append(candidate)
|
||||
skipped.append(
|
||||
{
|
||||
"repoId": candidate["repoId"],
|
||||
"targetGpu": candidate["targetGpu"],
|
||||
"reason": "all_accounts_at_old_model_queue_threshold",
|
||||
"reason": "age_policy_skipped",
|
||||
}
|
||||
)
|
||||
continue
|
||||
@@ -1428,6 +1438,7 @@ def run_submission(
|
||||
"marketIntelligence": market_summary,
|
||||
"candidatePreflight": preflight_summary,
|
||||
"agePolicy": {
|
||||
"mode": "admission_only",
|
||||
"recentModelDays": recent_model_days,
|
||||
"recentModelReserveSlots": recent_model_reserve_slots,
|
||||
"oldModelSubmitThresholds": old_model_queue_thresholds,
|
||||
|
||||
Reference in New Issue
Block a user