CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: reserve queue capacity for recent models

Browse Source
This commit is contained in:
CoolBoy
2026-08-11 01:29:54 +08:00
parent 38ab25fc3c
commit f12d96b138
13 changed files with 927 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
modelhub_submmit_api/main.py
View File

@@ -33,6 +33,7 @@ from modelhub_client import (
ModelHubAPIError,
ModelHubClient,
ModelHubClientPool,
OldModelQueuePolicyError,
is_duplicate_submission_error,
is_model_uniqueness_error,
)
@@ -107,6 +108,18 @@ def build_parser() -> argparse.ArgumentParser:
default=0,
help="Maximum tasks to submit in one run (0 means unlimited)",
)
parser.add_argument(
"--old-model-queue-threshold",
type=int,
default=int(os.getenv("MODELHUB_OLD_MODEL_QUEUE_THRESHOLD", "80")),
help=argparse.SUPPRESS,
)
parser.add_argument(
"--recent-model-days",
type=int,
default=int(os.getenv("MODELHUB_RECENT_MODEL_DAYS", "7")),
help=argparse.SUPPRESS,
)
parser.add_argument(
"--disable-candidate-preflight",
action="store_true",
@@ -548,9 +561,17 @@ def build_adaptive_scan_stages(
initial_updated_after,
initial_limit: int,
max_models: int = ADAPTIVE_SCAN_MAX_MODELS,
allow_older_than_recent_window: bool = True,
recent_model_days: int = 7,
) -> list[dict[str, Any]]:
max_models = max(1, min(ADAPTIVE_SCAN_MAX_MODELS, int(max_models)))
initial_limit = min(max_models, max(1, int(initial_limit)))
recent_cutoff = now - timedelta(days=max(1, int(recent_model_days)))
if not allow_older_than_recent_window and (
initial_updated_after is None or initial_updated_after < recent_cutoff
):
initial_updated_after = recent_cutoff
stages: list[dict[str, Any]] = [
{
"name": "configured_window",
@@ -559,17 +580,16 @@ def build_adaptive_scan_stages(
}
]
seven_days_ago = now - timedelta(days=7)
thirty_days_ago = now - timedelta(days=30)
if initial_updated_after is not None and initial_updated_after > seven_days_ago:
if initial_updated_after is not None and initial_updated_after > recent_cutoff:
stages.append(
{
"name": "last_7_days",
"updatedAfter": seven_days_ago,
"updatedAfter": recent_cutoff,
"limit": min(max_models, max(initial_limit, ADAPTIVE_SCAN_MIN_FALLBACK_MODELS)),
}
)
if initial_updated_after is not None and initial_updated_after > thirty_days_ago:
if allow_older_than_recent_window and initial_updated_after is not None and initial_updated_after > thirty_days_ago:
stages.append(
{
"name": "last_30_days",
@@ -577,7 +597,7 @@ def build_adaptive_scan_stages(
"limit": min(max_models, max(initial_limit, 1500)),
}
)
if initial_updated_after is not None or initial_limit < max_models:
if allow_older_than_recent_window and (initial_updated_after is not None or initial_limit < max_models):
stages.append(
{
"name": "all_history",
@@ -598,6 +618,13 @@ def build_adaptive_scan_stages(
return deduped
def is_recent_model(last_modified: Any, *, reference_time, recent_model_days: int = 7) -> bool:
parsed = parse_datetime(last_modified)
if parsed is None:
return False
return parsed >= reference_time - timedelta(days=max(1, int(recent_model_days)))
def collect_candidates_from_models(
*,
models: list[HFModelSummary],
@@ -702,7 +729,14 @@ def submit_candidate(
payload["strategyId"] = strategy_id
submit_time = utc_now()
try:
response = modelhub_client.add_task(payload)
if hasattr(modelhub_client, "add_task_for_model"):
response = modelhub_client.add_task_for_model(
payload,
model_last_modified=candidate.get("lastModified"),
submitted_at=submit_time,
)
else:
response = modelhub_client.add_task(payload)
task_id = extract_task_id_from_submit_response(response)
if task_id is None:
task_id = modelhub_client.find_recent_task_id(candidate["repoId"], candidate["targetGpu"], submit_time)
@@ -714,6 +748,12 @@ def submit_candidate(
"responseData": response.get("data"),
}
except ModelHubAPIError as exc:
if isinstance(exc, OldModelQueuePolicyError):
return {
"outcome": "old_model_policy_skipped",
"candidate": candidate,
"reason": "all_accounts_at_old_model_queue_threshold",
}
if is_model_uniqueness_error(exc):
print(
f"[submit] skipped repo={candidate['repoId']} gpu={candidate['targetGpu']} "
@@ -799,6 +839,8 @@ def run_submission(
clients,
capacity_probe_interval_cycles=max(0, int(getattr(args, "capacity_probe_interval_cycles", 3) or 0)),
capacity_state_path=Path(getattr(args, "capacity_state_path", DEFAULT_CAPACITY_STATE_PATH)),
old_model_queue_threshold=max(1, int(getattr(args, "old_model_queue_threshold", 80) or 80)),
recent_model_days=max(1, int(getattr(args, "recent_model_days", 7) or 7)),
)
if hasattr(modelhub_client, "begin_cycle"):
@@ -1065,11 +1107,25 @@ def run_submission(
pipeline_tags = pipeline_tags_for_task_types(selected_task_types)
explicit_scan_cap = max(0, int(getattr(args, "max_scan_models", 0) or 0))
recent_model_days = max(1, int(getattr(args, "recent_model_days", 7) or 7))
old_model_queue_threshold = max(1, int(getattr(args, "old_model_queue_threshold", 80) or 80))
old_model_slots_before_scan: int | None = None
if hasattr(modelhub_client, "old_model_submit_slots"):
old_model_slots_before_scan = int(modelhub_client.old_model_submit_slots())
allow_older_models_for_scan = old_model_slots_before_scan is None or old_model_slots_before_scan > 0
print(
f"[age-policy] recent_days={recent_model_days} account_threshold={old_model_queue_threshold} "
f"old_model_slots={old_model_slots_before_scan if old_model_slots_before_scan is not None else 'n/a'} "
f"scan_older={'on' if allow_older_models_for_scan else 'off'}",
flush=True,
)
for stage in build_adaptive_scan_stages(
now=now,
initial_updated_after=updated_after,
initial_limit=scan_limit,
max_models=explicit_scan_cap or ADAPTIVE_SCAN_MAX_MODELS,
allow_older_than_recent_window=allow_older_models_for_scan,
recent_model_days=recent_model_days,
):
if candidate_goal <= 0 or len(candidates) >= candidate_goal:
break
@@ -1088,6 +1144,16 @@ def run_submission(
)
except TypeError:
models = hf_discovery.list_recent_models(**query_kwargs)
if not allow_older_models_for_scan:
models = [
model
for model in models
if is_recent_model(
model.last_modified,
reference_time=now,
recent_model_days=recent_model_days,
)
]
stage_candidates, stage_skipped, stage_failed, processed_count = collect_candidates_from_models(
models=models,
@@ -1165,11 +1231,25 @@ def run_submission(
# already-scanned pool until the desired number of real submissions is
# reached or account capacity is genuinely exhausted.
while len(submitted) < target_submit_count and len(attempted_candidates) < max_submit_attempts:
allow_older_models_now = (
bool(modelhub_client.can_submit_old_models())
if hasattr(modelhub_client, "can_submit_old_models")
else True
)
submission_reference_time = utc_now()
remaining_candidates = [
candidate
for candidate in diversified_candidates
if candidate_key(candidate) not in attempted_keys
and not submission_exclusion_store.is_blocked(candidate["repoId"], candidate["targetGpu"])
and (
allow_older_models_now
or is_recent_model(
candidate.get("lastModified"),
reference_time=submission_reference_time,
recent_model_days=recent_model_days,
)
)
]
remaining_candidates = one_candidate_per_model(remaining_candidates)
desired_count = min(
@@ -1209,6 +1289,7 @@ def run_submission(
batch_submitted_candidates: list[dict[str, Any]] = []
batch_duplicate_candidates: list[dict[str, Any]] = []
batch_uniqueness_rejected_candidates: list[dict[str, Any]] = []
batch_policy_skipped_candidates: list[dict[str, Any]] = []
batch_failed_candidates: list[dict[str, Any]] = []
for index in range(len(batch_candidates)):
result = ordered_results.get(index)
@@ -1243,6 +1324,16 @@ def run_submission(
}
)
continue
if result["outcome"] == "old_model_policy_skipped":
batch_policy_skipped_candidates.append(candidate)
skipped.append(
{
"repoId": candidate["repoId"],
"targetGpu": candidate["targetGpu"],
"reason": "all_accounts_at_old_model_queue_threshold",
}
)
continue
if result["outcome"] in {"failed", "precheck_deferred"}:
batch_failed_candidates.append(candidate)
failed.append(
@@ -1290,7 +1381,7 @@ def run_submission(
claim_store.mark_submitted(
[*batch_submitted_candidates, *batch_duplicate_candidates, *batch_uniqueness_rejected_candidates]
)
claim_store.release(batch_failed_candidates)
claim_store.release([*batch_failed_candidates, *batch_policy_skipped_candidates])
if strategy_manager is not None:
strategy_manager.record_accepted(batch_submitted_candidates)
@@ -1332,6 +1423,12 @@ def run_submission(
"gpuStrategy": strategy_summary,
"marketIntelligence": market_summary,
"candidatePreflight": preflight_summary,
"agePolicy": {
"recentModelDays": recent_model_days,
"oldModelSubmitThreshold": old_model_queue_threshold,
"oldModelSlotsBeforeScan": old_model_slots_before_scan,
"olderHistoryScanEnabled": allow_older_models_for_scan,
},
"scanLimit": scan_limit,
"candidateGoal": candidate_goal,
"scanStages": scan_stages,