CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: calibrate account capacity weekly

Browse Source
This commit is contained in:
CoolBoy
2026-09-20 13:03:35 +08:00
parent 75c034d588
commit 4afce879dc
7 changed files with 466 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
modelhub_submmit_api/main.py
View File

@@ -258,7 +258,7 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument(
"--capacity-probe-interval-cycles",
type=int,
default=int(os.getenv("MODELHUB_CAPACITY_PROBE_INTERVAL_CYCLES", "3")),
default=int(os.getenv("MODELHUB_CAPACITY_PROBE_INTERVAL_CYCLES", "0")),
help=argparse.SUPPRESS,
)
parser.add_argument("--hf-base-url", default=os.getenv("MODELSCOPE_BASE_URL", "https://modelscope.cn"), help=argparse.SUPPRESS)
@@ -517,6 +517,9 @@ def resolve_submit_concurrency(
modelhub_client: ModelHubClient | ModelHubClientPool | Any,
planned_submit_count: int,
) -> int:
if hasattr(modelhub_client, "weekly_calibration_active") and modelhub_client.weekly_calibration_active():
# Pinning and capacity evidence require one in-flight task per account.
return 1
explicit_workers = int(getattr(args, "submit_concurrency", 0) or 0)
if explicit_workers > 0:
return explicit_workers
@@ -1050,6 +1053,17 @@ def submit_candidate(
"candidate": candidate,
"reason": "account_capacity_saturated",
}
if exc.code == 429:
print(
f"[submit] deferred repo={candidate['repoId']} gpu={candidate['targetGpu']} "
f"framework={candidate['framework']} reason=platform_rate_limited",
flush=True,
)
return {
"outcome": "rate_limit_deferred",
"candidate": candidate,
"reason": "platform_rate_limited",
}
print(
f"[submit] failed repo={candidate['repoId']} gpu={candidate['targetGpu']} "
f"framework={candidate['framework']} reason={exc}",
@@ -1150,11 +1164,12 @@ def run_submission(
clients = [ModelHubClient(token=token, base_url=args.modelhub_base_url) for token in token_values]
modelhub_client = ModelHubClientPool(
clients,
capacity_probe_interval_cycles=max(0, int(getattr(args, "capacity_probe_interval_cycles", 3) or 0)),
capacity_probe_interval_cycles=0,
capacity_state_path=Path(getattr(args, "capacity_state_path", DEFAULT_CAPACITY_STATE_PATH)),
recent_model_reserve_slots=max(0, int(getattr(args, "recent_model_reserve_slots", 5) or 0)),
recent_model_days=max(1, int(getattr(args, "recent_model_days", 7) or 7)),
)
modelhub_client.configure_weekly_calibration(now)
official_registry: OfficialCapabilityRegistry | None = None
official_summary: dict[str, Any] = {"enabled": False}
@@ -1499,10 +1514,16 @@ def run_submission(
old_model_queue_thresholds: list[int | None] | None = None
if hasattr(modelhub_client, "old_model_queue_thresholds"):
old_model_queue_thresholds = list(modelhub_client.old_model_queue_thresholds())
weekly_calibration = (
modelhub_client.weekly_calibration_status()
if hasattr(modelhub_client, "weekly_calibration_status") else {}
)
allow_older_models_for_scan = old_model_slots_before_scan is None or old_model_slots_before_scan > 0
print(
f"[age-policy] mode=admission_only recent_days={recent_model_days} "
f"[age-policy] mode={'weekly_capacity_calibration' if weekly_calibration.get('targetAccount') else 'admission_only'} "
f"recent_days={recent_model_days} "
f"reserve_recent_slots={recent_model_reserve_slots} "
f"calibration_account={weekly_calibration.get('targetAccount') or 'n/a'} "
f"account_thresholds={','.join('n/a' if value is None else str(value) for value in old_model_queue_thresholds) if old_model_queue_thresholds is not None else 'n/a'} "
f"old_model_slots={old_model_slots_before_scan if old_model_slots_before_scan is not None else 'n/a'} "
f"scan_older={'on' if allow_older_models_for_scan else 'off'}",
@@ -1743,7 +1764,7 @@ def run_submission(
}
)
continue
if result["outcome"] == "framework_evidence_deferred":
if result["outcome"] in {"framework_evidence_deferred", "rate_limit_deferred"}:
batch_policy_skipped_candidates.append(candidate)
skipped.append({
"repoId": candidate["repoId"],
@@ -1906,13 +1927,15 @@ def run_submission(
"marketIntelligence": market_summary,
"candidatePreflight": preflight_summary,
"agePolicy": {
"mode": "admission_only",
"mode": "weekly_capacity_calibration" if weekly_calibration.get("targetAccount") else "admission_only",
"recentModelDays": recent_model_days,
"recentModelReserveSlots": recent_model_reserve_slots,
"oldModelSubmitThresholds": old_model_queue_thresholds,
"oldModelSlotsBeforeScan": old_model_slots_before_scan,
"olderHistoryScanEnabled": allow_older_models_for_scan,
"calibrationAccount": weekly_calibration.get("targetAccount"),
},
"weeklyCalibration": weekly_calibration,
"scanLimit": scan_limit,
"rawCandidateGoal": raw_candidate_goal,
"candidatePoolLimit": candidate_pool_limit,