CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: calibrate account capacity weekly

Browse Source
This commit is contained in:
CoolBoy
2026-09-20 13:03:35 +08:00
parent 75c034d588
commit 4afce879dc
7 changed files with 466 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
modelhub_submmit_api/poll_runner.py
View File

@@ -218,7 +218,7 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument(
"--capacity-probe-interval-cycles",
type=int,
default=int(os.getenv("MODELHUB_CAPACITY_PROBE_INTERVAL_CYCLES", "3")),
default=int(os.getenv("MODELHUB_CAPACITY_PROBE_INTERVAL_CYCLES", "0")),
help=argparse.SUPPRESS,
)
parser.add_argument("--daily-runs-dir", default=str(DEFAULT_DAILY_RUNS_DIR), help=argparse.SUPPRESS)
@@ -278,7 +278,7 @@ def _build_modelhub_client(base_args: argparse.Namespace) -> ModelHubClientPool:
clients = [ModelHubClient(token=token, base_url=base_args.modelhub_base_url) for token in token_values]
return ModelHubClientPool(
clients,
capacity_probe_interval_cycles=max(0, int(getattr(base_args, "capacity_probe_interval_cycles", 3) or 0)),
capacity_probe_interval_cycles=0,
capacity_state_path=Path(getattr(base_args, "capacity_state_path", DEFAULT_CAPACITY_STATE_PATH)),
recent_model_reserve_slots=max(0, int(getattr(base_args, "recent_model_reserve_slots", 5) or 0)),
recent_model_days=max(1, int(getattr(base_args, "recent_model_days", 7) or 7)),
@@ -862,8 +862,8 @@ def run_poll_loop(
log(f"[poll] cycle={cycles} sleep={base_args.idle_interval_seconds}s reason=state_sync_unhealthy")
time.sleep(base_args.idle_interval_seconds)
continue
if hasattr(modelhub_client, "configure_capacity_probe"):
modelhub_client.configure_capacity_probe(cycles)
if hasattr(modelhub_client, "configure_weekly_calibration"):
modelhub_client.configure_weekly_calibration(utc_now())
outcome_synced_this_cycle = False
if (
@@ -1035,11 +1035,17 @@ def run_poll_loop(
active_counts = modelhub_client.active_task_counts() if hasattr(modelhub_client, "active_task_counts") else []
capacity_limits = modelhub_client.account_capacity_limits() if hasattr(modelhub_client, "account_capacity_limits") else []
capacity_probe = modelhub_client.capacity_probe_enabled() if hasattr(modelhub_client, "capacity_probe_enabled") else False
calibration = (
modelhub_client.weekly_calibration_status()
if hasattr(modelhub_client, "weekly_calibration_status") else {}
)
available_slots = modelhub_client.available_submit_slots() if hasattr(modelhub_client, "available_submit_slots") else None
log(
f"[poll] cycle={cycles} active_counts={','.join(str(count) for count in active_counts) if active_counts else 'n/a'} "
f"capacity_limits={','.join(str(limit) for limit in capacity_limits) if capacity_limits else 'n/a'} "
f"capacity_probe={'on' if capacity_probe else 'off'} "
f"weekly_calibration={'on' if calibration.get('targetAccount') else 'off'} "
f"calibration_account={calibration.get('targetAccount') or 'n/a'} "
f"available_slots={available_slots if available_slots is not None else 'n/a'}"
)
@@ -1079,6 +1085,22 @@ def run_poll_loop(
cycle_summaries.append(cycle_summary)
submitted_total += cycle_summary["submittedTotal"]
remaining_before_run = cycle_summary.get("remainingDailyQuotaBeforeRun")
if (
calibration.get("targetAccount")
and not cycle_summary.get("submittedTotal")
and cycle_summary.get("waveResults")
and not any(
(wave.get("summary") or {}).get("paused")
or (wave.get("summary") or {}).get("failedCount", 0)
for wave in (cycle_summary.get("waveResults") or [])
)
and not any(
(wave.get("summary") or {}).get("candidateCount", 0)
for wave in (cycle_summary.get("waveResults") or [])
)
and hasattr(modelhub_client, "pause_weekly_calibration_for_no_candidates")
):
modelhub_client.pause_weekly_calibration_for_no_candidates()
log(
f"[poll] cycle_done submitted_total={cycle_summary['submittedTotal']} "