feat: add adaptive GPU scheduling

modelhub_submmit_api/daily_runner.py

@@ -7,9 +7,10 @@ from pathlib import Path
 from typing import Any, Callable
 
 from common import utc_now, write_json
+from gpu_strategy import DEFAULT_GPU_STRATEGY_PATH
 from hf_discovery import HuggingFaceDiscovery
 from main import DEFAULT_LEDGER_PATH, DEFAULT_RUNS_DIR, make_run_dir, run_submission
-from modelhub_client import ModelHubClient, ModelHubClientPool
+from modelhub_client import DEFAULT_CAPACITY_STATE_PATH, ModelHubClient, ModelHubClientPool
 from outcome_tracker import OutcomeTracker
 from runner_common import DEFAULT_KEY_PATH, ensure_tokens
 from submission_claims import DEFAULT_CLAIMS_PATH
@@ -79,6 +80,13 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument("--skip-outcome-sync", action="store_true", help="Skip outcome sync from ModelHub before scanning")
     parser.add_argument("--skip-history-archive", action="store_true", help="Skip historical task archive download for this run")
     parser.add_argument("--dry-run", action="store_true", help="Plan the day without creating tasks")
+    parser.add_argument("--disable-gpu-strategy", action="store_true", help="Disable adaptive 50/30/20 GPU scheduling")
+    parser.add_argument(
+        "--gpu-strategy-refresh-submissions",
+        type=int,
+        default=200,
+        help="Recalculate adaptive GPU choices after this many accepted submissions",
+    )
     parser.add_argument("--key-path", default=str(DEFAULT_KEY_PATH), help="Path to KEY.md containing MODELSCOPE_TOKEN/XC_TOKEN")
     parser.add_argument("--runs-dir", default=str(DEFAULT_RUNS_DIR), help=argparse.SUPPRESS)
     parser.add_argument("--ledger-path", default=str(DEFAULT_LEDGER_PATH), help=argparse.SUPPRESS)
@@ -89,6 +97,24 @@ def build_parser() -> argparse.ArgumentParser:
     )
     parser.add_argument("--history-archive-path", default="history/platform_tasks.jsonl", help=argparse.SUPPRESS)
     parser.add_argument("--history-archive-limit", type=int, default=5000, help=argparse.SUPPRESS)
+    parser.add_argument(
+        "--gpu-strategy-state-path",
+        default=os.getenv("MODELHUB_GPU_STRATEGY_STATE_PATH", str(DEFAULT_GPU_STRATEGY_PATH)),
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument("--gpu-strategy-recent-window", type=int, default=1000, help=argparse.SUPPRESS)
+    parser.add_argument("--gpu-strategy-min-long-samples", type=int, default=100, help=argparse.SUPPRESS)
+    parser.add_argument(
+        "--capacity-state-path",
+        default=os.getenv("MODELHUB_CAPACITY_STATE_PATH", str(DEFAULT_CAPACITY_STATE_PATH)),
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--capacity-probe-interval-cycles",
+        type=int,
+        default=int(os.getenv("MODELHUB_CAPACITY_PROBE_INTERVAL_CYCLES", "3")),
+        help=argparse.SUPPRESS,
+    )
     parser.add_argument("--daily-runs-dir", default=str(DEFAULT_DAILY_RUNS_DIR), help=argparse.SUPPRESS)
     parser.add_argument("--hf-base-url", default=os.getenv("MODELSCOPE_BASE_URL", "https://modelscope.cn"), help=argparse.SUPPRESS)
     parser.add_argument("--modelhub-base-url", default=os.getenv("MODELHUB_BASE_URL", "https://modelhub.org.cn"), help=argparse.SUPPRESS)
@@ -118,6 +144,13 @@ def make_wave_namespace(base_args: argparse.Namespace, wave: WaveSpec) -> argpar
         scan_multiplier=getattr(base_args, "scan_multiplier", 4),
         skip_outcome_sync=getattr(base_args, "skip_outcome_sync", False),
         skip_history_archive=getattr(base_args, "skip_history_archive", False),
+        disable_gpu_strategy=getattr(base_args, "disable_gpu_strategy", False),
+        gpu_strategy_refresh_submissions=getattr(base_args, "gpu_strategy_refresh_submissions", 200),
+        gpu_strategy_state_path=getattr(base_args, "gpu_strategy_state_path", str(DEFAULT_GPU_STRATEGY_PATH)),
+        gpu_strategy_recent_window=getattr(base_args, "gpu_strategy_recent_window", 1000),
+        gpu_strategy_min_long_samples=getattr(base_args, "gpu_strategy_min_long_samples", 100),
+        capacity_state_path=getattr(base_args, "capacity_state_path", str(DEFAULT_CAPACITY_STATE_PATH)),
+        capacity_probe_interval_cycles=getattr(base_args, "capacity_probe_interval_cycles", 3),
         submit_concurrency=getattr(base_args, "submit_concurrency", 1),
         max_submits_per_run=getattr(base_args, "max_submits_per_run", 0),
         runs_dir=base_args.runs_dir,
@@ -149,7 +182,11 @@ def run_daily_batches(
         modelhub_tokens = list(getattr(base_args, "modelhub_tokens", None) or ([] if not base_args.modelhub_token else [base_args.modelhub_token]))
         token_values: list[str | None] = modelhub_tokens or [base_args.modelhub_token]
         clients = [ModelHubClient(token=token, base_url=base_args.modelhub_base_url) for token in token_values]
-        modelhub_client = ModelHubClientPool(clients)
+        modelhub_client = ModelHubClientPool(
+            clients,
+            capacity_probe_interval_cycles=max(0, int(getattr(base_args, "capacity_probe_interval_cycles", 3) or 0)),
+            capacity_state_path=Path(getattr(base_args, "capacity_state_path", DEFAULT_CAPACITY_STATE_PATH)),
+        )
     template_selector = template_selector or TemplateSelector()
 
     daily_run_dir = make_run_dir(Path(base_args.daily_runs_dir), now)
@@ -172,6 +209,11 @@ def run_daily_batches(
         log(f"[daily] round={round_index} start")
         for wave in waves:
             attempted_waves += 1
+            if (
+                hasattr(modelhub_client, "configure_capacity_probe")
+                and not bool(getattr(base_args, "capacity_probe_already_configured", False))
+            ):
+                modelhub_client.configure_capacity_probe(attempted_waves)
             wave_args = make_wave_namespace(base_args, wave)
             log(
                 f"[daily] round={round_index} wave={wave.name} "