from __future__ import annotations import argparse import os from dataclasses import asdict, dataclass from pathlib import Path from typing import Any, Callable from common import utc_now, write_json from gpu_strategy import DEFAULT_GPU_STRATEGY_PATH from hf_discovery import HuggingFaceDiscovery from main import DEFAULT_LEDGER_PATH, DEFAULT_RUNS_DIR, make_run_dir, run_submission from market_intelligence import ( DEFAULT_FETCH_WORKERS, DEFAULT_FRAMEWORK_MIN_SAMPLES, DEFAULT_FRAMEWORK_REFRESH_SECONDS, DEFAULT_MARKET_INTELLIGENCE_PATH, DEFAULT_QUEUE_REFRESH_SECONDS, DEFAULT_THROUGHPUT_WINDOW_HOURS, ) from modelhub_client import DEFAULT_CAPACITY_STATE_PATH, ModelHubClient, ModelHubClientPool from outcome_tracker import OutcomeTracker from runner_common import DEFAULT_KEY_PATH, ensure_tokens from submission_claims import DEFAULT_CLAIMS_PATH from template_selector import TemplateSelector DEFAULT_DAILY_RUNS_DIR = Path("daily_runs") @dataclass(frozen=True) class WaveSpec: name: str task_types: tuple[str, ...] gpus: str | None = None limit: int = 400 since_hours: int = 168 DEFAULT_WAVES: tuple[WaveSpec, ...] = ( WaveSpec("text_generation_core", ("text-generation",), gpus=None, limit=100000, since_hours=48), # WaveSpec("visual_multimodal_core", ("visual-multi-modal",), gpus=None, limit=400, since_hours=336), # WaveSpec("image_generation", ("text-to-image-generation",), gpus=None, limit=250, since_hours=720), # WaveSpec("asr", ("asr",), gpus=None, limit=250, since_hours=720), # WaveSpec("embedding_and_qa", ("feature_emb", "question_answering"), gpus=None, limit=200, since_hours=720), # WaveSpec("classification", ("text_classification", "vision_classification"), gpus=None, limit=200, since_hours=720), # WaveSpec("reinforcement_learning", ("reinforcement_learning",), gpus=None, limit=200, since_hours=720), ) def log(message: str) -> None: print(message, flush=True) def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="One-click daily ModelHub submission runner.") parser.add_argument("--daily-target", type=int, default=0, help="Total submissions to aim for per UTC day; 0 means unlimited") parser.add_argument("--rounds", type=int, default=3, help="How many wave cycles to run before stopping") parser.add_argument("--gpu", help="Single GPU alias or platform name, for example: k100") parser.add_argument("--gpus", help="Comma-separated GPU aliases/platform names. Omit to auto-use all safe GPUs.") parser.add_argument("--max-scan-models", type=int, default=0, help="Hard cap on total scanned models (0 means auto)") parser.add_argument( "--scan-multiplier", type=int, default=4, help="Multiplier used when auto-deriving scan limit from quota/queue capacity", ) parser.add_argument("--min-downloads", type=int, default=50, help="Minimum ModelScope download threshold") parser.add_argument( "--history-stats-threshold", type=int, default=500, help="Minimum local ledger records before using history stats to rank submissions", ) parser.add_argument("--read-concurrency", type=int, default=4, help="Concurrency for read-only remote calls") parser.add_argument( "--submit-concurrency", type=int, default=0, help="Concurrency for task submission calls (0 = auto based on token/client count)", ) parser.add_argument( "--max-submits-per-run", type=int, default=0, help="Maximum tasks to submit in one run (0 means unlimited)", ) parser.add_argument( "--recent-model-reserve-slots", type=int, default=int(os.getenv("MODELHUB_RECENT_MODEL_RESERVE_SLOTS", "10")), help=argparse.SUPPRESS, ) parser.add_argument( "--recent-model-days", type=int, default=int(os.getenv("MODELHUB_RECENT_MODEL_DAYS", "7")), help=argparse.SUPPRESS, ) parser.add_argument("--disable-candidate-preflight", action="store_true", help=argparse.SUPPRESS) parser.add_argument("--llm-classifier-endpoint", default=os.getenv("MODELHUB_LLM_CLASSIFIER_ENDPOINT"), help=argparse.SUPPRESS) parser.add_argument("--llm-classifier-model", default=os.getenv("MODELHUB_LLM_CLASSIFIER_MODEL"), help=argparse.SUPPRESS) parser.add_argument("--llm-classifier-api-key", default=os.getenv("MODELHUB_LLM_CLASSIFIER_API_KEY"), help=argparse.SUPPRESS) parser.add_argument("--llm-classifier-timeout-seconds", type=int, default=int(os.getenv("MODELHUB_LLM_CLASSIFIER_TIMEOUT_SECONDS", "20")), help=argparse.SUPPRESS) parser.add_argument("--llm-classifier-min-deny-confidence", type=float, default=float(os.getenv("MODELHUB_LLM_CLASSIFIER_MIN_DENY_CONFIDENCE", "0.85")), help=argparse.SUPPRESS) parser.add_argument("--llm-classifier-cache-path", default=os.getenv("MODELHUB_LLM_CLASSIFIER_CACHE_PATH", ".modelhub_state/llm_classifications.json"), help=argparse.SUPPRESS) parser.add_argument("--skip-outcome-sync", action="store_true", help="Skip outcome sync from ModelHub before scanning") parser.add_argument("--skip-history-archive", action="store_true", help="Skip historical task archive download for this run") parser.add_argument("--dry-run", action="store_true", help="Plan the day without creating tasks") parser.add_argument("--disable-gpu-strategy", action="store_true", help="Disable adaptive 70/30 proven-GPU scheduling") parser.add_argument( "--disable-market-intelligence", action="store_true", help="Disable live queue/throughput and public framework statistics", ) parser.add_argument( "--gpu-strategy-refresh-submissions", type=int, default=200, help="Recalculate adaptive GPU choices after this many accepted submissions", ) parser.add_argument("--key-path", default=str(DEFAULT_KEY_PATH), help="Path to KEY.md containing MODELSCOPE_TOKEN/XC_TOKEN") parser.add_argument("--runs-dir", default=str(DEFAULT_RUNS_DIR), help=argparse.SUPPRESS) parser.add_argument("--ledger-path", default=str(DEFAULT_LEDGER_PATH), help=argparse.SUPPRESS) parser.add_argument( "--claims-path", default=os.getenv("MODELHUB_AGENT_CLAIMS_PATH", str(DEFAULT_CLAIMS_PATH)), help=argparse.SUPPRESS, ) parser.add_argument( "--submission-exclusions-path", default=os.getenv("MODELHUB_SUBMISSION_EXCLUSIONS_PATH", ".modelhub_state/submission_exclusions.jsonl"), help=argparse.SUPPRESS, ) parser.add_argument("--history-archive-path", default="history/platform_tasks.jsonl", help=argparse.SUPPRESS) parser.add_argument("--history-archive-limit", type=int, default=5000, help=argparse.SUPPRESS) parser.add_argument( "--gpu-strategy-state-path", default=os.getenv("MODELHUB_GPU_STRATEGY_STATE_PATH", str(DEFAULT_GPU_STRATEGY_PATH)), help=argparse.SUPPRESS, ) parser.add_argument("--gpu-strategy-recent-window", type=int, default=1000, help=argparse.SUPPRESS) parser.add_argument("--gpu-strategy-min-long-samples", type=int, default=100, help=argparse.SUPPRESS) parser.add_argument( "--market-intelligence-state-path", default=os.getenv("MODELHUB_MARKET_INTELLIGENCE_PATH", str(DEFAULT_MARKET_INTELLIGENCE_PATH)), help=argparse.SUPPRESS, ) parser.add_argument( "--market-queue-refresh-seconds", type=int, default=int(os.getenv("MODELHUB_MARKET_QUEUE_REFRESH_SECONDS", str(DEFAULT_QUEUE_REFRESH_SECONDS))), help=argparse.SUPPRESS, ) parser.add_argument( "--market-framework-refresh-seconds", type=int, default=int(os.getenv("MODELHUB_MARKET_FRAMEWORK_REFRESH_SECONDS", str(DEFAULT_FRAMEWORK_REFRESH_SECONDS))), help=argparse.SUPPRESS, ) parser.add_argument( "--market-throughput-window-hours", type=int, default=int(os.getenv("MODELHUB_MARKET_THROUGHPUT_WINDOW_HOURS", str(DEFAULT_THROUGHPUT_WINDOW_HOURS))), help=argparse.SUPPRESS, ) parser.add_argument( "--market-fetch-workers", type=int, default=int(os.getenv("MODELHUB_MARKET_FETCH_WORKERS", str(DEFAULT_FETCH_WORKERS))), help=argparse.SUPPRESS, ) parser.add_argument( "--market-framework-min-samples", type=int, default=int(os.getenv("MODELHUB_MARKET_FRAMEWORK_MIN_SAMPLES", str(DEFAULT_FRAMEWORK_MIN_SAMPLES))), help=argparse.SUPPRESS, ) parser.add_argument( "--capacity-state-path", default=os.getenv("MODELHUB_CAPACITY_STATE_PATH", str(DEFAULT_CAPACITY_STATE_PATH)), help=argparse.SUPPRESS, ) parser.add_argument( "--capacity-probe-interval-cycles", type=int, default=int(os.getenv("MODELHUB_CAPACITY_PROBE_INTERVAL_CYCLES", "3")), help=argparse.SUPPRESS, ) parser.add_argument("--daily-runs-dir", default=str(DEFAULT_DAILY_RUNS_DIR), help=argparse.SUPPRESS) parser.add_argument("--hf-base-url", default=os.getenv("MODELSCOPE_BASE_URL", "https://modelscope.cn"), help=argparse.SUPPRESS) parser.add_argument("--modelhub-base-url", default=os.getenv("MODELHUB_BASE_URL", "https://modelhub.org.cn"), help=argparse.SUPPRESS) parser.add_argument("--modelhub-token", default=os.getenv("MODELHUB_XC_TOKEN") or os.getenv("XC_TOKEN"), help=argparse.SUPPRESS) parser.add_argument("--hf-token", default=os.getenv("HF_TOKEN"), help=argparse.SUPPRESS) parser.add_argument("--modelscope-token", default=os.getenv("MODELSCOPE_API_TOKEN") or os.getenv("MODELSCOPE_TOKEN"), help=argparse.SUPPRESS) return parser def make_wave_namespace(base_args: argparse.Namespace, wave: WaveSpec) -> argparse.Namespace: user_gpu = getattr(base_args, "gpu", None) user_gpus = getattr(base_args, "gpus", None) return argparse.Namespace( gpu=user_gpu, gpus=user_gpus if (user_gpus or user_gpu) else wave.gpus, task_types=",".join(wave.task_types), limit=wave.limit, min_downloads=base_args.min_downloads, daily_target=base_args.daily_target, dry_run=base_args.dry_run, since_hours=wave.since_hours, updated_after=None, stats_window_days=7, history_stats_threshold=base_args.history_stats_threshold, read_concurrency=base_args.read_concurrency, max_scan_models=getattr(base_args, "max_scan_models", 0), scan_multiplier=getattr(base_args, "scan_multiplier", 4), skip_outcome_sync=getattr(base_args, "skip_outcome_sync", False), skip_history_archive=getattr(base_args, "skip_history_archive", False), disable_gpu_strategy=getattr(base_args, "disable_gpu_strategy", False), disable_market_intelligence=getattr(base_args, "disable_market_intelligence", False), gpu_strategy_refresh_submissions=getattr(base_args, "gpu_strategy_refresh_submissions", 200), gpu_strategy_state_path=getattr(base_args, "gpu_strategy_state_path", str(DEFAULT_GPU_STRATEGY_PATH)), gpu_strategy_recent_window=getattr(base_args, "gpu_strategy_recent_window", 1000), gpu_strategy_min_long_samples=getattr(base_args, "gpu_strategy_min_long_samples", 100), market_intelligence_state_path=getattr( base_args, "market_intelligence_state_path", str(DEFAULT_MARKET_INTELLIGENCE_PATH), ), market_queue_refresh_seconds=getattr( base_args, "market_queue_refresh_seconds", DEFAULT_QUEUE_REFRESH_SECONDS, ), market_framework_refresh_seconds=getattr( base_args, "market_framework_refresh_seconds", DEFAULT_FRAMEWORK_REFRESH_SECONDS, ), market_throughput_window_hours=getattr( base_args, "market_throughput_window_hours", DEFAULT_THROUGHPUT_WINDOW_HOURS, ), market_fetch_workers=getattr(base_args, "market_fetch_workers", DEFAULT_FETCH_WORKERS), market_framework_min_samples=getattr( base_args, "market_framework_min_samples", DEFAULT_FRAMEWORK_MIN_SAMPLES, ), capacity_state_path=getattr(base_args, "capacity_state_path", str(DEFAULT_CAPACITY_STATE_PATH)), capacity_probe_interval_cycles=getattr(base_args, "capacity_probe_interval_cycles", 3), submit_concurrency=getattr(base_args, "submit_concurrency", 1), max_submits_per_run=getattr(base_args, "max_submits_per_run", 0), recent_model_reserve_slots=getattr(base_args, "recent_model_reserve_slots", 10), recent_model_days=getattr(base_args, "recent_model_days", 7), disable_candidate_preflight=getattr(base_args, "disable_candidate_preflight", False), llm_classifier_endpoint=getattr(base_args, "llm_classifier_endpoint", None), llm_classifier_model=getattr(base_args, "llm_classifier_model", None), llm_classifier_api_key=getattr(base_args, "llm_classifier_api_key", None), llm_classifier_timeout_seconds=getattr(base_args, "llm_classifier_timeout_seconds", 20), llm_classifier_min_deny_confidence=getattr(base_args, "llm_classifier_min_deny_confidence", 0.85), llm_classifier_cache_path=getattr( base_args, "llm_classifier_cache_path", ".modelhub_state/llm_classifications.json", ), runs_dir=base_args.runs_dir, ledger_path=base_args.ledger_path, outcomes_path=getattr(base_args, "outcomes_path", "outcomes/submissions.jsonl"), claims_path=getattr(base_args, "claims_path", str(DEFAULT_CLAIMS_PATH)), submission_exclusions_path=getattr( base_args, "submission_exclusions_path", ".modelhub_state/submission_exclusions.jsonl", ), history_archive_path=base_args.history_archive_path, history_archive_limit=base_args.history_archive_limit, hf_base_url=base_args.hf_base_url, modelhub_base_url=base_args.modelhub_base_url, modelhub_token=base_args.modelhub_token, ) def run_daily_batches( *, base_args: argparse.Namespace, waves: tuple[WaveSpec, ...] = DEFAULT_WAVES, now=None, run_fn: Callable[..., dict[str, Any]] = run_submission, hf_discovery: HuggingFaceDiscovery | None = None, modelhub_client: ModelHubClient | ModelHubClientPool | None = None, template_selector: TemplateSelector | None = None, outcome_tracker: OutcomeTracker | None = None, ) -> dict[str, Any]: now = now or utc_now() hf_discovery = hf_discovery or HuggingFaceDiscovery(base_url=base_args.hf_base_url) if modelhub_client is None: modelhub_tokens = list(getattr(base_args, "modelhub_tokens", None) or ([] if not base_args.modelhub_token else [base_args.modelhub_token])) token_values: list[str | None] = modelhub_tokens or [base_args.modelhub_token] clients = [ModelHubClient(token=token, base_url=base_args.modelhub_base_url) for token in token_values] modelhub_client = ModelHubClientPool( clients, capacity_probe_interval_cycles=max(0, int(getattr(base_args, "capacity_probe_interval_cycles", 3) or 0)), capacity_state_path=Path(getattr(base_args, "capacity_state_path", DEFAULT_CAPACITY_STATE_PATH)), ) template_selector = template_selector or TemplateSelector() daily_run_dir = make_run_dir(Path(base_args.daily_runs_dir), now) log(f"[daily] daily_run_dir={daily_run_dir}") log( f"[daily] target={base_args.daily_target} rounds={base_args.rounds} " f"dry_run={str(bool(base_args.dry_run)).lower()} waves={len(waves)}" ) if hasattr(modelhub_client, "active_task_counts"): active_counts = modelhub_client.active_task_counts() log(f"[daily] modelhub_accounts={len(active_counts)} active_counts={','.join(str(count) for count in active_counts)}") wave_results: list[dict[str, Any]] = [] submitted_total = 0 attempted_waves = 0 stopped_reason = "max_rounds_reached" for round_index in range(1, base_args.rounds + 1): round_submitted = 0 log(f"[daily] round={round_index} start") for wave in waves: attempted_waves += 1 if ( hasattr(modelhub_client, "configure_capacity_probe") and not bool(getattr(base_args, "capacity_probe_already_configured", False)) ): modelhub_client.configure_capacity_probe(attempted_waves) wave_args = make_wave_namespace(base_args, wave) log( f"[daily] round={round_index} wave={wave.name} " f"tasks={','.join(wave.task_types)} gpus={wave.gpus or 'auto'} " f"limit={wave.limit} since_hours={wave.since_hours}" ) summary = run_fn( wave_args, now=utc_now(), hf_discovery=hf_discovery, modelhub_client=modelhub_client, template_selector=template_selector, outcome_tracker=outcome_tracker, ) wave_result = { "round": round_index, "wave": asdict(wave), "summary": summary, } wave_results.append(wave_result) submitted_total += summary["submittedCount"] round_submitted += summary["submittedCount"] skip_reasons = summary.get("skipReasonCounts") or {} skip_reason_text = ",".join( f"{reason}:{count}" for reason, count in list(skip_reasons.items())[:4] ) or "none" log( f"[daily] wave_done name={wave.name} " f"candidates={summary['candidateCount']} planned={summary['plannedSubmitCount']} " f"submitted={summary['submittedCount']} skipped={summary['skippedCount']} " f"duplicates={summary.get('duplicateCount', 0)} " f"uniqueness_rejected={summary.get('modelGpuUniquenessRejectedCount', 0)} " f"failed={summary['failedCount']} " f"skip_reasons={skip_reason_text} " f"remaining_before_run={summary['remainingDailyQuotaBeforeRun']}" ) if summary.get("platformAvailableSlotsBeforeRun") == 0: stopped_reason = "platform_async_cap_reached" log(f"[daily] stop={stopped_reason}") return finalize_daily_run( daily_run_dir=daily_run_dir, now=now, base_args=base_args, wave_results=wave_results, submitted_total=submitted_total, attempted_waves=attempted_waves, stopped_reason=stopped_reason, ) if base_args.daily_target > 0 and summary["remainingDailyQuotaBeforeRun"] <= 0: stopped_reason = "daily_target_already_reached" log(f"[daily] stop={stopped_reason}") return finalize_daily_run( daily_run_dir=daily_run_dir, now=now, base_args=base_args, wave_results=wave_results, submitted_total=submitted_total, attempted_waves=attempted_waves, stopped_reason=stopped_reason, ) if base_args.daily_target > 0 and not base_args.dry_run and summary["plannedSubmitCount"] <= 0: stopped_reason = "daily_target_reached" log(f"[daily] stop={stopped_reason}") return finalize_daily_run( daily_run_dir=daily_run_dir, now=now, base_args=base_args, wave_results=wave_results, submitted_total=submitted_total, attempted_waves=attempted_waves, stopped_reason=stopped_reason, ) if round_submitted <= 0: stopped_reason = "no_new_submissions_in_round" log(f"[daily] round={round_index} submitted=0 stop={stopped_reason}") break log(f"[daily] round={round_index} submitted={round_submitted}") return finalize_daily_run( daily_run_dir=daily_run_dir, now=now, base_args=base_args, wave_results=wave_results, submitted_total=submitted_total, attempted_waves=attempted_waves, stopped_reason=stopped_reason, ) def finalize_daily_run( *, daily_run_dir: Path, now, base_args: argparse.Namespace, wave_results: list[dict[str, Any]], submitted_total: int, attempted_waves: int, stopped_reason: str, ) -> dict[str, Any]: last_wave_summary = wave_results[-1]["summary"] if wave_results else {} duplicate_total = sum( int(wave_result.get("summary", {}).get("duplicateCount", 0) or 0) for wave_result in wave_results ) failed_total = sum( int(wave_result.get("summary", {}).get("failedCount", 0) or 0) for wave_result in wave_results ) summary = { "generatedAt": now.isoformat(), "dryRun": bool(base_args.dry_run), "dailyTarget": base_args.daily_target, "unlimitedDailyTarget": base_args.daily_target <= 0, "rounds": base_args.rounds, "attemptedWaves": attempted_waves, "submittedTotal": submitted_total, "duplicateTotal": duplicate_total, "failedTotal": failed_total, "stoppedReason": stopped_reason, "dailyRunDir": str(daily_run_dir), "remainingDailyQuotaBeforeRun": last_wave_summary.get("remainingDailyQuotaBeforeRun"), "platformAvailableSlotsBeforeRun": last_wave_summary.get("platformAvailableSlotsBeforeRun"), "waveResults": wave_results, } write_json(daily_run_dir / "summary.json", summary) log( f"[daily] finished submitted_total={submitted_total} attempted_waves={attempted_waves} " f"stopped_reason={stopped_reason}" ) return summary def main(argv: list[str] | None = None) -> int: parser = build_parser() args = parser.parse_args(argv) ensure_tokens(args) log( f"[daily] modelscope_token={'set' if bool(args.modelscope_token) else 'missing'} " f"xc_token={'set' if bool(args.modelhub_token) else 'missing'} " f"xc_tokens={len(getattr(args, 'modelhub_tokens', []) or [])}" ) summary = run_daily_batches(base_args=args) print(f"daily_run_dir={summary['dailyRunDir']}") print(f"submitted_total={summary['submittedTotal']}") print(f"attempted_waves={summary['attemptedWaves']}") print(f"stopped_reason={summary['stoppedReason']}") return 0 if __name__ == "__main__": raise SystemExit(main())