from __future__ import annotations import argparse import json import os import sys import time from collections import deque from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import timedelta from pathlib import Path from typing import Any, Callable from common import read_jsonl, utc_now, write_json from daily_runner import DEFAULT_DAILY_RUNS_DIR, log, run_daily_batches from gpu_strategy import DEFAULT_GPU_STRATEGY_PATH from hf_discovery import HuggingFaceDiscovery from main import DEFAULT_LEDGER_PATH, DEFAULT_RUNS_DIR, make_run_dir from market_intelligence import ( DEFAULT_FETCH_WORKERS, DEFAULT_FRAMEWORK_MIN_SAMPLES, DEFAULT_FRAMEWORK_REFRESH_SECONDS, DEFAULT_MARKET_INTELLIGENCE_PATH, DEFAULT_QUEUE_REFRESH_SECONDS, DEFAULT_THROUGHPUT_WINDOW_HOURS, ) from modelhub_client import DEFAULT_CAPACITY_STATE_PATH, ModelHubClient, ModelHubClientPool from outcome_tracker import DEFAULT_OUTCOMES_PATH, OutcomeTracker from official_capabilities import DEFAULT_OFFICIAL_CAPABILITIES_PATH from queue_cleanup import cleanup_certain_oom_tasks from routing_engine import DEFAULT_ROUTING_STATE_PATH from runner_common import DEFAULT_KEY_PATH, ensure_tokens from state_sync import ( DEFAULT_BATCH_SIZE, DEFAULT_BRANCH, DEFAULT_REMOTE, StateGitSync, load_state_git_credentials, ) from submission_claims import DEFAULT_CLAIMS_PATH from template_selector import TemplateSelector from version import AGENT_VERSION DEFAULT_POLL_RUNS_DIR = Path("poll_runs") DEFAULT_ARCHITECTURE_BLACKLIST_PATH = Path( ".modelhub_state/architecture_compatibility_blacklist.json" ) def _env_int(name: str, default: int, *, minimum: int = 0, maximum: int = 100_000) -> int: try: value = int(os.getenv(name, str(default))) except ValueError: value = default return min(maximum, max(minimum, value)) def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Continuously poll ModelHub queue slots and refill submissions.") parser.add_argument("--daily-target", type=int, default=0, help="Total submissions to aim for per UTC day; 0 means unlimited") parser.add_argument("--gpu", help="Single GPU alias or platform name, for example: k100") parser.add_argument("--gpus", help="Comma-separated GPU aliases/platform names. Omit to auto-use all safe GPUs.") parser.add_argument("--min-downloads", type=int, default=50, help="Minimum ModelScope download threshold") parser.add_argument( "--history-stats-threshold", type=int, default=500, help="Minimum local ledger records before using history stats to rank submissions", ) parser.add_argument("--read-concurrency", type=int, default=4, help="Concurrency for read-only remote calls") parser.add_argument( "--submit-concurrency", type=int, default=0, help="Concurrency for task submission calls (0 = auto based on token/client count)", ) parser.add_argument( "--max-submits-per-run", type=int, default=0, help="Maximum tasks to submit in one cycle (0 means unlimited)", ) parser.add_argument( "--recent-model-reserve-slots", type=int, default=int(os.getenv("MODELHUB_RECENT_MODEL_RESERVE_SLOTS", "5")), help=argparse.SUPPRESS, ) parser.add_argument( "--recent-model-days", type=int, default=int(os.getenv("MODELHUB_RECENT_MODEL_DAYS", "7")), help=argparse.SUPPRESS, ) parser.add_argument("--disable-candidate-preflight", action="store_true", help=argparse.SUPPRESS) parser.add_argument("--llm-classifier-endpoint", default=os.getenv("MODELHUB_LLM_CLASSIFIER_ENDPOINT"), help=argparse.SUPPRESS) parser.add_argument("--llm-classifier-model", default=os.getenv("MODELHUB_LLM_CLASSIFIER_MODEL"), help=argparse.SUPPRESS) parser.add_argument("--llm-classifier-api-key", default=os.getenv("MODELHUB_LLM_CLASSIFIER_API_KEY"), help=argparse.SUPPRESS) parser.add_argument("--llm-classifier-timeout-seconds", type=int, default=int(os.getenv("MODELHUB_LLM_CLASSIFIER_TIMEOUT_SECONDS", "20")), help=argparse.SUPPRESS) parser.add_argument("--llm-classifier-min-deny-confidence", type=float, default=float(os.getenv("MODELHUB_LLM_CLASSIFIER_MIN_DENY_CONFIDENCE", "0.85")), help=argparse.SUPPRESS) parser.add_argument("--llm-classifier-cache-path", default=os.getenv("MODELHUB_LLM_CLASSIFIER_CACHE_PATH", ".modelhub_state/llm_classifications.json"), help=argparse.SUPPRESS) parser.add_argument("--max-scan-models", type=int, default=0, help="Hard cap on total scanned models (0 means auto)") parser.add_argument( "--scan-multiplier", type=int, default=4, help="Multiplier used when auto-deriving scan limit from quota/queue capacity", ) parser.add_argument("--skip-outcome-sync", action="store_true", help="Skip outcome sync from ModelHub before scanning") parser.add_argument("--skip-history-archive", action="store_true", help="Skip historical task archive download for this run") parser.add_argument("--dry-run", action="store_true", help="Plan the day without creating tasks") parser.add_argument("--disable-gpu-strategy", action="store_true", help="Disable adaptive 70/30 proven-GPU scheduling") parser.add_argument( "--disable-market-intelligence", action="store_true", help="Disable live queue/throughput and public framework statistics", ) parser.add_argument( "--gpu-strategy-refresh-submissions", type=int, default=200, help="Recalculate adaptive GPU choices after this many accepted submissions", ) parser.add_argument("--key-path", default=str(DEFAULT_KEY_PATH), help="Path to KEY.md containing MODELSCOPE_TOKEN/XC_TOKEN") parser.add_argument("--runs-dir", default=str(DEFAULT_RUNS_DIR), help=argparse.SUPPRESS) parser.add_argument("--ledger-path", default=str(DEFAULT_LEDGER_PATH), help=argparse.SUPPRESS) parser.add_argument( "--claims-path", default=os.getenv("MODELHUB_AGENT_CLAIMS_PATH", str(DEFAULT_CLAIMS_PATH)), help=argparse.SUPPRESS, ) parser.add_argument( "--submission-exclusions-path", default=os.getenv("MODELHUB_SUBMISSION_EXCLUSIONS_PATH", ".modelhub_state/submission_exclusions.jsonl"), help=argparse.SUPPRESS, ) parser.add_argument("--history-archive-path", default="history/platform_tasks.jsonl", help=argparse.SUPPRESS) parser.add_argument("--history-archive-limit", type=int, default=5000, help=argparse.SUPPRESS) parser.add_argument( "--gpu-strategy-state-path", default=os.getenv("MODELHUB_GPU_STRATEGY_STATE_PATH", str(DEFAULT_GPU_STRATEGY_PATH)), help=argparse.SUPPRESS, ) parser.add_argument( "--routing-state-path", default=os.getenv("MODELHUB_ROUTING_STATE_PATH", str(DEFAULT_ROUTING_STATE_PATH)), help=argparse.SUPPRESS, ) parser.add_argument( "--official-capabilities-path", default=os.getenv("MODELHUB_OFFICIAL_CAPABILITIES_PATH", str(DEFAULT_OFFICIAL_CAPABILITIES_PATH)), help=argparse.SUPPRESS, ) parser.add_argument("--state-sync", action="store_true", help=argparse.SUPPRESS) parser.add_argument( "--state-sync-remote", default=os.getenv("MODELHUB_STATE_SYNC_REMOTE", DEFAULT_REMOTE), help=argparse.SUPPRESS, ) parser.add_argument( "--state-sync-branch", default=os.getenv("MODELHUB_STATE_SYNC_BRANCH", DEFAULT_BRANCH), help=argparse.SUPPRESS, ) parser.add_argument( "--state-sync-batch-size", type=int, default=int(os.getenv("MODELHUB_STATE_SYNC_BATCH_SIZE", str(DEFAULT_BATCH_SIZE))), help=argparse.SUPPRESS, ) parser.add_argument("--gpu-strategy-recent-window", type=int, default=1000, help=argparse.SUPPRESS) parser.add_argument("--gpu-strategy-min-long-samples", type=int, default=100, help=argparse.SUPPRESS) parser.add_argument( "--market-intelligence-state-path", default=os.getenv("MODELHUB_MARKET_INTELLIGENCE_PATH", str(DEFAULT_MARKET_INTELLIGENCE_PATH)), help=argparse.SUPPRESS, ) parser.add_argument( "--market-queue-refresh-seconds", type=int, default=int(os.getenv("MODELHUB_MARKET_QUEUE_REFRESH_SECONDS", str(DEFAULT_QUEUE_REFRESH_SECONDS))), help=argparse.SUPPRESS, ) parser.add_argument( "--market-framework-refresh-seconds", type=int, default=int(os.getenv("MODELHUB_MARKET_FRAMEWORK_REFRESH_SECONDS", str(DEFAULT_FRAMEWORK_REFRESH_SECONDS))), help=argparse.SUPPRESS, ) parser.add_argument( "--market-throughput-window-hours", type=int, default=int(os.getenv("MODELHUB_MARKET_THROUGHPUT_WINDOW_HOURS", str(DEFAULT_THROUGHPUT_WINDOW_HOURS))), help=argparse.SUPPRESS, ) parser.add_argument( "--market-fetch-workers", type=int, default=int(os.getenv("MODELHUB_MARKET_FETCH_WORKERS", str(DEFAULT_FETCH_WORKERS))), help=argparse.SUPPRESS, ) parser.add_argument( "--market-framework-min-samples", type=int, default=int(os.getenv("MODELHUB_MARKET_FRAMEWORK_MIN_SAMPLES", str(DEFAULT_FRAMEWORK_MIN_SAMPLES))), help=argparse.SUPPRESS, ) parser.add_argument( "--capacity-state-path", default=os.getenv("MODELHUB_CAPACITY_STATE_PATH", str(DEFAULT_CAPACITY_STATE_PATH)), help=argparse.SUPPRESS, ) parser.add_argument( "--capacity-probe-interval-cycles", type=int, default=int(os.getenv("MODELHUB_CAPACITY_PROBE_INTERVAL_CYCLES", "3")), help=argparse.SUPPRESS, ) parser.add_argument("--daily-runs-dir", default=str(DEFAULT_DAILY_RUNS_DIR), help=argparse.SUPPRESS) parser.add_argument("--poll-runs-dir", default=str(DEFAULT_POLL_RUNS_DIR), help=argparse.SUPPRESS) parser.add_argument("--outcomes-path", default=str(DEFAULT_OUTCOMES_PATH), help=argparse.SUPPRESS) parser.add_argument("--hf-base-url", default="https://modelscope.cn", help=argparse.SUPPRESS) parser.add_argument("--modelhub-base-url", default="https://modelhub.org.cn", help=argparse.SUPPRESS) parser.add_argument("--modelhub-token", default=None, help=argparse.SUPPRESS) parser.add_argument("--hf-token", default=None, help=argparse.SUPPRESS) parser.add_argument("--modelscope-token", default=None, help=argparse.SUPPRESS) parser.add_argument("--poll-interval-seconds", type=int, default=15, help="Sleep between polling cycles when no slots are available") parser.add_argument("--idle-interval-seconds", type=int, default=60, help="Sleep between cycles when a scan submits nothing") parser.add_argument("--post-cycle-cooldown-seconds", type=int, default=2, help="Short sleep after a successful cycle") parser.add_argument("--max-cycles", type=int, default=0, help="Optional hard stop after N cycles; 0 means run until quota is reached") parser.add_argument("--print-stats", action="store_true", help="Load outcomes, sync, print stats report, and exit") parser.add_argument("--disable-queue-cleanup", action="store_true", help=argparse.SUPPRESS) parser.add_argument( "--queue-cleanup-interval-cycles", type=int, default=int(os.getenv("MODELHUB_QUEUE_CLEANUP_INTERVAL_CYCLES", "120")), help=argparse.SUPPRESS, ) parser.add_argument( "--queue-cleanup-read-concurrency", type=int, default=int(os.getenv("MODELHUB_QUEUE_CLEANUP_READ_CONCURRENCY", "6")), help=argparse.SUPPRESS, ) parser.add_argument( "--queue-cleanup-report-path", default=os.getenv("MODELHUB_QUEUE_CLEANUP_REPORT_PATH", ".modelhub_state/queue_cleanup_latest.json"), help=argparse.SUPPRESS, ) parser.add_argument( "--architecture-blacklist-path", default=os.getenv( "MODELHUB_ARCHITECTURE_BLACKLIST_PATH", str(DEFAULT_ARCHITECTURE_BLACKLIST_PATH), ), help=argparse.SUPPRESS, ) return parser def _make_cycle_args(base_args: argparse.Namespace) -> argparse.Namespace: cycle_args = argparse.Namespace(**vars(base_args)) cycle_args.rounds = 1 # Poll runner handles outcome sync on its own schedule (every OUTCOME_SYNC_INTERVAL cycles). # Skip the per-cycle sync inside run_submission to avoid redundant paginated API calls. cycle_args.skip_outcome_sync = True return cycle_args def _build_modelhub_client(base_args: argparse.Namespace) -> ModelHubClientPool: modelhub_tokens = list(getattr(base_args, "modelhub_tokens", None) or ([] if not base_args.modelhub_token else [base_args.modelhub_token])) token_values: list[str | None] = modelhub_tokens or [base_args.modelhub_token] clients = [ModelHubClient(token=token, base_url=base_args.modelhub_base_url) for token in token_values] return ModelHubClientPool( clients, capacity_probe_interval_cycles=max(0, int(getattr(base_args, "capacity_probe_interval_cycles", 3) or 0)), capacity_state_path=Path(getattr(base_args, "capacity_state_path", DEFAULT_CAPACITY_STATE_PATH)), recent_model_reserve_slots=max(0, int(getattr(base_args, "recent_model_reserve_slots", 5) or 0)), recent_model_days=max(1, int(getattr(base_args, "recent_model_days", 7) or 7)), ) def _persist_architecture_blacklist( report: dict[str, Any], *, path: Path, ) -> set[str]: blocks = report.get("architectureCompatibilityBlocks") or {} blocks = blocks if isinstance(blocks, dict) else {} write_json( path, { "generatedAt": report.get("generatedAt"), "summary": report.get("architectureCompatibilitySummary") or {}, "blocks": blocks, }, ) return set(str(key) for key in blocks) def _load_task_compatibility_contexts( outcome_tracker: OutcomeTracker, *, ledger_path: Path, ) -> dict[str, dict[str, Any]]: contexts = outcome_tracker.get_task_compatibility_contexts() for record in read_jsonl(ledger_path): task_id_value = record.get("taskId") if task_id_value is None: continue task_id = str(task_id_value) existing = contexts.get(task_id) if existing is None: contexts[task_id] = { "taskId": task_id, "modelId": str(record.get("modelId") or ""), "targetGpu": str(record.get("targetGpu") or ""), "framework": str(record.get("framework") or ""), "taskType": str(record.get("taskType") or ""), "modelProfile": {}, "submitTime": record.get("submitTime"), } continue for field in ("modelId", "targetGpu", "framework", "taskType", "submitTime"): if not existing.get(field) and record.get(field): existing[field] = record.get(field) return contexts def _load_complete_owned_history( modelhub_client: ModelHubClient | ModelHubClientPool, ) -> tuple[list[dict[str, Any]], list[int]]: clients = ( list(modelhub_client.clients) if isinstance(modelhub_client, ModelHubClientPool) else [modelhub_client] ) by_account: dict[int, list[dict[str, Any]]] = {} errors: list[int] = [] with ThreadPoolExecutor(max_workers=min(12, max(1, len(clients)))) as executor: futures = { executor.submit(client.list_tasks, page_size=100, only_mine=True): index for index, client in enumerate(clients, start=1) } for future in as_completed(futures): account_index = futures[future] try: by_account[account_index] = future.result() except Exception: errors.append(account_index) deduped: dict[str, dict[str, Any]] = {} anonymous: list[dict[str, Any]] = [] for account_index in sorted(by_account): for task in by_account[account_index]: if not isinstance(task, dict): continue task_id = task.get("taskId") if task_id is None: anonymous.append(task) continue deduped[str(task_id)] = task return [*deduped.values(), *anonymous], sorted(errors) def _bootstrap_architecture_history( *, modelhub_client: ModelHubClient | ModelHubClientPool, outcome_tracker: OutcomeTracker, ledger_path: Path, now, ) -> dict[str, Any]: """Prefer public failure evidence, then fall back to all owned history.""" probe_size = _env_int( "MODELHUB_ARCHITECTURE_COMMUNITY_PROBE_SIZE", 50, minimum=1, maximum=100, ) community_records: list[dict[str, Any]] = [] community_probe_error: str | None = None try: probe_payload = modelhub_client.list_tasks_page( current=1, page_size=probe_size, only_mine=False, status="success", verify_result=-1, ) probe_records = (probe_payload.get("data") or {}).get("records") or [] community_records = [record for record in probe_records if isinstance(record, dict)] except Exception as exc: community_probe_error = f"{type(exc).__name__}: {exc}" community_usable = [ record for record in community_records if record.get("logCosUrl") and record.get("modelId") and record.get("gpuType") ] log( f"[architecture-bootstrap] community_probe={len(community_records)} " f"usable_failure_details={len(community_usable)} " f"error={'none' if community_probe_error is None else community_probe_error}" ) source = "community_latest" listing_errors: list[int] = [] if community_usable: lookback_days = _env_int( "MODELHUB_ARCHITECTURE_COMMUNITY_LOOKBACK_DAYS", 30, minimum=1, maximum=365, ) latest_limit = _env_int( "MODELHUB_ARCHITECTURE_COMMUNITY_LATEST_LIMIT", 5000, minimum=1, maximum=50_000, ) try: history_tasks = modelhub_client.list_tasks( page_size=100, only_mine=False, begin_time=now - timedelta(days=lookback_days), end_time=now, status="success", verify_result=-1, max_records=latest_limit, ) history_tasks = [task for task in history_tasks if task.get("logCosUrl")] log( f"[architecture-bootstrap] source=community_latest " f"lookback_days={lookback_days} records={len(history_tasks)} limit={latest_limit}" ) except Exception as exc: source = "owned_full_history" log( f"[architecture-bootstrap] community_history_error={type(exc).__name__}: {exc} " "fallback=owned_full_history" ) history_tasks, listing_errors = _load_complete_owned_history(modelhub_client) else: source = "owned_full_history" history_tasks, listing_errors = _load_complete_owned_history(modelhub_client) account_count = ( len(modelhub_client.clients) if isinstance(modelhub_client, ModelHubClientPool) else 1 ) log( f"[architecture-bootstrap] source=owned_full_history records={len(history_tasks)} " f"accounts={account_count} listing_errors={','.join(map(str, listing_errors)) or 'none'}" ) contexts = _load_task_compatibility_contexts( outcome_tracker, ledger_path=ledger_path, ) summary = outcome_tracker.bootstrap_from_history_tasks( history_tasks, task_contexts=contexts, enrichment_limit=_env_int( "MODELHUB_ARCHITECTURE_BOOTSTRAP_MAX_LOGS", 0, minimum=0, maximum=100_000, ), enrichment_workers=_env_int( "MODELHUB_ARCHITECTURE_BOOTSTRAP_WORKERS", 8, minimum=1, maximum=16, ), log=log, ) feedback = outcome_tracker.get_stats_report() block_count = len(feedback.get("architectureCompatibilityBlocks") or {}) summary.update( { "source": source, "communityProbeRecords": len(community_records), "communityUsableFailureDetails": len(community_usable), "listingErrorAccounts": listing_errors, "architectureBlocks": block_count, } ) log( f"[architecture-bootstrap] finished source={source} " f"terminal={summary['terminalRecords']} failure_logs={summary['eligibleFailureLogs']} " f"explicit_architecture_failures={summary['explicitArchitectureFailures']} " f"recovered_frameworks={summary['recoveredFrameworks']} blocks={block_count}" ) return summary def run_poll_loop( *, base_args: argparse.Namespace, now=None, run_fn: Callable[..., dict[str, Any]] = run_daily_batches, hf_discovery: HuggingFaceDiscovery | None = None, modelhub_client: ModelHubClient | ModelHubClientPool | None = None, template_selector: TemplateSelector | None = None, outcome_tracker: OutcomeTracker | None = None, ) -> dict[str, Any]: now = now or utc_now() state_sync: StateGitSync | None = getattr(base_args, "_state_sync_manager", None) if state_sync is None and bool(getattr(base_args, "state_sync", False)): state_sync = StateGitSync( project_root=Path(__file__).resolve().parent.parent, credentials=load_state_git_credentials(), remote=str(getattr(base_args, "state_sync_remote", DEFAULT_REMOTE)), branch=str(getattr(base_args, "state_sync_branch", DEFAULT_BRANCH)), batch_size=max(1, int(getattr(base_args, "state_sync_batch_size", DEFAULT_BATCH_SIZE) or DEFAULT_BATCH_SIZE)), log_fn=log, ) try: state_sync.acquire_process_lock() except Exception as exc: state_sync.last_error = str(exc) state_sync.healthy = False else: state_sync.restore() base_args._state_sync_manager = state_sync if state_sync is not None: state_sync.write_readiness( ready=False, reason="startup_recovery" if state_sync.healthy else "state_sync_unhealthy", ) hf_discovery = hf_discovery or HuggingFaceDiscovery(base_url=base_args.hf_base_url) modelhub_client = modelhub_client or _build_modelhub_client(base_args) template_selector = template_selector or TemplateSelector() if state_sync is not None and state_sync.healthy: try: active_tasks = ( modelhub_client.list_active_tasks_by_account() if hasattr(modelhub_client, "list_active_tasks_by_account") else [] ) recovery = state_sync.reconcile_active_tasks(active_tasks) if not state_sync.sync("startup"): log("[cycle] paused reason=state_sync_unhealthy") else: log( f"[state-recovery] active={recovery['active']} " f"reconciled={recovery['reconciled']} unresolved={recovery['unresolved']}" ) except Exception as exc: state_sync.healthy = False state_sync.last_error = str(exc) log(f"[state-recovery] active_scan_failed reason={type(exc).__name__}: {exc}") poll_runs_dir = Path(base_args.poll_runs_dir) poll_runs_dir.mkdir(parents=True, exist_ok=True) poll_run_dir = make_run_dir(poll_runs_dir, now) outcome_tracker = outcome_tracker or OutcomeTracker(Path(base_args.outcomes_path)) recovered_contexts = _load_task_compatibility_contexts( outcome_tracker, ledger_path=Path(base_args.ledger_path), ) repaired_contexts = outcome_tracker.merge_task_contexts(recovered_contexts) if repaired_contexts: log(f"[outcome-recovery] metadata_repaired={repaired_contexts}") OUTCOME_SYNC_INTERVAL = 3 STATS_PRINT_INTERVAL = 10 log(f"[poll] version={AGENT_VERSION} poll_run_dir={poll_run_dir}") log( f"[poll] target={base_args.daily_target} dry_run={str(bool(base_args.dry_run)).lower()} " f"poll_interval={base_args.poll_interval_seconds}s idle_interval={base_args.idle_interval_seconds}s" ) architecture_bootstrap_summary: dict[str, Any] = {"enabled": False} if not getattr(base_args, "skip_outcome_sync", False): try: if outcome_tracker.has_durable_checkpoint: cached_feedback = outcome_tracker.get_stats_report() architecture_bootstrap_summary = { "source": "durable_checkpoint", "terminalRecords": int(cached_feedback.get("terminalRecords") or 0), "architectureBlocks": len(cached_feedback.get("architectureCompatibilityBlocks") or {}), "fullHistoryScanSkipped": True, } log( "[architecture-bootstrap] source=durable_checkpoint " f"terminal={architecture_bootstrap_summary['terminalRecords']} " f"blocks={architecture_bootstrap_summary['architectureBlocks']} " "full_history_scan=skipped" ) else: architecture_bootstrap_summary = _bootstrap_architecture_history( modelhub_client=modelhub_client, outcome_tracker=outcome_tracker, ledger_path=Path(base_args.ledger_path), now=now, ) architecture_bootstrap_summary["enabled"] = True _persist_architecture_blacklist( outcome_tracker.get_stats_report(), path=Path( getattr( base_args, "architecture_blacklist_path", DEFAULT_ARCHITECTURE_BLACKLIST_PATH, ) ), ) except Exception as exc: architecture_bootstrap_summary = { "enabled": True, "error": f"{type(exc).__name__}: {exc}", } log( f"[architecture-bootstrap] error={type(exc).__name__}: {exc} " "continue_polling=true" ) else: architecture_bootstrap_summary["reason"] = "outcome_sync_disabled" retained_cycles = max(10, int(os.getenv("MODELHUB_AGENT_RETAINED_CYCLE_SUMMARIES", "50"))) retained_cleanups = max(5, int(os.getenv("MODELHUB_AGENT_RETAINED_CLEANUP_SUMMARIES", "20"))) cycle_summaries: deque[dict[str, Any]] = deque(maxlen=retained_cycles) queue_cleanup_runs: deque[dict[str, Any]] = deque(maxlen=retained_cleanups) submitted_total = 0 cycles = 0 stopped_reason = "max_cycles_reached" pending_architecture_cleanup = False last_cleaned_architecture_blocks: set[str] = set() while True: if base_args.max_cycles and cycles >= base_args.max_cycles: stopped_reason = "max_cycles_reached" break cycles += 1 if state_sync is not None and not state_sync.healthy: recovered = ( state_sync.sync("retry") if state_sync._workspace is not None else state_sync.retry_restore() ) if not recovered: state_sync.write_readiness(ready=False, reason="state_sync_unhealthy") log(f"[poll] cycle={cycles} sleep={base_args.idle_interval_seconds}s reason=state_sync_unhealthy") time.sleep(base_args.idle_interval_seconds) continue if hasattr(modelhub_client, "configure_capacity_probe"): modelhub_client.configure_capacity_probe(cycles) outcome_synced_this_cycle = False if ( cycles % OUTCOME_SYNC_INTERVAL == 0 and not getattr(base_args, "skip_outcome_sync", False) ): try: synced = outcome_tracker.sync_from_api( modelhub_client, task_contexts=_load_task_compatibility_contexts( outcome_tracker, ledger_path=Path(base_args.ledger_path), ), ) outcome_synced_this_cycle = True if synced > 0: log(f"[poll] cycle={cycles} outcome_sync_updated={synced}") sync_feedback = outcome_tracker.get_stats_report() active_block_keys = _persist_architecture_blacklist( sync_feedback, path=Path( getattr( base_args, "architecture_blacklist_path", DEFAULT_ARCHITECTURE_BLACKLIST_PATH, ) ), ) new_block_keys = active_block_keys - last_cleaned_architecture_blocks if ( new_block_keys and not bool(getattr(base_args, "disable_queue_cleanup", False)) and isinstance(modelhub_client, ModelHubClientPool) ): pending_architecture_cleanup = True log( f"[queue-cleanup] dynamic_architecture_blocks_added={len(new_block_keys)} " f"cleanup_next=immediate" ) except Exception as exc: log(f"[poll] cycle={cycles} outcome_sync_error={exc}") cleanup_interval = max(0, int(getattr(base_args, "queue_cleanup_interval_cycles", 120) or 0)) scheduled_queue_cleanup = bool( cycles == 1 or (cleanup_interval > 0 and cycles % cleanup_interval == 0) ) architecture_only_cleanup = bool( pending_architecture_cleanup and not scheduled_queue_cleanup ) should_cleanup_queue = ( not bool(getattr(base_args, "disable_queue_cleanup", False)) and isinstance(modelhub_client, ModelHubClientPool) and ( scheduled_queue_cleanup or pending_architecture_cleanup ) ) if should_cleanup_queue: try: if ( not outcome_synced_this_cycle and not getattr(base_args, "skip_outcome_sync", False) ): synced_before_cleanup = outcome_tracker.sync_from_api( modelhub_client, task_contexts=_load_task_compatibility_contexts( outcome_tracker, ledger_path=Path(base_args.ledger_path), ), ) if synced_before_cleanup: log( f"[queue-cleanup] outcome_sync_updated={synced_before_cleanup}" ) cleanup_feedback = outcome_tracker.get_stats_report() cleanup_gpu_memory = cleanup_feedback.get("observedGpuMemoryGiB") or {} cleanup_architecture_blocks = ( cleanup_feedback.get("architectureCompatibilityBlocks") or {} ) active_architecture_block_keys = _persist_architecture_blacklist( cleanup_feedback, path=Path( getattr( base_args, "architecture_blacklist_path", DEFAULT_ARCHITECTURE_BLACKLIST_PATH, ) ), ) if active_architecture_block_keys - last_cleaned_architecture_blocks: pending_architecture_cleanup = True log( "[queue-cleanup] age_cleanup=disabled_admission_only " f"deterministic_cleanup={'architecture_only' if architecture_only_cleanup else 'full'}" ) cleanup_summary = cleanup_certain_oom_tasks( modelhub_client, hf_discovery, dry_run=bool(base_args.dry_run), read_concurrency=max(1, int(getattr(base_args, "queue_cleanup_read_concurrency", 6) or 6)), gpu_memory_gib=cleanup_gpu_memory if isinstance(cleanup_gpu_memory, dict) else None, architecture_compatibility_blocks=( cleanup_architecture_blocks if isinstance(cleanup_architecture_blocks, dict) else None ), task_compatibility_contexts=( _load_task_compatibility_contexts( outcome_tracker, ledger_path=Path(base_args.ledger_path), ) ), architecture_only=architecture_only_cleanup, log=log, ) policy_cancelled_recorded = outcome_tracker.mark_policy_cancellations( cleanup_summary["cancelledTasks"] ) if policy_cancelled_recorded: outcome_tracker.save() cleanup_summary["policyCancelledRecorded"] = policy_cancelled_recorded write_json( Path( getattr( base_args, "queue_cleanup_report_path", ".modelhub_state/queue_cleanup_latest.json", ) ), cleanup_summary, ) queue_cleanup_runs.append( { "cycle": cycles, "mode": ( "architecture_dynamic" if architecture_only_cleanup else "deterministic_full" ), "ageCleanupMode": "admission_only", "ageReservedSlots": max( 0, int(getattr(base_args, "recent_model_reserve_slots", 5) or 0), ), "ageQueueThresholds": cleanup_summary["oldModelQueueThresholds"], "activeScanned": cleanup_summary["activeScanned"], "certainOomCount": cleanup_summary["certainOomCount"], "architectureBlockCount": cleanup_summary[ "architectureBlockCount" ], "architectureIncompatibleCount": cleanup_summary[ "architectureIncompatibleCount" ], "officialCapabilityInvalidCount": cleanup_summary.get( "officialCapabilityInvalidCount", 0, ), "oldOverflowCount": cleanup_summary["oldOverflowCount"], "cancelledCount": cleanup_summary["cancelledCount"], "policyCancelledRecorded": policy_cancelled_recorded, "stopErrorCount": len(cleanup_summary["stopErrors"]), } ) pending_architecture_cleanup = False last_cleaned_architecture_blocks = active_architecture_block_keys except Exception as exc: log(f"[queue-cleanup] error={type(exc).__name__}: {exc} continue_polling=true") active_counts = modelhub_client.active_task_counts() if hasattr(modelhub_client, "active_task_counts") else [] capacity_limits = modelhub_client.account_capacity_limits() if hasattr(modelhub_client, "account_capacity_limits") else [] capacity_probe = modelhub_client.capacity_probe_enabled() if hasattr(modelhub_client, "capacity_probe_enabled") else False available_slots = modelhub_client.available_submit_slots() if hasattr(modelhub_client, "available_submit_slots") else None log( f"[poll] cycle={cycles} active_counts={','.join(str(count) for count in active_counts) if active_counts else 'n/a'} " f"capacity_limits={','.join(str(limit) for limit in capacity_limits) if capacity_limits else 'n/a'} " f"capacity_probe={'on' if capacity_probe else 'off'} " f"available_slots={available_slots if available_slots is not None else 'n/a'}" ) if available_slots is not None and available_slots <= 0: if state_sync is not None: try: if cycles % 3 == 0 and hasattr(modelhub_client, "list_active_tasks_by_account"): state_sync.reconcile_active_tasks(modelhub_client.list_active_tasks_by_account()) sync_ok = state_sync.sync("cycle_no_slots") state_sync.write_readiness( ready=sync_ok, reason=None if sync_ok else "state_sync_unhealthy", extra={"cycle": cycles}, ) except Exception as exc: state_sync.healthy = False state_sync.last_error = str(exc) state_sync.write_readiness(ready=False, reason="state_sync_unhealthy") log(f"[poll] cycle={cycles} sleep={base_args.poll_interval_seconds}s reason=no_available_slots") time.sleep(base_args.poll_interval_seconds) continue cycle_args = _make_cycle_args(base_args) cycle_args.capacity_probe_already_configured = True cycle_summary = run_fn( base_args=cycle_args, now=utc_now(), hf_discovery=hf_discovery, modelhub_client=modelhub_client, template_selector=template_selector, outcome_tracker=outcome_tracker, ) cycle_summaries.append(cycle_summary) submitted_total += cycle_summary["submittedTotal"] remaining_before_run = cycle_summary.get("remainingDailyQuotaBeforeRun") log( f"[poll] cycle_done submitted_total={cycle_summary['submittedTotal']} " f"duplicates={cycle_summary.get('duplicateTotal', 0)} " f"remaining_before_run={remaining_before_run if remaining_before_run is not None else 'n/a'} " f"stop={cycle_summary['stoppedReason']}" ) if state_sync is not None: try: if cycle_summary.get("submittedTotal", 0) > 0 or cycles % 3 == 0: active_tasks = ( modelhub_client.list_active_tasks_by_account() if hasattr(modelhub_client, "list_active_tasks_by_account") else [] ) state_sync.reconcile_active_tasks(active_tasks) sync_ok = state_sync.sync("cycle") official_paused = any( bool((wave_result.get("summary") or {}).get("paused")) for wave_result in (cycle_summary.get("waveResults") or []) ) state_sync.write_readiness( ready=sync_ok and not official_paused, reason=( "critical_official_signal_unavailable" if official_paused else (None if sync_ok else "state_sync_unhealthy") ), extra={"cycle": cycles}, ) except Exception as exc: state_sync.healthy = False state_sync.last_error = str(exc) state_sync.write_readiness(ready=False, reason="state_sync_unhealthy") if base_args.daily_target > 0 and remaining_before_run is not None and remaining_before_run <= 0: stopped_reason = "daily_target_already_reached" break if cycle_summary["submittedTotal"] <= 0: duplicate_total = int(cycle_summary.get("duplicateTotal", 0) or 0) if duplicate_total > 0 and (available_slots is None or available_slots > 0): retry_delay = max(1, int(getattr(base_args, "post_cycle_cooldown_seconds", 2) or 2)) log(f"[poll] cycle={cycles} sleep={retry_delay}s reason=duplicates_need_replacement_candidates") time.sleep(retry_delay) else: log(f"[poll] cycle={cycles} sleep={base_args.idle_interval_seconds}s reason=no_new_submissions") time.sleep(base_args.idle_interval_seconds) continue if cycles % STATS_PRINT_INTERVAL == 0: try: stats = outcome_tracker.get_stats_report() totals = stats.get("totals", {}) log( f"[poll] cycle={cycles} outcome_stats " f"terminal={totals.get('total', 0)} " f"success={totals.get('successCount', 0)} " f"failed={totals.get('failureCount', 0)} " f"success_rate={totals.get('successRate', 0):.3f} " f"failure_rate={totals.get('failureRate', 0):.3f} " f"architecture_blocks={len(stats.get('architectureCompatibilityBlocks') or {})}" ) except Exception as exc: log(f"[poll] cycle={cycles} outcome_stats_error={exc}") if hasattr(modelhub_client, "available_submit_slots") and modelhub_client.available_submit_slots() <= 0: log(f"[poll] cycle={cycles} sleep={base_args.poll_interval_seconds}s reason=queue_refilled") time.sleep(base_args.poll_interval_seconds) else: cooldown = getattr(base_args, "post_cycle_cooldown_seconds", 1) if cooldown > 0: time.sleep(cooldown) try: outcome_tracker.sync_from_api( modelhub_client, task_contexts=_load_task_compatibility_contexts( outcome_tracker, ledger_path=Path(base_args.ledger_path), ), ) except Exception: pass outcome_tracker.save() try: stats_report = outcome_tracker.get_stats_report() except Exception: stats_report = {} summary = { "generatedAt": now.isoformat(), "dryRun": bool(base_args.dry_run), "dailyTarget": base_args.daily_target, "unlimitedDailyTarget": base_args.daily_target <= 0, "cycles": cycles, "submittedTotal": submitted_total, "stoppedReason": stopped_reason, "pollRunDir": str(poll_run_dir), "cycleSummaries": list(cycle_summaries), "cycleSummariesRetained": len(cycle_summaries), "cycleSummariesRetentionLimit": retained_cycles, "queueCleanupRuns": list(queue_cleanup_runs), "queueCleanupRunsRetained": len(queue_cleanup_runs), "queueCleanupRunsRetentionLimit": retained_cleanups, "architectureBootstrap": architecture_bootstrap_summary, "outcomeStats": stats_report, } write_json(poll_run_dir / "summary.json", summary) log(f"[poll] finished submitted_total={submitted_total} cycles={cycles} stopped_reason={stopped_reason}") if state_sync is not None: state_sync.write_readiness(ready=False, reason="worker_stopped") state_sync.close() return summary def main(argv: list[str] | None = None) -> int: parser = build_parser() args = parser.parse_args(argv) ensure_tokens(args) if getattr(args, "print_stats", False): try: modelhub_client = _build_modelhub_client(args) tracker = OutcomeTracker(Path(args.outcomes_path)) tracker.sync_from_api(modelhub_client) report = tracker.get_stats_report() print(json.dumps(report, ensure_ascii=False, indent=2)) except Exception as exc: print(f"Failed to generate stats report: {exc}", file=sys.stderr) return 1 return 0 log( f"[poll] modelscope_token={'set' if bool(args.modelscope_token) else 'missing'} " f"xc_token={'set' if bool(args.modelhub_token) else 'missing'} " f"xc_tokens={len(getattr(args, 'modelhub_tokens', []) or [])}" ) summary = run_poll_loop(base_args=args) print(f"poll_run_dir={summary['pollRunDir']}") print(f"submitted_total={summary['submittedTotal']}") print(f"cycles={summary['cycles']}") print(f"stopped_reason={summary['stoppedReason']}") return 0 if __name__ == "__main__": raise SystemExit(main())