964362818e
Do not cache transient repository-list failures as empty weight sets. Deduplicate normalized file paths without lowering reported file sizes. Record whitelisted OOM evidence and bounded discovery diagnostics.
2065 lines
90 KiB
Python
2065 lines
90 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import os
|
|
from collections import Counter
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from datetime import timedelta
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from common import parse_datetime, runtime_instance_id, utc_now, write_json, write_jsonl
|
|
from candidate_preflight import CandidatePreflightAdvisor, MODEL_LOAD_OVERHEAD
|
|
from config_optimizer import SafeConfigOptimizer
|
|
from gpu_strategy import DEFAULT_GPU_STRATEGY_PATH
|
|
from hf_discovery import HuggingFaceDiscovery
|
|
from history_stats import (
|
|
append_ledger_entry,
|
|
build_empty_pre_submit_report,
|
|
count_submissions_for_day,
|
|
load_ledger,
|
|
)
|
|
from market_intelligence import (
|
|
DEFAULT_FETCH_WORKERS,
|
|
DEFAULT_FRAMEWORK_MIN_SAMPLES,
|
|
DEFAULT_FRAMEWORK_REFRESH_SECONDS,
|
|
DEFAULT_MARKET_INTELLIGENCE_PATH,
|
|
DEFAULT_QUEUE_REFRESH_SECONDS,
|
|
DEFAULT_THROUGHPUT_WINDOW_HOURS,
|
|
MarketIntelligenceManager,
|
|
)
|
|
from modelhub_client import (
|
|
DEFAULT_CAPACITY_STATE_PATH,
|
|
ModelHubAPIError,
|
|
ModelHubClient,
|
|
ModelHubClientPool,
|
|
OldModelQueuePolicyError,
|
|
is_capacity_error,
|
|
is_duplicate_submission_error,
|
|
is_framework_prerequisite_error,
|
|
is_model_uniqueness_error,
|
|
)
|
|
from models import CandidateModel, HFModelSummary, ModelInspection
|
|
from official_capabilities import (
|
|
DEFAULT_OFFICIAL_CAPABILITIES_PATH,
|
|
OfficialCapabilityRegistry,
|
|
OfficialCapabilityUnavailable,
|
|
)
|
|
from outcome_tracker import DEFAULT_OUTCOMES_PATH, OutcomeTracker
|
|
from routing_engine import DEFAULT_ROUTING_STATE_PATH, SuccessFirstRoutingEngine
|
|
from submission_claims import DEFAULT_CLAIMS_PATH, SubmissionClaimStore, candidate_key, diversify_candidates
|
|
from submission_exclusions import DEFAULT_SUBMISSION_EXCLUSIONS_PATH, SubmissionExclusionStore
|
|
from task_registry import (
|
|
TASK_SPEC_BY_TYPE,
|
|
all_task_types,
|
|
compatible_frameworks_for_task,
|
|
pipeline_tags_for_task_types,
|
|
register_dynamic_task_route,
|
|
task_specs_for_model,
|
|
)
|
|
from template_selector import TemplateSelector
|
|
|
|
|
|
DEFAULT_RUNS_DIR = Path("runs")
|
|
DEFAULT_LEDGER_PATH = Path("ledger/submissions.jsonl")
|
|
ADAPTIVE_SCAN_MAX_MODELS = 3000
|
|
ADAPTIVE_SCAN_MIN_FALLBACK_MODELS = 500
|
|
MAX_ROUTES_PER_MODEL = 6
|
|
MAX_PENDING_ROUTES_PER_MODEL = 2
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser(description="Auto discover and submit public ModelScope models to ModelHub.")
|
|
parser.add_argument("--gpu", help="Single GPU alias or platform name, for example: k100")
|
|
parser.add_argument("--gpus", help="Comma-separated GPU aliases/platform names. Omit to auto-use all safe GPUs from templates.")
|
|
parser.add_argument("--task-types", help="Comma-separated ModelHub task types. Omit to auto-enable all supported task types.")
|
|
parser.add_argument("--limit", type=int, default=300, help="Maximum number of ModelScope models to scan per pipeline tag")
|
|
parser.add_argument("--max-scan-models", type=int, default=0, help="Hard cap on total scanned models (0 means auto)")
|
|
parser.add_argument(
|
|
"--scan-multiplier",
|
|
type=int,
|
|
default=4,
|
|
help="Multiplier used when auto-deriving scan limit from quota/queue capacity",
|
|
)
|
|
parser.add_argument(
|
|
"--candidate-pool-limit",
|
|
type=int,
|
|
default=100,
|
|
help="Maximum prepared candidates retained before submitting; bounds memory in large queues",
|
|
)
|
|
parser.add_argument("--min-downloads", type=int, default=50, help="Minimum downloads threshold")
|
|
parser.add_argument("--daily-target", type=int, default=0, help="Daily submission target across all auto runs; 0 means unlimited")
|
|
parser.add_argument("--dry-run", action="store_true", help="Only write artifacts without creating tasks")
|
|
parser.add_argument(
|
|
"--disable-gpu-strategy",
|
|
action="store_true",
|
|
help="Disable adaptive 70/30 proven-GPU scheduling and keep the legacy candidate order",
|
|
)
|
|
parser.add_argument(
|
|
"--disable-market-intelligence",
|
|
action="store_true",
|
|
help="Disable live queue/throughput and public framework statistics",
|
|
)
|
|
parser.add_argument(
|
|
"--gpu-strategy-refresh-submissions",
|
|
type=int,
|
|
default=200,
|
|
help="Recalculate adaptive GPU choices after this many accepted submissions",
|
|
)
|
|
parser.add_argument("--skip-outcome-sync", action="store_true", help="Skip outcome sync from ModelHub before scanning")
|
|
parser.add_argument("--skip-history-archive", action="store_true", help="Skip historical task archive download for this run")
|
|
group = parser.add_mutually_exclusive_group()
|
|
group.add_argument("--since-hours", type=int, help="Only scan models updated within the last N hours")
|
|
group.add_argument("--updated-after", help="Only scan models updated after the given UTC time")
|
|
parser.add_argument("--stats-window-days", type=int, default=7, help="History window in days")
|
|
parser.add_argument(
|
|
"--history-stats-threshold",
|
|
type=int,
|
|
default=500,
|
|
help="Minimum local ledger records before using history stats to rank submissions",
|
|
)
|
|
parser.add_argument("--read-concurrency", type=int, default=4, help="Concurrency for read-only remote calls")
|
|
parser.add_argument(
|
|
"--submit-concurrency",
|
|
type=int,
|
|
default=0,
|
|
help="Concurrency for task submission calls (0 = auto based on token/client count)",
|
|
)
|
|
parser.add_argument(
|
|
"--max-submits-per-run",
|
|
type=int,
|
|
default=0,
|
|
help="Maximum tasks to submit in one run (0 means unlimited)",
|
|
)
|
|
parser.add_argument(
|
|
"--recent-model-reserve-slots",
|
|
type=int,
|
|
default=int(os.getenv("MODELHUB_RECENT_MODEL_RESERVE_SLOTS", "5")),
|
|
help=argparse.SUPPRESS,
|
|
)
|
|
parser.add_argument(
|
|
"--recent-model-days",
|
|
type=int,
|
|
default=int(os.getenv("MODELHUB_RECENT_MODEL_DAYS", "7")),
|
|
help=argparse.SUPPRESS,
|
|
)
|
|
parser.add_argument(
|
|
"--disable-candidate-preflight",
|
|
action="store_true",
|
|
help="Disable repository structure, memory, and context-length preflight checks",
|
|
)
|
|
parser.add_argument(
|
|
"--llm-classifier-endpoint",
|
|
default=os.getenv("MODELHUB_LLM_CLASSIFIER_ENDPOINT"),
|
|
help=argparse.SUPPRESS,
|
|
)
|
|
parser.add_argument(
|
|
"--llm-classifier-model",
|
|
default=os.getenv("MODELHUB_LLM_CLASSIFIER_MODEL"),
|
|
help=argparse.SUPPRESS,
|
|
)
|
|
parser.add_argument(
|
|
"--llm-classifier-api-key",
|
|
default=os.getenv("MODELHUB_LLM_CLASSIFIER_API_KEY"),
|
|
help=argparse.SUPPRESS,
|
|
)
|
|
parser.add_argument(
|
|
"--llm-classifier-timeout-seconds",
|
|
type=int,
|
|
default=int(os.getenv("MODELHUB_LLM_CLASSIFIER_TIMEOUT_SECONDS", "20")),
|
|
help=argparse.SUPPRESS,
|
|
)
|
|
parser.add_argument(
|
|
"--llm-classifier-min-deny-confidence",
|
|
type=float,
|
|
default=float(os.getenv("MODELHUB_LLM_CLASSIFIER_MIN_DENY_CONFIDENCE", "0.85")),
|
|
help=argparse.SUPPRESS,
|
|
)
|
|
parser.add_argument(
|
|
"--llm-classifier-cache-path",
|
|
default=os.getenv("MODELHUB_LLM_CLASSIFIER_CACHE_PATH", ".modelhub_state/llm_classifications.json"),
|
|
help=argparse.SUPPRESS,
|
|
)
|
|
parser.add_argument("--runs-dir", default=str(DEFAULT_RUNS_DIR), help=argparse.SUPPRESS)
|
|
parser.add_argument("--ledger-path", default=str(DEFAULT_LEDGER_PATH), help=argparse.SUPPRESS)
|
|
parser.add_argument("--outcomes-path", default=str(DEFAULT_OUTCOMES_PATH), help=argparse.SUPPRESS)
|
|
parser.add_argument(
|
|
"--claims-path",
|
|
default=os.getenv("MODELHUB_AGENT_CLAIMS_PATH", str(DEFAULT_CLAIMS_PATH)),
|
|
help=argparse.SUPPRESS,
|
|
)
|
|
parser.add_argument("--history-archive-path", default="history/platform_tasks.jsonl", help=argparse.SUPPRESS)
|
|
parser.add_argument("--history-archive-limit", type=int, default=5000, help=argparse.SUPPRESS)
|
|
parser.add_argument(
|
|
"--submission-exclusions-path",
|
|
default=os.getenv("MODELHUB_SUBMISSION_EXCLUSIONS_PATH", str(DEFAULT_SUBMISSION_EXCLUSIONS_PATH)),
|
|
help=argparse.SUPPRESS,
|
|
)
|
|
parser.add_argument(
|
|
"--market-intelligence-state-path",
|
|
default=os.getenv("MODELHUB_MARKET_INTELLIGENCE_PATH", str(DEFAULT_MARKET_INTELLIGENCE_PATH)),
|
|
help=argparse.SUPPRESS,
|
|
)
|
|
parser.add_argument(
|
|
"--market-queue-refresh-seconds",
|
|
type=int,
|
|
default=int(os.getenv("MODELHUB_MARKET_QUEUE_REFRESH_SECONDS", str(DEFAULT_QUEUE_REFRESH_SECONDS))),
|
|
help=argparse.SUPPRESS,
|
|
)
|
|
parser.add_argument(
|
|
"--market-framework-refresh-seconds",
|
|
type=int,
|
|
default=int(os.getenv("MODELHUB_MARKET_FRAMEWORK_REFRESH_SECONDS", str(DEFAULT_FRAMEWORK_REFRESH_SECONDS))),
|
|
help=argparse.SUPPRESS,
|
|
)
|
|
parser.add_argument(
|
|
"--market-throughput-window-hours",
|
|
type=int,
|
|
default=int(os.getenv("MODELHUB_MARKET_THROUGHPUT_WINDOW_HOURS", str(DEFAULT_THROUGHPUT_WINDOW_HOURS))),
|
|
help=argparse.SUPPRESS,
|
|
)
|
|
parser.add_argument(
|
|
"--market-fetch-workers",
|
|
type=int,
|
|
default=int(os.getenv("MODELHUB_MARKET_FETCH_WORKERS", str(DEFAULT_FETCH_WORKERS))),
|
|
help=argparse.SUPPRESS,
|
|
)
|
|
parser.add_argument(
|
|
"--market-framework-min-samples",
|
|
type=int,
|
|
default=int(os.getenv("MODELHUB_MARKET_FRAMEWORK_MIN_SAMPLES", str(DEFAULT_FRAMEWORK_MIN_SAMPLES))),
|
|
help=argparse.SUPPRESS,
|
|
)
|
|
parser.add_argument(
|
|
"--gpu-strategy-state-path",
|
|
default=os.getenv("MODELHUB_GPU_STRATEGY_STATE_PATH", str(DEFAULT_GPU_STRATEGY_PATH)),
|
|
help=argparse.SUPPRESS,
|
|
)
|
|
parser.add_argument(
|
|
"--routing-state-path",
|
|
default=os.getenv("MODELHUB_ROUTING_STATE_PATH", str(DEFAULT_ROUTING_STATE_PATH)),
|
|
help=argparse.SUPPRESS,
|
|
)
|
|
parser.add_argument(
|
|
"--official-capabilities-path",
|
|
default=os.getenv(
|
|
"MODELHUB_OFFICIAL_CAPABILITIES_PATH",
|
|
str(DEFAULT_OFFICIAL_CAPABILITIES_PATH),
|
|
),
|
|
help=argparse.SUPPRESS,
|
|
)
|
|
parser.add_argument("--gpu-strategy-recent-window", type=int, default=1000, help=argparse.SUPPRESS)
|
|
parser.add_argument("--gpu-strategy-min-long-samples", type=int, default=100, help=argparse.SUPPRESS)
|
|
parser.add_argument(
|
|
"--capacity-state-path",
|
|
default=os.getenv("MODELHUB_CAPACITY_STATE_PATH", str(DEFAULT_CAPACITY_STATE_PATH)),
|
|
help=argparse.SUPPRESS,
|
|
)
|
|
parser.add_argument(
|
|
"--capacity-probe-interval-cycles",
|
|
type=int,
|
|
default=int(os.getenv("MODELHUB_CAPACITY_PROBE_INTERVAL_CYCLES", "0")),
|
|
help=argparse.SUPPRESS,
|
|
)
|
|
parser.add_argument("--hf-base-url", default=os.getenv("MODELSCOPE_BASE_URL", "https://modelscope.cn"), help=argparse.SUPPRESS)
|
|
parser.add_argument("--modelhub-base-url", default=os.getenv("MODELHUB_BASE_URL", "https://modelhub.org.cn"), help=argparse.SUPPRESS)
|
|
parser.add_argument("--modelhub-token", default=os.getenv("MODELHUB_XC_TOKEN") or os.getenv("XC_TOKEN"), help=argparse.SUPPRESS)
|
|
return parser
|
|
|
|
|
|
def determine_updated_after(args: argparse.Namespace, now) -> Any:
|
|
if args.since_hours is not None:
|
|
return now - timedelta(hours=args.since_hours)
|
|
if args.updated_after:
|
|
parsed = parse_datetime(args.updated_after)
|
|
if parsed is None:
|
|
raise ValueError(f"Invalid --updated-after value: {args.updated_after}")
|
|
return parsed
|
|
return None
|
|
|
|
|
|
def resolve_task_types(args: argparse.Namespace) -> list[str]:
|
|
if not args.task_types:
|
|
return all_task_types()
|
|
selected = [value.strip() for value in args.task_types.split(",") if value.strip()]
|
|
unknown = [task_type for task_type in selected if task_type not in TASK_SPEC_BY_TYPE]
|
|
if unknown:
|
|
raise ValueError(f"Unknown task types: {', '.join(sorted(unknown))}")
|
|
return selected
|
|
|
|
|
|
def resolve_target_gpus(args: argparse.Namespace, selector: TemplateSelector, task_types: list[str]) -> list[str]:
|
|
raw_values: list[str] = []
|
|
if args.gpus:
|
|
raw_values.extend(value.strip() for value in args.gpus.split(",") if value.strip())
|
|
if args.gpu:
|
|
raw_values.append(args.gpu)
|
|
|
|
if raw_values:
|
|
normalized: list[str] = []
|
|
for value in raw_values:
|
|
target_gpu = selector.normalize_gpu(value)
|
|
if target_gpu not in normalized:
|
|
normalized.append(target_gpu)
|
|
return normalized
|
|
|
|
discovered: list[str] = []
|
|
for task_type in task_types:
|
|
for target_gpu in selector.supported_target_gpus(task_type, auto_only=True):
|
|
if target_gpu not in discovered:
|
|
discovered.append(target_gpu)
|
|
return discovered
|
|
|
|
|
|
def resolve_scan_limit(
|
|
args: argparse.Namespace,
|
|
*,
|
|
target_gpu_count: int,
|
|
remaining_daily_quota: int,
|
|
platform_available_slots: int | None,
|
|
) -> int:
|
|
base_limit = max(1, args.limit)
|
|
explicit_max = max(0, int(getattr(args, "max_scan_models", 0) or 0))
|
|
if explicit_max > 0:
|
|
return min(base_limit, explicit_max)
|
|
|
|
multiplier = max(1, int(getattr(args, "scan_multiplier", 4) or 1))
|
|
if args.daily_target > 0:
|
|
needed = max(1, remaining_daily_quota)
|
|
elif platform_available_slots is not None and platform_available_slots > 0:
|
|
needed = max(1, platform_available_slots)
|
|
else:
|
|
needed = max(200, target_gpu_count * 20)
|
|
|
|
max_submits_per_run = max(0, int(getattr(args, "max_submits_per_run", 0) or 0))
|
|
if max_submits_per_run > 0:
|
|
needed = min(needed, max_submits_per_run)
|
|
|
|
return min(base_limit, needed * multiplier)
|
|
|
|
|
|
def choose_candidate_for_gpu(
|
|
*,
|
|
model: HFModelSummary,
|
|
inspection: ModelInspection,
|
|
template_selector: TemplateSelector,
|
|
task_types: list[str],
|
|
target_gpu: str,
|
|
market_intelligence: MarketIntelligenceManager | None = None,
|
|
preflight_advisor: CandidatePreflightAdvisor | None = None,
|
|
require_official_config: bool = False,
|
|
config_optimizer: SafeConfigOptimizer | None = None,
|
|
) -> CandidateModel | None:
|
|
candidate, _reason = choose_candidate_for_gpu_detailed(
|
|
model=model,
|
|
inspection=inspection,
|
|
template_selector=template_selector,
|
|
task_types=task_types,
|
|
target_gpu=target_gpu,
|
|
market_intelligence=market_intelligence,
|
|
preflight_advisor=preflight_advisor,
|
|
require_official_config=require_official_config,
|
|
config_optimizer=config_optimizer,
|
|
)
|
|
return candidate
|
|
|
|
|
|
def choose_candidate_for_gpu_detailed(
|
|
*,
|
|
model: HFModelSummary,
|
|
inspection: ModelInspection,
|
|
template_selector: TemplateSelector,
|
|
task_types: list[str],
|
|
target_gpu: str,
|
|
market_intelligence: MarketIntelligenceManager | None = None,
|
|
preflight_advisor: CandidatePreflightAdvisor | None = None,
|
|
require_official_config: bool = False,
|
|
config_optimizer: SafeConfigOptimizer | None = None,
|
|
excluded_frameworks: set[str] | None = None,
|
|
rejection_details: list[dict[str, Any]] | None = None,
|
|
) -> tuple[CandidateModel | None, str | None]:
|
|
last_rejection_reason: str | None = None
|
|
for task_type in task_types:
|
|
supported_frameworks = template_selector.supported_frameworks_for_auto(task_type, target_gpu)
|
|
compatible_frameworks: list[str] = []
|
|
try:
|
|
if supported_frameworks:
|
|
compatible_frameworks = compatible_frameworks_for_task(
|
|
task_type,
|
|
target_gpu,
|
|
supported_frameworks,
|
|
inspection,
|
|
)
|
|
except ValueError:
|
|
pass
|
|
if market_intelligence is not None:
|
|
compatible_frameworks = market_intelligence.selectable_frameworks(
|
|
task_type=task_type,
|
|
target_gpu=target_gpu,
|
|
incumbent_frameworks=compatible_frameworks,
|
|
inspection=inspection,
|
|
)
|
|
if excluded_frameworks:
|
|
available_frameworks = [
|
|
framework for framework in compatible_frameworks
|
|
if framework not in excluded_frameworks
|
|
]
|
|
if compatible_frameworks and not available_frameworks:
|
|
last_rejection_reason = "prior_failed_framework_no_alternative"
|
|
compatible_frameworks = available_frameworks
|
|
if not compatible_frameworks:
|
|
continue
|
|
|
|
for framework in compatible_frameworks:
|
|
config_params = (
|
|
market_intelligence.official_config(
|
|
task_type=task_type,
|
|
target_gpu=target_gpu,
|
|
framework=framework,
|
|
gguf_filename=inspection.selected_gguf,
|
|
)
|
|
if market_intelligence is not None
|
|
else None
|
|
)
|
|
if config_params is not None:
|
|
template_id = f"modelhub-live-{task_type}-{framework}-{target_gpu}".lower().replace("_", "-")
|
|
elif market_intelligence is not None and require_official_config:
|
|
# A live-discovered route without a valid official config is a
|
|
# critical capability gap, not permission to use a stale hand template.
|
|
last_rejection_reason = "official_build_config_unavailable"
|
|
continue
|
|
else:
|
|
try:
|
|
template = template_selector.select_template(task_type, framework, target_gpu)
|
|
config_params = template_selector.render_config(
|
|
template,
|
|
gguf_filename=inspection.selected_gguf if framework == "llamacpp" else None,
|
|
)
|
|
template_id = template.template_id
|
|
except (KeyError, ValueError):
|
|
continue
|
|
framework_metadata = (
|
|
market_intelligence.framework_metadata(task_type, target_gpu, framework)
|
|
if market_intelligence is not None
|
|
else {}
|
|
)
|
|
score = float(framework_metadata.get("frameworkCombinedScore") or 0.0)
|
|
warnings: list[str] = []
|
|
config_optimization: dict[str, Any] = {"source": "official", "applied": False}
|
|
if config_optimizer is not None and config_params is not None:
|
|
config_params, config_optimization = config_optimizer.optimize(
|
|
task_type=task_type,
|
|
target_gpu=target_gpu,
|
|
framework=framework,
|
|
official_config=config_params,
|
|
official_lower_bound=float(
|
|
framework_metadata.get("frameworkMarketWilsonLowerBound") or 0.0
|
|
),
|
|
)
|
|
if config_optimization.get("applied"):
|
|
warnings.append("validated_success_config_patch_applied")
|
|
if bool(framework_metadata.get("frameworkEvidenceQualified", False)):
|
|
warnings.append("framework_selected_from_success_evidence")
|
|
if config_params is not None and bool(framework_metadata.get("frameworkOfficialConfigValid", False)):
|
|
warnings.append("official_build_config_synced")
|
|
preflight_metadata: dict[str, Any] = {}
|
|
if preflight_advisor is not None:
|
|
assessment = preflight_advisor.assess(
|
|
inspection=inspection,
|
|
task_type=task_type,
|
|
target_gpu=target_gpu,
|
|
framework=framework,
|
|
config_params=config_params,
|
|
)
|
|
if not assessment.allowed:
|
|
last_rejection_reason = assessment.reason
|
|
if rejection_details is not None:
|
|
rejection_details.append({
|
|
"taskType": task_type,
|
|
"framework": framework,
|
|
"reason": assessment.reason,
|
|
"metadata": {
|
|
key: assessment.metadata[key]
|
|
for key in (
|
|
"modelType", "architectures", "quantizationMethod",
|
|
"estimatedLoadBytes", "repositoryOnDiskBytes",
|
|
"recursiveRepositoryBytes", "publishedSizeBytes",
|
|
"gpuMemoryGiB", "gpuMemoryEvidence",
|
|
"estimatedRequiredGiB", "memorySizingBasis",
|
|
"maximumRepositorySizeGiB",
|
|
) if key in assessment.metadata
|
|
},
|
|
})
|
|
continue
|
|
config_params = assessment.config_params
|
|
warnings.extend(assessment.warnings)
|
|
preflight_metadata = assessment.metadata
|
|
preflight_metadata = {
|
|
**preflight_metadata,
|
|
"configOptimization": config_optimization,
|
|
"modelscopeParams": model.params,
|
|
"modelscopeFileSize": model.file_size,
|
|
"modelscopeTasks": list(model.tasks),
|
|
"modelscopeTags": list(model.tags),
|
|
"modelscopeLicense": model.license,
|
|
"modelCard": dict(inspection.model_card_metadata),
|
|
}
|
|
repository_size = inspection.repository_size_bytes
|
|
if model.file_size and repository_size and model.file_size > repository_size:
|
|
warnings.append("modelscope_published_size_used_as_conservative_upper_bound")
|
|
preflight_metadata["conservativeRepositorySizeBytes"] = model.file_size
|
|
spec = TASK_SPEC_BY_TYPE[task_type]
|
|
return CandidateModel(
|
|
repo_id=model.repo_id,
|
|
model_address=model.model_address,
|
|
pipeline_tag=model.pipeline_tag,
|
|
modality=spec.modality,
|
|
task_type=task_type,
|
|
target_gpu=target_gpu,
|
|
framework=framework,
|
|
template_id=template_id,
|
|
config_params=config_params,
|
|
downloads=model.downloads,
|
|
last_modified=model.last_modified,
|
|
gguf_filename=inspection.selected_gguf,
|
|
score=score,
|
|
warnings=warnings,
|
|
preflight_metadata=preflight_metadata,
|
|
), None
|
|
return None, last_rejection_reason
|
|
|
|
|
|
def resolve_submit_concurrency(
|
|
args: argparse.Namespace,
|
|
*,
|
|
modelhub_client: ModelHubClient | ModelHubClientPool | Any,
|
|
planned_submit_count: int,
|
|
) -> int:
|
|
if hasattr(modelhub_client, "weekly_calibration_active") and modelhub_client.weekly_calibration_active():
|
|
# Pinning and capacity evidence require one in-flight task per account.
|
|
return 1
|
|
explicit_workers = int(getattr(args, "submit_concurrency", 0) or 0)
|
|
if explicit_workers > 0:
|
|
return explicit_workers
|
|
|
|
clients = getattr(modelhub_client, "clients", None)
|
|
if isinstance(clients, list) and clients:
|
|
if planned_submit_count > 0:
|
|
return max(1, min(len(clients), planned_submit_count))
|
|
return len(clients)
|
|
|
|
return 1
|
|
|
|
|
|
def resolve_max_submit_count(
|
|
args: argparse.Namespace,
|
|
planned_count: int,
|
|
remaining_daily_quota: int,
|
|
) -> int:
|
|
explicit_limit = int(getattr(args, "max_submits_per_run", 0) or 0)
|
|
planned_submit_count = min(planned_count, max(0, remaining_daily_quota))
|
|
if explicit_limit > 0:
|
|
planned_submit_count = min(planned_submit_count, explicit_limit)
|
|
return planned_submit_count
|
|
|
|
|
|
def process_model_for_candidates(
|
|
*,
|
|
model: HFModelSummary,
|
|
hf_discovery: HuggingFaceDiscovery,
|
|
modelhub_client: ModelHubClient,
|
|
template_selector: TemplateSelector,
|
|
target_gpus: list[str],
|
|
allowed_task_types: list[str],
|
|
outcome_tracker: OutcomeTracker | None = None,
|
|
submission_exclusion_store: SubmissionExclusionStore | None = None,
|
|
market_intelligence: MarketIntelligenceManager | None = None,
|
|
preflight_advisor: CandidatePreflightAdvisor | None = None,
|
|
official_registry: OfficialCapabilityRegistry | None = None,
|
|
config_optimizer: SafeConfigOptimizer | None = None,
|
|
allow_dynamic_tasks: bool = False,
|
|
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]:
|
|
if model.private or model.gated:
|
|
reason = "modelscope_private" if model.private else "modelscope_gated"
|
|
return [], [{"repoId": model.repo_id, "reason": reason}], []
|
|
specs = [spec for spec in task_specs_for_model(model) if spec.task_type in allowed_task_types]
|
|
if not specs and official_registry is None:
|
|
return [], [{"repoId": model.repo_id, "reason": f"unsupported_pipeline_tag:{model.pipeline_tag or 'unknown'}"}], []
|
|
|
|
try:
|
|
if hasattr(modelhub_client, "model_submission_precheck"):
|
|
precheck = modelhub_client.model_submission_precheck(model.repo_id)
|
|
processed_gpus = set(precheck.get("processedGpus") or set())
|
|
successful_gpus = set(precheck.get("successfulGpus") or set())
|
|
failed_frameworks_by_gpu = precheck.get("failedFrameworksByGpu") or {}
|
|
else:
|
|
processed_gpus = modelhub_client.processed_gpus_for_model(model.repo_id)
|
|
successful_gpus = set()
|
|
failed_frameworks_by_gpu = {}
|
|
except Exception as exc:
|
|
return (
|
|
[],
|
|
[{"repoId": model.repo_id, "reason": "community_precheck_unavailable_fail_closed"}],
|
|
[{"repoId": model.repo_id, "reason": f"community_precheck_error:{exc}"}],
|
|
)
|
|
if len(processed_gpus) >= MAX_ROUTES_PER_MODEL:
|
|
return (
|
|
[],
|
|
[{
|
|
"repoId": model.repo_id,
|
|
"reason": "model_route_limit_reached",
|
|
"processedGpuCount": len(processed_gpus),
|
|
}],
|
|
[],
|
|
)
|
|
pending_route_count = (
|
|
outcome_tracker.pending_route_count_for_model(model.repo_id)
|
|
if outcome_tracker is not None
|
|
else 0
|
|
)
|
|
if pending_route_count >= MAX_PENDING_ROUTES_PER_MODEL:
|
|
return (
|
|
[],
|
|
[{
|
|
"repoId": model.repo_id,
|
|
"reason": "model_pending_feedback_limit",
|
|
"pendingRouteCount": pending_route_count,
|
|
}],
|
|
[],
|
|
)
|
|
candidates: list[dict[str, Any]] = []
|
|
skipped: list[dict[str, Any]] = []
|
|
failed: list[dict[str, Any]] = []
|
|
allowed_task_types_set = [spec.task_type for spec in specs]
|
|
# Preserve the routing order: the first six GPUs are the preferred
|
|
# tranche. Lower-ranked GPUs are a fallback only when the preferred
|
|
# tranche produces no candidate for this model. This keeps queue/quality
|
|
# prioritization deterministic while still recovering from a model being
|
|
# already processed, too large, or unsupported on every preferred GPU.
|
|
preferred_target_gpus = target_gpus[:MAX_ROUTES_PER_MODEL]
|
|
fallback_target_gpus = target_gpus[MAX_ROUTES_PER_MODEL:]
|
|
|
|
def prepare_pending_routes(route_target_gpus: list[str]) -> list[tuple[str, list[str], set[str]]]:
|
|
pending: list[tuple[str, list[str], set[str]]] = []
|
|
for target_gpu in route_target_gpus:
|
|
if submission_exclusion_store is not None and submission_exclusion_store.is_blocked(model.repo_id, target_gpu):
|
|
skipped.append(
|
|
{"repoId": model.repo_id, "targetGpu": target_gpu, "reason": "model_gpu_uniqueness_blocklist"}
|
|
)
|
|
continue
|
|
if target_gpu in processed_gpus:
|
|
skipped.append({"repoId": model.repo_id, "targetGpu": target_gpu, "reason": "already_processed_for_gpu"})
|
|
continue
|
|
if target_gpu in failed_frameworks_by_gpu:
|
|
skipped.append({
|
|
"repoId": model.repo_id,
|
|
"targetGpu": target_gpu,
|
|
"reason": "prior_failed_gpu_closed_accuracy_first",
|
|
})
|
|
continue
|
|
failed_frameworks = set(failed_frameworks_by_gpu.get(target_gpu) or set())
|
|
if outcome_tracker and outcome_tracker.is_model_gpu_failed(model.repo_id, target_gpu):
|
|
skipped.append({"repoId": model.repo_id, "targetGpu": target_gpu, "reason": "local_failure_cooldown_24h"})
|
|
continue
|
|
if preflight_advisor is not None and model.file_size:
|
|
memory_gib = preflight_advisor.gpu_memory_gib.get(target_gpu)
|
|
if memory_gib and model.file_size / (1024**3) * MODEL_LOAD_OVERHEAD > memory_gib:
|
|
skipped.append({
|
|
"repoId": model.repo_id,
|
|
"targetGpu": target_gpu,
|
|
"reason": "preflight_predicted_oom",
|
|
"preflightDetails": {
|
|
"stage": "published_size_precheck",
|
|
"publishedSizeBytes": model.file_size,
|
|
"estimatedRequiredGiB": model.file_size / (1024**3) * MODEL_LOAD_OVERHEAD,
|
|
"gpuMemoryGiB": memory_gib,
|
|
"gpuMemoryEvidence": preflight_advisor.gpu_memory_evidence.get(target_gpu),
|
|
"loadOverhead": MODEL_LOAD_OVERHEAD,
|
|
},
|
|
})
|
|
continue
|
|
|
|
exact_task_types = list(allowed_task_types_set)
|
|
if official_registry is not None:
|
|
exact_task_types = official_registry.task_types_for(
|
|
modelhub_client,
|
|
model_address=model.model_address,
|
|
model_last_modified=model.last_modified.isoformat() if model.last_modified else None,
|
|
gpu=target_gpu,
|
|
)
|
|
route_task_types = (
|
|
list(exact_task_types)
|
|
if allow_dynamic_tasks
|
|
else [task_type for task_type in allowed_task_types_set if task_type in exact_task_types]
|
|
)
|
|
compatible_task_types = []
|
|
for task_type in route_task_types:
|
|
register_dynamic_task_route(task_type, model.pipeline_tag or task_type)
|
|
if market_intelligence is not None and not market_intelligence.has_framework_route(task_type, target_gpu):
|
|
try:
|
|
market_intelligence.ensure_framework_route(modelhub_client, task_type, target_gpu)
|
|
except Exception as exc:
|
|
raise OfficialCapabilityUnavailable(
|
|
f"framework capability unavailable for {task_type}/{target_gpu}: {type(exc).__name__}: {exc}"
|
|
) from exc
|
|
if template_selector.supported_frameworks_for_auto(task_type, target_gpu):
|
|
compatible_task_types.append(task_type)
|
|
continue
|
|
if market_intelligence is not None and market_intelligence.has_framework_route(task_type, target_gpu):
|
|
compatible_task_types.append(task_type)
|
|
if not compatible_task_types:
|
|
skipped.append(
|
|
{
|
|
"repoId": model.repo_id,
|
|
"targetGpu": target_gpu,
|
|
"reason": "no_compatible_auto_template_or_framework",
|
|
}
|
|
)
|
|
continue
|
|
pending.append((
|
|
target_gpu,
|
|
compatible_task_types,
|
|
set(failed_frameworks_by_gpu.get(target_gpu) or set()),
|
|
))
|
|
return pending
|
|
|
|
pending_task_types_by_gpu = prepare_pending_routes(preferred_target_gpus)
|
|
used_fallback_routes = False
|
|
if not pending_task_types_by_gpu and fallback_target_gpus:
|
|
pending_task_types_by_gpu = prepare_pending_routes(fallback_target_gpus)
|
|
used_fallback_routes = True
|
|
|
|
if not pending_task_types_by_gpu:
|
|
return candidates, skipped, failed
|
|
|
|
try:
|
|
inspection = hf_discovery.inspect_model(model)
|
|
except Exception as exc:
|
|
return [], skipped, [{"repoId": model.repo_id, "reason": str(exc)}]
|
|
|
|
def evaluate_pending_routes(pending_routes: list[tuple[str, list[str], set[str]]]) -> int:
|
|
before_count = len(candidates)
|
|
for target_gpu, task_types, excluded_frameworks in pending_routes:
|
|
if len(candidates) - before_count >= MAX_ROUTES_PER_MODEL:
|
|
break
|
|
rejection_details: list[dict[str, Any]] = []
|
|
best, preflight_reason = choose_candidate_for_gpu_detailed(
|
|
model=model,
|
|
inspection=inspection,
|
|
template_selector=template_selector,
|
|
task_types=task_types,
|
|
target_gpu=target_gpu,
|
|
market_intelligence=market_intelligence,
|
|
preflight_advisor=preflight_advisor,
|
|
require_official_config=official_registry is not None,
|
|
config_optimizer=config_optimizer,
|
|
excluded_frameworks=excluded_frameworks,
|
|
rejection_details=rejection_details,
|
|
)
|
|
if best is None:
|
|
reason = (
|
|
preflight_reason
|
|
or (
|
|
"no_publicly_vetted_compatible_framework"
|
|
if market_intelligence is not None
|
|
else "no_compatible_auto_template_or_framework"
|
|
)
|
|
)
|
|
skipped.append({
|
|
"repoId": model.repo_id, "targetGpu": target_gpu, "reason": reason,
|
|
**({"preflightRejections": rejection_details} if rejection_details else {}),
|
|
})
|
|
continue
|
|
record = candidate_to_record(best)
|
|
if excluded_frameworks:
|
|
record["previousFailedFrameworks"] = sorted(excluded_frameworks)
|
|
record["communitySuccessfulGpuCount"] = len(successful_gpus)
|
|
if successful_gpus:
|
|
record["communitySuccessfulGpus"] = sorted(successful_gpus)
|
|
if outcome_tracker is not None:
|
|
record["transformersPrerequisiteSatisfied"] = (
|
|
outcome_tracker.has_non_transformers_failure(model.repo_id)
|
|
)
|
|
if market_intelligence is not None:
|
|
record.update(market_intelligence.gpu_metadata(target_gpu))
|
|
record.update(market_intelligence.framework_metadata(best.task_type, target_gpu, best.framework))
|
|
optimization = (record.get("preflightMetadata") or {}).get("configOptimization") or {}
|
|
if optimization.get("applied"):
|
|
record["frameworkConfigSource"] = "official_plus_learned_patch"
|
|
candidates.append(record)
|
|
return len(candidates) - before_count
|
|
|
|
evaluate_pending_routes(pending_task_types_by_gpu)
|
|
# If preferred GPUs had routes but every one failed candidate preflight,
|
|
# give the remaining eligible GPUs one chance. Once a
|
|
# preferred candidate exists, lower-ranked GPUs stay untouched for this
|
|
# model and remain available for later scans.
|
|
if not candidates and not used_fallback_routes and fallback_target_gpus:
|
|
fallback_pending = prepare_pending_routes(fallback_target_gpus)
|
|
if fallback_pending:
|
|
evaluate_pending_routes(fallback_pending)
|
|
|
|
return candidates, skipped, failed
|
|
|
|
|
|
def build_adaptive_scan_stages(
|
|
*,
|
|
now,
|
|
initial_updated_after,
|
|
initial_limit: int,
|
|
max_models: int = ADAPTIVE_SCAN_MAX_MODELS,
|
|
allow_older_than_recent_window: bool = True,
|
|
recent_model_days: int = 7,
|
|
) -> list[dict[str, Any]]:
|
|
max_models = max(1, min(ADAPTIVE_SCAN_MAX_MODELS, int(max_models)))
|
|
initial_limit = min(max_models, max(1, int(initial_limit)))
|
|
recent_cutoff = now - timedelta(days=max(1, int(recent_model_days)))
|
|
if not allow_older_than_recent_window and (
|
|
initial_updated_after is None or initial_updated_after < recent_cutoff
|
|
):
|
|
initial_updated_after = recent_cutoff
|
|
|
|
stages: list[dict[str, Any]] = [
|
|
{
|
|
"name": "configured_window",
|
|
"updatedAfter": initial_updated_after,
|
|
"limit": initial_limit,
|
|
}
|
|
]
|
|
|
|
thirty_days_ago = now - timedelta(days=30)
|
|
if initial_updated_after is not None and initial_updated_after > recent_cutoff:
|
|
stages.append(
|
|
{
|
|
"name": "last_7_days",
|
|
"updatedAfter": recent_cutoff,
|
|
"limit": min(max_models, max(initial_limit, ADAPTIVE_SCAN_MIN_FALLBACK_MODELS)),
|
|
}
|
|
)
|
|
if allow_older_than_recent_window and initial_updated_after is not None and initial_updated_after > thirty_days_ago:
|
|
stages.append(
|
|
{
|
|
"name": "last_30_days",
|
|
"updatedAfter": thirty_days_ago,
|
|
"limit": min(max_models, max(initial_limit, 1500)),
|
|
}
|
|
)
|
|
if allow_older_than_recent_window and (initial_updated_after is not None or initial_limit < max_models):
|
|
stages.append(
|
|
{
|
|
"name": "all_history",
|
|
"updatedAfter": None,
|
|
"limit": max_models,
|
|
}
|
|
)
|
|
if not allow_older_than_recent_window:
|
|
# Reserved positions still need a deep search of eligible recent
|
|
# models. Widening the date window alone repeatedly scans the same
|
|
# short prefix (often already processed) on every poll cycle.
|
|
stages.append(
|
|
{
|
|
"name": "recent_window_full",
|
|
"updatedAfter": recent_cutoff,
|
|
"limit": max_models,
|
|
}
|
|
)
|
|
deduped: list[dict[str, Any]] = []
|
|
seen: set[tuple[str | None, int, str]] = set()
|
|
for stage in stages:
|
|
updated_after = stage["updatedAfter"]
|
|
signature = (
|
|
updated_after.isoformat() if updated_after is not None else None,
|
|
int(stage["limit"]),
|
|
str(stage.get("sort") or "last_modified"),
|
|
)
|
|
if signature in seen:
|
|
continue
|
|
seen.add(signature)
|
|
deduped.append(stage)
|
|
return deduped
|
|
|
|
|
|
def is_recent_model(last_modified: Any, *, reference_time, recent_model_days: int = 7) -> bool:
|
|
parsed = parse_datetime(last_modified)
|
|
if parsed is None:
|
|
return False
|
|
return parsed >= reference_time - timedelta(days=max(1, int(recent_model_days)))
|
|
|
|
|
|
def collect_candidates_from_models(
|
|
*,
|
|
models: list[HFModelSummary],
|
|
seen_model_ids: set[str],
|
|
candidate_goal: int,
|
|
hf_discovery: HuggingFaceDiscovery,
|
|
modelhub_client: ModelHubClient,
|
|
template_selector: TemplateSelector,
|
|
target_gpus: list[str],
|
|
selected_task_types: list[str],
|
|
outcome_tracker: OutcomeTracker | None,
|
|
submission_exclusion_store: SubmissionExclusionStore | None,
|
|
read_concurrency: int,
|
|
market_intelligence: MarketIntelligenceManager | None = None,
|
|
preflight_advisor: CandidatePreflightAdvisor | None = None,
|
|
official_registry: OfficialCapabilityRegistry | None = None,
|
|
config_optimizer: SafeConfigOptimizer | None = None,
|
|
allow_dynamic_tasks: bool = False,
|
|
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]], int]:
|
|
candidates: list[dict[str, Any]] = []
|
|
candidate_models: set[str] = set()
|
|
skipped: list[dict[str, Any]] = []
|
|
failed: list[dict[str, Any]] = []
|
|
processed_count = 0
|
|
workers = max(1, int(read_concurrency))
|
|
chunk_size = max(16, workers * 8)
|
|
new_models = [model for model in models if model.repo_id not in seen_model_ids]
|
|
|
|
for offset in range(0, len(new_models), chunk_size):
|
|
if len(candidate_models) >= candidate_goal:
|
|
break
|
|
chunk = new_models[offset : offset + chunk_size]
|
|
for model in chunk:
|
|
seen_model_ids.add(model.repo_id)
|
|
|
|
with ThreadPoolExecutor(max_workers=workers) as executor:
|
|
futures = {
|
|
executor.submit(
|
|
process_model_for_candidates,
|
|
model=model,
|
|
hf_discovery=hf_discovery,
|
|
modelhub_client=modelhub_client,
|
|
template_selector=template_selector,
|
|
target_gpus=target_gpus,
|
|
allowed_task_types=selected_task_types,
|
|
outcome_tracker=outcome_tracker,
|
|
submission_exclusion_store=submission_exclusion_store,
|
|
market_intelligence=market_intelligence,
|
|
preflight_advisor=preflight_advisor,
|
|
official_registry=official_registry,
|
|
config_optimizer=config_optimizer,
|
|
allow_dynamic_tasks=allow_dynamic_tasks,
|
|
): index
|
|
for index, model in enumerate(chunk)
|
|
}
|
|
ordered_results: dict[int, tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]] = {}
|
|
for future in as_completed(futures):
|
|
index = futures[future]
|
|
model = chunk[index]
|
|
try:
|
|
ordered_results[index] = future.result()
|
|
except OfficialCapabilityUnavailable:
|
|
raise
|
|
except Exception as exc:
|
|
ordered_results[index] = ([], [], [{"repoId": model.repo_id, "reason": str(exc)}])
|
|
|
|
for index in range(len(chunk)):
|
|
model_candidates, model_skipped, model_failed = ordered_results.get(index, ([], [], []))
|
|
candidates.extend(model_candidates)
|
|
if model_candidates:
|
|
model_id = str(model_candidates[0].get("repoId") or model_candidates[0].get("modelAddress") or "")
|
|
if model_id:
|
|
candidate_models.add(model_id)
|
|
skipped.extend(model_skipped)
|
|
failed.extend(model_failed)
|
|
processed_count += len(chunk)
|
|
|
|
return candidates, skipped, failed, processed_count
|
|
|
|
|
|
def submit_candidate(
|
|
candidate: dict[str, Any],
|
|
modelhub_client: ModelHubClient,
|
|
) -> dict[str, Any]:
|
|
if hasattr(modelhub_client, "model_submission_precheck"):
|
|
try:
|
|
precheck = modelhub_client.model_submission_precheck(candidate["repoId"], force_refresh=True)
|
|
except Exception as exc:
|
|
print(
|
|
f"[submit] deferred repo={candidate['repoId']} gpu={candidate['targetGpu']} "
|
|
f"framework={candidate['framework']} reason=community_precheck_unavailable",
|
|
flush=True,
|
|
)
|
|
return {
|
|
"outcome": "precheck_deferred",
|
|
"candidate": candidate,
|
|
"reason": f"community_precheck_error:{exc}",
|
|
}
|
|
if candidate["targetGpu"] in set(precheck.get("processedGpus") or set()):
|
|
return {
|
|
"outcome": "duplicate",
|
|
"candidate": candidate,
|
|
"reason": "already_processed_for_gpu",
|
|
}
|
|
if len(set(precheck.get("processedGpus") or set())) >= MAX_ROUTES_PER_MODEL:
|
|
return {
|
|
"outcome": "duplicate",
|
|
"candidate": candidate,
|
|
"reason": "model_route_limit_reached",
|
|
}
|
|
failed_frameworks = set(
|
|
(precheck.get("failedFrameworksByGpu") or {}).get(candidate["targetGpu"]) or set()
|
|
)
|
|
if failed_frameworks:
|
|
return {
|
|
"outcome": "duplicate",
|
|
"candidate": candidate,
|
|
"reason": "prior_failed_gpu_closed_accuracy_first",
|
|
}
|
|
|
|
payload = {
|
|
"modelAddress": candidate["modelAddress"],
|
|
"taskType": candidate["taskType"],
|
|
"targetGpu": candidate["targetGpu"],
|
|
"framework": candidate["framework"],
|
|
"configParams": candidate["configParams"],
|
|
}
|
|
strategy_id = os.getenv("STRATEGY_ID")
|
|
if strategy_id:
|
|
payload["strategyId"] = strategy_id
|
|
submit_time = utc_now()
|
|
try:
|
|
if hasattr(modelhub_client, "add_task_for_model"):
|
|
response = modelhub_client.add_task_for_model(
|
|
payload,
|
|
model_last_modified=candidate.get("lastModified"),
|
|
submitted_at=submit_time,
|
|
)
|
|
else:
|
|
response = modelhub_client.add_task(payload)
|
|
task_id = extract_task_id_from_submit_response(response)
|
|
if task_id is None:
|
|
task_id = modelhub_client.find_recent_task_id(candidate["repoId"], candidate["targetGpu"], submit_time)
|
|
return {
|
|
"outcome": "submitted",
|
|
"candidate": candidate,
|
|
"submitTime": submit_time.isoformat(),
|
|
"taskId": task_id,
|
|
"responseData": response.get("data"),
|
|
}
|
|
except ModelHubAPIError as exc:
|
|
if isinstance(exc, OldModelQueuePolicyError):
|
|
print(
|
|
f"[submit] deferred repo={candidate['repoId']} gpu={candidate['targetGpu']} "
|
|
f"framework={candidate['framework']} reason=age_policy_skipped",
|
|
flush=True,
|
|
)
|
|
return {
|
|
"outcome": "age_policy_deferred",
|
|
"candidate": candidate,
|
|
"reason": "age_policy_skipped",
|
|
}
|
|
if is_framework_prerequisite_error(exc):
|
|
print(
|
|
f"[submit] deferred repo={candidate['repoId']} gpu={candidate['targetGpu']} "
|
|
f"framework={candidate['framework']} reason=framework_prerequisite",
|
|
flush=True,
|
|
)
|
|
return {
|
|
"outcome": "framework_prerequisite_deferred",
|
|
"candidate": candidate,
|
|
"reason": "framework_prerequisite_deferred",
|
|
}
|
|
if is_model_uniqueness_error(exc):
|
|
print(
|
|
f"[submit] skipped repo={candidate['repoId']} gpu={candidate['targetGpu']} "
|
|
f"framework={candidate['framework']} reason=model_uniqueness_rejected",
|
|
flush=True,
|
|
)
|
|
return {
|
|
"outcome": "uniqueness_rejected",
|
|
"candidate": candidate,
|
|
"reason": str(exc),
|
|
}
|
|
if is_duplicate_submission_error(exc):
|
|
print(
|
|
f"[submit] skipped repo={candidate['repoId']} gpu={candidate['targetGpu']} "
|
|
f"framework={candidate['framework']} reason=already_validating",
|
|
flush=True,
|
|
)
|
|
return {
|
|
"outcome": "duplicate",
|
|
"candidate": candidate,
|
|
"reason": str(exc),
|
|
}
|
|
if is_capacity_error(exc):
|
|
print(
|
|
f"[submit] deferred repo={candidate['repoId']} gpu={candidate['targetGpu']} "
|
|
f"framework={candidate['framework']} reason=account_capacity_saturated",
|
|
flush=True,
|
|
)
|
|
return {
|
|
"outcome": "capacity_deferred",
|
|
"candidate": candidate,
|
|
"reason": "account_capacity_saturated",
|
|
}
|
|
if exc.code == 429:
|
|
print(
|
|
f"[submit] deferred repo={candidate['repoId']} gpu={candidate['targetGpu']} "
|
|
f"framework={candidate['framework']} reason=platform_rate_limited",
|
|
flush=True,
|
|
)
|
|
return {
|
|
"outcome": "rate_limit_deferred",
|
|
"candidate": candidate,
|
|
"reason": "platform_rate_limited",
|
|
}
|
|
print(
|
|
f"[submit] failed repo={candidate['repoId']} gpu={candidate['targetGpu']} "
|
|
f"framework={candidate['framework']} reason={exc}",
|
|
flush=True,
|
|
)
|
|
return {
|
|
"outcome": "failed",
|
|
"candidate": candidate,
|
|
"reason": str(exc),
|
|
}
|
|
|
|
|
|
def make_run_dir(runs_dir: Path, now) -> Path:
|
|
runs_dir.mkdir(parents=True, exist_ok=True)
|
|
base_name = now.strftime("%Y%m%dT%H%M%SZ")
|
|
for suffix in range(0, 1000):
|
|
candidate = runs_dir / (base_name if suffix == 0 else f"{base_name}.{suffix:03d}")
|
|
try:
|
|
candidate.mkdir(exist_ok=False)
|
|
return candidate
|
|
except FileExistsError:
|
|
continue
|
|
raise RuntimeError(f"Unable to allocate a unique run directory under {runs_dir}")
|
|
|
|
|
|
def one_candidate_per_model(candidates: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
selected: list[dict[str, Any]] = []
|
|
seen_models: set[str] = set()
|
|
for candidate in candidates:
|
|
model_id = str(candidate.get("repoId") or candidate.get("modelAddress") or "")
|
|
if not model_id or model_id in seen_models:
|
|
continue
|
|
seen_models.add(model_id)
|
|
selected.append(candidate)
|
|
return selected
|
|
|
|
|
|
def limit_routes_per_model(
|
|
candidates: list[dict[str, Any]],
|
|
*,
|
|
max_routes: int = MAX_ROUTES_PER_MODEL,
|
|
) -> list[dict[str, Any]]:
|
|
selected: list[dict[str, Any]] = []
|
|
counts: Counter[str] = Counter()
|
|
for candidate in candidates:
|
|
model_id = str(candidate.get("repoId") or candidate.get("modelAddress") or "")
|
|
if not model_id or counts[model_id] >= max(1, int(max_routes)):
|
|
continue
|
|
counts[model_id] += 1
|
|
selected.append(candidate)
|
|
return selected
|
|
|
|
|
|
def candidate_model_count(candidates: list[dict[str, Any]]) -> int:
|
|
return len(
|
|
{
|
|
str(candidate.get("repoId") or candidate.get("modelAddress") or "")
|
|
for candidate in candidates
|
|
if candidate.get("repoId") or candidate.get("modelAddress")
|
|
}
|
|
)
|
|
|
|
|
|
def limit_candidate_models(candidates: list[dict[str, Any]], *, max_models: int) -> list[dict[str, Any]]:
|
|
selected: list[dict[str, Any]] = []
|
|
models: set[str] = set()
|
|
for candidate in candidates:
|
|
model_id = str(candidate.get("repoId") or candidate.get("modelAddress") or "")
|
|
if not model_id:
|
|
continue
|
|
if model_id not in models and len(models) >= max(0, int(max_models)):
|
|
continue
|
|
models.add(model_id)
|
|
selected.append(candidate)
|
|
return selected
|
|
|
|
|
|
def run_submission(
|
|
args: argparse.Namespace,
|
|
*,
|
|
now=None,
|
|
hf_discovery: HuggingFaceDiscovery | None = None,
|
|
modelhub_client: ModelHubClient | ModelHubClientPool | None = None,
|
|
template_selector: TemplateSelector | None = None,
|
|
outcome_tracker: OutcomeTracker | None = None,
|
|
) -> dict[str, Any]:
|
|
now = now or utc_now()
|
|
template_selector = template_selector or TemplateSelector()
|
|
selected_task_types = resolve_task_types(args)
|
|
configured_target_gpus = resolve_target_gpus(args, template_selector, selected_task_types)
|
|
if not configured_target_gpus:
|
|
raise RuntimeError("No auto-submittable GPUs are available for the selected task types")
|
|
|
|
hf_discovery = hf_discovery or HuggingFaceDiscovery(base_url=args.hf_base_url)
|
|
if modelhub_client is None:
|
|
modelhub_tokens = list(getattr(args, "modelhub_tokens", None) or ([] if not args.modelhub_token else [args.modelhub_token]))
|
|
token_values: list[str | None] = modelhub_tokens or [args.modelhub_token]
|
|
clients = [ModelHubClient(token=token, base_url=args.modelhub_base_url) for token in token_values]
|
|
modelhub_client = ModelHubClientPool(
|
|
clients,
|
|
capacity_probe_interval_cycles=0,
|
|
capacity_state_path=Path(getattr(args, "capacity_state_path", DEFAULT_CAPACITY_STATE_PATH)),
|
|
recent_model_reserve_slots=max(0, int(getattr(args, "recent_model_reserve_slots", 5) or 0)),
|
|
recent_model_days=max(1, int(getattr(args, "recent_model_days", 7) or 7)),
|
|
)
|
|
modelhub_client.configure_weekly_calibration(now)
|
|
|
|
official_registry: OfficialCapabilityRegistry | None = None
|
|
official_summary: dict[str, Any] = {"enabled": False}
|
|
official_capable = all(
|
|
hasattr(modelhub_client, name)
|
|
for name in ("list_machine_info", "list_task_levels", "list_model_task_types")
|
|
)
|
|
if official_capable:
|
|
official_registry = OfficialCapabilityRegistry(
|
|
Path(
|
|
getattr(
|
|
args,
|
|
"official_capabilities_path",
|
|
DEFAULT_OFFICIAL_CAPABILITIES_PATH,
|
|
)
|
|
)
|
|
)
|
|
official_registry.prepare(
|
|
modelhub_client,
|
|
fallback_gpus=configured_target_gpus,
|
|
task_types=selected_task_types,
|
|
now=now,
|
|
)
|
|
if not official_registry.ready:
|
|
raise OfficialCapabilityUnavailable(
|
|
official_registry.pause_reason or "critical_official_signal_unavailable"
|
|
)
|
|
requested_gpus = configured_target_gpus if (getattr(args, "gpu", None) or getattr(args, "gpus", None)) else None
|
|
target_gpus = official_registry.eligible_gpus(requested_gpus)
|
|
official_summary = official_registry.summary()
|
|
else:
|
|
# Compatibility path for injected test clients and offline dry-runs.
|
|
target_gpus = list(configured_target_gpus)
|
|
if not target_gpus:
|
|
raise OfficialCapabilityUnavailable("official catalog contains no eligible GPU")
|
|
submission_target_gpus = list(target_gpus)
|
|
|
|
if hasattr(modelhub_client, "begin_cycle"):
|
|
modelhub_client.begin_cycle()
|
|
|
|
runs_dir = Path(args.runs_dir)
|
|
ledger_path = Path(args.ledger_path)
|
|
history_archive_path = Path(args.history_archive_path)
|
|
run_dir = make_run_dir(runs_dir, now)
|
|
ledger_path.parent.mkdir(parents=True, exist_ok=True)
|
|
history_archive_path.parent.mkdir(parents=True, exist_ok=True)
|
|
ledger_entries = load_ledger(ledger_path)
|
|
|
|
outcome_tracker = outcome_tracker or OutcomeTracker(Path(args.outcomes_path))
|
|
# Online decisions are deliberately deterministic. The optional classifier
|
|
# module remains available for offline, human-reviewed analysis only.
|
|
outcome_tracker.set_failure_llm_classifier(None)
|
|
submission_exclusion_store = SubmissionExclusionStore(
|
|
Path(getattr(args, "submission_exclusions_path", DEFAULT_SUBMISSION_EXCLUSIONS_PATH))
|
|
)
|
|
preflight_advisor: CandidatePreflightAdvisor | None = None
|
|
disable_preflight = bool(getattr(args, "disable_candidate_preflight", False)) or os.getenv(
|
|
"MODELHUB_DISABLE_CANDIDATE_PREFLIGHT", ""
|
|
).strip().lower() in {"1", "true", "yes"}
|
|
if not disable_preflight:
|
|
preflight_advisor = CandidatePreflightAdvisor(llm_classifier=None)
|
|
preflight_summary: dict[str, Any] = (
|
|
preflight_advisor.summary() if preflight_advisor is not None else {"enabled": False}
|
|
)
|
|
config_optimizer = SafeConfigOptimizer(
|
|
intents_path=Path(".modelhub_state/recovery_intents.jsonl"),
|
|
outcomes_path=Path(args.outcomes_path),
|
|
)
|
|
strategy_manager: SuccessFirstRoutingEngine | None = None
|
|
strategy_summary: dict[str, Any] = {"enabled": False}
|
|
market_intelligence: MarketIntelligenceManager | None = None
|
|
market_summary: dict[str, Any] = {"enabled": False}
|
|
strategy_enabled = (
|
|
not bool(getattr(args, "disable_gpu_strategy", False))
|
|
and not bool(getattr(args, "gpu", None) or getattr(args, "gpus", None))
|
|
and len(target_gpus) > 1
|
|
)
|
|
synced_count = 0
|
|
if not getattr(args, "skip_outcome_sync", False):
|
|
try:
|
|
task_contexts = outcome_tracker.get_task_compatibility_contexts()
|
|
for entry in ledger_entries:
|
|
task_id = str(entry.get("taskId") or "")
|
|
if not task_id:
|
|
continue
|
|
context = task_contexts.setdefault(task_id, {})
|
|
for field, source in (
|
|
("modelId", "modelId"),
|
|
("targetGpu", "targetGpu"),
|
|
("framework", "framework"),
|
|
("taskType", "taskType"),
|
|
("submitTime", "submitTime"),
|
|
):
|
|
if not context.get(field) and entry.get(source):
|
|
context[field] = entry.get(source)
|
|
synced_count = outcome_tracker.sync_from_api(
|
|
modelhub_client,
|
|
task_contexts=task_contexts,
|
|
)
|
|
except Exception:
|
|
pass
|
|
if preflight_advisor is not None:
|
|
try:
|
|
preflight_advisor.set_feedback_stats(outcome_tracker.get_stats_report())
|
|
except Exception:
|
|
preflight_advisor.set_feedback_stats(None)
|
|
|
|
updated_after = determine_updated_after(args, now)
|
|
day_start = now.replace(hour=0, minute=0, second=0, microsecond=0)
|
|
# Count today's submissions from the local ledger (avoids expensive paginated API call)
|
|
daily_snapshot = count_submissions_for_day(tasks=[], ledger_entries=ledger_entries, day_start=day_start, day_end=now)
|
|
# A configured daily target must include every pooled account and every
|
|
# concurrent worker, not only the first token's first page.
|
|
if args.daily_target > 0:
|
|
list_kwargs: dict[str, Any] = {
|
|
"page_size": 100,
|
|
"only_mine": True,
|
|
"begin_time": day_start,
|
|
"end_time": now,
|
|
}
|
|
if isinstance(modelhub_client, ModelHubClientPool):
|
|
list_kwargs["_fanout_all"] = True
|
|
today_tasks = modelhub_client.list_tasks(**list_kwargs)
|
|
daily_snapshot = count_submissions_for_day(
|
|
tasks=today_tasks,
|
|
ledger_entries=ledger_entries,
|
|
day_start=day_start,
|
|
day_end=now,
|
|
)
|
|
# In unlimited mode, keep the inexpensive one-page fallback for an empty ledger.
|
|
elif daily_snapshot["totalCount"] <= 0:
|
|
today_tasks_page = modelhub_client.list_tasks_page(
|
|
current=1,
|
|
page_size=20,
|
|
only_mine=True,
|
|
begin_time=day_start,
|
|
end_time=now,
|
|
)
|
|
today_tasks = (today_tasks_page.get("data") or {}).get("records") or []
|
|
daily_snapshot = count_submissions_for_day(tasks=today_tasks, ledger_entries=ledger_entries, day_start=day_start, day_end=now)
|
|
platform_available_slots = None
|
|
if hasattr(modelhub_client, "available_submit_slots"):
|
|
platform_available_slots = max(0, int(modelhub_client.available_submit_slots()))
|
|
unlimited_daily_target = args.daily_target <= 0
|
|
if unlimited_daily_target:
|
|
remaining_daily_quota = platform_available_slots if platform_available_slots is not None else 1_000_000_000
|
|
else:
|
|
remaining_daily_quota = max(0, args.daily_target - daily_snapshot["totalCount"])
|
|
if platform_available_slots is not None:
|
|
remaining_daily_quota = min(remaining_daily_quota, platform_available_slots)
|
|
|
|
history_report_reason = "decision_state_only"
|
|
if args.daily_target > 0 and remaining_daily_quota <= 0:
|
|
archived_history: list[dict[str, Any]] = []
|
|
report = build_empty_pre_submit_report(
|
|
window_days=args.stats_window_days,
|
|
generated_at=now,
|
|
reason="daily_target_reached_before_scan",
|
|
ledger_entries=len(ledger_entries),
|
|
)
|
|
write_json(run_dir / "pre_submit_report.json", report)
|
|
return {
|
|
"generatedAt": now.isoformat(),
|
|
"dryRun": bool(args.dry_run),
|
|
"selectedTaskTypes": selected_task_types,
|
|
"targetGpus": target_gpus,
|
|
"submissionEligibleGpus": submission_target_gpus,
|
|
"dailyTarget": args.daily_target,
|
|
"unlimitedDailyTarget": unlimited_daily_target,
|
|
"submittedTodayBeforeRun": daily_snapshot["totalCount"],
|
|
"remainingDailyQuotaBeforeRun": remaining_daily_quota,
|
|
"platformAvailableSlotsBeforeRun": platform_available_slots,
|
|
"historyStatsEnabled": False,
|
|
"historyStatsThreshold": getattr(args, "history_stats_threshold", 500),
|
|
"historyArchivePath": str(history_archive_path),
|
|
"historyArchiveRecordCount": len(archived_history),
|
|
"gpuStrategy": strategy_summary,
|
|
"marketIntelligence": market_summary,
|
|
"candidatePreflight": preflight_summary,
|
|
"scanLimit": 0,
|
|
"candidateGoal": 0,
|
|
"scanStages": [],
|
|
"scannedModels": 0,
|
|
"candidateCount": 0,
|
|
"candidateFrameworkCounts": {},
|
|
"targetSubmitCount": 0,
|
|
"maxSubmitAttempts": 0,
|
|
"plannedSubmitCount": 0,
|
|
"submittedCount": 0,
|
|
"submittedGpuCounts": {},
|
|
"submittedFrameworkCounts": {},
|
|
"duplicateCount": 0,
|
|
"modelGpuUniquenessRejectedCount": 0,
|
|
"skippedCount": 0,
|
|
"skipReasonCounts": {},
|
|
"failedCount": 0,
|
|
"failureReasonCounts": {},
|
|
"submissionExclusionsPath": str(submission_exclusion_store.path),
|
|
"warnings": report.get("warnings", []),
|
|
"runDir": str(run_dir),
|
|
"outcomeSyncCount": synced_count,
|
|
}
|
|
|
|
market_capable = (
|
|
hasattr(modelhub_client, "list_machine_info")
|
|
and hasattr(modelhub_client, "list_framework_stats")
|
|
and hasattr(modelhub_client, "list_tasks_page")
|
|
)
|
|
if not bool(getattr(args, "disable_market_intelligence", False)) and market_capable:
|
|
market_intelligence = MarketIntelligenceManager(
|
|
Path(getattr(args, "market_intelligence_state_path", DEFAULT_MARKET_INTELLIGENCE_PATH)),
|
|
queue_refresh_seconds=max(
|
|
60,
|
|
int(getattr(args, "market_queue_refresh_seconds", DEFAULT_QUEUE_REFRESH_SECONDS) or DEFAULT_QUEUE_REFRESH_SECONDS),
|
|
),
|
|
framework_refresh_seconds=max(
|
|
300,
|
|
int(
|
|
getattr(args, "market_framework_refresh_seconds", DEFAULT_FRAMEWORK_REFRESH_SECONDS)
|
|
or DEFAULT_FRAMEWORK_REFRESH_SECONDS
|
|
),
|
|
),
|
|
throughput_window_hours=max(
|
|
1,
|
|
int(
|
|
getattr(args, "market_throughput_window_hours", DEFAULT_THROUGHPUT_WINDOW_HOURS)
|
|
or DEFAULT_THROUGHPUT_WINDOW_HOURS
|
|
),
|
|
),
|
|
fetch_workers=max(
|
|
1,
|
|
int(getattr(args, "market_fetch_workers", DEFAULT_FETCH_WORKERS) or DEFAULT_FETCH_WORKERS),
|
|
),
|
|
framework_min_samples=max(
|
|
1,
|
|
int(
|
|
getattr(args, "market_framework_min_samples", DEFAULT_FRAMEWORK_MIN_SAMPLES)
|
|
or DEFAULT_FRAMEWORK_MIN_SAMPLES
|
|
),
|
|
),
|
|
)
|
|
market_intelligence.prepare(
|
|
modelhub_client,
|
|
supported_gpus=target_gpus,
|
|
task_types=selected_task_types,
|
|
now=now,
|
|
)
|
|
try:
|
|
market_intelligence.set_local_outcome_stats(outcome_tracker.get_stats_report())
|
|
except Exception:
|
|
market_intelligence.set_local_outcome_stats(None)
|
|
market_summary = market_intelligence.summary()
|
|
framework_state = (market_intelligence.state or {}).get("frameworkStats") or {}
|
|
framework_updated_at = parse_datetime(market_summary.get("frameworkUpdatedAt"))
|
|
framework_cache_expired = (
|
|
framework_updated_at is None
|
|
or now - framework_updated_at > timedelta(hours=24)
|
|
)
|
|
if market_summary.get("frameworkError") and (not framework_state or framework_cache_expired):
|
|
raise OfficialCapabilityUnavailable("critical official framework catalog is unavailable")
|
|
submission_target_gpus = market_intelligence.eligible_gpus(target_gpus)
|
|
market_summary["eligibleGpus"] = submission_target_gpus
|
|
market_summary["shadowOnlyGpus"] = [gpu for gpu in target_gpus if gpu not in submission_target_gpus]
|
|
print(
|
|
f"[market] eligible_gpus={','.join(submission_target_gpus) or 'none'} "
|
|
f"shadow_only={','.join(market_summary['shadowOnlyGpus']) or 'none'}",
|
|
flush=True,
|
|
)
|
|
|
|
if strategy_enabled:
|
|
routing_path = getattr(args, "routing_state_path", None)
|
|
legacy_strategy_path = getattr(args, "gpu_strategy_state_path", None)
|
|
if (
|
|
not routing_path
|
|
or (
|
|
str(routing_path) == str(DEFAULT_ROUTING_STATE_PATH)
|
|
and legacy_strategy_path
|
|
and str(legacy_strategy_path) != str(DEFAULT_GPU_STRATEGY_PATH)
|
|
)
|
|
):
|
|
routing_path = legacy_strategy_path or DEFAULT_ROUTING_STATE_PATH
|
|
strategy_manager = SuccessFirstRoutingEngine(
|
|
Path(routing_path),
|
|
outcome_stats=outcome_tracker.get_stats_report(),
|
|
)
|
|
strategy_summary = strategy_manager.summary()
|
|
|
|
archived_history = []
|
|
|
|
report = build_empty_pre_submit_report(
|
|
window_days=args.stats_window_days,
|
|
generated_at=now,
|
|
reason=history_report_reason,
|
|
ledger_entries=len(ledger_entries),
|
|
)
|
|
write_json(run_dir / "pre_submit_report.json", report)
|
|
|
|
scan_limit = resolve_scan_limit(
|
|
args,
|
|
target_gpu_count=len(submission_target_gpus),
|
|
remaining_daily_quota=remaining_daily_quota,
|
|
platform_available_slots=platform_available_slots,
|
|
)
|
|
|
|
candidates: list[dict[str, Any]] = []
|
|
skipped: list[dict[str, Any]] = []
|
|
failed: list[dict[str, Any]] = []
|
|
scan_stages: list[dict[str, Any]] = []
|
|
seen_model_ids: set[str] = set()
|
|
scanned_model_count = 0
|
|
explicit_submit_limit = max(0, int(getattr(args, "max_submits_per_run", 0) or 0))
|
|
if platform_available_slots is None and unlimited_daily_target:
|
|
submission_goal = scan_limit
|
|
else:
|
|
submission_goal = max(0, int(remaining_daily_quota))
|
|
if explicit_submit_limit > 0:
|
|
submission_goal = min(submission_goal, explicit_submit_limit)
|
|
if strategy_manager is not None:
|
|
submission_goal = min(submission_goal, strategy_manager.submissions_until_refresh)
|
|
attempt_multiplier = max(1, int(getattr(args, "scan_multiplier", 4) or 1))
|
|
raw_candidate_goal = max(submission_goal, submission_goal * attempt_multiplier)
|
|
candidate_pool_limit = max(20, int(getattr(args, "candidate_pool_limit", 100) or 100))
|
|
candidate_goal = min(raw_candidate_goal, candidate_pool_limit)
|
|
print(
|
|
f"[scan] candidate_pool_limit={candidate_pool_limit} "
|
|
f"raw_goal={raw_candidate_goal} effective_goal={candidate_goal}",
|
|
flush=True,
|
|
)
|
|
|
|
pipeline_tags = pipeline_tags_for_task_types(selected_task_types)
|
|
if official_registry is not None and not getattr(args, "task_types", None):
|
|
for task_name in official_registry.state.get("discoveredTaskTypes") or []:
|
|
normalized = str(task_name or "").strip()
|
|
if normalized and normalized not in pipeline_tags:
|
|
pipeline_tags.append(normalized)
|
|
explicit_scan_cap = max(0, int(getattr(args, "max_scan_models", 0) or 0))
|
|
recent_model_days = max(1, int(getattr(args, "recent_model_days", 7) or 7))
|
|
recent_model_reserve_slots = max(0, int(getattr(args, "recent_model_reserve_slots", 5) or 0))
|
|
old_model_slots_before_scan: int | None = None
|
|
if hasattr(modelhub_client, "old_model_submit_slots"):
|
|
old_model_slots_before_scan = int(modelhub_client.old_model_submit_slots())
|
|
old_model_queue_thresholds: list[int | None] | None = None
|
|
if hasattr(modelhub_client, "old_model_queue_thresholds"):
|
|
old_model_queue_thresholds = list(modelhub_client.old_model_queue_thresholds())
|
|
weekly_calibration = (
|
|
modelhub_client.weekly_calibration_status()
|
|
if hasattr(modelhub_client, "weekly_calibration_status") else {}
|
|
)
|
|
allow_older_models_for_scan = old_model_slots_before_scan is None or old_model_slots_before_scan > 0
|
|
print(
|
|
f"[age-policy] mode={'weekly_capacity_calibration' if weekly_calibration.get('targetAccount') else 'admission_only'} "
|
|
f"recent_days={recent_model_days} "
|
|
f"reserve_recent_slots={recent_model_reserve_slots} "
|
|
f"calibration_account={weekly_calibration.get('targetAccount') or 'n/a'} "
|
|
f"account_thresholds={','.join('n/a' if value is None else str(value) for value in old_model_queue_thresholds) if old_model_queue_thresholds is not None else 'n/a'} "
|
|
f"old_model_slots={old_model_slots_before_scan if old_model_slots_before_scan is not None else 'n/a'} "
|
|
f"scan_older={'on' if allow_older_models_for_scan else 'off'}",
|
|
flush=True,
|
|
)
|
|
for stage in build_adaptive_scan_stages(
|
|
now=now,
|
|
initial_updated_after=updated_after,
|
|
initial_limit=scan_limit,
|
|
max_models=explicit_scan_cap or ADAPTIVE_SCAN_MAX_MODELS,
|
|
allow_older_than_recent_window=allow_older_models_for_scan,
|
|
recent_model_days=recent_model_days,
|
|
):
|
|
current_candidate_models = candidate_model_count(candidates)
|
|
if candidate_goal <= 0 or current_candidate_models >= candidate_goal:
|
|
break
|
|
skipped_before_stage = len(skipped)
|
|
stage_updated_after = stage["updatedAfter"]
|
|
stage_limit = int(stage["limit"])
|
|
query_kwargs = {
|
|
"pipeline_tags": pipeline_tags,
|
|
"limit": stage_limit,
|
|
"min_downloads": args.min_downloads,
|
|
"updated_after": stage_updated_after,
|
|
}
|
|
if stage.get("sort"):
|
|
query_kwargs["sort"] = stage["sort"]
|
|
if stage.get("rotatePages"):
|
|
query_kwargs["rotate_pages"] = True
|
|
try:
|
|
models = hf_discovery.list_recent_models(
|
|
**query_kwargs,
|
|
read_concurrency=max(1, args.read_concurrency),
|
|
)
|
|
except TypeError:
|
|
models = hf_discovery.list_recent_models(**query_kwargs)
|
|
if not allow_older_models_for_scan:
|
|
recent_models: list[HFModelSummary] = []
|
|
for model in models:
|
|
if is_recent_model(
|
|
model.last_modified,
|
|
reference_time=now,
|
|
recent_model_days=recent_model_days,
|
|
):
|
|
recent_models.append(model)
|
|
continue
|
|
if model.repo_id in seen_model_ids:
|
|
continue
|
|
seen_model_ids.add(model.repo_id)
|
|
skipped.append(
|
|
{
|
|
"repoId": model.repo_id,
|
|
"reason": "age_policy_skipped",
|
|
"deferred": True,
|
|
}
|
|
)
|
|
models = recent_models
|
|
|
|
stage_candidates, stage_skipped, stage_failed, processed_count = collect_candidates_from_models(
|
|
models=models,
|
|
seen_model_ids=seen_model_ids,
|
|
candidate_goal=max(1, candidate_goal - current_candidate_models),
|
|
hf_discovery=hf_discovery,
|
|
modelhub_client=modelhub_client,
|
|
template_selector=template_selector,
|
|
target_gpus=submission_target_gpus,
|
|
selected_task_types=selected_task_types,
|
|
outcome_tracker=outcome_tracker,
|
|
submission_exclusion_store=submission_exclusion_store,
|
|
read_concurrency=max(1, args.read_concurrency),
|
|
market_intelligence=market_intelligence,
|
|
preflight_advisor=preflight_advisor,
|
|
official_registry=official_registry,
|
|
config_optimizer=config_optimizer,
|
|
allow_dynamic_tasks=not bool(getattr(args, "task_types", None)),
|
|
)
|
|
stage_candidates = limit_routes_per_model(stage_candidates, max_routes=MAX_ROUTES_PER_MODEL)
|
|
remaining_candidate_capacity = max(0, candidate_goal - current_candidate_models)
|
|
stage_candidates = limit_candidate_models(
|
|
stage_candidates,
|
|
max_models=remaining_candidate_capacity,
|
|
)
|
|
candidates.extend(stage_candidates)
|
|
skipped.extend(stage_skipped)
|
|
failed.extend(stage_failed)
|
|
scanned_model_count += processed_count
|
|
stage_summary = {
|
|
"name": stage["name"],
|
|
"sort": stage.get("sort") or "last_modified",
|
|
"updatedAfter": stage_updated_after.isoformat() if stage_updated_after is not None else None,
|
|
"requestedLimit": stage_limit,
|
|
"discoveredModels": len(models),
|
|
"newModelsProcessed": processed_count,
|
|
"candidatesAdded": len(stage_candidates),
|
|
"candidateCountAfterStage": len(candidates),
|
|
"candidateModelsAfterStage": candidate_model_count(candidates),
|
|
"skippedAdded": len(skipped) - skipped_before_stage,
|
|
"failedAdded": len(stage_failed),
|
|
"discoveryQueries": list(getattr(hf_discovery, "last_listing_diagnostics", [])),
|
|
}
|
|
scan_stages.append(stage_summary)
|
|
print(
|
|
f"[scan] stage={stage['name']} discovered={len(models)} new_processed={processed_count} "
|
|
f"routes_added={len(stage_candidates)} routes_total={len(candidates)} "
|
|
f"candidate_models={candidate_model_count(candidates)}/{candidate_goal} "
|
|
f"skipped_added={len(skipped) - skipped_before_stage}",
|
|
flush=True,
|
|
)
|
|
|
|
submitted: list[dict[str, Any]] = []
|
|
duplicate_candidates: list[dict[str, Any]] = []
|
|
uniqueness_rejected_candidates: list[dict[str, Any]] = []
|
|
target_submit_count = resolve_max_submit_count(
|
|
args=args,
|
|
planned_count=candidate_model_count(candidates),
|
|
remaining_daily_quota=remaining_daily_quota,
|
|
)
|
|
if strategy_manager is not None:
|
|
target_submit_count = min(target_submit_count, strategy_manager.submissions_until_refresh)
|
|
instance_id = runtime_instance_id()
|
|
diversified_candidates = diversify_candidates(candidates, instance_id=instance_id)
|
|
if strategy_manager is not None:
|
|
diversified_candidates = strategy_manager.order_candidates(diversified_candidates)
|
|
write_jsonl(run_dir / "candidates.jsonl", diversified_candidates)
|
|
claim_store: SubmissionClaimStore | None = None
|
|
attempted_candidates: list[dict[str, Any]] = []
|
|
submit_workers = 1
|
|
state_sync = getattr(args, "_state_sync_manager", None)
|
|
state_sync_paused = False
|
|
capacity_saturated = False
|
|
|
|
if args.dry_run:
|
|
attempted_candidates = diversified_candidates[:target_submit_count]
|
|
else:
|
|
claim_store = SubmissionClaimStore(
|
|
Path(getattr(args, "claims_path", DEFAULT_CLAIMS_PATH)),
|
|
owner_id=instance_id,
|
|
)
|
|
attempted_keys: set[str] = set()
|
|
attempt_multiplier = max(1, int(getattr(args, "scan_multiplier", 4) or 1))
|
|
max_submit_attempts = min(
|
|
len(diversified_candidates),
|
|
max(target_submit_count, target_submit_count * attempt_multiplier),
|
|
)
|
|
|
|
# A batch can contain candidates another machine has already submitted.
|
|
# Keep those duplicate claims and immediately draw replacements from the
|
|
# already-scanned pool until the desired number of real submissions is
|
|
# reached or account capacity is genuinely exhausted.
|
|
while len(submitted) < target_submit_count and len(attempted_candidates) < max_submit_attempts:
|
|
remaining_candidates = [
|
|
candidate
|
|
for candidate in diversified_candidates
|
|
if candidate_key(candidate) not in attempted_keys
|
|
and not submission_exclusion_store.is_blocked(candidate["repoId"], candidate["targetGpu"])
|
|
]
|
|
remaining_candidates = one_candidate_per_model(remaining_candidates)
|
|
desired_count = min(
|
|
target_submit_count - len(submitted),
|
|
max_submit_attempts - len(attempted_candidates),
|
|
)
|
|
if state_sync is not None:
|
|
desired_count = min(desired_count, int(state_sync.batch_size))
|
|
batch_candidates = claim_store.claim(remaining_candidates, limit=desired_count)
|
|
if not batch_candidates:
|
|
break
|
|
attempted_candidates.extend(batch_candidates)
|
|
attempted_keys.update(candidate_key(candidate) for candidate in batch_candidates)
|
|
|
|
state_batch_id: str | None = None
|
|
if state_sync is not None:
|
|
state_batch_id = state_sync.begin_batch(batch_candidates)
|
|
if state_batch_id is None:
|
|
claim_store.release(batch_candidates)
|
|
state_sync_paused = True
|
|
print("[cycle] paused reason=state_sync_unhealthy phase=intent", flush=True)
|
|
break
|
|
|
|
batch_workers = resolve_submit_concurrency(
|
|
args,
|
|
modelhub_client=modelhub_client,
|
|
planned_submit_count=len(batch_candidates),
|
|
)
|
|
submit_workers = max(submit_workers, batch_workers)
|
|
with ThreadPoolExecutor(max_workers=batch_workers) as executor:
|
|
futures = {
|
|
executor.submit(submit_candidate, candidate, modelhub_client): index
|
|
for index, candidate in enumerate(batch_candidates)
|
|
}
|
|
ordered_results: dict[int, dict[str, Any]] = {}
|
|
for future in as_completed(futures):
|
|
index = futures[future]
|
|
try:
|
|
ordered_results[index] = future.result()
|
|
except Exception as exc:
|
|
candidate = batch_candidates[index]
|
|
ordered_results[index] = {
|
|
"outcome": "failed",
|
|
"candidate": candidate,
|
|
"reason": str(exc),
|
|
}
|
|
|
|
batch_submitted_candidates: list[dict[str, Any]] = []
|
|
batch_duplicate_candidates: list[dict[str, Any]] = []
|
|
batch_uniqueness_rejected_candidates: list[dict[str, Any]] = []
|
|
batch_policy_skipped_candidates: list[dict[str, Any]] = []
|
|
batch_capacity_deferred_candidates: list[dict[str, Any]] = []
|
|
batch_failed_candidates: list[dict[str, Any]] = []
|
|
for index in range(len(batch_candidates)):
|
|
result = ordered_results.get(index)
|
|
if result is None:
|
|
continue
|
|
candidate = result["candidate"]
|
|
if result["outcome"] == "uniqueness_rejected":
|
|
duplicate_candidates.append(candidate)
|
|
uniqueness_rejected_candidates.append(candidate)
|
|
batch_uniqueness_rejected_candidates.append(candidate)
|
|
submission_exclusion_store.block(
|
|
candidate["repoId"],
|
|
candidate["targetGpu"],
|
|
reason=result.get("reason", "model_uniqueness_rejected"),
|
|
)
|
|
skipped.append(
|
|
{
|
|
"repoId": candidate["repoId"],
|
|
"targetGpu": candidate["targetGpu"],
|
|
"reason": "model_uniqueness_rejected",
|
|
}
|
|
)
|
|
continue
|
|
if result["outcome"] == "duplicate":
|
|
duplicate_candidates.append(candidate)
|
|
batch_duplicate_candidates.append(candidate)
|
|
skipped.append(
|
|
{
|
|
"repoId": candidate["repoId"],
|
|
"targetGpu": candidate["targetGpu"],
|
|
"reason": result.get("reason") or "already_validating_on_platform",
|
|
}
|
|
)
|
|
continue
|
|
if result["outcome"] in {"framework_evidence_deferred", "rate_limit_deferred"}:
|
|
batch_policy_skipped_candidates.append(candidate)
|
|
skipped.append({
|
|
"repoId": candidate["repoId"],
|
|
"targetGpu": candidate["targetGpu"],
|
|
"framework": candidate["framework"],
|
|
"reason": result["reason"],
|
|
"deferred": True,
|
|
})
|
|
continue
|
|
if result["outcome"] == "age_policy_deferred":
|
|
batch_policy_skipped_candidates.append(candidate)
|
|
skipped.append(
|
|
{
|
|
"repoId": candidate["repoId"],
|
|
"targetGpu": candidate["targetGpu"],
|
|
"reason": "age_policy_skipped",
|
|
}
|
|
)
|
|
continue
|
|
if result["outcome"] == "capacity_deferred":
|
|
capacity_saturated = True
|
|
batch_capacity_deferred_candidates.append(candidate)
|
|
skipped.append(
|
|
{
|
|
"repoId": candidate["repoId"],
|
|
"targetGpu": candidate["targetGpu"],
|
|
"reason": "account_capacity_saturated",
|
|
"deferred": True,
|
|
}
|
|
)
|
|
continue
|
|
if result["outcome"] == "framework_prerequisite_deferred":
|
|
batch_policy_skipped_candidates.append(candidate)
|
|
skipped.append(
|
|
{
|
|
"repoId": candidate["repoId"],
|
|
"targetGpu": candidate["targetGpu"],
|
|
"framework": candidate["framework"],
|
|
"reason": "framework_prerequisite_deferred",
|
|
"deferred": True,
|
|
}
|
|
)
|
|
continue
|
|
if result["outcome"] in {"failed", "precheck_deferred"}:
|
|
batch_failed_candidates.append(candidate)
|
|
failed.append(
|
|
{
|
|
"repoId": candidate["repoId"],
|
|
"targetGpu": candidate["targetGpu"],
|
|
"framework": candidate["framework"],
|
|
"taskType": candidate["taskType"],
|
|
"reason": result.get("reason", "submission_failed"),
|
|
}
|
|
)
|
|
continue
|
|
|
|
batch_submitted_candidates.append(candidate)
|
|
submitted_record = {
|
|
**candidate,
|
|
"submitTime": result["submitTime"],
|
|
"taskId": result["taskId"],
|
|
"responseData": result["responseData"],
|
|
}
|
|
submitted.append(submitted_record)
|
|
append_ledger_entry(
|
|
ledger_path,
|
|
{
|
|
"modelId": candidate["repoId"],
|
|
"modelAddress": candidate["modelAddress"],
|
|
"targetGpu": candidate["targetGpu"],
|
|
"framework": candidate["framework"],
|
|
"templateId": candidate["templateId"],
|
|
"taskId": result["taskId"],
|
|
"taskType": candidate["taskType"],
|
|
"submitTime": result["submitTime"],
|
|
},
|
|
)
|
|
outcome_tracker.record_submission(
|
|
model_id=candidate["repoId"],
|
|
target_gpu=candidate["targetGpu"],
|
|
framework=candidate["framework"],
|
|
task_type=candidate["taskType"],
|
|
task_id=result["taskId"],
|
|
submit_time=result["submitTime"],
|
|
model_profile={
|
|
**dict(candidate.get("preflightMetadata") or {}),
|
|
"configFingerprint": hashlib.sha256(
|
|
str(candidate.get("configParams") or "").encode("utf-8")
|
|
).hexdigest(),
|
|
"configSource": candidate.get("frameworkConfigSource") or "official",
|
|
},
|
|
)
|
|
|
|
claim_store.mark_submitted(
|
|
[*batch_submitted_candidates, *batch_duplicate_candidates, *batch_uniqueness_rejected_candidates]
|
|
)
|
|
claim_store.release(
|
|
[
|
|
*batch_failed_candidates,
|
|
*batch_policy_skipped_candidates,
|
|
*batch_capacity_deferred_candidates,
|
|
]
|
|
)
|
|
if strategy_manager is not None:
|
|
strategy_manager.record_accepted(batch_submitted_candidates)
|
|
outcome_tracker.save()
|
|
if state_sync is not None and state_batch_id is not None:
|
|
ordered_batch_results = [
|
|
ordered_results[index]
|
|
for index in range(len(batch_candidates))
|
|
if index in ordered_results
|
|
]
|
|
if not state_sync.finish_batch(state_batch_id, ordered_batch_results):
|
|
state_sync_paused = True
|
|
print("[cycle] paused reason=state_sync_unhealthy phase=result", flush=True)
|
|
break
|
|
|
|
if hasattr(modelhub_client, "available_submit_slots") and modelhub_client.available_submit_slots() <= 0:
|
|
break
|
|
if capacity_saturated:
|
|
# Every account was tried by the pool. More attempts in this
|
|
# cycle only duplicate the same capacity response and pollute
|
|
# durable logs; refresh capacity on the next poll instead.
|
|
break
|
|
|
|
write_jsonl(run_dir / "submitted.jsonl", submitted)
|
|
write_jsonl(run_dir / "skipped.jsonl", skipped)
|
|
write_jsonl(run_dir / "failed.jsonl", failed)
|
|
|
|
if strategy_manager is not None:
|
|
strategy_summary = strategy_manager.summary()
|
|
if preflight_advisor is not None:
|
|
preflight_summary = preflight_advisor.summary()
|
|
|
|
outcome_tracker.save()
|
|
skip_reason_counts = Counter(str(item.get("reason") or "unknown") for item in skipped)
|
|
failure_reason_counts = Counter(str(item.get("reason") or "unknown") for item in failed)
|
|
candidate_framework_counts = Counter(str(item.get("framework") or "unknown") for item in candidates)
|
|
submitted_gpu_counts = Counter(str(item.get("targetGpu") or "unknown") for item in submitted)
|
|
submitted_framework_counts = Counter(str(item.get("framework") or "unknown") for item in submitted)
|
|
|
|
summary = {
|
|
"generatedAt": now.isoformat(),
|
|
"dryRun": bool(args.dry_run),
|
|
"selectedTaskTypes": selected_task_types,
|
|
"targetGpus": target_gpus,
|
|
"submissionEligibleGpus": submission_target_gpus,
|
|
"maxSubmitsPerRun": int(getattr(args, "max_submits_per_run", 0) or 0),
|
|
"dailyTarget": args.daily_target,
|
|
"unlimitedDailyTarget": unlimited_daily_target,
|
|
"submittedTodayBeforeRun": daily_snapshot["totalCount"],
|
|
"remainingDailyQuotaBeforeRun": remaining_daily_quota,
|
|
"platformAvailableSlotsBeforeRun": platform_available_slots,
|
|
"historyStatsEnabled": False,
|
|
"historyStatsThreshold": getattr(args, "history_stats_threshold", 500),
|
|
"historyArchivePath": str(history_archive_path),
|
|
"historyArchiveRecordCount": len(archived_history),
|
|
"gpuStrategy": strategy_summary,
|
|
"officialCapabilities": official_summary,
|
|
"marketIntelligence": market_summary,
|
|
"candidatePreflight": preflight_summary,
|
|
"agePolicy": {
|
|
"mode": "weekly_capacity_calibration" if weekly_calibration.get("targetAccount") else "admission_only",
|
|
"recentModelDays": recent_model_days,
|
|
"recentModelReserveSlots": recent_model_reserve_slots,
|
|
"oldModelSubmitThresholds": old_model_queue_thresholds,
|
|
"oldModelSlotsBeforeScan": old_model_slots_before_scan,
|
|
"olderHistoryScanEnabled": allow_older_models_for_scan,
|
|
"calibrationAccount": weekly_calibration.get("targetAccount"),
|
|
},
|
|
"weeklyCalibration": weekly_calibration,
|
|
"scanLimit": scan_limit,
|
|
"rawCandidateGoal": raw_candidate_goal,
|
|
"candidatePoolLimit": candidate_pool_limit,
|
|
"candidateGoal": candidate_goal,
|
|
"scanStages": scan_stages,
|
|
"scannedModels": scanned_model_count,
|
|
"candidateCount": len(candidates),
|
|
"candidateModelCount": candidate_model_count(candidates),
|
|
"candidateFrameworkCounts": dict(candidate_framework_counts.most_common()),
|
|
"targetSubmitCount": target_submit_count,
|
|
"maxSubmitAttempts": 0 if args.dry_run else min(
|
|
len(diversified_candidates),
|
|
max(target_submit_count, target_submit_count * max(1, int(getattr(args, "scan_multiplier", 4) or 1))),
|
|
),
|
|
"plannedSubmitCount": len(attempted_candidates),
|
|
"duplicateCount": len(duplicate_candidates),
|
|
"modelGpuUniquenessRejectedCount": len(uniqueness_rejected_candidates),
|
|
"submittedCount": len(submitted),
|
|
"submittedGpuCounts": dict(submitted_gpu_counts.most_common()),
|
|
"submittedFrameworkCounts": dict(submitted_framework_counts.most_common()),
|
|
"skippedCount": len(skipped),
|
|
"skipReasonCounts": dict(skip_reason_counts.most_common()),
|
|
"failedCount": len(failed),
|
|
"failureReasonCounts": dict(failure_reason_counts.most_common()),
|
|
"submissionExclusionsPath": str(submission_exclusion_store.path),
|
|
"submitConcurrencyUsed": submit_workers,
|
|
"warnings": report.get("warnings", []),
|
|
"runDir": str(run_dir),
|
|
"outcomeSyncCount": synced_count,
|
|
"stateSync": {
|
|
"enabled": state_sync is not None,
|
|
"healthy": bool(getattr(state_sync, "healthy", True)),
|
|
"paused": state_sync_paused,
|
|
"generation": int(getattr(state_sync, "generation", 0) or 0),
|
|
},
|
|
}
|
|
write_json(run_dir / "summary.json", summary)
|
|
return summary
|
|
|
|
|
|
def candidate_to_record(candidate: CandidateModel) -> dict[str, Any]:
|
|
return {
|
|
"repoId": candidate.repo_id,
|
|
"modelAddress": candidate.model_address,
|
|
"pipelineTag": candidate.pipeline_tag,
|
|
"modality": candidate.modality,
|
|
"taskType": candidate.task_type,
|
|
"taskPriority": TASK_SPEC_BY_TYPE[candidate.task_type].priority,
|
|
"targetGpu": candidate.target_gpu,
|
|
"framework": candidate.framework,
|
|
"templateId": candidate.template_id,
|
|
"configParams": candidate.config_params,
|
|
"downloads": candidate.downloads,
|
|
"lastModified": candidate.last_modified.isoformat() if candidate.last_modified else None,
|
|
"ggufFilename": candidate.gguf_filename,
|
|
"score": candidate.score,
|
|
"warnings": candidate.warnings,
|
|
"preflightMetadata": candidate.preflight_metadata,
|
|
}
|
|
|
|
|
|
def extract_task_id_from_submit_response(response: dict[str, Any]) -> str | None:
|
|
data = response.get("data")
|
|
if isinstance(data, dict) and data.get("id") is not None:
|
|
return str(data["id"])
|
|
return None
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = build_parser()
|
|
args = parser.parse_args(argv)
|
|
summary = run_submission(args)
|
|
print(f"run_dir={summary['runDir']}")
|
|
print(f"daily_target={summary['dailyTarget']} remaining_before_run={summary['remainingDailyQuotaBeforeRun']}")
|
|
print(f"task_types={','.join(summary['selectedTaskTypes'])}")
|
|
print(f"gpus={','.join(summary['targetGpus'])}")
|
|
print(
|
|
f"candidates={summary['candidateCount']} planned={summary['plannedSubmitCount']} "
|
|
f"submitted={summary['submittedCount']} skipped={summary['skippedCount']} failed={summary['failedCount']}"
|
|
)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|