CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: add durable success-first modelhub agent

Browse Source
This commit is contained in:
CoolBoy
2026-08-15 19:24:26 +08:00
parent 91d1d3d87d
commit 4260793c03
22 changed files with 2420 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

256
modelhub_submmit_api/main.py
View File

@@ -1,6 +1,7 @@
from __future__ import annotations
import argparse
import hashlib
import os
from collections import Counter
from concurrent.futures import ThreadPoolExecutor, as_completed
@@ -10,7 +11,8 @@ from typing import Any
from common import parse_datetime, runtime_instance_id, utc_now, write_json, write_jsonl
from candidate_preflight import CandidatePreflightAdvisor
from gpu_strategy import DEFAULT_GPU_STRATEGY_PATH, GPUStrategyManager
from config_optimizer import SafeConfigOptimizer
from gpu_strategy import DEFAULT_GPU_STRATEGY_PATH
from hf_discovery import HuggingFaceDiscovery
from history_stats import (
append_ledger_entry,
@@ -38,10 +40,23 @@ from modelhub_client import (
is_model_uniqueness_error,
)
from models import CandidateModel, HFModelSummary, ModelInspection
from official_capabilities import (
DEFAULT_OFFICIAL_CAPABILITIES_PATH,
OfficialCapabilityRegistry,
OfficialCapabilityUnavailable,
)
from outcome_tracker import DEFAULT_OUTCOMES_PATH, OutcomeTracker
from routing_engine import DEFAULT_ROUTING_STATE_PATH, SuccessFirstRoutingEngine
from submission_claims import DEFAULT_CLAIMS_PATH, SubmissionClaimStore, candidate_key, diversify_candidates
from submission_exclusions import DEFAULT_SUBMISSION_EXCLUSIONS_PATH, SubmissionExclusionStore
from task_registry import TASK_SPEC_BY_TYPE, all_task_types, compatible_frameworks_for_task, pipeline_tags_for_task_types, task_specs_for_model
from task_registry import (
TASK_SPEC_BY_TYPE,
all_task_types,
compatible_frameworks_for_task,
pipeline_tags_for_task_types,
register_dynamic_task_route,
task_specs_for_model,
)
from template_selector import TemplateSelector
@@ -212,6 +227,19 @@ def build_parser() -> argparse.ArgumentParser:
default=os.getenv("MODELHUB_GPU_STRATEGY_STATE_PATH", str(DEFAULT_GPU_STRATEGY_PATH)),
help=argparse.SUPPRESS,
)
parser.add_argument(
"--routing-state-path",
default=os.getenv("MODELHUB_ROUTING_STATE_PATH", str(DEFAULT_ROUTING_STATE_PATH)),
help=argparse.SUPPRESS,
)
parser.add_argument(
"--official-capabilities-path",
default=os.getenv(
"MODELHUB_OFFICIAL_CAPABILITIES_PATH",
str(DEFAULT_OFFICIAL_CAPABILITIES_PATH),
),
help=argparse.SUPPRESS,
)
parser.add_argument("--gpu-strategy-recent-window", type=int, default=1000, help=argparse.SUPPRESS)
parser.add_argument("--gpu-strategy-min-long-samples", type=int, default=100, help=argparse.SUPPRESS)
parser.add_argument(
@@ -311,6 +339,8 @@ def choose_candidate_for_gpu(
target_gpu: str,
market_intelligence: MarketIntelligenceManager | None = None,
preflight_advisor: CandidatePreflightAdvisor | None = None,
require_official_config: bool = False,
config_optimizer: SafeConfigOptimizer | None = None,
) -> CandidateModel | None:
candidate, _reason = choose_candidate_for_gpu_detailed(
model=model,
@@ -320,6 +350,8 @@ def choose_candidate_for_gpu(
target_gpu=target_gpu,
market_intelligence=market_intelligence,
preflight_advisor=preflight_advisor,
require_official_config=require_official_config,
config_optimizer=config_optimizer,
)
return candidate
@@ -333,6 +365,8 @@ def choose_candidate_for_gpu_detailed(
target_gpu: str,
market_intelligence: MarketIntelligenceManager | None = None,
preflight_advisor: CandidatePreflightAdvisor | None = None,
require_official_config: bool = False,
config_optimizer: SafeConfigOptimizer | None = None,
) -> tuple[CandidateModel | None, str | None]:
last_rejection_reason: str | None = None
for task_type in task_types:
@@ -371,6 +405,11 @@ def choose_candidate_for_gpu_detailed(
)
if config_params is not None:
template_id = f"modelhub-live-{task_type}-{framework}-{target_gpu}".lower().replace("_", "-")
elif market_intelligence is not None and require_official_config:
# A live-discovered route without a valid official config is a
# critical capability gap, not permission to use a stale hand template.
last_rejection_reason = "official_build_config_unavailable"
continue
else:
try:
template = template_selector.select_template(task_type, framework, target_gpu)
@@ -388,6 +427,19 @@ def choose_candidate_for_gpu_detailed(
)
score = float(framework_metadata.get("frameworkCombinedScore") or 0.0)
warnings: list[str] = []
config_optimization: dict[str, Any] = {"source": "official", "applied": False}
if config_optimizer is not None and config_params is not None:
config_params, config_optimization = config_optimizer.optimize(
task_type=task_type,
target_gpu=target_gpu,
framework=framework,
official_config=config_params,
official_lower_bound=float(
framework_metadata.get("frameworkMarketWilsonLowerBound") or 0.0
),
)
if config_optimization.get("applied"):
warnings.append("validated_success_config_patch_applied")
if bool(framework_metadata.get("frameworkEvidenceQualified", False)):
warnings.append("framework_selected_from_success_evidence")
if config_params is not None and bool(framework_metadata.get("frameworkOfficialConfigValid", False)):
@@ -407,6 +459,20 @@ def choose_candidate_for_gpu_detailed(
config_params = assessment.config_params
warnings.extend(assessment.warnings)
preflight_metadata = assessment.metadata
preflight_metadata = {
**preflight_metadata,
"configOptimization": config_optimization,
"modelscopeParams": model.params,
"modelscopeFileSize": model.file_size,
"modelscopeTasks": list(model.tasks),
"modelscopeTags": list(model.tags),
"modelscopeLicense": model.license,
"modelCard": dict(inspection.model_card_metadata),
}
repository_size = inspection.repository_size_bytes
if model.file_size and repository_size and model.file_size > repository_size:
warnings.append("modelscope_published_size_used_as_conservative_upper_bound")
preflight_metadata["conservativeRepositorySizeBytes"] = model.file_size
spec = TASK_SPEC_BY_TYPE[task_type]
return CandidateModel(
repo_id=model.repo_id,
@@ -471,9 +537,15 @@ def process_model_for_candidates(
submission_exclusion_store: SubmissionExclusionStore | None = None,
market_intelligence: MarketIntelligenceManager | None = None,
preflight_advisor: CandidatePreflightAdvisor | None = None,
official_registry: OfficialCapabilityRegistry | None = None,
config_optimizer: SafeConfigOptimizer | None = None,
allow_dynamic_tasks: bool = False,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]:
if model.private or model.gated:
reason = "modelscope_private" if model.private else "modelscope_gated"
return [], [{"repoId": model.repo_id, "reason": reason}], []
specs = [spec for spec in task_specs_for_model(model) if spec.task_type in allowed_task_types]
if not specs:
if not specs and official_registry is None:
return [], [{"repoId": model.repo_id, "reason": f"unsupported_pipeline_tag:{model.pipeline_tag or 'unknown'}"}], []
try:
@@ -494,7 +566,8 @@ def process_model_for_candidates(
allowed_task_types_set = [spec.task_type for spec in specs]
pending_task_types_by_gpu: list[tuple[str, list[str]]] = []
for target_gpu in target_gpus:
route_target_gpus = target_gpus[:6] if official_registry is not None else target_gpus
for target_gpu in route_target_gpus:
if submission_exclusion_store is not None and submission_exclusion_store.is_blocked(model.repo_id, target_gpu):
skipped.append(
{"repoId": model.repo_id, "targetGpu": target_gpu, "reason": "model_gpu_uniqueness_blocklist"}
@@ -507,11 +580,34 @@ def process_model_for_candidates(
skipped.append({"repoId": model.repo_id, "targetGpu": target_gpu, "reason": "local_failure_cooldown_24h"})
continue
compatible_task_types = [
task_type
for task_type in allowed_task_types_set
if template_selector.supported_frameworks_for_auto(task_type, target_gpu)
]
exact_task_types = list(allowed_task_types_set)
if official_registry is not None:
exact_task_types = official_registry.task_types_for(
modelhub_client,
model_address=model.model_address,
model_last_modified=model.last_modified.isoformat() if model.last_modified else None,
gpu=target_gpu,
)
route_task_types = (
list(exact_task_types)
if allow_dynamic_tasks
else [task_type for task_type in allowed_task_types_set if task_type in exact_task_types]
)
compatible_task_types = []
for task_type in route_task_types:
register_dynamic_task_route(task_type, model.pipeline_tag or task_type)
if market_intelligence is not None and not market_intelligence.has_framework_route(task_type, target_gpu):
try:
market_intelligence.ensure_framework_route(modelhub_client, task_type, target_gpu)
except Exception as exc:
raise OfficialCapabilityUnavailable(
f"framework capability unavailable for {task_type}/{target_gpu}: {type(exc).__name__}: {exc}"
) from exc
if template_selector.supported_frameworks_for_auto(task_type, target_gpu):
compatible_task_types.append(task_type)
continue
if market_intelligence is not None and market_intelligence.has_framework_route(task_type, target_gpu):
compatible_task_types.append(task_type)
if not compatible_task_types:
skipped.append({"repoId": model.repo_id, "targetGpu": target_gpu, "reason": "no_compatible_auto_template_or_framework"})
continue
@@ -534,6 +630,8 @@ def process_model_for_candidates(
target_gpu=target_gpu,
market_intelligence=market_intelligence,
preflight_advisor=preflight_advisor,
require_official_config=official_registry is not None,
config_optimizer=config_optimizer,
)
if best is None:
reason = (
@@ -550,6 +648,9 @@ def process_model_for_candidates(
if market_intelligence is not None:
record.update(market_intelligence.gpu_metadata(target_gpu))
record.update(market_intelligence.framework_metadata(best.task_type, target_gpu, best.framework))
optimization = (record.get("preflightMetadata") or {}).get("configOptimization") or {}
if optimization.get("applied"):
record["frameworkConfigSource"] = "official_plus_learned_patch"
candidates.append(record)
return candidates, skipped, failed
@@ -640,6 +741,9 @@ def collect_candidates_from_models(
read_concurrency: int,
market_intelligence: MarketIntelligenceManager | None = None,
preflight_advisor: CandidatePreflightAdvisor | None = None,
official_registry: OfficialCapabilityRegistry | None = None,
config_optimizer: SafeConfigOptimizer | None = None,
allow_dynamic_tasks: bool = False,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]], int]:
candidates: list[dict[str, Any]] = []
skipped: list[dict[str, Any]] = []
@@ -670,6 +774,9 @@ def collect_candidates_from_models(
submission_exclusion_store=submission_exclusion_store,
market_intelligence=market_intelligence,
preflight_advisor=preflight_advisor,
official_registry=official_registry,
config_optimizer=config_optimizer,
allow_dynamic_tasks=allow_dynamic_tasks,
): index
for index, model in enumerate(chunk)
}
@@ -679,6 +786,8 @@ def collect_candidates_from_models(
model = chunk[index]
try:
ordered_results[index] = future.result()
except OfficialCapabilityUnavailable:
raise
except Exception as exc:
ordered_results[index] = ([], [], [{"repoId": model.repo_id, "reason": str(exc)}])
@@ -830,10 +939,9 @@ def run_submission(
now = now or utc_now()
template_selector = template_selector or TemplateSelector()
selected_task_types = resolve_task_types(args)
target_gpus = resolve_target_gpus(args, template_selector, selected_task_types)
if not target_gpus:
configured_target_gpus = resolve_target_gpus(args, template_selector, selected_task_types)
if not configured_target_gpus:
raise RuntimeError("No auto-submittable GPUs are available for the selected task types")
submission_target_gpus = list(target_gpus)
hf_discovery = hf_discovery or HuggingFaceDiscovery(base_url=args.hf_base_url)
if modelhub_client is None:
@@ -848,6 +956,42 @@ def run_submission(
recent_model_days=max(1, int(getattr(args, "recent_model_days", 7) or 7)),
)
official_registry: OfficialCapabilityRegistry | None = None
official_summary: dict[str, Any] = {"enabled": False}
official_capable = all(
hasattr(modelhub_client, name)
for name in ("list_machine_info", "list_task_levels", "list_model_task_types")
)
if official_capable:
official_registry = OfficialCapabilityRegistry(
Path(
getattr(
args,
"official_capabilities_path",
DEFAULT_OFFICIAL_CAPABILITIES_PATH,
)
)
)
official_registry.prepare(
modelhub_client,
fallback_gpus=configured_target_gpus,
task_types=selected_task_types,
now=now,
)
if not official_registry.ready:
raise OfficialCapabilityUnavailable(
official_registry.pause_reason or "critical_official_signal_unavailable"
)
requested_gpus = configured_target_gpus if (getattr(args, "gpu", None) or getattr(args, "gpus", None)) else None
target_gpus = official_registry.eligible_gpus(requested_gpus)
official_summary = official_registry.summary()
else:
# Compatibility path for injected test clients and offline dry-runs.
target_gpus = list(configured_target_gpus)
if not target_gpus:
raise OfficialCapabilityUnavailable("official catalog contains no eligible GPU")
submission_target_gpus = list(target_gpus)
if hasattr(modelhub_client, "begin_cycle"):
modelhub_client.begin_cycle()
@@ -874,7 +1018,11 @@ def run_submission(
preflight_summary: dict[str, Any] = (
preflight_advisor.summary() if preflight_advisor is not None else {"enabled": False}
)
strategy_manager: GPUStrategyManager | None = None
config_optimizer = SafeConfigOptimizer(
intents_path=Path(".modelhub_state/recovery_intents.jsonl"),
outcomes_path=Path(args.outcomes_path),
)
strategy_manager: SuccessFirstRoutingEngine | None = None
strategy_summary: dict[str, Any] = {"enabled": False}
market_intelligence: MarketIntelligenceManager | None = None
market_summary: dict[str, Any] = {"enabled": False}
@@ -1042,6 +1190,14 @@ def run_submission(
except Exception:
market_intelligence.set_local_outcome_stats(None)
market_summary = market_intelligence.summary()
framework_state = (market_intelligence.state or {}).get("frameworkStats") or {}
framework_updated_at = parse_datetime(market_summary.get("frameworkUpdatedAt"))
framework_cache_expired = (
framework_updated_at is None
or now - framework_updated_at > timedelta(hours=24)
)
if market_summary.get("frameworkError") and (not framework_state or framework_cache_expired):
raise OfficialCapabilityUnavailable("critical official framework catalog is unavailable")
submission_target_gpus = market_intelligence.eligible_gpus(target_gpus)
market_summary["eligibleGpus"] = submission_target_gpus
market_summary["shadowOnlyGpus"] = [gpu for gpu in target_gpus if gpu not in submission_target_gpus]
@@ -1052,19 +1208,20 @@ def run_submission(
)
if strategy_enabled:
strategy_manager = GPUStrategyManager(
Path(getattr(args, "gpu_strategy_state_path", DEFAULT_GPU_STRATEGY_PATH)),
refresh_submissions=max(1, int(getattr(args, "gpu_strategy_refresh_submissions", 200) or 200)),
recent_terminal_window=max(1, int(getattr(args, "gpu_strategy_recent_window", 1000) or 1000)),
long_term_min_samples=max(1, int(getattr(args, "gpu_strategy_min_long_samples", 100) or 100)),
market_intelligence=market_intelligence,
)
strategy_history_records = outcome_tracker.get_strategy_history_records()
strategy_manager.prepare(
modelhub_client,
supported_gpus=target_gpus,
now=now,
history_records=strategy_history_records,
routing_path = getattr(args, "routing_state_path", None)
legacy_strategy_path = getattr(args, "gpu_strategy_state_path", None)
if (
not routing_path
or (
str(routing_path) == str(DEFAULT_ROUTING_STATE_PATH)
and legacy_strategy_path
and str(legacy_strategy_path) != str(DEFAULT_GPU_STRATEGY_PATH)
)
):
routing_path = legacy_strategy_path or DEFAULT_ROUTING_STATE_PATH
strategy_manager = SuccessFirstRoutingEngine(
Path(routing_path),
outcome_stats=outcome_tracker.get_stats_report(),
)
strategy_summary = strategy_manager.summary()
@@ -1117,6 +1274,11 @@ def run_submission(
candidate_goal = max(submission_goal, submission_goal * attempt_multiplier)
pipeline_tags = pipeline_tags_for_task_types(selected_task_types)
if official_registry is not None and not getattr(args, "task_types", None):
for task_name in official_registry.state.get("discoveredTaskTypes") or []:
normalized = str(task_name or "").strip()
if normalized and normalized not in pipeline_tags:
pipeline_tags.append(normalized)
explicit_scan_cap = max(0, int(getattr(args, "max_scan_models", 0) or 0))
recent_model_days = max(1, int(getattr(args, "recent_model_days", 7) or 7))
recent_model_reserve_slots = max(0, int(getattr(args, "recent_model_reserve_slots", 5) or 0))
@@ -1197,6 +1359,9 @@ def run_submission(
read_concurrency=max(1, args.read_concurrency),
market_intelligence=market_intelligence,
preflight_advisor=preflight_advisor,
official_registry=official_registry,
config_optimizer=config_optimizer,
allow_dynamic_tasks=not bool(getattr(args, "task_types", None)),
)
candidates.extend(stage_candidates)
skipped.extend(stage_skipped)
@@ -1239,6 +1404,8 @@ def run_submission(
claim_store: SubmissionClaimStore | None = None
attempted_candidates: list[dict[str, Any]] = []
submit_workers = 1
state_sync = getattr(args, "_state_sync_manager", None)
state_sync_paused = False
if args.dry_run:
attempted_candidates = diversified_candidates[:target_submit_count]
@@ -1270,12 +1437,23 @@ def run_submission(
target_submit_count - len(submitted),
max_submit_attempts - len(attempted_candidates),
)
if state_sync is not None:
desired_count = min(desired_count, int(state_sync.batch_size))
batch_candidates = claim_store.claim(remaining_candidates, limit=desired_count)
if not batch_candidates:
break
attempted_candidates.extend(batch_candidates)
attempted_keys.update(candidate_key(candidate) for candidate in batch_candidates)
state_batch_id: str | None = None
if state_sync is not None:
state_batch_id = state_sync.begin_batch(batch_candidates)
if state_batch_id is None:
claim_store.release(batch_candidates)
state_sync_paused = True
print("[cycle] paused reason=state_sync_unhealthy phase=intent", flush=True)
break
batch_workers = resolve_submit_concurrency(
args,
modelhub_client=modelhub_client,
@@ -1389,7 +1567,13 @@ def run_submission(
task_type=candidate["taskType"],
task_id=result["taskId"],
submit_time=result["submitTime"],
model_profile=candidate.get("preflightMetadata") or {},
model_profile={
**dict(candidate.get("preflightMetadata") or {}),
"configFingerprint": hashlib.sha256(
str(candidate.get("configParams") or "").encode("utf-8")
).hexdigest(),
"configSource": candidate.get("frameworkConfigSource") or "official",
},
)
claim_store.mark_submitted(
@@ -1398,6 +1582,17 @@ def run_submission(
claim_store.release([*batch_failed_candidates, *batch_policy_skipped_candidates])
if strategy_manager is not None:
strategy_manager.record_accepted(batch_submitted_candidates)
outcome_tracker.save()
if state_sync is not None and state_batch_id is not None:
ordered_batch_results = [
ordered_results[index]
for index in range(len(batch_candidates))
if index in ordered_results
]
if not state_sync.finish_batch(state_batch_id, ordered_batch_results):
state_sync_paused = True
print("[cycle] paused reason=state_sync_unhealthy phase=result", flush=True)
break
if hasattr(modelhub_client, "available_submit_slots") and modelhub_client.available_submit_slots() <= 0:
break
@@ -1435,6 +1630,7 @@ def run_submission(
"historyArchivePath": str(history_archive_path),
"historyArchiveRecordCount": len(archived_history),
"gpuStrategy": strategy_summary,
"officialCapabilities": official_summary,
"marketIntelligence": market_summary,
"candidatePreflight": preflight_summary,
"agePolicy": {
@@ -1471,6 +1667,12 @@ def run_submission(
"warnings": report.get("warnings", []),
"runDir": str(run_dir),
"outcomeSyncCount": synced_count,
"stateSync": {
"enabled": state_sync is not None,
"healthy": bool(getattr(state_sync, "healthy", True)),
"paused": state_sync_paused,
"generation": int(getattr(state_sync, "generation", 0) or 0),
},
}
write_json(run_dir / "summary.json", summary)
return summary