CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: add durable success-first modelhub agent

Browse Source
This commit is contained in:
CoolBoy
2026-08-15 19:24:26 +08:00
parent 91d1d3d87d
commit 4260793c03
22 changed files with 2420 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
modelhub_submmit_api/daily_runner.py
View File

@@ -20,6 +20,7 @@ from market_intelligence import (
)
from modelhub_client import DEFAULT_CAPACITY_STATE_PATH, ModelHubClient, ModelHubClientPool
from outcome_tracker import OutcomeTracker
from official_capabilities import OfficialCapabilityUnavailable
from runner_common import DEFAULT_KEY_PATH, ensure_tokens
from submission_claims import DEFAULT_CLAIMS_PATH
from template_selector import TemplateSelector
@@ -220,6 +221,12 @@ def make_wave_namespace(base_args: argparse.Namespace, wave: WaveSpec) -> argpar
disable_market_intelligence=getattr(base_args, "disable_market_intelligence", False),
gpu_strategy_refresh_submissions=getattr(base_args, "gpu_strategy_refresh_submissions", 200),
gpu_strategy_state_path=getattr(base_args, "gpu_strategy_state_path", str(DEFAULT_GPU_STRATEGY_PATH)),
routing_state_path=getattr(base_args, "routing_state_path", ".modelhub_state/routing_intelligence.json"),
official_capabilities_path=getattr(
base_args,
"official_capabilities_path",
".modelhub_state/official_capabilities.json",
),
gpu_strategy_recent_window=getattr(base_args, "gpu_strategy_recent_window", 1000),
gpu_strategy_min_long_samples=getattr(base_args, "gpu_strategy_min_long_samples", 100),
market_intelligence_state_path=getattr(
@@ -279,6 +286,7 @@ def make_wave_namespace(base_args: argparse.Namespace, wave: WaveSpec) -> argpar
hf_base_url=base_args.hf_base_url,
modelhub_base_url=base_args.modelhub_base_url,
modelhub_token=base_args.modelhub_token,
_state_sync_manager=getattr(base_args, "_state_sync_manager", None),
)
@@ -337,14 +345,33 @@ def run_daily_batches(
f"tasks={','.join(wave.task_types)} gpus={wave.gpus or 'auto'} "
f"limit={wave.limit} since_hours={wave.since_hours}"
)
summary = run_fn(
wave_args,
now=utc_now(),
hf_discovery=hf_discovery,
modelhub_client=modelhub_client,
template_selector=template_selector,
outcome_tracker=outcome_tracker,
)
try:
summary = run_fn(
wave_args,
now=utc_now(),
hf_discovery=hf_discovery,
modelhub_client=modelhub_client,
template_selector=template_selector,
outcome_tracker=outcome_tracker,
)
except OfficialCapabilityUnavailable as exc:
reason = str(exc) or "critical_official_signal_unavailable"
log(f"[cycle] paused reason=critical_official_signal_unavailable detail={reason}")
summary = {
"candidateCount": 0,
"plannedSubmitCount": 0,
"submittedCount": 0,
"skippedCount": 0,
"duplicateCount": 0,
"modelGpuUniquenessRejectedCount": 0,
"failedCount": 0,
"skipReasonCounts": {},
"remainingDailyQuotaBeforeRun": None,
"platformAvailableSlotsBeforeRun": None,
"paused": True,
"pauseReason": "critical_official_signal_unavailable",
"pauseDetail": reason,
}
wave_result = {
"round": round_index,
"wave": asdict(wave),
@@ -382,7 +409,11 @@ def run_daily_batches(
stopped_reason=stopped_reason,
)
if base_args.daily_target > 0 and summary["remainingDailyQuotaBeforeRun"] <= 0:
if (
not summary.get("paused")
and base_args.daily_target > 0
and summary["remainingDailyQuotaBeforeRun"] <= 0
):
stopped_reason = "daily_target_already_reached"
log(f"[daily] stop={stopped_reason}")
return finalize_daily_run(
@@ -394,7 +425,12 @@ def run_daily_batches(
attempted_waves=attempted_waves,
stopped_reason=stopped_reason,
)
if base_args.daily_target > 0 and not base_args.dry_run and summary["plannedSubmitCount"] <= 0:
if (
not summary.get("paused")
and base_args.daily_target > 0
and not base_args.dry_run
and summary["plannedSubmitCount"] <= 0
):
stopped_reason = "daily_target_reached"
log(f"[daily] stop={stopped_reason}")
return finalize_daily_run(