CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: add durable success-first modelhub agent

Browse Source
This commit is contained in:
CoolBoy
2026-08-15 19:24:26 +08:00
parent 91d1d3d87d
commit 4260793c03
22 changed files with 2420 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
modelhub_submmit_api/poll_runner.py
View File

@@ -25,8 +25,17 @@ from market_intelligence import (
)
from modelhub_client import DEFAULT_CAPACITY_STATE_PATH, ModelHubClient, ModelHubClientPool
from outcome_tracker import DEFAULT_OUTCOMES_PATH, OutcomeTracker
from official_capabilities import DEFAULT_OFFICIAL_CAPABILITIES_PATH
from queue_cleanup import cleanup_certain_oom_tasks
from routing_engine import DEFAULT_ROUTING_STATE_PATH
from runner_common import DEFAULT_KEY_PATH, ensure_tokens
from state_sync import (
DEFAULT_BATCH_SIZE,
DEFAULT_BRANCH,
DEFAULT_REMOTE,
StateGitSync,
load_state_git_credentials,
)
from submission_claims import DEFAULT_CLAIMS_PATH
from template_selector import TemplateSelector
from version import AGENT_VERSION
@@ -132,6 +141,33 @@ def build_parser() -> argparse.ArgumentParser:
default=os.getenv("MODELHUB_GPU_STRATEGY_STATE_PATH", str(DEFAULT_GPU_STRATEGY_PATH)),
help=argparse.SUPPRESS,
)
parser.add_argument(
"--routing-state-path",
default=os.getenv("MODELHUB_ROUTING_STATE_PATH", str(DEFAULT_ROUTING_STATE_PATH)),
help=argparse.SUPPRESS,
)
parser.add_argument(
"--official-capabilities-path",
default=os.getenv("MODELHUB_OFFICIAL_CAPABILITIES_PATH", str(DEFAULT_OFFICIAL_CAPABILITIES_PATH)),
help=argparse.SUPPRESS,
)
parser.add_argument("--state-sync", action="store_true", help=argparse.SUPPRESS)
parser.add_argument(
"--state-sync-remote",
default=os.getenv("MODELHUB_STATE_SYNC_REMOTE", DEFAULT_REMOTE),
help=argparse.SUPPRESS,
)
parser.add_argument(
"--state-sync-branch",
default=os.getenv("MODELHUB_STATE_SYNC_BRANCH", DEFAULT_BRANCH),
help=argparse.SUPPRESS,
)
parser.add_argument(
"--state-sync-batch-size",
type=int,
default=int(os.getenv("MODELHUB_STATE_SYNC_BATCH_SIZE", str(DEFAULT_BATCH_SIZE))),
help=argparse.SUPPRESS,
)
parser.add_argument("--gpu-strategy-recent-window", type=int, default=1000, help=argparse.SUPPRESS)
parser.add_argument("--gpu-strategy-min-long-samples", type=int, default=100, help=argparse.SUPPRESS)
parser.add_argument(
@@ -471,10 +507,53 @@ def run_poll_loop(
outcome_tracker: OutcomeTracker | None = None,
) -> dict[str, Any]:
now = now or utc_now()
state_sync: StateGitSync | None = getattr(base_args, "_state_sync_manager", None)
if state_sync is None and bool(getattr(base_args, "state_sync", False)):
state_sync = StateGitSync(
project_root=Path(__file__).resolve().parent.parent,
credentials=load_state_git_credentials(),
remote=str(getattr(base_args, "state_sync_remote", DEFAULT_REMOTE)),
branch=str(getattr(base_args, "state_sync_branch", DEFAULT_BRANCH)),
batch_size=max(1, int(getattr(base_args, "state_sync_batch_size", DEFAULT_BATCH_SIZE) or DEFAULT_BATCH_SIZE)),
log_fn=log,
)
try:
state_sync.acquire_process_lock()
except Exception as exc:
state_sync.last_error = str(exc)
state_sync.healthy = False
else:
state_sync.restore()
base_args._state_sync_manager = state_sync
if state_sync is not None:
state_sync.write_readiness(
ready=False,
reason="startup_recovery" if state_sync.healthy else "state_sync_unhealthy",
)
hf_discovery = hf_discovery or HuggingFaceDiscovery(base_url=base_args.hf_base_url)
modelhub_client = modelhub_client or _build_modelhub_client(base_args)
template_selector = template_selector or TemplateSelector()
if state_sync is not None and state_sync.healthy:
try:
active_tasks = (
modelhub_client.list_active_tasks_by_account()
if hasattr(modelhub_client, "list_active_tasks_by_account")
else []
)
recovery = state_sync.reconcile_active_tasks(active_tasks)
if not state_sync.sync("startup"):
log("[cycle] paused reason=state_sync_unhealthy")
else:
log(
f"[state-recovery] active={recovery['active']} "
f"reconciled={recovery['reconciled']} unresolved={recovery['unresolved']}"
)
except Exception as exc:
state_sync.healthy = False
state_sync.last_error = str(exc)
log(f"[state-recovery] active_scan_failed reason={type(exc).__name__}: {exc}")
poll_runs_dir = Path(base_args.poll_runs_dir)
poll_runs_dir.mkdir(parents=True, exist_ok=True)
poll_run_dir = make_run_dir(poll_runs_dir, now)
@@ -535,6 +614,17 @@ def run_poll_loop(
break
cycles += 1
if state_sync is not None and not state_sync.healthy:
recovered = (
state_sync.sync("retry")
if state_sync._workspace is not None
else state_sync.retry_restore()
)
if not recovered:
state_sync.write_readiness(ready=False, reason="state_sync_unhealthy")
log(f"[poll] cycle={cycles} sleep={base_args.idle_interval_seconds}s reason=state_sync_unhealthy")
time.sleep(base_args.idle_interval_seconds)
continue
if hasattr(modelhub_client, "configure_capacity_probe"):
modelhub_client.configure_capacity_probe(cycles)
@@ -678,6 +768,10 @@ def run_poll_loop(
"architectureIncompatibleCount": cleanup_summary[
"architectureIncompatibleCount"
],
"officialCapabilityInvalidCount": cleanup_summary.get(
"officialCapabilityInvalidCount",
0,
),
"oldOverflowCount": cleanup_summary["oldOverflowCount"],
"cancelledCount": cleanup_summary["cancelledCount"],
"policyCancelledRecorded": policy_cancelled_recorded,
@@ -701,6 +795,20 @@ def run_poll_loop(
)
if available_slots is not None and available_slots <= 0:
if state_sync is not None:
try:
if cycles % 3 == 0 and hasattr(modelhub_client, "list_active_tasks_by_account"):
state_sync.reconcile_active_tasks(modelhub_client.list_active_tasks_by_account())
sync_ok = state_sync.sync("cycle_no_slots")
state_sync.write_readiness(
ready=sync_ok,
reason=None if sync_ok else "state_sync_unhealthy",
extra={"cycle": cycles},
)
except Exception as exc:
state_sync.healthy = False
state_sync.last_error = str(exc)
state_sync.write_readiness(ready=False, reason="state_sync_unhealthy")
log(f"[poll] cycle={cycles} sleep={base_args.poll_interval_seconds}s reason=no_available_slots")
time.sleep(base_args.poll_interval_seconds)
continue
@@ -726,6 +834,34 @@ def run_poll_loop(
f"stop={cycle_summary['stoppedReason']}"
)
if state_sync is not None:
try:
if cycle_summary.get("submittedTotal", 0) > 0 or cycles % 3 == 0:
active_tasks = (
modelhub_client.list_active_tasks_by_account()
if hasattr(modelhub_client, "list_active_tasks_by_account")
else []
)
state_sync.reconcile_active_tasks(active_tasks)
sync_ok = state_sync.sync("cycle")
official_paused = any(
bool((wave_result.get("summary") or {}).get("paused"))
for wave_result in (cycle_summary.get("waveResults") or [])
)
state_sync.write_readiness(
ready=sync_ok and not official_paused,
reason=(
"critical_official_signal_unavailable"
if official_paused
else (None if sync_ok else "state_sync_unhealthy")
),
extra={"cycle": cycles},
)
except Exception as exc:
state_sync.healthy = False
state_sync.last_error = str(exc)
state_sync.write_readiness(ready=False, reason="state_sync_unhealthy")
if base_args.daily_target > 0 and remaining_before_run is not None and remaining_before_run <= 0:
stopped_reason = "daily_target_already_reached"
break
@@ -791,6 +927,9 @@ def run_poll_loop(
}
write_json(poll_run_dir / "summary.json", summary)
log(f"[poll] finished submitted_total={submitted_total} cycles={cycles} stopped_reason={stopped_reason}")
if state_sync is not None:
state_sync.write_readiness(ready=False, reason="worker_stopped")
state_sync.close()
return summary