feat: add durable success-first modelhub agent
This commit is contained in:
CoolBoy
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import math
|
||||
import re
|
||||
import statistics
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
@@ -20,6 +21,9 @@ DEFAULT_FETCH_WORKERS = 4
|
||||
DEFAULT_FRAMEWORK_MIN_SAMPLES = 300
|
||||
DEFAULT_GPU_MIN_RECENT_TERMINALS = 20
|
||||
DEFAULT_FRAMEWORK_MIN_WILSON = 0.05
|
||||
DEFAULT_COMMUNITY_WINDOW_DAYS = 30
|
||||
DEFAULT_COMMUNITY_MAX_MODELS = 2000
|
||||
DEFAULT_COMMUNITY_HYDRATE_PER_REFRESH = 50
|
||||
NEW_FRAMEWORK_PROMOTION_MARGIN = 1.10
|
||||
ERROR_RETRY_SECONDS = 300
|
||||
|
||||
@@ -143,6 +147,7 @@ class MarketIntelligenceManager:
|
||||
self.state: dict[str, Any] | None = None
|
||||
self.local_outcome_stats: dict[str, Any] = {}
|
||||
self._last_framework_request_errors: set[tuple[str, str]] = set()
|
||||
self._route_refresh_lock = threading.Lock()
|
||||
|
||||
def set_local_outcome_stats(self, report: dict[str, Any] | None) -> None:
|
||||
self.local_outcome_stats = report if isinstance(report, dict) else {}
|
||||
@@ -214,6 +219,9 @@ class MarketIntelligenceManager:
|
||||
"frameworkStats": {},
|
||||
"queueError": None,
|
||||
"frameworkError": None,
|
||||
"communitySample": {},
|
||||
"communityUpdatedAt": None,
|
||||
"communityError": None,
|
||||
}
|
||||
|
||||
def prepare(
|
||||
@@ -233,6 +241,9 @@ class MarketIntelligenceManager:
|
||||
elif int((loaded or {}).get("version") or 0) == 2 and self._catalog_compatible(loaded, gpus, tasks):
|
||||
state = dict(loaded or {})
|
||||
state["version"] = MARKET_STATE_VERSION
|
||||
state.setdefault("communitySample", {})
|
||||
state.setdefault("communityUpdatedAt", now.isoformat())
|
||||
state.setdefault("communityError", None)
|
||||
self._rescore_cached_gpu_stats(state.get("gpuStats") or {})
|
||||
else:
|
||||
state = self._base_state(gpus, tasks, now)
|
||||
@@ -314,6 +325,31 @@ class MarketIntelligenceManager:
|
||||
state["frameworkStats"] = {}
|
||||
self.log(f"[market] framework_refresh_error reason={state['frameworkError']}")
|
||||
|
||||
community_due = not _fresh(
|
||||
state.get("communityUpdatedAt"),
|
||||
now=now,
|
||||
ttl_seconds=900,
|
||||
)
|
||||
if state.get("communityError") and _fresh(
|
||||
state.get("communityAttemptedAt"),
|
||||
now=now,
|
||||
ttl_seconds=300,
|
||||
):
|
||||
community_due = False
|
||||
if community_due:
|
||||
state["communityAttemptedAt"] = now.isoformat()
|
||||
try:
|
||||
state["communitySample"] = self._refresh_community_sample(
|
||||
client,
|
||||
previous=state.get("communitySample") or {},
|
||||
now=now,
|
||||
)
|
||||
state["communityUpdatedAt"] = now.isoformat()
|
||||
state["communityError"] = None
|
||||
except Exception as exc:
|
||||
state["communityError"] = f"{type(exc).__name__}: {exc}"
|
||||
self.log(f"[market] community_refresh_error reason={state['communityError']}")
|
||||
|
||||
state["generatedAt"] = now.isoformat()
|
||||
state["throughputWindowHours"] = self.throughput_window_hours
|
||||
self.state = state
|
||||
@@ -321,6 +357,72 @@ class MarketIntelligenceManager:
|
||||
self._log_snapshot("refreshed" if queue_due or framework_due else "loaded")
|
||||
return state
|
||||
|
||||
def _refresh_community_sample(
|
||||
self,
|
||||
client: Any,
|
||||
*,
|
||||
previous: dict[str, Any],
|
||||
now: datetime,
|
||||
) -> dict[str, Any]:
|
||||
cutoff = now - timedelta(days=DEFAULT_COMMUNITY_WINDOW_DAYS)
|
||||
sample = {
|
||||
str(model_id): dict(item)
|
||||
for model_id, item in previous.items()
|
||||
if isinstance(item, dict)
|
||||
and (parse_datetime(item.get("updateTime")) or cutoff) >= cutoff
|
||||
}
|
||||
payload = client.list_tasks_page(
|
||||
current=1,
|
||||
page_size=200,
|
||||
only_mine=False,
|
||||
begin_time=cutoff,
|
||||
end_time=now,
|
||||
)
|
||||
records = ((payload.get("data") or {}).get("records") or []) if isinstance(payload, dict) else []
|
||||
candidates: list[tuple[str, dict[str, Any]]] = []
|
||||
for record in records:
|
||||
if not isinstance(record, dict):
|
||||
continue
|
||||
model_id = str(record.get("modelId") or "").strip()
|
||||
if model_id and model_id not in sample:
|
||||
candidates.append((model_id, record))
|
||||
if len(candidates) >= DEFAULT_COMMUNITY_HYDRATE_PER_REFRESH:
|
||||
break
|
||||
for model_id, record in candidates:
|
||||
evidence: list[dict[str, Any]] = []
|
||||
try:
|
||||
verify = client.get_verify_result_map(model_id)
|
||||
except Exception:
|
||||
verify = {}
|
||||
if isinstance(verify, dict):
|
||||
for gpu, gpu_result in verify.items():
|
||||
rows = gpu_result.get("records") if isinstance(gpu_result, dict) else []
|
||||
for row in rows or []:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
evidence.append(
|
||||
{
|
||||
"gpu": gpu,
|
||||
"framework": row.get("framework"),
|
||||
"verifyResult": row.get("verifyResult"),
|
||||
"taskId": row.get("taskId") or row.get("contestTaskId"),
|
||||
}
|
||||
)
|
||||
sample[model_id] = {
|
||||
"modelId": model_id,
|
||||
"updateTime": record.get("updateTime") or now.isoformat(),
|
||||
"taskType": record.get("taskType") or record.get("modelTaskLevel"),
|
||||
"evidence": evidence,
|
||||
}
|
||||
if len(sample) > DEFAULT_COMMUNITY_MAX_MODELS:
|
||||
ordered = sorted(
|
||||
sample.items(),
|
||||
key=lambda item: parse_datetime(item[1].get("updateTime")) or cutoff,
|
||||
reverse=True,
|
||||
)
|
||||
sample = dict(ordered[:DEFAULT_COMMUNITY_MAX_MODELS])
|
||||
return sample
|
||||
|
||||
def _fetch_gpu_stats(self, client: Any, *, gpus: list[str], now: datetime) -> dict[str, dict[str, Any]]:
|
||||
local_end = now.astimezone()
|
||||
local_begin = local_end - timedelta(hours=self.throughput_window_hours)
|
||||
@@ -740,6 +842,33 @@ class MarketIntelligenceManager:
|
||||
compatible.append(framework)
|
||||
return compatible
|
||||
|
||||
def has_framework_route(self, task_type: str, target_gpu: str) -> bool:
|
||||
rows = (
|
||||
(((self.state or {}).get("frameworkStats") or {}).get(task_type) or {}).get(target_gpu)
|
||||
or {}
|
||||
)
|
||||
return any(bool(item.get("officialConfigValid", False)) for item in rows.values())
|
||||
|
||||
def ensure_framework_route(self, client: Any, task_type: str, target_gpu: str) -> None:
|
||||
if self.has_framework_route(task_type, target_gpu):
|
||||
return
|
||||
with self._route_refresh_lock:
|
||||
if self.has_framework_route(task_type, target_gpu):
|
||||
return
|
||||
fetched = self._fetch_framework_stats(
|
||||
client,
|
||||
gpus=[target_gpu],
|
||||
task_types=[task_type],
|
||||
)
|
||||
rows = ((fetched.get(task_type) or {}).get(target_gpu) or {})
|
||||
if not rows:
|
||||
return
|
||||
state = self.state or self._base_state([target_gpu], [task_type], utc_now())
|
||||
state.setdefault("frameworkStats", {}).setdefault(task_type, {})[target_gpu] = rows
|
||||
state["frameworkUpdatedAt"] = utc_now().isoformat()
|
||||
self.state = state
|
||||
write_json(self.path, state)
|
||||
|
||||
def official_config(
|
||||
self,
|
||||
*,
|
||||
@@ -814,5 +943,8 @@ class MarketIntelligenceManager:
|
||||
"throughputWindowHours": int(self.state.get("throughputWindowHours") or self.throughput_window_hours),
|
||||
"queueError": self.state.get("queueError"),
|
||||
"frameworkError": self.state.get("frameworkError"),
|
||||
"communityUpdatedAt": self.state.get("communityUpdatedAt"),
|
||||
"communityError": self.state.get("communityError"),
|
||||
"communityModelCount": len(self.state.get("communitySample") or {}),
|
||||
"gpuStats": dict(self.state.get("gpuStats") or {}),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user