CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: prioritize proven GPU framework combinations

Browse Source
This commit is contained in:
CoolBoy
2026-08-05 18:21:59 +08:00
parent 882479e43e
commit 5e47d9e695
11 changed files with 431 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

184
modelhub_submmit_api/market_intelligence.py
View File

@@ -12,12 +12,15 @@ from common import parse_datetime, read_json, utc_now, write_json
DEFAULT_MARKET_INTELLIGENCE_PATH = Path(".modelhub_state/market_intelligence.json")
MARKET_STATE_VERSION = 2
MARKET_STATE_VERSION = 3
DEFAULT_QUEUE_REFRESH_SECONDS = 600
DEFAULT_FRAMEWORK_REFRESH_SECONDS = 21_600
DEFAULT_THROUGHPUT_WINDOW_HOURS = 6
DEFAULT_FETCH_WORKERS = 4
DEFAULT_FRAMEWORK_MIN_SAMPLES = 100
DEFAULT_FRAMEWORK_MIN_SAMPLES = 300
DEFAULT_GPU_MIN_RECENT_TERMINALS = 20
DEFAULT_FRAMEWORK_MIN_WILSON = 0.05
NEW_FRAMEWORK_PROMOTION_MARGIN = 1.10
ERROR_RETRY_SECONDS = 300
@@ -73,6 +76,7 @@ def _neutral_gpu_stats(gpus: list[str]) -> dict[str, dict[str, Any]]:
"healthFactor": 1.0,
"queueWeight": 1.0,
"selectionWeight": 1.0,
"submissionEligible": None,
"error": "market_data_unavailable",
}
for gpu in gpus
@@ -156,6 +160,45 @@ class MarketIntelligenceManager:
return False
return list(state.get("supportedGpus") or []) == gpus and list(state.get("taskTypes") or []) == task_types
@staticmethod
def _catalog_compatible(state: dict[str, Any] | None, gpus: list[str], task_types: list[str]) -> bool:
if not state:
return False
return list(state.get("supportedGpus") or []) == gpus and list(state.get("taskTypes") or []) == task_types
@staticmethod
def _rescore_cached_gpu_stats(gpu_stats: dict[str, dict[str, Any]]) -> None:
usable = [
item
for item in gpu_stats.values()
if item.get("backlogHours") is not None and item.get("recentWilsonLowerBound") is not None
]
finite_backlogs = [
float(item["backlogHours"])
for item in usable
if float(item["backlogHours"]) < 9_999.0
]
median_backlog = statistics.median(finite_backlogs) if finite_backlogs else 24.0
max_wilson = max((float(item.get("recentWilsonLowerBound") or 0.0) for item in usable), default=0.0)
for item in usable:
backlog = max(0.25, float(item.get("backlogHours") or 9_999.0))
wilson = float(item.get("recentWilsonLowerBound") or 0.0)
health_factor = float(item.get("healthFactor") or 1.0)
queue_factor = _clamp((max(0.25, median_backlog) / backlog) ** 0.15, 0.70, 1.30)
quality_ratio = wilson / max_wilson if max_wilson > 0 else 0.0
quality_factor = _clamp(2.5 * (quality_ratio**2.2), 0.05, 2.5)
item["queueFactor"] = queue_factor
item["qualityFactor"] = quality_factor
item["queueWeight"] = _clamp(queue_factor * health_factor, 0.05, 2.0)
item["selectionWeight"] = _clamp(queue_factor * quality_factor * health_factor, 0.02, 3.0)
item["submissionEligible"] = bool(
item.get("canVerify") is not False
and health_factor >= 0.5
and int(item.get("recentTerminal") or 0) >= DEFAULT_GPU_MIN_RECENT_TERMINALS
and int(item.get("recentSuccess") or 0) > 0
and wilson >= DEFAULT_FRAMEWORK_MIN_WILSON
)
def _base_state(self, gpus: list[str], task_types: list[str], now: datetime) -> dict[str, Any]:
return {
"version": MARKET_STATE_VERSION,
@@ -185,7 +228,14 @@ class MarketIntelligenceManager:
gpus = list(dict.fromkeys(gpu for gpu in supported_gpus if gpu))
tasks = list(dict.fromkeys(task for task in task_types if task))
loaded = self._load()
state = loaded if self._compatible(loaded, gpus, tasks) else self._base_state(gpus, tasks, now)
if self._compatible(loaded, gpus, tasks):
state = loaded
elif int((loaded or {}).get("version") or 0) == 2 and self._catalog_compatible(loaded, gpus, tasks):
state = dict(loaded or {})
state["version"] = MARKET_STATE_VERSION
self._rescore_cached_gpu_stats(state.get("gpuStats") or {})
else:
state = self._base_state(gpus, tasks, now)
queue_due = not _fresh(
state.get("queueUpdatedAt"),
@@ -355,9 +405,16 @@ class MarketIntelligenceManager:
result = _neutral_gpu_stats(gpus)
for gpu, stats in fetched.items():
backlog = max(0.25, float(stats["backlogHours"]))
queue_factor = _clamp(math.sqrt(max(0.25, median_backlog) / backlog), 0.35, 1.75)
# Queue pressure is now a tie-breaker between proven GPUs, not a way
# for a fast but unreliable pool to outrank a successful one.
queue_factor = _clamp(
(max(0.25, median_backlog) / backlog) ** 0.15,
0.70,
1.30,
)
wilson = float(stats["recentWilsonLowerBound"])
quality_factor = 1.0 if max_wilson <= 0 else 0.6 + 1.4 * (wilson / max_wilson)
quality_ratio = wilson / max_wilson if max_wilson > 0 else 0.0
quality_factor = _clamp(2.5 * (quality_ratio**2.2), 0.05, 2.5)
health_factor = 1.0
if stats.get("canVerify") is False:
health_factor = 0.05
@@ -379,7 +436,14 @@ class MarketIntelligenceManager:
stats["qualityFactor"] = quality_factor
stats["healthFactor"] = health_factor
stats["queueWeight"] = _clamp(queue_factor * health_factor, 0.05, 2.0)
stats["selectionWeight"] = _clamp(queue_factor * quality_factor * health_factor, 0.05, 3.0)
stats["selectionWeight"] = _clamp(queue_factor * quality_factor * health_factor, 0.02, 3.0)
stats["submissionEligible"] = bool(
stats.get("canVerify") is not False
and health_factor >= 0.5
and int(stats.get("recentTerminal") or 0) >= DEFAULT_GPU_MIN_RECENT_TERMINALS
and int(stats.get("recentSuccess") or 0) > 0
and wilson >= DEFAULT_FRAMEWORK_MIN_WILSON
)
result[gpu] = stats
for gpu, error in errors.items():
@@ -472,10 +536,22 @@ class MarketIntelligenceManager:
return result
def gpu_weight(self, gpu: str, *, category: str) -> float:
del category
stats = ((self.state or {}).get("gpuStats") or {}).get(gpu) or {}
if category == "all_supported":
return max(0.05, float(stats.get("queueWeight") or 1.0))
return max(0.05, float(stats.get("selectionWeight") or 1.0))
return max(0.02, float(stats.get("selectionWeight") or 1.0))
def eligible_gpus(self, supported_gpus: list[str]) -> list[str]:
stats = (self.state or {}).get("gpuStats") or {}
ranked: list[tuple[float, int, str]] = []
for index, gpu in enumerate(supported_gpus):
item = stats.get(gpu) or {}
eligibility = item.get("submissionEligible")
# Missing live data falls back to the scheduler's already-proven
# long/recent pools. Explicitly failed public gates never do.
if eligibility is False:
continue
ranked.append((-float(item.get("selectionWeight") or 1.0), index, gpu))
return [gpu for _weight, _index, gpu in sorted(ranked)]
def gpu_metadata(self, gpu: str) -> dict[str, Any]:
stats = ((self.state or {}).get("gpuStats") or {}).get(gpu) or {}
@@ -487,6 +563,7 @@ class MarketIntelligenceManager:
"queueBacklogHours": stats.get("backlogHours"),
"publicRecentSuccessRate": stats.get("recentSuccessRate"),
"publicThroughputPerHour": stats.get("throughputPerHour"),
"marketSubmissionEligible": stats.get("submissionEligible"),
"marketDataStale": bool(stats.get("stale", False)),
}
@@ -529,17 +606,41 @@ class MarketIntelligenceManager:
local_samples = local_success + local_failure
local_score = _wilson_lower_bound(local_success, local_samples)
public_qualified = public_samples >= self.framework_min_samples
recent_item = (self.local_outcome_stats.get("recentCombinationStats") or {}).get(local_key) or {}
recent_success = max(0, int(recent_item.get("successCount") or 0))
recent_failure = max(0, int(recent_item.get("failureCount") or 0))
recent_samples = recent_success + recent_failure
recent_rate = recent_success / recent_samples if recent_samples else None
consecutive_failures = max(0, int(recent_item.get("consecutiveFailures") or 0))
last_terminal_at = parse_datetime(recent_item.get("lastTerminalAt"))
circuit_reason = None
circuit_until = None
if last_terminal_at is not None and consecutive_failures >= 5:
circuit_reason = "five_consecutive_local_failures"
circuit_until = last_terminal_at + timedelta(hours=12)
elif last_terminal_at is not None and recent_samples >= 20 and recent_rate is not None and recent_rate < 0.20:
circuit_reason = "recent_local_success_below_20_percent"
circuit_until = last_terminal_at + timedelta(hours=6)
circuit_open = bool(circuit_until is not None and circuit_until > utc_now())
if not circuit_open:
circuit_reason = None
circuit_until = None
public_qualified = (
public_samples >= self.framework_min_samples
and public_score >= DEFAULT_FRAMEWORK_MIN_WILSON
)
local_qualified = local_samples >= 20
if public_qualified:
local_weight = min(0.35, local_samples / (local_samples + 50.0)) if local_samples >= 5 else 0.0
combined = public_score * (1.0 - local_weight) + local_score * local_weight
elif local_qualified:
combined = local_score
evidence_success = recent_success if recent_samples >= 5 else local_success
evidence_samples = recent_samples if recent_samples >= 5 else local_samples
evidence_score = _wilson_lower_bound(evidence_success, evidence_samples)
local_weight = min(0.60, evidence_samples / (evidence_samples + 100.0)) if evidence_samples >= 5 else 0.0
combined = public_score * (1.0 - local_weight) + evidence_score * local_weight
else:
combined = 0.0
return {
"qualified": public_qualified or local_qualified,
"qualified": public_qualified and not circuit_open,
"publicQualified": public_qualified,
"localQualified": local_qualified,
"combinedScore": combined,
@@ -548,8 +649,55 @@ class MarketIntelligenceManager:
"localSamples": local_samples,
"localSuccessRate": local_success / local_samples if local_samples else None,
"localScore": local_score if local_samples else None,
"recentLocalSamples": recent_samples,
"recentLocalSuccessRate": recent_rate,
"consecutiveLocalFailures": consecutive_failures,
"circuitOpen": circuit_open,
"circuitReason": circuit_reason,
"circuitUntil": circuit_until.isoformat() if circuit_until else None,
}
def selectable_frameworks(
self,
*,
task_type: str,
target_gpu: str,
incumbent_frameworks: list[str],
inspection: Any,
) -> list[str]:
discovered = self.compatible_discovered_frameworks(
task_type=task_type,
target_gpu=target_gpu,
inspection=inspection,
)
candidates = list(dict.fromkeys([*incumbent_frameworks, *discovered]))
vetted = [
framework
for framework in candidates
if self._framework_evidence(task_type, target_gpu, framework)["qualified"]
]
incumbent_scores = [
float(self._framework_evidence(task_type, target_gpu, framework)["combinedScore"])
for framework in vetted
if framework in incumbent_frameworks
]
incumbent_best = max(incumbent_scores, default=0.0)
promoted: list[str] = []
for framework in vetted:
if framework in incumbent_frameworks:
promoted.append(framework)
continue
evidence = self._framework_evidence(task_type, target_gpu, framework)
if incumbent_best > 0 and float(evidence["combinedScore"]) < incumbent_best * NEW_FRAMEWORK_PROMOTION_MARGIN:
continue
promoted.append(framework)
return self.rank_frameworks(
task_type=task_type,
target_gpu=target_gpu,
compatible_frameworks=promoted,
)
def compatible_discovered_frameworks(
self,
*,
@@ -609,10 +757,16 @@ class MarketIntelligenceManager:
"frameworkMarketWilsonLowerBound": item.get("wilsonLowerBound"),
"frameworkLocalSamples": evidence["localSamples"],
"frameworkLocalSuccessRate": evidence["localSuccessRate"],
"frameworkRecentLocalSamples": evidence["recentLocalSamples"],
"frameworkRecentLocalSuccessRate": evidence["recentLocalSuccessRate"],
"frameworkConsecutiveLocalFailures": evidence["consecutiveLocalFailures"],
"frameworkCombinedScore": evidence["combinedScore"],
"frameworkMarketQualified": evidence["publicQualified"],
"frameworkLocalQualified": evidence["localQualified"],
"frameworkEvidenceQualified": evidence["qualified"],
"frameworkCircuitOpen": evidence["circuitOpen"],
"frameworkCircuitReason": evidence["circuitReason"],
"frameworkCircuitUntil": evidence["circuitUntil"],
"frameworkOfficialConfigValid": bool(item.get("officialConfigValid", False)),
"frameworkOfficialConfigStale": bool(item.get("officialConfigStale", False)),
"frameworkConfigSource": "modelhub_live" if item.get("officialConfigValid") else "local_template",