CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: add failure-aware preflight and Qwen review

Browse Source
This commit is contained in:
CoolBoy
2026-08-10 21:44:42 +08:00
parent 5e47d9e695
commit 3d15f60284
21 changed files with 2672 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
modelhub_submmit_api/market_intelligence.py
View File

@@ -602,20 +602,33 @@ class MarketIntelligenceManager:
local_key = f"{target_gpu}|{framework}|{task_type}"
local_item = (self.local_outcome_stats.get("combinationStats") or {}).get(local_key) or {}
local_success = max(0, int(local_item.get("successCount") or 0))
local_failure = max(0, int(local_item.get("failureCount") or 0))
local_failure = max(
0,
int(local_item.get("attributableFailureCount", local_item.get("failureCount") or 0)),
)
local_samples = local_success + local_failure
local_score = _wilson_lower_bound(local_success, local_samples)
recent_item = (self.local_outcome_stats.get("recentCombinationStats") or {}).get(local_key) or {}
recent_success = max(0, int(recent_item.get("successCount") or 0))
recent_failure = max(0, int(recent_item.get("failureCount") or 0))
recent_failure = max(
0,
int(recent_item.get("attributableFailureCount", recent_item.get("failureCount") or 0)),
)
recent_samples = recent_success + recent_failure
recent_rate = recent_success / recent_samples if recent_samples else None
consecutive_failures = max(0, int(recent_item.get("consecutiveFailures") or 0))
consecutive_platform_failures = max(
0, int(recent_item.get("consecutivePlatformFailures") or 0)
)
last_terminal_at = parse_datetime(recent_item.get("lastTerminalAt"))
last_platform_failure_at = parse_datetime(recent_item.get("lastPlatformFailureAt"))
circuit_reason = None
circuit_until = None
if last_terminal_at is not None and consecutive_failures >= 5:
if last_platform_failure_at is not None and consecutive_platform_failures >= 3:
circuit_reason = "three_consecutive_platform_failures"
circuit_until = last_platform_failure_at + timedelta(minutes=30)
elif last_terminal_at is not None and consecutive_failures >= 5:
circuit_reason = "five_consecutive_local_failures"
circuit_until = last_terminal_at + timedelta(hours=12)
elif last_terminal_at is not None and recent_samples >= 20 and recent_rate is not None and recent_rate < 0.20:
@@ -652,6 +665,7 @@ class MarketIntelligenceManager:
"recentLocalSamples": recent_samples,
"recentLocalSuccessRate": recent_rate,
"consecutiveLocalFailures": consecutive_failures,
"consecutivePlatformFailures": consecutive_platform_failures,
"circuitOpen": circuit_open,
"circuitReason": circuit_reason,
"circuitUntil": circuit_until.isoformat() if circuit_until else None,