feat: prioritize proven GPU framework combinations

This commit is contained in:
CoolBoy
2026-08-05 18:21:59 +08:00
parent 882479e43e
commit 5e47d9e695
11 changed files with 431 additions and 86 deletions

View File

@@ -8,7 +8,7 @@ It currently supports:
- continuous queue refill via `run_poll.sh`
- multiple ModelHub tokens read from `KEY.md` and `KEYS.md`
- automatic task/framework/template selection across the supported GPU catalog
- adaptive long-term/exploration/recent GPU scheduling with a persistent local snapshot
- adaptive long-term/recent GPU exploitation with a persistent local snapshot
- live queue/throughput-aware GPU weighting and confidence-ranked framework selection
## Layout
@@ -74,19 +74,21 @@ bash run_poll.sh --dry-run
## Behavior
- The runner auto-discovers all safe GPU/template combinations from the public submit catalog.
- Automatic GPU selection uses exact 50/30/20 accepted-task scheduling: long-term
Wilson-ranked top 3 GPUs, all supported GPUs, and the top GPUs from the latest
1,000 terminal tasks.
- Automatic GPU selection uses exact 70/30 accepted-task scheduling: long-term
Wilson-ranked top 3 GPUs and the top GPUs from the latest 1,000 terminal tasks.
There is no all-GPU exploration category.
- Within each category, weighted-fair scheduling uses estimated queue backlog hours,
recent public throughput/success, machine availability, and worker concurrency.
Unavailable or stalled GPU pools are circuit-broken instead of continuing to absorb work.
- Compatible frameworks are ranked by ModelHub public aggregate success statistics
plus capped local GPU+framework evidence, with a 100-sample public minimum and
Wilson confidence bounds. The legacy safe order is retained whenever evidence
is missing or too small.
plus capped local GPU+framework evidence, with a 300-sample public minimum and
Wilson confidence bounds. Missing or undersized public evidence receives zero
traffic rather than falling back to exploration.
- New frameworks are discovered from the live catalog but get no novelty bonus.
They are eligible only with a complete official build config that passes local
validation; cached local templates remain the fallback if live config sync fails.
validation and a confidence score at least 10% above the best incumbent.
- Five consecutive local failures pause a GPU/framework pair for 12 hours; a
sub-20% rate over the latest 20 terminal tasks pauses it for 6 hours.
- A strategy generation lasts exactly 200 platform-accepted submissions. Rejected API calls and
duplicates do not advance it. The next cycle refreshes platform history before submitting again.
- Strategy state is stored in `.modelhub_state/gpu_strategy.json`; a generation never recalculates
@@ -163,7 +165,7 @@ Each run typically includes:
Persistent local scheduler state is written under `.modelhub_state/`:
- `gpu_strategy.json`: GPU ranks, generation progress, and 50/30/20 accepted counters
- `gpu_strategy.json`: GPU ranks, generation progress, and 70/30 accepted counters
- `market_intelligence.json`: cached public queue, throughput, health, and framework statistics
- `account_capacity.json`: learned per-account active-task limits
- `submission_exclusions.jsonl`: non-retryable model/GPU uniqueness rejections

View File

@@ -88,7 +88,7 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument("--skip-outcome-sync", action="store_true", help="Skip outcome sync from ModelHub before scanning")
parser.add_argument("--skip-history-archive", action="store_true", help="Skip historical task archive download for this run")
parser.add_argument("--dry-run", action="store_true", help="Plan the day without creating tasks")
parser.add_argument("--disable-gpu-strategy", action="store_true", help="Disable adaptive 50/30/20 GPU scheduling")
parser.add_argument("--disable-gpu-strategy", action="store_true", help="Disable adaptive 70/30 proven-GPU scheduling")
parser.add_argument(
"--disable-market-intelligence",
action="store_true",

View File

@@ -17,13 +17,12 @@ DEFAULT_GPU_STRATEGY_PATH = Path(".modelhub_state/gpu_strategy.json")
DEFAULT_REFRESH_SUBMISSIONS = 200
DEFAULT_RECENT_TERMINAL_WINDOW = 1000
DEFAULT_LONG_TERM_MIN_SAMPLES = 100
STRATEGY_STATE_VERSION = 2
STRATEGY_STATE_VERSION = 3
LONG_TERM = "long_term"
ALL_SUPPORTED = "all_supported"
RECENT = "recent"
CATEGORIES = (LONG_TERM, ALL_SUPPORTED, RECENT)
CATEGORY_WEIGHTS = {LONG_TERM: 5, ALL_SUPPORTED: 3, RECENT: 2}
CATEGORIES = (LONG_TERM, RECENT)
CATEGORY_WEIGHTS = {LONG_TERM: 7, RECENT: 3}
def _empty_category_counts() -> dict[str, int]:
return {category: 0 for category in CATEGORIES}
@@ -354,7 +353,7 @@ class GPUStrategyManager:
self.log(
f"[strategy] {action} generation={self.state.get('generation', 0)} "
f"accepted={self.state.get('acceptedSinceRefresh', 0)}/{self.state.get('refreshSubmissions', self.refresh_submissions)} "
f"categories={counts.get(LONG_TERM, 0)},{counts.get(ALL_SUPPORTED, 0)},{counts.get(RECENT, 0)} "
f"categories={counts.get(LONG_TERM, 0)},{counts.get(RECENT, 0)} "
f"long={','.join(self.state.get('longTermGpus') or [])} recent={self.state.get('recentGpu') or 'n/a'}"
)
@@ -377,20 +376,25 @@ class GPUStrategyManager:
recent = [recent_gpu]
if category == LONG_TERM:
eligible = long_term
eligible = list(long_term)
base_pattern = (5.0, 3.0, 2.0)
elif category == RECENT:
eligible = recent
base_pattern = (6.0, 3.0, 1.0)
else:
eligible = supported
base_pattern = tuple(1.0 for _ in eligible)
eligible = list(recent)
base_pattern = (6.0, 3.0, 1.0)
if self.market_intelligence is not None:
try:
vetted = self.market_intelligence.eligible_gpus(supported)
except Exception:
vetted = supported
eligible = [gpu for gpu in eligible if gpu in vetted]
eligible.extend(gpu for gpu in vetted if gpu not in eligible)
if not eligible:
return []
weights: dict[str, float] = {}
for index, gpu in enumerate(eligible):
base = base_pattern[min(index, len(base_pattern) - 1)]
base = base_pattern[index] if index < len(base_pattern) else 0.5
market_weight = 1.0
if self.market_intelligence is not None:
try:
@@ -460,9 +464,13 @@ class GPUStrategyManager:
break
if selected is None:
# With market intelligence enabled, an empty vetted pool means
# stop instead of silently turning fallback into exploration.
if self.market_intelligence is not None:
break
# Unknown/custom GPUs can only appear when callers bypass the normal resolver.
selected = next((candidate for candidate in candidates if candidate_key(candidate) not in used), None)
actual_category = ALL_SUPPORTED
actual_category = planned_category
if selected is None:
break
@@ -508,9 +516,9 @@ class GPUStrategyManager:
for category in CATEGORIES
}
for candidate in candidates:
category = str(candidate.get("strategyCategory") or ALL_SUPPORTED)
category = str(candidate.get("strategyCategory") or LONG_TERM)
if category not in category_counts:
category = ALL_SUPPORTED
category = LONG_TERM
category_counts[category] += 1
gpu = str(candidate.get("targetGpu") or "")
if gpu in gpu_category_counts[category]:

View File

@@ -68,7 +68,7 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument(
"--disable-gpu-strategy",
action="store_true",
help="Disable adaptive 50/30/20 GPU scheduling and keep the legacy candidate order",
help="Disable adaptive 70/30 proven-GPU scheduling and keep the legacy candidate order",
)
parser.add_argument(
"--disable-market-intelligence",
@@ -274,22 +274,14 @@ def choose_candidate_for_gpu(
except ValueError:
pass
if market_intelligence is not None:
discovered = market_intelligence.compatible_discovered_frameworks(
compatible_frameworks = market_intelligence.selectable_frameworks(
task_type=task_type,
target_gpu=target_gpu,
incumbent_frameworks=compatible_frameworks,
inspection=inspection,
)
compatible_frameworks.extend(
framework for framework in discovered if framework not in compatible_frameworks
)
if not compatible_frameworks:
continue
if market_intelligence is not None:
compatible_frameworks = market_intelligence.rank_frameworks(
task_type=task_type,
target_gpu=target_gpu,
compatible_frameworks=compatible_frameworks,
)
for framework in compatible_frameworks:
config_params = (
@@ -451,7 +443,12 @@ def process_model_for_candidates(
market_intelligence=market_intelligence,
)
if best is None:
skipped.append({"repoId": model.repo_id, "targetGpu": target_gpu, "reason": "no_compatible_auto_template_or_framework"})
reason = (
"no_publicly_vetted_compatible_framework"
if market_intelligence is not None
else "no_compatible_auto_template_or_framework"
)
skipped.append({"repoId": model.repo_id, "targetGpu": target_gpu, "reason": reason})
continue
record = candidate_to_record(best)
if market_intelligence is not None:
@@ -706,6 +703,7 @@ def run_submission(
target_gpus = resolve_target_gpus(args, template_selector, selected_task_types)
if not target_gpus:
raise RuntimeError("No auto-submittable GPUs are available for the selected task types")
submission_target_gpus = list(target_gpus)
hf_discovery = hf_discovery or HuggingFaceDiscovery(base_url=args.hf_base_url)
if modelhub_client is None:
@@ -809,6 +807,7 @@ def run_submission(
"dryRun": bool(args.dry_run),
"selectedTaskTypes": selected_task_types,
"targetGpus": target_gpus,
"submissionEligibleGpus": submission_target_gpus,
"dailyTarget": args.daily_target,
"unlimitedDailyTarget": unlimited_daily_target,
"submittedTodayBeforeRun": daily_snapshot["totalCount"],
@@ -893,6 +892,14 @@ def run_submission(
except Exception:
market_intelligence.set_local_outcome_stats(None)
market_summary = market_intelligence.summary()
submission_target_gpus = market_intelligence.eligible_gpus(target_gpus)
market_summary["eligibleGpus"] = submission_target_gpus
market_summary["shadowOnlyGpus"] = [gpu for gpu in target_gpus if gpu not in submission_target_gpus]
print(
f"[market] eligible_gpus={','.join(submission_target_gpus) or 'none'} "
f"shadow_only={','.join(market_summary['shadowOnlyGpus']) or 'none'}",
flush=True,
)
if strategy_enabled:
strategy_manager = GPUStrategyManager(
@@ -930,7 +937,7 @@ def run_submission(
scan_limit = resolve_scan_limit(
args,
target_gpu_count=len(target_gpus),
target_gpu_count=len(submission_target_gpus),
remaining_daily_quota=remaining_daily_quota,
platform_available_slots=platform_available_slots,
)
@@ -986,7 +993,7 @@ def run_submission(
hf_discovery=hf_discovery,
modelhub_client=modelhub_client,
template_selector=template_selector,
target_gpus=target_gpus,
target_gpus=submission_target_gpus,
selected_task_types=selected_task_types,
outcome_tracker=outcome_tracker,
submission_exclusion_store=submission_exclusion_store,
@@ -1204,6 +1211,7 @@ def run_submission(
"dryRun": bool(args.dry_run),
"selectedTaskTypes": selected_task_types,
"targetGpus": target_gpus,
"submissionEligibleGpus": submission_target_gpus,
"maxSubmitsPerRun": int(getattr(args, "max_submits_per_run", 0) or 0),
"dailyTarget": args.daily_target,
"unlimitedDailyTarget": unlimited_daily_target,

View File

@@ -12,12 +12,15 @@ from common import parse_datetime, read_json, utc_now, write_json
DEFAULT_MARKET_INTELLIGENCE_PATH = Path(".modelhub_state/market_intelligence.json")
MARKET_STATE_VERSION = 2
MARKET_STATE_VERSION = 3
DEFAULT_QUEUE_REFRESH_SECONDS = 600
DEFAULT_FRAMEWORK_REFRESH_SECONDS = 21_600
DEFAULT_THROUGHPUT_WINDOW_HOURS = 6
DEFAULT_FETCH_WORKERS = 4
DEFAULT_FRAMEWORK_MIN_SAMPLES = 100
DEFAULT_FRAMEWORK_MIN_SAMPLES = 300
DEFAULT_GPU_MIN_RECENT_TERMINALS = 20
DEFAULT_FRAMEWORK_MIN_WILSON = 0.05
NEW_FRAMEWORK_PROMOTION_MARGIN = 1.10
ERROR_RETRY_SECONDS = 300
@@ -73,6 +76,7 @@ def _neutral_gpu_stats(gpus: list[str]) -> dict[str, dict[str, Any]]:
"healthFactor": 1.0,
"queueWeight": 1.0,
"selectionWeight": 1.0,
"submissionEligible": None,
"error": "market_data_unavailable",
}
for gpu in gpus
@@ -156,6 +160,45 @@ class MarketIntelligenceManager:
return False
return list(state.get("supportedGpus") or []) == gpus and list(state.get("taskTypes") or []) == task_types
@staticmethod
def _catalog_compatible(state: dict[str, Any] | None, gpus: list[str], task_types: list[str]) -> bool:
if not state:
return False
return list(state.get("supportedGpus") or []) == gpus and list(state.get("taskTypes") or []) == task_types
@staticmethod
def _rescore_cached_gpu_stats(gpu_stats: dict[str, dict[str, Any]]) -> None:
usable = [
item
for item in gpu_stats.values()
if item.get("backlogHours") is not None and item.get("recentWilsonLowerBound") is not None
]
finite_backlogs = [
float(item["backlogHours"])
for item in usable
if float(item["backlogHours"]) < 9_999.0
]
median_backlog = statistics.median(finite_backlogs) if finite_backlogs else 24.0
max_wilson = max((float(item.get("recentWilsonLowerBound") or 0.0) for item in usable), default=0.0)
for item in usable:
backlog = max(0.25, float(item.get("backlogHours") or 9_999.0))
wilson = float(item.get("recentWilsonLowerBound") or 0.0)
health_factor = float(item.get("healthFactor") or 1.0)
queue_factor = _clamp((max(0.25, median_backlog) / backlog) ** 0.15, 0.70, 1.30)
quality_ratio = wilson / max_wilson if max_wilson > 0 else 0.0
quality_factor = _clamp(2.5 * (quality_ratio**2.2), 0.05, 2.5)
item["queueFactor"] = queue_factor
item["qualityFactor"] = quality_factor
item["queueWeight"] = _clamp(queue_factor * health_factor, 0.05, 2.0)
item["selectionWeight"] = _clamp(queue_factor * quality_factor * health_factor, 0.02, 3.0)
item["submissionEligible"] = bool(
item.get("canVerify") is not False
and health_factor >= 0.5
and int(item.get("recentTerminal") or 0) >= DEFAULT_GPU_MIN_RECENT_TERMINALS
and int(item.get("recentSuccess") or 0) > 0
and wilson >= DEFAULT_FRAMEWORK_MIN_WILSON
)
def _base_state(self, gpus: list[str], task_types: list[str], now: datetime) -> dict[str, Any]:
return {
"version": MARKET_STATE_VERSION,
@@ -185,7 +228,14 @@ class MarketIntelligenceManager:
gpus = list(dict.fromkeys(gpu for gpu in supported_gpus if gpu))
tasks = list(dict.fromkeys(task for task in task_types if task))
loaded = self._load()
state = loaded if self._compatible(loaded, gpus, tasks) else self._base_state(gpus, tasks, now)
if self._compatible(loaded, gpus, tasks):
state = loaded
elif int((loaded or {}).get("version") or 0) == 2 and self._catalog_compatible(loaded, gpus, tasks):
state = dict(loaded or {})
state["version"] = MARKET_STATE_VERSION
self._rescore_cached_gpu_stats(state.get("gpuStats") or {})
else:
state = self._base_state(gpus, tasks, now)
queue_due = not _fresh(
state.get("queueUpdatedAt"),
@@ -355,9 +405,16 @@ class MarketIntelligenceManager:
result = _neutral_gpu_stats(gpus)
for gpu, stats in fetched.items():
backlog = max(0.25, float(stats["backlogHours"]))
queue_factor = _clamp(math.sqrt(max(0.25, median_backlog) / backlog), 0.35, 1.75)
# Queue pressure is now a tie-breaker between proven GPUs, not a way
# for a fast but unreliable pool to outrank a successful one.
queue_factor = _clamp(
(max(0.25, median_backlog) / backlog) ** 0.15,
0.70,
1.30,
)
wilson = float(stats["recentWilsonLowerBound"])
quality_factor = 1.0 if max_wilson <= 0 else 0.6 + 1.4 * (wilson / max_wilson)
quality_ratio = wilson / max_wilson if max_wilson > 0 else 0.0
quality_factor = _clamp(2.5 * (quality_ratio**2.2), 0.05, 2.5)
health_factor = 1.0
if stats.get("canVerify") is False:
health_factor = 0.05
@@ -379,7 +436,14 @@ class MarketIntelligenceManager:
stats["qualityFactor"] = quality_factor
stats["healthFactor"] = health_factor
stats["queueWeight"] = _clamp(queue_factor * health_factor, 0.05, 2.0)
stats["selectionWeight"] = _clamp(queue_factor * quality_factor * health_factor, 0.05, 3.0)
stats["selectionWeight"] = _clamp(queue_factor * quality_factor * health_factor, 0.02, 3.0)
stats["submissionEligible"] = bool(
stats.get("canVerify") is not False
and health_factor >= 0.5
and int(stats.get("recentTerminal") or 0) >= DEFAULT_GPU_MIN_RECENT_TERMINALS
and int(stats.get("recentSuccess") or 0) > 0
and wilson >= DEFAULT_FRAMEWORK_MIN_WILSON
)
result[gpu] = stats
for gpu, error in errors.items():
@@ -472,10 +536,22 @@ class MarketIntelligenceManager:
return result
def gpu_weight(self, gpu: str, *, category: str) -> float:
del category
stats = ((self.state or {}).get("gpuStats") or {}).get(gpu) or {}
if category == "all_supported":
return max(0.05, float(stats.get("queueWeight") or 1.0))
return max(0.05, float(stats.get("selectionWeight") or 1.0))
return max(0.02, float(stats.get("selectionWeight") or 1.0))
def eligible_gpus(self, supported_gpus: list[str]) -> list[str]:
stats = (self.state or {}).get("gpuStats") or {}
ranked: list[tuple[float, int, str]] = []
for index, gpu in enumerate(supported_gpus):
item = stats.get(gpu) or {}
eligibility = item.get("submissionEligible")
# Missing live data falls back to the scheduler's already-proven
# long/recent pools. Explicitly failed public gates never do.
if eligibility is False:
continue
ranked.append((-float(item.get("selectionWeight") or 1.0), index, gpu))
return [gpu for _weight, _index, gpu in sorted(ranked)]
def gpu_metadata(self, gpu: str) -> dict[str, Any]:
stats = ((self.state or {}).get("gpuStats") or {}).get(gpu) or {}
@@ -487,6 +563,7 @@ class MarketIntelligenceManager:
"queueBacklogHours": stats.get("backlogHours"),
"publicRecentSuccessRate": stats.get("recentSuccessRate"),
"publicThroughputPerHour": stats.get("throughputPerHour"),
"marketSubmissionEligible": stats.get("submissionEligible"),
"marketDataStale": bool(stats.get("stale", False)),
}
@@ -529,17 +606,41 @@ class MarketIntelligenceManager:
local_samples = local_success + local_failure
local_score = _wilson_lower_bound(local_success, local_samples)
public_qualified = public_samples >= self.framework_min_samples
recent_item = (self.local_outcome_stats.get("recentCombinationStats") or {}).get(local_key) or {}
recent_success = max(0, int(recent_item.get("successCount") or 0))
recent_failure = max(0, int(recent_item.get("failureCount") or 0))
recent_samples = recent_success + recent_failure
recent_rate = recent_success / recent_samples if recent_samples else None
consecutive_failures = max(0, int(recent_item.get("consecutiveFailures") or 0))
last_terminal_at = parse_datetime(recent_item.get("lastTerminalAt"))
circuit_reason = None
circuit_until = None
if last_terminal_at is not None and consecutive_failures >= 5:
circuit_reason = "five_consecutive_local_failures"
circuit_until = last_terminal_at + timedelta(hours=12)
elif last_terminal_at is not None and recent_samples >= 20 and recent_rate is not None and recent_rate < 0.20:
circuit_reason = "recent_local_success_below_20_percent"
circuit_until = last_terminal_at + timedelta(hours=6)
circuit_open = bool(circuit_until is not None and circuit_until > utc_now())
if not circuit_open:
circuit_reason = None
circuit_until = None
public_qualified = (
public_samples >= self.framework_min_samples
and public_score >= DEFAULT_FRAMEWORK_MIN_WILSON
)
local_qualified = local_samples >= 20
if public_qualified:
local_weight = min(0.35, local_samples / (local_samples + 50.0)) if local_samples >= 5 else 0.0
combined = public_score * (1.0 - local_weight) + local_score * local_weight
elif local_qualified:
combined = local_score
evidence_success = recent_success if recent_samples >= 5 else local_success
evidence_samples = recent_samples if recent_samples >= 5 else local_samples
evidence_score = _wilson_lower_bound(evidence_success, evidence_samples)
local_weight = min(0.60, evidence_samples / (evidence_samples + 100.0)) if evidence_samples >= 5 else 0.0
combined = public_score * (1.0 - local_weight) + evidence_score * local_weight
else:
combined = 0.0
return {
"qualified": public_qualified or local_qualified,
"qualified": public_qualified and not circuit_open,
"publicQualified": public_qualified,
"localQualified": local_qualified,
"combinedScore": combined,
@@ -548,8 +649,55 @@ class MarketIntelligenceManager:
"localSamples": local_samples,
"localSuccessRate": local_success / local_samples if local_samples else None,
"localScore": local_score if local_samples else None,
"recentLocalSamples": recent_samples,
"recentLocalSuccessRate": recent_rate,
"consecutiveLocalFailures": consecutive_failures,
"circuitOpen": circuit_open,
"circuitReason": circuit_reason,
"circuitUntil": circuit_until.isoformat() if circuit_until else None,
}
def selectable_frameworks(
self,
*,
task_type: str,
target_gpu: str,
incumbent_frameworks: list[str],
inspection: Any,
) -> list[str]:
discovered = self.compatible_discovered_frameworks(
task_type=task_type,
target_gpu=target_gpu,
inspection=inspection,
)
candidates = list(dict.fromkeys([*incumbent_frameworks, *discovered]))
vetted = [
framework
for framework in candidates
if self._framework_evidence(task_type, target_gpu, framework)["qualified"]
]
incumbent_scores = [
float(self._framework_evidence(task_type, target_gpu, framework)["combinedScore"])
for framework in vetted
if framework in incumbent_frameworks
]
incumbent_best = max(incumbent_scores, default=0.0)
promoted: list[str] = []
for framework in vetted:
if framework in incumbent_frameworks:
promoted.append(framework)
continue
evidence = self._framework_evidence(task_type, target_gpu, framework)
if incumbent_best > 0 and float(evidence["combinedScore"]) < incumbent_best * NEW_FRAMEWORK_PROMOTION_MARGIN:
continue
promoted.append(framework)
return self.rank_frameworks(
task_type=task_type,
target_gpu=target_gpu,
compatible_frameworks=promoted,
)
def compatible_discovered_frameworks(
self,
*,
@@ -609,10 +757,16 @@ class MarketIntelligenceManager:
"frameworkMarketWilsonLowerBound": item.get("wilsonLowerBound"),
"frameworkLocalSamples": evidence["localSamples"],
"frameworkLocalSuccessRate": evidence["localSuccessRate"],
"frameworkRecentLocalSamples": evidence["recentLocalSamples"],
"frameworkRecentLocalSuccessRate": evidence["recentLocalSuccessRate"],
"frameworkConsecutiveLocalFailures": evidence["consecutiveLocalFailures"],
"frameworkCombinedScore": evidence["combinedScore"],
"frameworkMarketQualified": evidence["publicQualified"],
"frameworkLocalQualified": evidence["localQualified"],
"frameworkEvidenceQualified": evidence["qualified"],
"frameworkCircuitOpen": evidence["circuitOpen"],
"frameworkCircuitReason": evidence["circuitReason"],
"frameworkCircuitUntil": evidence["circuitUntil"],
"frameworkOfficialConfigValid": bool(item.get("officialConfigValid", False)),
"frameworkOfficialConfigStale": bool(item.get("officialConfigStale", False)),
"frameworkConfigSource": "modelhub_live" if item.get("officialConfigValid") else "local_template",

View File

@@ -153,6 +153,28 @@ class OutcomeTracker:
f"{gpu}|{fw}|{tt}": {"targetGpu": gpu, "framework": fw, "taskType": tt, **_summarize(records)}
for (gpu, fw, tt), records in combo_groups.items()
}
recent_combination_stats: dict[str, dict[str, Any]] = {}
for (gpu, fw, tt), records in combo_groups.items():
recent = sorted(records, key=_outcome_record_timestamp, reverse=True)[:20]
consecutive_failures = 0
for record in recent:
if record.get("outcome") != "failed":
break
consecutive_failures += 1
last_terminal_at = None
if recent:
last_terminal_at = (
parse_datetime(recent[0].get("lastSyncTime"))
or parse_datetime(recent[0].get("submitTime"))
)
recent_combination_stats[f"{gpu}|{fw}|{tt}"] = {
"targetGpu": gpu,
"framework": fw,
"taskType": tt,
**_summarize(recent),
"consecutiveFailures": consecutive_failures,
"lastTerminalAt": last_terminal_at.isoformat() if last_terminal_at else None,
}
warnings: list[str] = []
for gpu, summary in gpu_summaries.items():
@@ -172,6 +194,7 @@ class OutcomeTracker:
"gpuSummaries": gpu_summaries,
"frameworkSummaries": framework_summaries,
"combinationStats": combination_stats,
"recentCombinationStats": recent_combination_stats,
"totals": _summarize(terminal),
"warnings": warnings,
}
@@ -275,6 +298,11 @@ def _summarize(records: list[dict[str, Any]]) -> dict[str, Any]:
}
def _outcome_record_timestamp(record: dict[str, Any]) -> float:
timestamp = parse_datetime(record.get("lastSyncTime")) or parse_datetime(record.get("submitTime"))
return timestamp.timestamp() if timestamp else 0.0
def _outcome_record_key(record: dict[str, Any]) -> str:
task_id = record.get("taskId")
if task_id is not None:

View File

@@ -67,7 +67,7 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument("--skip-outcome-sync", action="store_true", help="Skip outcome sync from ModelHub before scanning")
parser.add_argument("--skip-history-archive", action="store_true", help="Skip historical task archive download for this run")
parser.add_argument("--dry-run", action="store_true", help="Plan the day without creating tasks")
parser.add_argument("--disable-gpu-strategy", action="store_true", help="Disable adaptive 50/30/20 GPU scheduling")
parser.add_argument("--disable-gpu-strategy", action="store_true", help="Disable adaptive 70/30 proven-GPU scheduling")
parser.add_argument(
"--disable-market-intelligence",
action="store_true",

View File

@@ -1 +1 @@
AGENT_VERSION = "2026.08.04.1"
AGENT_VERSION = "2026.08.05.1"