819 lines
36 KiB
Python
819 lines
36 KiB
Python
from __future__ import annotations
|
|
|
|
import math
|
|
import re
|
|
import statistics
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
from common import parse_datetime, read_json, utc_now, write_json
|
|
|
|
|
|
DEFAULT_MARKET_INTELLIGENCE_PATH = Path(".modelhub_state/market_intelligence.json")
|
|
MARKET_STATE_VERSION = 3
|
|
DEFAULT_QUEUE_REFRESH_SECONDS = 600
|
|
DEFAULT_FRAMEWORK_REFRESH_SECONDS = 21_600
|
|
DEFAULT_THROUGHPUT_WINDOW_HOURS = 6
|
|
DEFAULT_FETCH_WORKERS = 4
|
|
DEFAULT_FRAMEWORK_MIN_SAMPLES = 300
|
|
DEFAULT_GPU_MIN_RECENT_TERMINALS = 20
|
|
DEFAULT_FRAMEWORK_MIN_WILSON = 0.05
|
|
NEW_FRAMEWORK_PROMOTION_MARGIN = 1.10
|
|
ERROR_RETRY_SECONDS = 300
|
|
|
|
|
|
def _clamp(value: float, minimum: float, maximum: float) -> float:
|
|
return max(minimum, min(maximum, value))
|
|
|
|
|
|
def _wilson_lower_bound(successes: int, total: int, *, z: float = 1.96) -> float:
|
|
if total <= 0:
|
|
return 0.0
|
|
probability = successes / total
|
|
z_squared = z * z
|
|
denominator = 1.0 + z_squared / total
|
|
centre = probability + z_squared / (2.0 * total)
|
|
margin = z * math.sqrt(
|
|
(probability * (1.0 - probability) / total) + z_squared / (4.0 * total * total)
|
|
)
|
|
return max(0.0, (centre - margin) / denominator)
|
|
|
|
|
|
def _fresh(timestamp: Any, *, now: datetime, ttl_seconds: int) -> bool:
|
|
parsed = parse_datetime(timestamp)
|
|
if parsed is None:
|
|
return False
|
|
return (now - parsed).total_seconds() < max(1, ttl_seconds)
|
|
|
|
|
|
def _page_total(client: Any, **kwargs: Any) -> int:
|
|
payload = client.list_tasks_page(current=1, page_size=1, only_mine=False, **kwargs)
|
|
data = payload.get("data") or {}
|
|
if not isinstance(data, dict) or "total" not in data:
|
|
raise RuntimeError("Public task count response is incomplete")
|
|
return max(0, int(data.get("total") or 0))
|
|
|
|
|
|
def _neutral_gpu_stats(gpus: list[str]) -> dict[str, dict[str, Any]]:
|
|
return {
|
|
gpu: {
|
|
"gpu": gpu,
|
|
"available": True,
|
|
"canVerify": None,
|
|
"maxConcurrentTasks": None,
|
|
"waiting": None,
|
|
"running": None,
|
|
"recentSuccess": None,
|
|
"recentTerminal": None,
|
|
"recentSuccessRate": None,
|
|
"recentWilsonLowerBound": None,
|
|
"throughputPerHour": None,
|
|
"backlogHours": None,
|
|
"queueFactor": 1.0,
|
|
"qualityFactor": 1.0,
|
|
"healthFactor": 1.0,
|
|
"queueWeight": 1.0,
|
|
"selectionWeight": 1.0,
|
|
"submissionEligible": None,
|
|
"error": "market_data_unavailable",
|
|
}
|
|
for gpu in gpus
|
|
}
|
|
|
|
|
|
def _validated_official_config(config: str, *, framework: str, target_gpu: str) -> tuple[bool, str | None]:
|
|
if not isinstance(config, str) or not config.strip():
|
|
return False, "empty_config"
|
|
if len(config) > 200_000:
|
|
return False, "config_too_large"
|
|
if "sut_config:" not in config or "ref_config:" not in config:
|
|
return False, "missing_sut_or_ref_config"
|
|
if any(marker in config for marker in ("{{", "}}", "PLACEHOLDER")):
|
|
return False, "unresolved_placeholder"
|
|
|
|
declared = re.search(r"(?m)^\s*framework:\s*['\"]?([^'\"\s]+)", config)
|
|
if declared is None or declared.group(1).strip() != framework:
|
|
return False, "framework_mismatch"
|
|
|
|
if target_gpu == "Biren_166m":
|
|
gpu_counts = [int(value) for value in re.findall(r"\bgpu_num:\s*['\"]?(\d+)", config)]
|
|
parallel_counts = [
|
|
int(value)
|
|
for value in re.findall(
|
|
r"(?:-tp|--tensor-parallel-size)(?:\s+|,\s*|\n\s*-\s*)['\"]?(\d+)",
|
|
config,
|
|
)
|
|
]
|
|
if any(value > 1 for value in [*gpu_counts, *parallel_counts]):
|
|
return False, "biren_parallelism_exceeds_one"
|
|
return True, None
|
|
|
|
|
|
def _render_official_config(config: str, *, gguf_filename: str | None) -> str:
|
|
if not gguf_filename:
|
|
return config
|
|
return re.sub(
|
|
r"(?i)(/model/)[^,\]\s'\"]+\.gguf",
|
|
lambda match: f"{match.group(1)}{gguf_filename}",
|
|
config,
|
|
)
|
|
|
|
|
|
class MarketIntelligenceManager:
|
|
def __init__(
|
|
self,
|
|
path: Path | str = DEFAULT_MARKET_INTELLIGENCE_PATH,
|
|
*,
|
|
queue_refresh_seconds: int = DEFAULT_QUEUE_REFRESH_SECONDS,
|
|
framework_refresh_seconds: int = DEFAULT_FRAMEWORK_REFRESH_SECONDS,
|
|
throughput_window_hours: int = DEFAULT_THROUGHPUT_WINDOW_HOURS,
|
|
fetch_workers: int = DEFAULT_FETCH_WORKERS,
|
|
framework_min_samples: int = DEFAULT_FRAMEWORK_MIN_SAMPLES,
|
|
log_fn: Callable[[str], None] | None = None,
|
|
) -> None:
|
|
self.path = Path(path)
|
|
self.queue_refresh_seconds = max(60, int(queue_refresh_seconds))
|
|
self.framework_refresh_seconds = max(300, int(framework_refresh_seconds))
|
|
self.throughput_window_hours = max(1, int(throughput_window_hours))
|
|
self.fetch_workers = max(1, min(8, int(fetch_workers)))
|
|
self.framework_min_samples = max(1, int(framework_min_samples))
|
|
self.log = log_fn or (lambda message: print(message, flush=True))
|
|
self.state: dict[str, Any] | None = None
|
|
self.local_outcome_stats: dict[str, Any] = {}
|
|
self._last_framework_request_errors: set[tuple[str, str]] = set()
|
|
|
|
def set_local_outcome_stats(self, report: dict[str, Any] | None) -> None:
|
|
self.local_outcome_stats = report if isinstance(report, dict) else {}
|
|
|
|
def _load(self) -> dict[str, Any] | None:
|
|
try:
|
|
payload = read_json(self.path)
|
|
except (FileNotFoundError, ValueError):
|
|
return None
|
|
return payload if isinstance(payload, dict) else None
|
|
|
|
@staticmethod
|
|
def _compatible(state: dict[str, Any] | None, gpus: list[str], task_types: list[str]) -> bool:
|
|
if not state or int(state.get("version") or 0) != MARKET_STATE_VERSION:
|
|
return False
|
|
return list(state.get("supportedGpus") or []) == gpus and list(state.get("taskTypes") or []) == task_types
|
|
|
|
@staticmethod
|
|
def _catalog_compatible(state: dict[str, Any] | None, gpus: list[str], task_types: list[str]) -> bool:
|
|
if not state:
|
|
return False
|
|
return list(state.get("supportedGpus") or []) == gpus and list(state.get("taskTypes") or []) == task_types
|
|
|
|
@staticmethod
|
|
def _rescore_cached_gpu_stats(gpu_stats: dict[str, dict[str, Any]]) -> None:
|
|
usable = [
|
|
item
|
|
for item in gpu_stats.values()
|
|
if item.get("backlogHours") is not None and item.get("recentWilsonLowerBound") is not None
|
|
]
|
|
finite_backlogs = [
|
|
float(item["backlogHours"])
|
|
for item in usable
|
|
if float(item["backlogHours"]) < 9_999.0
|
|
]
|
|
median_backlog = statistics.median(finite_backlogs) if finite_backlogs else 24.0
|
|
max_wilson = max((float(item.get("recentWilsonLowerBound") or 0.0) for item in usable), default=0.0)
|
|
for item in usable:
|
|
backlog = max(0.25, float(item.get("backlogHours") or 9_999.0))
|
|
wilson = float(item.get("recentWilsonLowerBound") or 0.0)
|
|
health_factor = float(item.get("healthFactor") or 1.0)
|
|
queue_factor = _clamp((max(0.25, median_backlog) / backlog) ** 0.15, 0.70, 1.30)
|
|
quality_ratio = wilson / max_wilson if max_wilson > 0 else 0.0
|
|
quality_factor = _clamp(2.5 * (quality_ratio**2.2), 0.05, 2.5)
|
|
item["queueFactor"] = queue_factor
|
|
item["qualityFactor"] = quality_factor
|
|
item["queueWeight"] = _clamp(queue_factor * health_factor, 0.05, 2.0)
|
|
item["selectionWeight"] = _clamp(queue_factor * quality_factor * health_factor, 0.02, 3.0)
|
|
item["submissionEligible"] = bool(
|
|
item.get("canVerify") is not False
|
|
and health_factor >= 0.5
|
|
and int(item.get("recentTerminal") or 0) >= DEFAULT_GPU_MIN_RECENT_TERMINALS
|
|
and int(item.get("recentSuccess") or 0) > 0
|
|
and wilson >= DEFAULT_FRAMEWORK_MIN_WILSON
|
|
)
|
|
|
|
def _base_state(self, gpus: list[str], task_types: list[str], now: datetime) -> dict[str, Any]:
|
|
return {
|
|
"version": MARKET_STATE_VERSION,
|
|
"generatedAt": now.isoformat(),
|
|
"supportedGpus": gpus,
|
|
"taskTypes": task_types,
|
|
"queueUpdatedAt": None,
|
|
"frameworkUpdatedAt": None,
|
|
"queueAttemptedAt": None,
|
|
"frameworkAttemptedAt": None,
|
|
"throughputWindowHours": self.throughput_window_hours,
|
|
"gpuStats": _neutral_gpu_stats(gpus),
|
|
"frameworkStats": {},
|
|
"queueError": None,
|
|
"frameworkError": None,
|
|
}
|
|
|
|
def prepare(
|
|
self,
|
|
client: Any,
|
|
*,
|
|
supported_gpus: list[str],
|
|
task_types: list[str],
|
|
now: datetime | None = None,
|
|
) -> dict[str, Any]:
|
|
now = now or utc_now()
|
|
gpus = list(dict.fromkeys(gpu for gpu in supported_gpus if gpu))
|
|
tasks = list(dict.fromkeys(task for task in task_types if task))
|
|
loaded = self._load()
|
|
if self._compatible(loaded, gpus, tasks):
|
|
state = loaded
|
|
elif int((loaded or {}).get("version") or 0) == 2 and self._catalog_compatible(loaded, gpus, tasks):
|
|
state = dict(loaded or {})
|
|
state["version"] = MARKET_STATE_VERSION
|
|
self._rescore_cached_gpu_stats(state.get("gpuStats") or {})
|
|
else:
|
|
state = self._base_state(gpus, tasks, now)
|
|
|
|
queue_due = not _fresh(
|
|
state.get("queueUpdatedAt"),
|
|
now=now,
|
|
ttl_seconds=self.queue_refresh_seconds,
|
|
)
|
|
framework_due = not _fresh(
|
|
state.get("frameworkUpdatedAt"),
|
|
now=now,
|
|
ttl_seconds=self.framework_refresh_seconds,
|
|
)
|
|
if state.get("queueError") and _fresh(
|
|
state.get("queueAttemptedAt"),
|
|
now=now,
|
|
ttl_seconds=min(ERROR_RETRY_SECONDS, self.queue_refresh_seconds),
|
|
):
|
|
queue_due = False
|
|
if state.get("frameworkError") and _fresh(
|
|
state.get("frameworkAttemptedAt"),
|
|
now=now,
|
|
ttl_seconds=min(ERROR_RETRY_SECONDS, self.framework_refresh_seconds),
|
|
):
|
|
framework_due = False
|
|
|
|
if queue_due:
|
|
state["queueAttemptedAt"] = now.isoformat()
|
|
try:
|
|
previous_gpu_stats = state.get("gpuStats") or {}
|
|
fresh_gpu_stats = self._fetch_gpu_stats(client, gpus=gpus, now=now)
|
|
for gpu, fresh_item in fresh_gpu_stats.items():
|
|
previous_item = previous_gpu_stats.get(gpu) or {}
|
|
if fresh_item.get("error") and previous_item and not previous_item.get("error"):
|
|
stale_item = dict(previous_item)
|
|
stale_item["stale"] = True
|
|
stale_item["refreshError"] = fresh_item.get("error")
|
|
fresh_gpu_stats[gpu] = stale_item
|
|
else:
|
|
fresh_item["stale"] = False
|
|
state["gpuStats"] = fresh_gpu_stats
|
|
state["queueUpdatedAt"] = now.isoformat()
|
|
state["queueError"] = None
|
|
except Exception as exc:
|
|
state["queueError"] = f"{type(exc).__name__}: {exc}"
|
|
if not state.get("gpuStats"):
|
|
state["gpuStats"] = _neutral_gpu_stats(gpus)
|
|
self.log(f"[market] queue_refresh_error reason={state['queueError']}")
|
|
|
|
if framework_due:
|
|
state["frameworkAttemptedAt"] = now.isoformat()
|
|
try:
|
|
previous_framework_stats = state.get("frameworkStats") or {}
|
|
fresh_framework_stats = self._fetch_framework_stats(client, gpus=gpus, task_types=tasks)
|
|
for task_type, gpu in self._last_framework_request_errors:
|
|
previous_rows = ((previous_framework_stats.get(task_type) or {}).get(gpu) or {})
|
|
if previous_rows:
|
|
fresh_framework_stats[task_type][gpu] = {
|
|
framework: {**dict(item), "stale": True}
|
|
for framework, item in previous_rows.items()
|
|
}
|
|
for task_type, by_gpu in fresh_framework_stats.items():
|
|
for gpu, rows in by_gpu.items():
|
|
previous_rows = ((previous_framework_stats.get(task_type) or {}).get(gpu) or {})
|
|
for framework, item in rows.items():
|
|
previous_item = previous_rows.get(framework) or {}
|
|
if not item.get("officialConfigValid") and previous_item.get("officialConfigValid"):
|
|
item["officialConfigValid"] = True
|
|
item["officialConfig"] = previous_item.get("officialConfig")
|
|
item["officialConfigStale"] = True
|
|
item["officialConfigRefreshError"] = item.get("officialConfigError")
|
|
state["frameworkStats"] = fresh_framework_stats
|
|
state["frameworkUpdatedAt"] = now.isoformat()
|
|
state["frameworkError"] = None
|
|
except Exception as exc:
|
|
state["frameworkError"] = f"{type(exc).__name__}: {exc}"
|
|
if not state.get("frameworkStats"):
|
|
state["frameworkStats"] = {}
|
|
self.log(f"[market] framework_refresh_error reason={state['frameworkError']}")
|
|
|
|
state["generatedAt"] = now.isoformat()
|
|
state["throughputWindowHours"] = self.throughput_window_hours
|
|
self.state = state
|
|
write_json(self.path, state)
|
|
self._log_snapshot("refreshed" if queue_due or framework_due else "loaded")
|
|
return state
|
|
|
|
def _fetch_gpu_stats(self, client: Any, *, gpus: list[str], now: datetime) -> dict[str, dict[str, Any]]:
|
|
local_end = now.astimezone()
|
|
local_begin = local_end - timedelta(hours=self.throughput_window_hours)
|
|
begin_text = local_begin.strftime("%Y-%m-%d %H:%M:%S")
|
|
end_text = local_end.strftime("%Y-%m-%d %H:%M:%S")
|
|
|
|
machine_by_gpu: dict[str, dict[str, Any]] = {}
|
|
for item in client.list_machine_info():
|
|
gpu = str(item.get("gpuType") or "")
|
|
if gpu:
|
|
machine_by_gpu[gpu] = item
|
|
|
|
def fetch_one(gpu: str) -> tuple[str, dict[str, Any]]:
|
|
waiting = _page_total(client, status="waiting", gpu_type=gpu)
|
|
running = _page_total(client, status="running", gpu_type=gpu)
|
|
completed = _page_total(
|
|
client,
|
|
status="success",
|
|
gpu_type=gpu,
|
|
begin_time=begin_text,
|
|
end_time=end_text,
|
|
)
|
|
success = _page_total(
|
|
client,
|
|
status="success",
|
|
verify_result=1,
|
|
gpu_type=gpu,
|
|
begin_time=begin_text,
|
|
end_time=end_text,
|
|
)
|
|
abnormal = _page_total(
|
|
client,
|
|
status="failed",
|
|
gpu_type=gpu,
|
|
begin_time=begin_text,
|
|
end_time=end_text,
|
|
)
|
|
terminal = max(success, completed + abnormal)
|
|
throughput = terminal / self.throughput_window_hours
|
|
backlog_hours = waiting / throughput if throughput > 0 else 9_999.0
|
|
success_rate = success / terminal if terminal else 0.0
|
|
machine = machine_by_gpu.get(gpu) or {}
|
|
return gpu, {
|
|
"gpu": gpu,
|
|
"available": machine.get("canVerify") is not False,
|
|
"canVerify": machine.get("canVerify"),
|
|
"maxConcurrentTasks": machine.get("maxConcurrentTasks"),
|
|
"waiting": waiting,
|
|
"running": running,
|
|
"recentSuccess": success,
|
|
"recentTerminal": terminal,
|
|
"recentSuccessRate": success_rate,
|
|
"recentWilsonLowerBound": _wilson_lower_bound(success, terminal),
|
|
"throughputPerHour": throughput,
|
|
"backlogHours": backlog_hours,
|
|
"error": None,
|
|
}
|
|
|
|
fetched: dict[str, dict[str, Any]] = {}
|
|
errors: dict[str, str] = {}
|
|
with ThreadPoolExecutor(max_workers=min(self.fetch_workers, max(1, len(gpus)))) as executor:
|
|
futures = {executor.submit(fetch_one, gpu): gpu for gpu in gpus}
|
|
for future in as_completed(futures):
|
|
gpu = futures[future]
|
|
try:
|
|
name, stats = future.result()
|
|
fetched[name] = stats
|
|
except Exception as exc:
|
|
errors[gpu] = f"{type(exc).__name__}: {exc}"
|
|
|
|
if not fetched and gpus:
|
|
raise RuntimeError("all GPU queue-stat requests failed")
|
|
|
|
finite_backlogs = [
|
|
float(stats["backlogHours"])
|
|
for stats in fetched.values()
|
|
if float(stats["backlogHours"]) < 9_999.0
|
|
]
|
|
median_backlog = statistics.median(finite_backlogs) if finite_backlogs else 24.0
|
|
max_wilson = max((float(stats["recentWilsonLowerBound"]) for stats in fetched.values()), default=0.0)
|
|
|
|
result = _neutral_gpu_stats(gpus)
|
|
for gpu, stats in fetched.items():
|
|
backlog = max(0.25, float(stats["backlogHours"]))
|
|
# Queue pressure is now a tie-breaker between proven GPUs, not a way
|
|
# for a fast but unreliable pool to outrank a successful one.
|
|
queue_factor = _clamp(
|
|
(max(0.25, median_backlog) / backlog) ** 0.15,
|
|
0.70,
|
|
1.30,
|
|
)
|
|
wilson = float(stats["recentWilsonLowerBound"])
|
|
quality_ratio = wilson / max_wilson if max_wilson > 0 else 0.0
|
|
quality_factor = _clamp(2.5 * (quality_ratio**2.2), 0.05, 2.5)
|
|
health_factor = 1.0
|
|
if stats.get("canVerify") is False:
|
|
health_factor = 0.05
|
|
elif (
|
|
int(stats.get("running") or 0) == 0
|
|
and int(stats.get("waiting") or 0) > 0
|
|
and int(stats.get("maxConcurrentTasks") or 0) <= 0
|
|
and int(stats.get("recentSuccess") or 0) == 0
|
|
):
|
|
health_factor = 0.10
|
|
elif (
|
|
int(stats.get("running") or 0) == 0
|
|
and int(stats.get("waiting") or 0) > 0
|
|
and int(stats.get("recentTerminal") or 0) == 0
|
|
):
|
|
health_factor = 0.25
|
|
|
|
stats["queueFactor"] = queue_factor
|
|
stats["qualityFactor"] = quality_factor
|
|
stats["healthFactor"] = health_factor
|
|
stats["queueWeight"] = _clamp(queue_factor * health_factor, 0.05, 2.0)
|
|
stats["selectionWeight"] = _clamp(queue_factor * quality_factor * health_factor, 0.02, 3.0)
|
|
stats["submissionEligible"] = bool(
|
|
stats.get("canVerify") is not False
|
|
and health_factor >= 0.5
|
|
and int(stats.get("recentTerminal") or 0) >= DEFAULT_GPU_MIN_RECENT_TERMINALS
|
|
and int(stats.get("recentSuccess") or 0) > 0
|
|
and wilson >= DEFAULT_FRAMEWORK_MIN_WILSON
|
|
)
|
|
result[gpu] = stats
|
|
|
|
for gpu, error in errors.items():
|
|
result[gpu]["error"] = error
|
|
return result
|
|
|
|
def _fetch_framework_stats(
|
|
self,
|
|
client: Any,
|
|
*,
|
|
gpus: list[str],
|
|
task_types: list[str],
|
|
) -> dict[str, dict[str, dict[str, dict[str, Any]]]]:
|
|
self._last_framework_request_errors = set()
|
|
result: dict[str, dict[str, dict[str, dict[str, Any]]]] = {
|
|
task_type: {gpu: {} for gpu in gpus}
|
|
for task_type in task_types
|
|
}
|
|
|
|
def fetch_one(task_type: str, gpu: str) -> tuple[str, str, list[dict[str, Any]]]:
|
|
return task_type, gpu, client.list_framework_stats(task_type, gpu)
|
|
|
|
with ThreadPoolExecutor(max_workers=self.fetch_workers) as executor:
|
|
futures = {
|
|
executor.submit(fetch_one, task_type, gpu): (task_type, gpu)
|
|
for task_type in task_types
|
|
for gpu in gpus
|
|
}
|
|
successful_requests = 0
|
|
errors: list[str] = []
|
|
for future in as_completed(futures):
|
|
task_type, gpu = futures[future]
|
|
try:
|
|
_, _, rows = future.result()
|
|
successful_requests += 1
|
|
except Exception as exc:
|
|
errors.append(f"{task_type}/{gpu}: {type(exc).__name__}: {exc}")
|
|
self._last_framework_request_errors.add((task_type, gpu))
|
|
continue
|
|
for row in rows:
|
|
framework = str(row.get("framework") or "").strip()
|
|
if not framework:
|
|
continue
|
|
total = max(0, int(row.get("modelCount") or 0))
|
|
success = max(0, min(total, int(row.get("successCount") or 0)))
|
|
result[task_type][gpu][framework] = {
|
|
"framework": framework,
|
|
"modelCount": total,
|
|
"successCount": success,
|
|
"successRate": success / total if total else 0.0,
|
|
"wilsonLowerBound": _wilson_lower_bound(success, total),
|
|
}
|
|
if futures and successful_requests <= 0:
|
|
detail = errors[0] if errors else "unknown error"
|
|
raise RuntimeError(f"all framework-stat requests failed ({detail})")
|
|
|
|
if hasattr(client, "get_build_config"):
|
|
config_targets = [
|
|
(task_type, gpu, framework)
|
|
for task_type, by_gpu in result.items()
|
|
for gpu, by_framework in by_gpu.items()
|
|
for framework in by_framework
|
|
]
|
|
|
|
def fetch_config(task_type: str, gpu: str, framework: str) -> tuple[str, str, str, str]:
|
|
return task_type, gpu, framework, client.get_build_config(task_type, gpu, framework)
|
|
|
|
with ThreadPoolExecutor(max_workers=self.fetch_workers) as executor:
|
|
config_futures = {
|
|
executor.submit(fetch_config, task_type, gpu, framework): (task_type, gpu, framework)
|
|
for task_type, gpu, framework in config_targets
|
|
}
|
|
for future in as_completed(config_futures):
|
|
task_type, gpu, framework = config_futures[future]
|
|
item = result[task_type][gpu][framework]
|
|
try:
|
|
_, _, _, config = future.result()
|
|
valid, reason = _validated_official_config(
|
|
config,
|
|
framework=framework,
|
|
target_gpu=gpu,
|
|
)
|
|
item["officialConfigValid"] = valid
|
|
item["officialConfigError"] = reason
|
|
if valid:
|
|
item["officialConfig"] = config
|
|
except Exception as exc:
|
|
item["officialConfigValid"] = False
|
|
item["officialConfigError"] = f"{type(exc).__name__}: {exc}"
|
|
return result
|
|
|
|
def gpu_weight(self, gpu: str, *, category: str) -> float:
|
|
del category
|
|
stats = ((self.state or {}).get("gpuStats") or {}).get(gpu) or {}
|
|
return max(0.02, float(stats.get("selectionWeight") or 1.0))
|
|
|
|
def eligible_gpus(self, supported_gpus: list[str]) -> list[str]:
|
|
stats = (self.state or {}).get("gpuStats") or {}
|
|
ranked: list[tuple[float, int, str]] = []
|
|
for index, gpu in enumerate(supported_gpus):
|
|
item = stats.get(gpu) or {}
|
|
eligibility = item.get("submissionEligible")
|
|
# Missing live data falls back to the scheduler's already-proven
|
|
# long/recent pools. Explicitly failed public gates never do.
|
|
if eligibility is False:
|
|
continue
|
|
ranked.append((-float(item.get("selectionWeight") or 1.0), index, gpu))
|
|
return [gpu for _weight, _index, gpu in sorted(ranked)]
|
|
|
|
def gpu_metadata(self, gpu: str) -> dict[str, Any]:
|
|
stats = ((self.state or {}).get("gpuStats") or {}).get(gpu) or {}
|
|
return {
|
|
"marketWeight": float(stats.get("selectionWeight") or 1.0),
|
|
"queueWeight": float(stats.get("queueWeight") or 1.0),
|
|
"queueWaiting": stats.get("waiting"),
|
|
"queueRunning": stats.get("running"),
|
|
"queueBacklogHours": stats.get("backlogHours"),
|
|
"publicRecentSuccessRate": stats.get("recentSuccessRate"),
|
|
"publicThroughputPerHour": stats.get("throughputPerHour"),
|
|
"marketSubmissionEligible": stats.get("submissionEligible"),
|
|
"marketDataStale": bool(stats.get("stale", False)),
|
|
}
|
|
|
|
def rank_frameworks(
|
|
self,
|
|
*,
|
|
task_type: str,
|
|
target_gpu: str,
|
|
compatible_frameworks: list[str],
|
|
) -> list[str]:
|
|
stats = (
|
|
(((self.state or {}).get("frameworkStats") or {}).get(task_type) or {}).get(target_gpu)
|
|
or {}
|
|
)
|
|
legacy_index = {framework: index for index, framework in enumerate(compatible_frameworks)}
|
|
|
|
def rank_key(framework: str) -> tuple[int, float, int, int]:
|
|
evidence = self._framework_evidence(task_type, target_gpu, framework)
|
|
return (
|
|
1 if evidence["qualified"] else 0,
|
|
float(evidence["combinedScore"]) if evidence["qualified"] else 0.0,
|
|
int(evidence["publicSamples"]) + int(evidence["localSamples"]),
|
|
-legacy_index[framework],
|
|
)
|
|
|
|
return sorted(compatible_frameworks, key=rank_key, reverse=True)
|
|
|
|
def _framework_evidence(self, task_type: str, target_gpu: str, framework: str) -> dict[str, Any]:
|
|
public_item = (
|
|
(((self.state or {}).get("frameworkStats") or {}).get(task_type) or {}).get(target_gpu)
|
|
or {}
|
|
).get(framework) or {}
|
|
public_samples = max(0, int(public_item.get("modelCount") or 0))
|
|
public_score = float(public_item.get("wilsonLowerBound") or 0.0)
|
|
|
|
local_key = f"{target_gpu}|{framework}|{task_type}"
|
|
local_item = (self.local_outcome_stats.get("combinationStats") or {}).get(local_key) or {}
|
|
local_success = max(0, int(local_item.get("successCount") or 0))
|
|
local_failure = max(
|
|
0,
|
|
int(local_item.get("attributableFailureCount", local_item.get("failureCount") or 0)),
|
|
)
|
|
local_samples = local_success + local_failure
|
|
local_score = _wilson_lower_bound(local_success, local_samples)
|
|
|
|
recent_item = (self.local_outcome_stats.get("recentCombinationStats") or {}).get(local_key) or {}
|
|
recent_success = max(0, int(recent_item.get("successCount") or 0))
|
|
recent_failure = max(
|
|
0,
|
|
int(recent_item.get("attributableFailureCount", recent_item.get("failureCount") or 0)),
|
|
)
|
|
recent_samples = recent_success + recent_failure
|
|
recent_rate = recent_success / recent_samples if recent_samples else None
|
|
consecutive_failures = max(0, int(recent_item.get("consecutiveFailures") or 0))
|
|
consecutive_platform_failures = max(
|
|
0, int(recent_item.get("consecutivePlatformFailures") or 0)
|
|
)
|
|
last_terminal_at = parse_datetime(recent_item.get("lastTerminalAt"))
|
|
last_platform_failure_at = parse_datetime(recent_item.get("lastPlatformFailureAt"))
|
|
circuit_reason = None
|
|
circuit_until = None
|
|
if last_platform_failure_at is not None and consecutive_platform_failures >= 3:
|
|
circuit_reason = "three_consecutive_platform_failures"
|
|
circuit_until = last_platform_failure_at + timedelta(minutes=30)
|
|
elif last_terminal_at is not None and consecutive_failures >= 5:
|
|
circuit_reason = "five_consecutive_local_failures"
|
|
circuit_until = last_terminal_at + timedelta(hours=12)
|
|
elif last_terminal_at is not None and recent_samples >= 20 and recent_rate is not None and recent_rate < 0.20:
|
|
circuit_reason = "recent_local_success_below_20_percent"
|
|
circuit_until = last_terminal_at + timedelta(hours=6)
|
|
circuit_open = bool(circuit_until is not None and circuit_until > utc_now())
|
|
if not circuit_open:
|
|
circuit_reason = None
|
|
circuit_until = None
|
|
|
|
public_qualified = (
|
|
public_samples >= self.framework_min_samples
|
|
and public_score >= DEFAULT_FRAMEWORK_MIN_WILSON
|
|
)
|
|
local_qualified = local_samples >= 20
|
|
if public_qualified:
|
|
evidence_success = recent_success if recent_samples >= 5 else local_success
|
|
evidence_samples = recent_samples if recent_samples >= 5 else local_samples
|
|
evidence_score = _wilson_lower_bound(evidence_success, evidence_samples)
|
|
local_weight = min(0.60, evidence_samples / (evidence_samples + 100.0)) if evidence_samples >= 5 else 0.0
|
|
combined = public_score * (1.0 - local_weight) + evidence_score * local_weight
|
|
else:
|
|
combined = 0.0
|
|
return {
|
|
"qualified": public_qualified and not circuit_open,
|
|
"publicQualified": public_qualified,
|
|
"localQualified": local_qualified,
|
|
"combinedScore": combined,
|
|
"publicSamples": public_samples,
|
|
"publicScore": public_score,
|
|
"localSamples": local_samples,
|
|
"localSuccessRate": local_success / local_samples if local_samples else None,
|
|
"localScore": local_score if local_samples else None,
|
|
"recentLocalSamples": recent_samples,
|
|
"recentLocalSuccessRate": recent_rate,
|
|
"consecutiveLocalFailures": consecutive_failures,
|
|
"consecutivePlatformFailures": consecutive_platform_failures,
|
|
"circuitOpen": circuit_open,
|
|
"circuitReason": circuit_reason,
|
|
"circuitUntil": circuit_until.isoformat() if circuit_until else None,
|
|
}
|
|
|
|
def selectable_frameworks(
|
|
self,
|
|
*,
|
|
task_type: str,
|
|
target_gpu: str,
|
|
incumbent_frameworks: list[str],
|
|
inspection: Any,
|
|
) -> list[str]:
|
|
discovered = self.compatible_discovered_frameworks(
|
|
task_type=task_type,
|
|
target_gpu=target_gpu,
|
|
inspection=inspection,
|
|
)
|
|
candidates = list(dict.fromkeys([*incumbent_frameworks, *discovered]))
|
|
vetted = [
|
|
framework
|
|
for framework in candidates
|
|
if self._framework_evidence(task_type, target_gpu, framework)["qualified"]
|
|
]
|
|
incumbent_scores = [
|
|
float(self._framework_evidence(task_type, target_gpu, framework)["combinedScore"])
|
|
for framework in vetted
|
|
if framework in incumbent_frameworks
|
|
]
|
|
incumbent_best = max(incumbent_scores, default=0.0)
|
|
|
|
promoted: list[str] = []
|
|
for framework in vetted:
|
|
if framework in incumbent_frameworks:
|
|
promoted.append(framework)
|
|
continue
|
|
evidence = self._framework_evidence(task_type, target_gpu, framework)
|
|
if incumbent_best > 0 and float(evidence["combinedScore"]) < incumbent_best * NEW_FRAMEWORK_PROMOTION_MARGIN:
|
|
continue
|
|
promoted.append(framework)
|
|
return self.rank_frameworks(
|
|
task_type=task_type,
|
|
target_gpu=target_gpu,
|
|
compatible_frameworks=promoted,
|
|
)
|
|
|
|
def compatible_discovered_frameworks(
|
|
self,
|
|
*,
|
|
task_type: str,
|
|
target_gpu: str,
|
|
inspection: Any,
|
|
) -> list[str]:
|
|
stats = (
|
|
(((self.state or {}).get("frameworkStats") or {}).get(task_type) or {}).get(target_gpu)
|
|
or {}
|
|
)
|
|
compatible: list[str] = []
|
|
for framework, item in stats.items():
|
|
if not bool(item.get("officialConfigValid", False)):
|
|
continue
|
|
normalized = framework.lower()
|
|
if "llamacpp" in normalized or "gguf" in normalized:
|
|
usable = bool(getattr(inspection, "has_gguf", False))
|
|
elif "onnx" in normalized or "sherpa" in normalized:
|
|
usable = bool(getattr(inspection, "has_onnx_weights", False))
|
|
elif task_type in {"text-generation", "visual-multi-modal", "reinforcement_learning"}:
|
|
usable = bool(getattr(inspection, "has_vllm_weights", False))
|
|
else:
|
|
usable = bool(getattr(inspection, "has_standard_weights", False))
|
|
if usable:
|
|
compatible.append(framework)
|
|
return compatible
|
|
|
|
def official_config(
|
|
self,
|
|
*,
|
|
task_type: str,
|
|
target_gpu: str,
|
|
framework: str,
|
|
gguf_filename: str | None = None,
|
|
) -> str | None:
|
|
item = (
|
|
(((self.state or {}).get("frameworkStats") or {}).get(task_type) or {}).get(target_gpu)
|
|
or {}
|
|
).get(framework) or {}
|
|
if not bool(item.get("officialConfigValid", False)):
|
|
return None
|
|
config = item.get("officialConfig")
|
|
if not isinstance(config, str) or not config:
|
|
return None
|
|
return _render_official_config(config, gguf_filename=gguf_filename)
|
|
|
|
def framework_metadata(self, task_type: str, target_gpu: str, framework: str) -> dict[str, Any]:
|
|
item = (
|
|
(((self.state or {}).get("frameworkStats") or {}).get(task_type) or {}).get(target_gpu)
|
|
or {}
|
|
).get(framework) or {}
|
|
evidence = self._framework_evidence(task_type, target_gpu, framework)
|
|
return {
|
|
"frameworkMarketSamples": int(item.get("modelCount") or 0),
|
|
"frameworkMarketSuccessRate": item.get("successRate"),
|
|
"frameworkMarketWilsonLowerBound": item.get("wilsonLowerBound"),
|
|
"frameworkLocalSamples": evidence["localSamples"],
|
|
"frameworkLocalSuccessRate": evidence["localSuccessRate"],
|
|
"frameworkRecentLocalSamples": evidence["recentLocalSamples"],
|
|
"frameworkRecentLocalSuccessRate": evidence["recentLocalSuccessRate"],
|
|
"frameworkConsecutiveLocalFailures": evidence["consecutiveLocalFailures"],
|
|
"frameworkCombinedScore": evidence["combinedScore"],
|
|
"frameworkMarketQualified": evidence["publicQualified"],
|
|
"frameworkLocalQualified": evidence["localQualified"],
|
|
"frameworkEvidenceQualified": evidence["qualified"],
|
|
"frameworkCircuitOpen": evidence["circuitOpen"],
|
|
"frameworkCircuitReason": evidence["circuitReason"],
|
|
"frameworkCircuitUntil": evidence["circuitUntil"],
|
|
"frameworkOfficialConfigValid": bool(item.get("officialConfigValid", False)),
|
|
"frameworkOfficialConfigStale": bool(item.get("officialConfigStale", False)),
|
|
"frameworkConfigSource": "modelhub_live" if item.get("officialConfigValid") else "local_template",
|
|
}
|
|
|
|
def _log_snapshot(self, action: str) -> None:
|
|
if self.state is None:
|
|
return
|
|
stats = self.state.get("gpuStats") or {}
|
|
ranked = sorted(
|
|
stats.values(),
|
|
key=lambda item: -float(item.get("selectionWeight") or 0.0),
|
|
)
|
|
leaders = ",".join(
|
|
f"{item.get('gpu')}:{float(item.get('selectionWeight') or 0.0):.2f}"
|
|
for item in ranked[:3]
|
|
)
|
|
self.log(
|
|
f"[market] {action} queue_at={self.state.get('queueUpdatedAt') or 'n/a'} "
|
|
f"framework_at={self.state.get('frameworkUpdatedAt') or 'n/a'} leaders={leaders or 'n/a'}"
|
|
)
|
|
|
|
def summary(self) -> dict[str, Any]:
|
|
if self.state is None:
|
|
return {"enabled": False}
|
|
return {
|
|
"enabled": True,
|
|
"statePath": str(self.path),
|
|
"queueUpdatedAt": self.state.get("queueUpdatedAt"),
|
|
"frameworkUpdatedAt": self.state.get("frameworkUpdatedAt"),
|
|
"throughputWindowHours": int(self.state.get("throughputWindowHours") or self.throughput_window_hours),
|
|
"queueError": self.state.get("queueError"),
|
|
"frameworkError": self.state.get("frameworkError"),
|
|
"gpuStats": dict(self.state.get("gpuStats") or {}),
|
|
}
|