submmit/modelhub_submmit_api/gpu_strategy.py

from __future__ import annotations

import math
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any, Callable

from common import parse_datetime, read_json, utc_now, write_json
from history_stats import is_failure, is_success
from modelhub_client import ModelHubClientPool
from submission_claims import candidate_key


DEFAULT_GPU_STRATEGY_PATH = Path(".modelhub_state/gpu_strategy.json")
DEFAULT_REFRESH_SUBMISSIONS = 200
DEFAULT_RECENT_TERMINAL_WINDOW = 1000
DEFAULT_LONG_TERM_MIN_SAMPLES = 100
STRATEGY_STATE_VERSION = 4

LONG_TERM = "long_term"
RECENT = "recent"
CATEGORIES = (LONG_TERM, RECENT)
CATEGORY_WEIGHTS = {LONG_TERM: 7, RECENT: 3}

def _empty_category_counts() -> dict[str, int]:
    return {category: 0 for category in CATEGORIES}


def _empty_gpu_category_counts(supported_gpus: list[str]) -> dict[str, dict[str, int]]:
    return {
        category: {gpu: 0 for gpu in supported_gpus}
        for category in CATEGORIES
    }


def _gpu_name(record: dict[str, Any]) -> str:
    return str(record.get("gpuType") or record.get("targetGpu") or "").strip()


def _terminal_outcome(record: dict[str, Any]) -> str | None:
    try:
        if is_success(record):
            return "success"
        if is_failure(record):
            return "failure"
    except TypeError:
        # Be tolerant of APIs that serialize verifyResult as a string.
        copied = dict(record)
        try:
            copied["verifyResult"] = float(record.get("verifyResult"))
        except (TypeError, ValueError):
            copied["verifyResult"] = None
        if is_success(copied):
            return "success"
        if is_failure(copied):
            return "failure"
    return None


def _task_sort_key(record: dict[str, Any]) -> tuple[float, str]:
    timestamp = (
        parse_datetime(record.get("updateTime"))
        or parse_datetime(record.get("createTime"))
        or parse_datetime(record.get("submitTime"))
    )
    return (timestamp.timestamp() if timestamp else 0.0, str(record.get("taskId") or ""))


def wilson_lower_bound(successes: int, total: int, *, z: float = 1.96) -> float:
    if total <= 0:
        return 0.0
    probability = successes / total
    z_squared = z * z
    denominator = 1.0 + z_squared / total
    centre = probability + z_squared / (2.0 * total)
    margin = z * math.sqrt(
        (probability * (1.0 - probability) / total) + z_squared / (4.0 * total * total)
    )
    return max(0.0, (centre - margin) / denominator)


def _summarize_gpu_records(
    records: list[dict[str, Any]],
    supported_gpus: list[str],
) -> dict[str, dict[str, Any]]:
    supported = set(supported_gpus)
    counts: dict[str, dict[str, int]] = {
        gpu: {"success": 0, "failure": 0}
        for gpu in supported_gpus
    }
    for record in records:
        gpu = _gpu_name(record)
        if gpu not in supported:
            continue
        outcome = _terminal_outcome(record)
        if outcome is not None:
            counts[gpu][outcome] += 1

    summaries: dict[str, dict[str, Any]] = {}
    for gpu in supported_gpus:
        success = counts[gpu]["success"]
        failure = counts[gpu]["failure"]
        terminal = success + failure
        summaries[gpu] = {
            "gpu": gpu,
            "success": success,
            "failure": failure,
            "terminal": terminal,
            "successRate": success / terminal if terminal else 0.0,
            "wilsonLowerBound": wilson_lower_bound(success, terminal),
        }
    return summaries


def _rank_gpu_summaries(summaries: dict[str, dict[str, Any]]) -> list[dict[str, Any]]:
    return sorted(
        summaries.values(),
        key=lambda item: (
            -float(item["wilsonLowerBound"]),
            -float(item["successRate"]),
            -int(item["terminal"]),
            str(item["gpu"]),
        ),
    )


def build_strategy_snapshot(
    tasks: list[dict[str, Any]],
    *,
    supported_gpus: list[str],
    generated_at: datetime | None = None,
    generation: int = 0,
    recent_terminal_window: int = DEFAULT_RECENT_TERMINAL_WINDOW,
    long_term_min_samples: int = DEFAULT_LONG_TERM_MIN_SAMPLES,
    refresh_submissions: int = DEFAULT_REFRESH_SUBMISSIONS,
) -> dict[str, Any]:
    generated_at = generated_at or utc_now()
    supported_gpus = list(dict.fromkeys(gpu for gpu in supported_gpus if gpu))
    if not supported_gpus:
        raise ValueError("At least one supported GPU is required")

    all_time = _summarize_gpu_records(tasks, supported_gpus)
    all_time_ranked = _rank_gpu_summaries(all_time)
    qualified_long = [
        item for item in all_time_ranked
        if int(item["terminal"]) >= max(1, int(long_term_min_samples))
    ]
    long_term_gpus = [str(item["gpu"]) for item in qualified_long[:3]]
    if len(long_term_gpus) < min(3, len(supported_gpus)):
        for item in all_time_ranked:
            gpu = str(item["gpu"])
            if gpu not in long_term_gpus:
                long_term_gpus.append(gpu)
            if len(long_term_gpus) >= min(3, len(supported_gpus)):
                break

    terminal_tasks = [record for record in tasks if _terminal_outcome(record) is not None]
    terminal_tasks.sort(key=_task_sort_key, reverse=True)
    recent_tasks = terminal_tasks[: max(1, int(recent_terminal_window))]
    recent = _summarize_gpu_records(recent_tasks, supported_gpus)
    recent_ranked = _rank_gpu_summaries(recent)
    recent_qualified = [item for item in recent_ranked if int(item["terminal"]) >= 20]
    recent_source = recent_qualified if recent_qualified else (recent_ranked if recent_tasks else [])
    recent_gpus = [str(item["gpu"]) for item in recent_source[: min(3, len(supported_gpus))]]
    if not recent_gpus:
        recent_gpus = list(long_term_gpus[: min(3, len(supported_gpus))])
    recent_gpu = recent_gpus[0]

    return {
        "version": STRATEGY_STATE_VERSION,
        "generation": max(0, int(generation)),
        "generatedAt": generated_at.isoformat(),
        "historyReady": True,
        "historyTaskCount": len(tasks),
        "supportedGpus": supported_gpus,
        "longTermGpus": long_term_gpus,
        "recentGpu": recent_gpu,
        "recentGpus": recent_gpus,
        "recentTerminalCount": len(recent_tasks),
        "refreshSubmissions": max(1, int(refresh_submissions)),
        "recentTerminalWindow": max(1, int(recent_terminal_window)),
        "longTermMinSamples": max(1, int(long_term_min_samples)),
        "acceptedSinceRefresh": 0,
        "acceptedTotal": 0,
        "acceptedByCategory": _empty_category_counts(),
        "acceptedByGpuCategory": _empty_gpu_category_counts(supported_gpus),
        "longTermStats": all_time_ranked,
        "recentStats": recent_ranked,
        "lastRefreshError": None,
        "refreshRetryAfter": None,
    }


def choose_next_category(counts: dict[str, int]) -> str:
    normalized = {category: max(0, int(counts.get(category, 0))) for category in CATEGORIES}
    next_total = sum(normalized.values()) + 1

    def deficit(category: str) -> int:
        return CATEGORY_WEIGHTS[category] * next_total - normalized[category] * sum(CATEGORY_WEIGHTS.values())

    return max(CATEGORIES, key=lambda category: (deficit(category), -CATEGORIES.index(category)))


class GPUStrategyManager:
    def __init__(
        self,
        path: Path | str = DEFAULT_GPU_STRATEGY_PATH,
        *,
        refresh_submissions: int = DEFAULT_REFRESH_SUBMISSIONS,
        recent_terminal_window: int = DEFAULT_RECENT_TERMINAL_WINDOW,
        long_term_min_samples: int = DEFAULT_LONG_TERM_MIN_SAMPLES,
        market_intelligence: Any | None = None,
        log_fn: Callable[[str], None] | None = None,
    ) -> None:
        self.path = Path(path)
        self.refresh_submissions = max(1, int(refresh_submissions))
        self.recent_terminal_window = max(1, int(recent_terminal_window))
        self.long_term_min_samples = max(1, int(long_term_min_samples))
        self.market_intelligence = market_intelligence
        self.log = log_fn or (lambda message: print(message, flush=True))
        self.state: dict[str, Any] | None = None

    def set_market_intelligence(self, market_intelligence: Any | None) -> None:
        self.market_intelligence = market_intelligence

    def _load(self) -> dict[str, Any] | None:
        try:
            value = read_json(self.path)
        except (FileNotFoundError, ValueError):
            return None
        return value if isinstance(value, dict) else None

    def _is_compatible(self, state: dict[str, Any] | None, supported_gpus: list[str]) -> bool:
        if not state or int(state.get("version") or 0) != STRATEGY_STATE_VERSION:
            return False
        return list(state.get("supportedGpus") or []) == supported_gpus

    def _refresh_due(self, state: dict[str, Any] | None, supported_gpus: list[str], now: datetime) -> tuple[bool, str]:
        if not self._is_compatible(state, supported_gpus):
            return True, "initial_or_gpu_catalog_changed"
        retry_after = parse_datetime(state.get("refreshRetryAfter")) if state is not None else None
        if state is not None and state.get("lastRefreshError") and retry_after is not None and retry_after > now:
            return False, "refresh_error_backoff"
        if not bool(state.get("historyReady", False)):
            return (retry_after is None or retry_after <= now), "history_not_ready"
        accepted = int(state.get("acceptedSinceRefresh") or 0)
        refresh_every = max(1, int(state.get("refreshSubmissions") or self.refresh_submissions))
        return accepted >= refresh_every, "accepted_submission_threshold"

    @staticmethod
    def _load_platform_history(client: Any) -> list[dict[str, Any]]:
        if isinstance(client, ModelHubClientPool):
            by_account: dict[int, list[dict[str, Any]]] = {}
            errors: list[int] = []
            with ThreadPoolExecutor(max_workers=min(len(client.clients), 12)) as executor:
                futures = {
                    executor.submit(account_client.list_tasks, page_size=100, only_mine=True): index
                    for index, account_client in enumerate(client.clients, start=1)
                }
                for future in as_completed(futures):
                    index = futures[future]
                    try:
                        by_account[index] = future.result()
                    except Exception:
                        errors.append(index)
            if errors:
                joined = ",".join(str(index) for index in sorted(errors))
                raise RuntimeError(f"ModelHub history fetch failed for account indexes: {joined}")
            tasks = [task for index in sorted(by_account) for task in by_account[index]]
        else:
            tasks = client.list_tasks(page_size=100, only_mine=True)

        deduped: list[dict[str, Any]] = []
        seen_task_ids: set[str] = set()
        for task in tasks:
            if not isinstance(task, dict):
                continue
            task_id = str(task.get("taskId")) if task.get("taskId") is not None else None
            if task_id and task_id in seen_task_ids:
                continue
            if task_id:
                seen_task_ids.add(task_id)
            deduped.append(task)
        return deduped

    def prepare(
        self,
        client: Any,
        *,
        supported_gpus: list[str],
        now: datetime | None = None,
        history_records: list[dict[str, Any]] | None = None,
    ) -> dict[str, Any]:
        now = now or utc_now()
        supported_gpus = list(dict.fromkeys(gpu for gpu in supported_gpus if gpu))
        state = self._load()
        refresh_due, reason = self._refresh_due(state, supported_gpus, now)
        if not refresh_due and state is not None:
            self.state = state
            self._log_state("loaded")
            return state

        previous_generation = int(state.get("generation", -1)) if state is not None else -1
        self.log(f"[strategy] refresh_start reason={reason} supported_gpus={len(supported_gpus)}")
        try:
            tasks = (
                list(history_records)
                if history_records is not None
                else self._load_platform_history(client)
            )
            refreshed = build_strategy_snapshot(
                tasks,
                supported_gpus=supported_gpus,
                generated_at=now,
                generation=previous_generation + 1,
                recent_terminal_window=self.recent_terminal_window,
                long_term_min_samples=self.long_term_min_samples,
                refresh_submissions=self.refresh_submissions,
            )
            refreshed["acceptedTotal"] = int((state or {}).get("acceptedTotal") or 0)
            refreshed["historySource"] = (
                "classified_attributable_outcomes"
                if history_records is not None
                else "raw_platform_history"
            )
            self.state = refreshed
            write_json(self.path, refreshed)
            self._log_state("refreshed")
            return refreshed
        except Exception as exc:
            self.log(f"[strategy] refresh_error reason={type(exc).__name__}: {exc}")
            if self._is_compatible(state, supported_gpus):
                assert state is not None
                state["lastRefreshError"] = str(exc)
                state["refreshRetryAfter"] = (now + timedelta(minutes=5)).isoformat()
                self.state = state
                write_json(self.path, state)
                return state

            fallback = build_strategy_snapshot(
                [],
                supported_gpus=supported_gpus,
                generated_at=now,
                generation=0,
                recent_terminal_window=self.recent_terminal_window,
                long_term_min_samples=self.long_term_min_samples,
                refresh_submissions=self.refresh_submissions,
            )
            fallback["historyReady"] = False
            fallback["lastRefreshError"] = str(exc)
            fallback["refreshRetryAfter"] = (now + timedelta(minutes=5)).isoformat()
            self.state = fallback
            write_json(self.path, fallback)
            self._log_state("fallback")
            return fallback

    def _log_state(self, action: str) -> None:
        if self.state is None:
            return
        counts = self.state.get("acceptedByCategory") or {}
        self.log(
            f"[strategy] {action} generation={self.state.get('generation', 0)} "
            f"accepted={self.state.get('acceptedSinceRefresh', 0)}/{self.state.get('refreshSubmissions', self.refresh_submissions)} "
            f"categories={counts.get(LONG_TERM, 0)},{counts.get(RECENT, 0)} "
            f"long={','.join(self.state.get('longTermGpus') or [])} recent={self.state.get('recentGpu') or 'n/a'}"
        )

    @property
    def submissions_until_refresh(self) -> int:
        if self.state is None:
            return self.refresh_submissions
        refresh_every = max(1, int(self.state.get("refreshSubmissions") or self.refresh_submissions))
        accepted = max(0, int(self.state.get("acceptedSinceRefresh") or 0))
        return max(0, refresh_every - accepted)

    def _category_gpu_order(self, category: str, gpu_counts: dict[str, int]) -> list[str]:
        assert self.state is not None
        supported = list(self.state.get("supportedGpus") or [])
        long_term = [gpu for gpu in self.state.get("longTermGpus") or [] if gpu in supported]
        recent = [gpu for gpu in self.state.get("recentGpus") or [] if gpu in supported]
        if not recent:
            recent_gpu = str(self.state.get("recentGpu") or "")
            if recent_gpu in supported:
                recent = [recent_gpu]

        if category == LONG_TERM:
            eligible = list(long_term)
            base_pattern = (5.0, 3.0, 2.0)
        else:
            eligible = list(recent)
            base_pattern = (6.0, 3.0, 1.0)

        if self.market_intelligence is not None:
            try:
                vetted = self.market_intelligence.eligible_gpus(supported)
            except Exception:
                vetted = supported
            eligible = [gpu for gpu in eligible if gpu in vetted]
            eligible.extend(gpu for gpu in vetted if gpu not in eligible)
        if not eligible:
            return []

        weights: dict[str, float] = {}
        for index, gpu in enumerate(eligible):
            base = base_pattern[index] if index < len(base_pattern) else 0.5
            market_weight = 1.0
            if self.market_intelligence is not None:
                try:
                    market_weight = float(self.market_intelligence.gpu_weight(gpu, category=category))
                except Exception:
                    market_weight = 1.0
            weights[gpu] = max(0.05, base * market_weight)

        total_weight = sum(weights.values()) or 1.0
        next_total = sum(max(0, int(gpu_counts.get(gpu, 0))) for gpu in eligible) + 1
        rank_index = {gpu: index for index, gpu in enumerate(eligible)}
        return sorted(
            eligible,
            key=lambda gpu: (
                -((weights[gpu] / total_weight) * next_total - max(0, int(gpu_counts.get(gpu, 0)))),
                rank_index[gpu],
            ),
        )

    def order_candidates(self, candidates: list[dict[str, Any]]) -> list[dict[str, Any]]:
        if self.state is None or not candidates:
            return list(candidates)

        by_gpu: dict[str, list[dict[str, Any]]] = defaultdict(list)
        for candidate in candidates:
            by_gpu[str(candidate.get("targetGpu") or "")].append(candidate)
        gpu_indexes: dict[str, int] = defaultdict(int)
        used: set[str] = set()
        ordered: list[dict[str, Any]] = []
        virtual_counts = {
            category: max(0, int((self.state.get("acceptedByCategory") or {}).get(category, 0)))
            for category in CATEGORIES
        }
        stored_gpu_counts = self.state.get("acceptedByGpuCategory") or {}
        virtual_gpu_counts = {
            category: {
                gpu: max(0, int((stored_gpu_counts.get(category) or {}).get(gpu, 0)))
                for gpu in self.state.get("supportedGpus") or []
            }
            for category in CATEGORIES
        }

        def take_from_gpu(gpu: str) -> dict[str, Any] | None:
            pool = by_gpu.get(gpu) or []
            index = gpu_indexes[gpu]
            while index < len(pool):
                candidate = pool[index]
                index += 1
                gpu_indexes[gpu] = index
                if candidate_key(candidate) not in used:
                    return candidate
            gpu_indexes[gpu] = index
            return None

        while len(ordered) < len(candidates):
            planned_category = choose_next_category(virtual_counts)
            selected: dict[str, Any] | None = None
            actual_category = planned_category
            categories_to_try = [planned_category, *(category for category in CATEGORIES if category != planned_category)]
            for category in categories_to_try:
                for gpu in self._category_gpu_order(category, virtual_gpu_counts[category]):
                    selected = take_from_gpu(gpu)
                    if selected is not None:
                        actual_category = category
                        break
                if selected is not None:
                    break

            if selected is None:
                # With market intelligence enabled, an empty vetted pool means
                # stop instead of silently turning fallback into exploration.
                if self.market_intelligence is not None:
                    break
                # Unknown/custom GPUs can only appear when callers bypass the normal resolver.
                selected = next((candidate for candidate in candidates if candidate_key(candidate) not in used), None)
                actual_category = planned_category
            if selected is None:
                break

            key = candidate_key(selected)
            used.add(key)
            annotated = dict(selected)
            annotated["strategyCategory"] = actual_category
            annotated["strategyPlannedCategory"] = planned_category
            annotated["strategyGeneration"] = int(self.state.get("generation") or 0)
            if self.market_intelligence is not None:
                try:
                    annotated.update(self.market_intelligence.gpu_metadata(str(annotated.get("targetGpu") or "")))
                except Exception:
                    pass
            ordered.append(annotated)
            virtual_counts[actual_category] += 1
            selected_gpu = str(annotated.get("targetGpu") or "")
            if selected_gpu:
                virtual_gpu_counts[actual_category][selected_gpu] = (
                    virtual_gpu_counts[actual_category].get(selected_gpu, 0) + 1
                )

        return ordered

    def record_accepted(self, candidates: list[dict[str, Any]]) -> dict[str, Any] | None:
        if not candidates:
            return self.state
        state = self._load() or self.state
        if state is None:
            return None

        category_counts = {
            category: max(0, int((state.get("acceptedByCategory") or {}).get(category, 0)))
            for category in CATEGORIES
        }
        supported = list(state.get("supportedGpus") or [])
        stored_gpu_counts = state.get("acceptedByGpuCategory") or {}
        gpu_category_counts = {
            category: {
                gpu: max(0, int((stored_gpu_counts.get(category) or {}).get(gpu, 0)))
                for gpu in supported
            }
            for category in CATEGORIES
        }
        for candidate in candidates:
            category = str(candidate.get("strategyCategory") or LONG_TERM)
            if category not in category_counts:
                category = LONG_TERM
            category_counts[category] += 1
            gpu = str(candidate.get("targetGpu") or "")
            if gpu in gpu_category_counts[category]:
                gpu_category_counts[category][gpu] += 1

        accepted_count = len(candidates)
        state["acceptedSinceRefresh"] = int(state.get("acceptedSinceRefresh") or 0) + accepted_count
        state["acceptedTotal"] = int(state.get("acceptedTotal") or 0) + accepted_count
        state["acceptedByCategory"] = category_counts
        state["acceptedByGpuCategory"] = gpu_category_counts
        self.state = state
        write_json(self.path, state)
        self._log_state("progress")
        return state

    def summary(self) -> dict[str, Any]:
        if self.state is None:
            return {"enabled": False}
        return {
            "enabled": True,
            "statePath": str(self.path),
            "generation": int(self.state.get("generation") or 0),
            "generatedAt": self.state.get("generatedAt"),
            "historyReady": bool(self.state.get("historyReady", False)),
            "historyTaskCount": int(self.state.get("historyTaskCount") or 0),
            "acceptedSinceRefresh": int(self.state.get("acceptedSinceRefresh") or 0),
            "refreshSubmissions": int(self.state.get("refreshSubmissions") or self.refresh_submissions),
            "acceptedByCategory": dict(self.state.get("acceptedByCategory") or {}),
            "acceptedByGpuCategory": dict(self.state.get("acceptedByGpuCategory") or {}),
            "longTermGpus": list(self.state.get("longTermGpus") or []),
            "recentGpu": self.state.get("recentGpu"),
            "recentGpus": list(self.state.get("recentGpus") or []),
            "recentTerminalCount": int(self.state.get("recentTerminalCount") or 0),
            "refreshDue": self.submissions_until_refresh <= 0,
        }