fix: refill from expanded candidate history

modelhub_submmit_api/outcome_tracker.py

@@ -23,7 +23,7 @@ class OutcomeTracker:
         self._records: list[dict[str, Any]] = []
         self._by_task_id: dict[str, dict[str, Any]] = {}
         self._by_model_gpu: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
-        self._failed_model_gpus: set[tuple[str, str]] = set()
+        self._failed_model_gpus: dict[tuple[str, str], datetime] = {}
 
         self._records = read_jsonl(self.path)
         self._rebuild_indexes()
@@ -117,8 +117,19 @@ class OutcomeTracker:
 
         return updated_count
 
-    def is_model_gpu_failed(self, model_id: str, target_gpu: str) -> bool:
-        return (model_id, target_gpu) in self._failed_model_gpus
+    def is_model_gpu_failed(
+        self,
+        model_id: str,
+        target_gpu: str,
+        *,
+        cooldown_hours: int = 24,
+        now: datetime | None = None,
+    ) -> bool:
+        failed_at = self._failed_model_gpus.get((model_id, target_gpu))
+        if failed_at is None:
+            return False
+        now = now or utc_now()
+        return failed_at >= now - timedelta(hours=max(0, int(cooldown_hours)))
 
     def get_stats_report(self) -> dict[str, Any]:
         now = _now_iso()
@@ -186,12 +197,21 @@ class OutcomeTracker:
 
     def _rebuild_failed_index(self) -> None:
         self._failed_model_gpus.clear()
+        latest_by_combo: dict[tuple[str, str], tuple[datetime, dict[str, Any]]] = {}
         for record in self._records:
+            model_id = record.get("modelId") or ""
+            target_gpu = record.get("targetGpu") or ""
+            event_time = parse_datetime(record.get("lastSyncTime")) or parse_datetime(record.get("submitTime"))
+            if not model_id or not target_gpu or event_time is None:
+                continue
+            key = (model_id, target_gpu)
+            current = latest_by_combo.get(key)
+            if current is None or event_time >= current[0]:
+                latest_by_combo[key] = (event_time, record)
+
+        for key, (event_time, record) in latest_by_combo.items():
             if record.get("outcome") == "failed":
-                model_id = record.get("modelId") or ""
-                target_gpu = record.get("targetGpu") or ""
-                if model_id and target_gpu:
-                    self._failed_model_gpus.add((model_id, target_gpu))
+                self._failed_model_gpus[key] = event_time
 
     @staticmethod
     def _update_record_from_task(record: dict[str, Any], task: dict[str, Any]) -> None: