feat: add durable success-first modelhub agent

modelhub_submmit_api/modelhub_client.py

@@ -39,6 +39,8 @@ def parse_model_submission_precheck(payload: Any) -> dict[str, Any]:
     return {
         "isInDB": data.get("isInDB") is True,
         "processedGpus": set(str(gpu) for gpu in verify_result),
+        "verifyResult": verify_result,
+        "modelInfo": data.get("modelInfo") if isinstance(data.get("modelInfo"), dict) else {},
     }
 
 
@@ -124,6 +126,16 @@ class ModelHubClient:
             raise ModelHubAPIError("Machine info response is invalid", payload=payload)
         return [item for item in data if isinstance(item, dict)]
 
+    def list_task_levels(self) -> dict[str, Any]:
+        return self._request("GET", "/api/computility/task-levels/tree")
+
+    def list_model_task_types(self, target_gpu: str, model_address: str) -> dict[str, Any]:
+        return self._request(
+            "GET",
+            "/api/computility/driver/images/task-types/by-gpu",
+            query={"gpuTypeName": target_gpu, "modelAddress": model_address},
+        )
+
     def list_framework_stats(self, task_type: str, target_gpu: str) -> list[dict[str, Any]]:
         payload = self._request(
             "GET",
@@ -942,6 +954,8 @@ class ModelHubClientPool:
             self._finish_reservation(selected_index, reservation_id, succeeded=True)
             if capacity_probe:
                 self._promote_account_capacity(selected_index)
+            if isinstance(response, dict):
+                response.setdefault("_accountKey", self._account_keys[selected_index])
             return response
 
     def list_tasks_page(self, **kwargs):  # noqa: ANN003, ANN001
@@ -951,6 +965,12 @@ class ModelHubClientPool:
     def list_machine_info(self) -> list[dict[str, Any]]:
         return self._reader.list_machine_info()
 
+    def list_task_levels(self) -> dict[str, Any]:
+        return self._reader.list_task_levels()
+
+    def list_model_task_types(self, target_gpu: str, model_address: str) -> dict[str, Any]:
+        return self._reader.list_model_task_types(target_gpu, model_address)
+
     def list_framework_stats(self, task_type: str, target_gpu: str) -> list[dict[str, Any]]:
         return self._reader.list_framework_stats(task_type, target_gpu)
 
@@ -963,3 +983,30 @@ class ModelHubClientPool:
             if task_id is not None:
                 return task_id
         return None
+
+    def list_active_tasks_by_account(self) -> list[dict[str, Any]]:
+        """Return active tasks with a stable, non-secret account fingerprint."""
+        def fetch_account(index: int, client: ModelHubClient) -> tuple[int, list[dict[str, Any]]]:
+            tasks: list[dict[str, Any]] = []
+            seen: set[str] = set()
+            for status in ("waiting", "running"):
+                for task in client.list_tasks(page_size=100, only_mine=True, status=status):
+                    task_id = str(task.get("taskId") or "")
+                    if task_id and task_id in seen:
+                        continue
+                    if task_id:
+                        seen.add(task_id)
+                    if isinstance(task, dict) and is_active_task(task):
+                        tasks.append(task)
+            return index, tasks
+
+        merged: list[dict[str, Any]] = []
+        with ThreadPoolExecutor(max_workers=min(12, len(self.clients))) as executor:
+            futures = {
+                executor.submit(fetch_account, index, client): index
+                for index, client in enumerate(self.clients)
+            }
+            for future in as_completed(futures):
+                index, tasks = future.result()
+                merged.extend({**task, "accountKey": self._account_keys[index]} for task in tasks)
+        return merged