feat: add failure-aware preflight and Qwen review

modelhub_submmit_api/hf_discovery.py

@@ -69,6 +69,8 @@ class HuggingFaceDiscovery:
         )
         self._repo_tree_cache: dict[str, list[dict[str, Any]]] = {}
         self._repo_tree_lock = threading.Lock()
+        self._model_config_cache: dict[str, tuple[dict[str, Any], str | None]] = {}
+        self._model_config_lock = threading.Lock()
         self._model_page_cache: dict[tuple[str, int, int], tuple[float, list[dict[str, Any]]]] = {}
         self._model_page_cache_ttl = max(
             0.0,
@@ -197,7 +199,43 @@ class HuggingFaceDiscovery:
 
     def inspect_model(self, model: HFModelSummary) -> ModelInspection:
         entries = self.list_repo_tree(model.repo_id)
-        return inspect_repo_tree(model.repo_id, entries)
+        inspection = inspect_repo_tree(model.repo_id, entries)
+        if not inspection.has_root_config:
+            return inspection
+        model_config, config_error = self.get_model_config(model.repo_id)
+        return ModelInspection(
+            repo_id=inspection.repo_id,
+            file_paths=inspection.file_paths,
+            file_sizes=inspection.file_sizes,
+            gguf_files=inspection.gguf_files,
+            selected_gguf=inspection.selected_gguf,
+            weight_files=inspection.weight_files,
+            onnx_files=inspection.onnx_files,
+            model_config=model_config,
+            config_fetch_error=config_error,
+        )
+
+    def get_model_config(self, repo_id: str) -> tuple[dict[str, Any], str | None]:
+        with self._model_config_lock:
+            cached = self._model_config_cache.get(repo_id)
+        if cached is not None:
+            return dict(cached[0]), cached[1]
+
+        encoded_repo_id = "/".join(quote(part, safe="") for part in repo_id.split("/"))
+        try:
+            payload = self.legacy_http_client.request_json(
+                "GET",
+                f"/models/{encoded_repo_id}/resolve/master/config.json",
+            )
+            if not isinstance(payload, dict):
+                raise ValueError("config.json did not contain a JSON object")
+            result = (dict(payload), None)
+        except Exception as exc:
+            result = ({}, f"{type(exc).__name__}: {exc}")
+
+        with self._model_config_lock:
+            self._model_config_cache[repo_id] = result
+        return dict(result[0]), result[1]
 
     def list_repo_tree(self, repo_id: str) -> list[dict[str, Any]]:
         with self._repo_tree_lock:
@@ -279,6 +317,7 @@ class HuggingFaceDiscovery:
 
 def inspect_repo_tree(repo_id: str, entries: list[dict[str, Any]]) -> ModelInspection:
     file_paths: list[str] = []
+    file_sizes: dict[str, int] = {}
     gguf_files: list[str] = []
     vllm_weight_files: list[str] = []
     onnx_files: list[str] = []
@@ -290,7 +329,23 @@ def inspect_repo_tree(repo_id: str, entries: list[dict[str, Any]]) -> ModelInspe
         entry_type = (entry.get("type") or entry.get("Type") or "").lower()
         if entry_type in {"directory", "dir", "folder"}:
             continue
+        path = str(path)
+        if path.startswith("./"):
+            path = path[2:]
+        path = path.lstrip("/")
         file_paths.append(path)
+        size_value: Any = None
+        size_present = False
+        for size_key in ("Size", "size"):
+            if size_key in entry:
+                size_value = entry[size_key]
+                size_present = True
+                break
+        if size_present:
+            try:
+                file_sizes[path] = max(0, int(size_value))
+            except (TypeError, ValueError):
+                pass
         filename = PurePosixPath(path).name.lower()
         if any(filename.endswith(suffix) for suffix in GGUF_PRIORITY):
             gguf_files.append(path)
@@ -303,6 +358,7 @@ def inspect_repo_tree(repo_id: str, entries: list[dict[str, Any]]) -> ModelInspe
     return ModelInspection(
         repo_id=repo_id,
         file_paths=sorted(file_paths),
+        file_sizes=file_sizes,
         gguf_files=sorted(gguf_files),
         selected_gguf=PurePosixPath(selected_gguf).name if selected_gguf else None,
         weight_files=sorted(vllm_weight_files),