feat: add durable success-first modelhub agent

modelhub_submmit_api/hf_discovery.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
 
 import os
+import re
 import threading
 import time
+from dataclasses import replace
 from datetime import datetime, timezone
 from pathlib import PurePosixPath
 from typing import Any
@@ -92,6 +94,9 @@ class HuggingFaceDiscovery:
             ),
         )
         self._last_model_page_request_at = 0.0
+        self._unsupported_task_filters: dict[str, float] = {}
+        self._model_card_cache: dict[str, dict[str, Any]] = {}
+        self._model_card_lock = threading.Lock()
 
     def list_recent_models(
         self,
@@ -108,10 +113,12 @@ class HuggingFaceDiscovery:
         deduped: dict[str, HFModelSummary] = {}
         del read_concurrency
 
+        per_tag_limit = limit if len(pipeline_tags) <= 1 else max(10, (limit + len(pipeline_tags) - 1) // len(pipeline_tags))
+
         for pipeline_tag in pipeline_tags:
             for model in self._query_recent_models(
                 pipeline_tag=pipeline_tag,
-                limit=limit,
+                limit=per_tag_limit,
                 min_downloads=min_downloads,
                 updated_after=updated_after,
             ):
@@ -122,7 +129,7 @@ class HuggingFaceDiscovery:
                     deduped[model.repo_id] = model
         models = list(deduped.values())
         models.sort(key=lambda item: item.last_modified or parse_datetime("1970-01-01"), reverse=True)
-        return models
+        return models[: max(1, int(limit))]
 
     def _query_recent_models(
         self,
@@ -136,9 +143,10 @@ class HuggingFaceDiscovery:
         page_size = min(max(1, limit), 50)
         max_items = min(max(1, limit), 3000)
         task_tag = MODELSCOPE_TASK_TAGS.get(pipeline_tag, pipeline_tag)
+        filter_disabled = self._unsupported_task_filters.get(task_tag, 0.0) > time.monotonic()
         models: list[HFModelSummary] = []
         for page_number in range(1, (max_items + page_size - 1) // page_size + 1):
-            cache_key = (task_tag, page_number, page_size)
+            cache_key = ("*" if filter_disabled else task_tag, page_number, page_size)
             cached = self._model_page_cache.get(cache_key)
             if cached is not None and time.monotonic() - cached[0] < self._model_page_cache_ttl:
                 items = list(cached[1])
@@ -147,19 +155,34 @@ class HuggingFaceDiscovery:
                 if self._last_model_page_request_at > 0 and elapsed < self._page_interval_seconds:
                     time.sleep(self._page_interval_seconds - elapsed)
                 try:
+                    query = {
+                        "page_number": page_number,
+                        "page_size": page_size,
+                        "sort": "last_modified",
+                    }
+                    if not filter_disabled:
+                        query["filter.task"] = task_tag
                     payload = self.http_client.request_json(
                         "GET",
                         "/models",
-                        query={
-                            "page_number": page_number,
-                            "page_size": page_size,
-                            "sort": "last_modified",
-                            "filter.task": task_tag,
-                        },
+                        query=query,
                     )
                     self._last_model_page_request_at = time.monotonic()
                 except HttpJsonError as exc:
                     self._last_model_page_request_at = time.monotonic()
+                    if exc.status_code == 400 and not filter_disabled:
+                        self._unsupported_task_filters[task_tag] = time.monotonic() + 86_400
+                        print(
+                            f"[modelscope] task_filter_unsupported task={task_tag} "
+                            "fallback=unfiltered ttl=86400s",
+                            flush=True,
+                        )
+                        return self._query_recent_models(
+                            pipeline_tag=pipeline_tag,
+                            limit=limit,
+                            min_downloads=min_downloads,
+                            updated_after=updated_after,
+                        )
                     print(
                         f"[modelscope] list_models_error task={task_tag} page={page_number} "
                         f"partial_models={len(models)} retry_next_cycle=true error={exc}",
@@ -204,8 +227,9 @@ class HuggingFaceDiscovery:
         entries = self.list_repo_tree(model.repo_id)
         inspection = inspect_repo_tree(model.repo_id, entries)
         if not inspection.has_root_config:
-            return inspection
+            return replace(inspection, published_size_bytes=model.file_size)
         model_config, config_error = self.get_model_config(model.repo_id)
+        model_card_metadata = self.get_model_card_metadata(model.repo_id)
         return ModelInspection(
             repo_id=inspection.repo_id,
             file_paths=inspection.file_paths,
@@ -216,8 +240,31 @@ class HuggingFaceDiscovery:
             onnx_files=inspection.onnx_files,
             model_config=model_config,
             config_fetch_error=config_error,
+            model_card_metadata=model_card_metadata,
+            published_size_bytes=model.file_size,
         )
 
+    def get_model_card_metadata(self, repo_id: str) -> dict[str, Any]:
+        with self._model_card_lock:
+            cached = self._model_card_cache.get(repo_id)
+        if cached is not None:
+            return dict(cached)
+        encoded_repo_id = "/".join(quote(part, safe="") for part in repo_id.split("/"))
+        try:
+            payload = self.legacy_http_client.request_json("GET", f"/api/v1/models/{encoded_repo_id}")
+            data = payload.get("Data") if isinstance(payload, dict) else None
+            if not isinstance(data, dict) and isinstance(payload, dict):
+                data = payload.get("data")
+            readme = ""
+            if isinstance(data, dict):
+                readme = str(data.get("ReadMe") or data.get("readme") or data.get("README") or "")
+            result = parse_model_card_front_matter(readme)
+        except Exception:
+            result = {}
+        with self._model_card_lock:
+            self._model_card_cache[repo_id] = dict(result)
+        return result
+
     def get_model_config(self, repo_id: str) -> tuple[dict[str, Any], str | None]:
         with self._model_config_lock:
             cached = self._model_config_cache.get(repo_id)
@@ -356,9 +403,71 @@ class HuggingFaceDiscovery:
             last_modified=last_modified,
             pipeline_tag=pipeline_tag,
             created_at=parse_datetime(item.get("created_at") or item.get("CreatedAt")),
+            params=_optional_int(item.get("params") or item.get("Params") or item.get("parameter_count")),
+            file_size=_optional_int(item.get("file_size") or item.get("FileSize") or item.get("size")),
+            tags=_string_tuple(item.get("tags") or item.get("Tags")),
+            tasks=_string_tuple(item.get("tasks") or item.get("Tasks")),
+            license=str(item.get("license") or item.get("License") or "").strip() or None,
+            gated=bool(item.get("gated") or item.get("Gated")),
+            private=bool(item.get("private") or item.get("Private")),
+            likes=int(item.get("likes") or item.get("Likes") or 0),
         )
 
 
+def _optional_int(value: Any) -> int | None:
+    try:
+        parsed = int(value)
+    except (TypeError, ValueError):
+        return None
+    return parsed if parsed >= 0 else None
+
+
+def _string_tuple(value: Any) -> tuple[str, ...]:
+    if isinstance(value, str):
+        values = [part.strip() for part in value.split(",")]
+    elif isinstance(value, (list, tuple, set)):
+        values = [str(part).strip() for part in value]
+    else:
+        values = []
+    return tuple(dict.fromkeys(part for part in values if part))
+
+
+def parse_model_card_front_matter(readme: str) -> dict[str, Any]:
+    text = str(readme or "")[:65_536]
+    if not text.startswith("---"):
+        return {}
+    match = re.match(r"^---\s*\n(.*?)\n---(?:\s*\n|$)", text, flags=re.DOTALL)
+    if match is None:
+        return {}
+    front_matter = match.group(1)
+    try:
+        import yaml  # type: ignore
+
+        parsed = yaml.safe_load(front_matter)
+        data = parsed if isinstance(parsed, dict) else {}
+    except (ImportError, ValueError, TypeError):
+        data = {}
+        current_list: str | None = None
+        for raw_line in front_matter.splitlines():
+            if re.match(r"^\s+-\s+", raw_line) and current_list:
+                value = re.sub(r"^\s+-\s+", "", raw_line).strip().strip('"\'')
+                data.setdefault(current_list, []).append(value)
+                continue
+            if ":" not in raw_line or raw_line[:1].isspace():
+                continue
+            key, value = raw_line.split(":", 1)
+            key = key.strip()
+            value = value.strip().strip('"\'')
+            if not value:
+                data[key] = []
+                current_list = key
+            else:
+                data[key] = value
+                current_list = None
+    allowed = ("base_model", "base_model_relation", "frameworks", "tasks", "new_version")
+    return {key: data[key] for key in allowed if key in data}
+
+
 def inspect_repo_tree(repo_id: str, entries: list[dict[str, Any]]) -> ModelInspection:
     file_paths: list[str] = []
     file_sizes: dict[str, int] = {}