Switch agent discovery to ModelScope

.dockerignore

@@ -15,6 +15,12 @@ modelhub_submmit_api/.ipynb_checkpoints/KEY-checkpoint.md
 modelhub_submmit_api/.ipynb_checkpoints/KEYS-checkpoint.md
 
 # Runtime artifacts are regenerated by the deployed strategy.
+runs/
+daily_runs/
+poll_runs/
+ledger/
+outcomes/
+history/
 modelhub_submmit_api/runs/
 modelhub_submmit_api/daily_runs/
 modelhub_submmit_api/poll_runs/

.gitignore

@@ -12,6 +12,12 @@ modelhub_submmit_api/.ipynb_checkpoints/KEY-checkpoint.md
 modelhub_submmit_api/.ipynb_checkpoints/KEYS-checkpoint.md
 
 # Runtime artifacts.
+runs/
+daily_runs/
+poll_runs/
+ledger/
+outcomes/
+history/
 modelhub_submmit_api/runs/
 modelhub_submmit_api/daily_runs/
 modelhub_submmit_api/poll_runs/

README.md

@@ -15,13 +15,13 @@ submission poller in a child process.
 
 ## Runtime Environment
 
-The image includes single-account ModelHub and Hugging Face token fallbacks for
+The image includes single-account ModelHub and ModelScope token fallbacks for
 the agent platform. Environment variables can override them without rebuilding
 the image.
 
 - `MODELHUB_XC_TOKEN`, `XC_TOKEN`, or `MODELHUB_TOKEN` for ModelHub API authentication
 - `MODELHUB_JWT_TOKEN` or `JWT_TOKEN` can be used instead when the platform provides a JWT
-- `HF_TOKEN` optional override for the embedded Hugging Face fallback token
+- `MODELSCOPE_API_TOKEN` or `MODELSCOPE_TOKEN` optional override for the embedded ModelScope fallback token
 - `STRATEGY_ID` is expected to be injected by the ModelHub agent platform and is attached to submissions for strategy attribution; it is not an API authentication token
 
 Optional tuning:
@@ -40,6 +40,6 @@ Optional tuning:
 Create a tag and submit the repository URL plus tag in "我的适配智能体".
 
 ```bash
-git tag agent-v1
-git push origin agent-v1
+git tag agent-v4
+git push origin agent-v4
 ```

main.py

@@ -9,7 +9,7 @@ import time
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
 
-from modelhub_submmit_api.defaults import EMBEDDED_HF_TOKEN, EMBEDDED_MODELHUB_XC_TOKEN
+from modelhub_submmit_api.defaults import EMBEDDED_MODELHUB_XC_TOKEN, EMBEDDED_MODELSCOPE_TOKEN
 
 
 HOST = "0.0.0.0"
@@ -75,7 +75,7 @@ def _worker_command() -> list[str]:
 def _config() -> dict[str, object]:
     return {
         "strategy_id_present": bool(os.getenv("STRATEGY_ID")),
-        "hf_token_present": bool(os.getenv("HF_TOKEN") or EMBEDDED_HF_TOKEN),
+        "modelscope_token_present": bool(os.getenv("MODELSCOPE_API_TOKEN") or os.getenv("MODELSCOPE_TOKEN") or EMBEDDED_MODELSCOPE_TOKEN),
         "modelhub_auth_present": _has_modelhub_auth(),
         "config_error": config_error,
         "worker_running": worker is not None and worker.poll() is None,

modelhub_submmit_api/README.md

@@ -1,6 +1,6 @@
 # ModelHub Submission Runner
 
-This package automates Hugging Face model discovery and ModelHub submission.
+This package automates ModelScope model discovery and ModelHub submission.
 It currently supports:
 
 - one-shot submission planning via `main.py`
@@ -15,7 +15,7 @@ It currently supports:
 - `daily_runner.py`: daily wave orchestration
 - `poll_runner.py`: long-running queue refiller
 - `runner_common.py`: shared token / key file loading
-- `hf_discovery.py`: Hugging Face model discovery and inspection
+- `modelscope_discovery.py`: ModelScope model discovery and inspection
 - `modelhub_client.py`: ModelHub API client
 - `history_stats.py`: online history aggregation, ranking, and warnings
 - `template_selector.py`: template lookup and GPU normalization
@@ -24,7 +24,7 @@ It currently supports:
 
 ## Key Files
 
-- `KEY.md`: optional local Hugging Face and ModelHub tokens
+- `KEY.md`: optional local ModelScope and ModelHub tokens
 - `templates/public_submit/adapt_task_templates.jsonl`: public submit templates
 
 The agent platform deployment should use environment variables instead of key
@@ -81,9 +81,9 @@ bash run_poll.sh --dry-run
 Common flags:
 
 - `--daily-target`: total target submissions for the day; `0` means unlimited
-- `--min-downloads`: Hugging Face download floor
+- `--min-downloads`: ModelScope download floor
 - `--history-stats-threshold`: local ledger threshold before using online history stats
-- `--max-scan-models`: hard cap on scanned HF models for a run (0 = auto)
+- `--max-scan-models`: hard cap on scanned ModelScope models for a run (0 = auto)
 - `--scan-multiplier`: multiplier used for auto scan cap derivation from quota/queue capacity
 - `--read-concurrency`: concurrent HTTP reads while scanning model candidates (default 4)
 - `--max-submits-per-run`: max tasks to submit per run cycle (0 = unlimited)
@@ -97,7 +97,7 @@ Common flags:
 
 - `--poll-interval-seconds`: sleep when the account has no available slots
 - `--idle-interval-seconds`: sleep when a cycle submits nothing
-- `--max-scan-models`: hard cap on scanned HF models for this cycle (0 = auto)
+- `--max-scan-models`: hard cap on scanned ModelScope models for this cycle (0 = auto)
 - `--scan-multiplier`: multiplier used for auto scan cap derivation from quota/queue capacity
 - `--max-submits-per-run`: max tasks to submit per poll cycle (0 = unlimited)
 - `--skip-outcome-sync`: skip outcome sync before scanning

modelhub_submmit_api/daily_runner.py

@@ -7,9 +7,9 @@ from pathlib import Path
 from typing import Any, Callable
 
 from common import utc_now, write_json
-from hf_discovery import HuggingFaceDiscovery
 from main import DEFAULT_LEDGER_PATH, DEFAULT_RUNS_DIR, run_submission
-from modelhub_client import ModelHubClient
+from modelhub_client import ModelHubClient, MultiModelHubClient
+from modelscope_discovery import ModelScopeDiscovery
 from outcome_tracker import OutcomeTracker
 from runner_common import DEFAULT_KEY_PATH, ensure_tokens
 from template_selector import TemplateSelector
@@ -55,7 +55,7 @@ def build_parser() -> argparse.ArgumentParser:
         default=4,
         help="Multiplier used when auto-deriving scan limit from quota/queue capacity",
     )
-    parser.add_argument("--min-downloads", type=int, default=50, help="Minimum Hugging Face download threshold")
+    parser.add_argument("--min-downloads", type=int, default=50, help="Minimum ModelScope download threshold")
     parser.add_argument(
         "--history-stats-threshold",
         type=int,
@@ -78,16 +78,18 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument("--skip-outcome-sync", action="store_true", help="Skip outcome sync from ModelHub before scanning")
     parser.add_argument("--skip-history-archive", action="store_true", help="Skip historical task archive download for this run")
     parser.add_argument("--dry-run", action="store_true", help="Plan the day without creating tasks")
-    parser.add_argument("--key-path", default=str(DEFAULT_KEY_PATH), help="Path to KEY.md containing HF_TOKEN/XC_TOKEN")
+    parser.add_argument("--key-path", default=str(DEFAULT_KEY_PATH), help="Path to KEY.md containing MODELSCOPE_TOKEN/XC_TOKEN")
     parser.add_argument("--runs-dir", default=str(DEFAULT_RUNS_DIR), help=argparse.SUPPRESS)
     parser.add_argument("--ledger-path", default=str(DEFAULT_LEDGER_PATH), help=argparse.SUPPRESS)
     parser.add_argument("--history-archive-path", default="history/platform_tasks.jsonl", help=argparse.SUPPRESS)
     parser.add_argument("--history-archive-limit", type=int, default=5000, help=argparse.SUPPRESS)
     parser.add_argument("--daily-runs-dir", default=str(DEFAULT_DAILY_RUNS_DIR), help=argparse.SUPPRESS)
-    parser.add_argument("--hf-base-url", default=os.getenv("HF_BASE_URL", "https://huggingface.co"), help=argparse.SUPPRESS)
+    parser.add_argument("--hf-base-url", default=os.getenv("MODELSCOPE_BASE_URL", "https://modelscope.cn"), help=argparse.SUPPRESS)
+    parser.add_argument("--modelscope-base-url", default=os.getenv("MODELSCOPE_BASE_URL", "https://modelscope.cn"), help=argparse.SUPPRESS)
     parser.add_argument("--modelhub-base-url", default=os.getenv("MODELHUB_BASE_URL", "https://modelhub.org.cn"), help=argparse.SUPPRESS)
     parser.add_argument("--modelhub-token", default=os.getenv("MODELHUB_XC_TOKEN") or os.getenv("XC_TOKEN"), help=argparse.SUPPRESS)
     parser.add_argument("--hf-token", default=os.getenv("HF_TOKEN"), help=argparse.SUPPRESS)
+    parser.add_argument("--modelscope-token", default=os.getenv("MODELSCOPE_API_TOKEN") or os.getenv("MODELSCOPE_TOKEN"), help=argparse.SUPPRESS)
     return parser
 
 
@@ -119,6 +121,7 @@ def make_wave_namespace(base_args: argparse.Namespace, wave: WaveSpec) -> argpar
         history_archive_path=base_args.history_archive_path,
         history_archive_limit=base_args.history_archive_limit,
         hf_base_url=base_args.hf_base_url,
+        modelscope_base_url=getattr(base_args, "modelscope_base_url", base_args.hf_base_url),
         modelhub_base_url=base_args.modelhub_base_url,
         modelhub_token=base_args.modelhub_token,
     )
@@ -130,14 +133,19 @@ def run_daily_batches(
     waves: tuple[WaveSpec, ...] = DEFAULT_WAVES,
     now=None,
     run_fn: Callable[..., dict[str, Any]] = run_submission,
-    hf_discovery: HuggingFaceDiscovery | None = None,
+    hf_discovery: ModelScopeDiscovery | None = None,
     modelhub_client: ModelHubClient | None = None,
     template_selector: TemplateSelector | None = None,
     outcome_tracker: OutcomeTracker | None = None,
 ) -> dict[str, Any]:
     now = now or utc_now()
-    hf_discovery = hf_discovery or HuggingFaceDiscovery(base_url=base_args.hf_base_url)
+    discovery_base_url = getattr(base_args, "modelscope_base_url", None) or base_args.hf_base_url
+    hf_discovery = hf_discovery or ModelScopeDiscovery(base_url=discovery_base_url)
     if modelhub_client is None:
+        modelhub_tokens = list(getattr(base_args, "modelhub_tokens", []) or [])
+        if len(modelhub_tokens) > 1:
+            modelhub_client = MultiModelHubClient(tokens=modelhub_tokens, base_url=base_args.modelhub_base_url)
+        else:
             modelhub_client = ModelHubClient(token=base_args.modelhub_token, base_url=base_args.modelhub_base_url)
     template_selector = template_selector or TemplateSelector()
 
@@ -284,8 +292,8 @@ def main(argv: list[str] | None = None) -> int:
     args = parser.parse_args(argv)
     ensure_tokens(args)
     log(
-        f"[daily] hf_token={'set' if bool(args.hf_token) else 'missing'} "
-        f"xc_token={'set' if bool(args.modelhub_token) else 'missing'}"
+        f"[daily] modelscope_token={'set' if bool(args.modelscope_token) else 'missing'} "
+        f"xc_tokens={len(getattr(args, 'modelhub_tokens', []) or [])}"
     )
     summary = run_daily_batches(base_args=args)
     print(f"daily_run_dir={summary['dailyRunDir']}")

modelhub_submmit_api/defaults.py

@@ -1,2 +1,17 @@
-EMBEDDED_MODELHUB_XC_TOKEN = "a14776f6e7ad4c04a1710260613c294c"
+EMBEDDED_MODELHUB_XC_TOKENS = [
+    "a14776f6e7ad4c04a1710260613c294c",
+    "8408eb2f73034fa09e1919406774d111",
+    "b0b3add66b4641d5afdc7487589905f9",
+    "edff358739594aa4b9f26beed6ab8799",
+    "d260bdd83c4b4c6faa063a16685b8908",
+    "fc72d989bfa84d529fff6ea33e073945",
+    "30d137aa08944ded8b86aac942904cc0",
+    "fd60e08b313048efa413f627badfe0b3",
+    "9bd1431410ba4558bc730481dff74d49",
+    "a96aea9071d84576a963c706dc71dfac",
+    "be09a04434fb4e379344b0554aaf6e6a",
+    "6032b66717e041d3adb0217e86103c6d",
+]
+EMBEDDED_MODELHUB_XC_TOKEN = EMBEDDED_MODELHUB_XC_TOKENS[0]
 EMBEDDED_HF_TOKEN = "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"
+EMBEDDED_MODELSCOPE_TOKEN = "ms-b4918c83-7eb3-4034-8635-f154938ed3f0"

modelhub_submmit_api/hf_discovery.py

@@ -192,10 +192,10 @@ def inspect_repo_tree(repo_id: str, entries: list[dict[str, Any]]) -> ModelInspe
     onnx_files: list[str] = []
 
     for entry in entries:
-        path = entry.get("path") or entry.get("rfilename") or entry.get("name")
+        path = entry.get("path") or entry.get("Path") or entry.get("rfilename") or entry.get("name") or entry.get("Name")
         if not path:
             continue
-        entry_type = (entry.get("type") or "").lower()
+        entry_type = (entry.get("type") or entry.get("Type") or "").lower()
         if entry_type in {"directory", "dir", "folder"}:
             continue
         file_paths.append(path)

modelhub_submmit_api/main.py

@@ -8,7 +8,6 @@ from pathlib import Path
 from typing import Any
 
 from common import parse_datetime, utc_now, write_json, write_jsonl
-from hf_discovery import HuggingFaceDiscovery
 from history_stats import (
     append_ledger_entry,
     build_empty_pre_submit_report,
@@ -16,8 +15,9 @@ from history_stats import (
     load_ledger,
     update_history_archive,
 )
-from modelhub_client import ModelHubAPIError, ModelHubClient
+from modelhub_client import ModelHubAPIError, ModelHubClient, MultiModelHubClient
 from models import CandidateModel, HFModelSummary, ModelInspection
+from modelscope_discovery import ModelScopeDiscovery
 from outcome_tracker import DEFAULT_OUTCOMES_PATH, OutcomeTracker
 from task_registry import TASK_SPEC_BY_TYPE, all_task_types, choose_framework_for_task, choose_text_generation_framework, pipeline_tags_for_task_types, task_specs_for_model
 from template_selector import TemplateSelector
@@ -28,11 +28,11 @@ DEFAULT_LEDGER_PATH = Path("ledger/submissions.jsonl")
 
 
 def build_parser() -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser(description="Auto discover and submit public Hugging Face models to ModelHub.")
+    parser = argparse.ArgumentParser(description="Auto discover and submit public ModelScope models to ModelHub.")
     parser.add_argument("--gpu", help="Single GPU alias or platform name, for example: k100")
     parser.add_argument("--gpus", help="Comma-separated GPU aliases/platform names. Omit to auto-use all safe GPUs from templates.")
     parser.add_argument("--task-types", help="Comma-separated ModelHub task types. Omit to auto-enable all supported task types.")
-    parser.add_argument("--limit", type=int, default=300, help="Maximum number of Hugging Face models to scan per pipeline tag")
+    parser.add_argument("--limit", type=int, default=300, help="Maximum number of ModelScope models to scan per pipeline tag")
     parser.add_argument("--max-scan-models", type=int, default=0, help="Hard cap on total scanned models (0 means auto)")
     parser.add_argument(
         "--scan-multiplier",
@@ -73,7 +73,8 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument("--outcomes-path", default=str(DEFAULT_OUTCOMES_PATH), help=argparse.SUPPRESS)
     parser.add_argument("--history-archive-path", default="history/platform_tasks.jsonl", help=argparse.SUPPRESS)
     parser.add_argument("--history-archive-limit", type=int, default=5000, help=argparse.SUPPRESS)
-    parser.add_argument("--hf-base-url", default=os.getenv("HF_BASE_URL", "https://huggingface.co"), help=argparse.SUPPRESS)
+    parser.add_argument("--hf-base-url", default=os.getenv("MODELSCOPE_BASE_URL", "https://modelscope.cn"), help=argparse.SUPPRESS)
+    parser.add_argument("--modelscope-base-url", default=os.getenv("MODELSCOPE_BASE_URL", "https://modelscope.cn"), help=argparse.SUPPRESS)
     parser.add_argument("--modelhub-base-url", default=os.getenv("MODELHUB_BASE_URL", "https://modelhub.org.cn"), help=argparse.SUPPRESS)
     parser.add_argument("--modelhub-token", default=os.getenv("MODELHUB_XC_TOKEN") or os.getenv("XC_TOKEN"), help=argparse.SUPPRESS)
     return parser
@@ -227,7 +228,7 @@ def resolve_max_submit_count(
 def process_model_for_candidates(
     *,
     model: HFModelSummary,
-    hf_discovery: HuggingFaceDiscovery,
+    hf_discovery: ModelScopeDiscovery,
     modelhub_client: ModelHubClient,
     template_selector: TemplateSelector,
     target_gpus: list[str],
@@ -339,7 +340,7 @@ def run_submission(
     args: argparse.Namespace,
     *,
     now=None,
-    hf_discovery: HuggingFaceDiscovery | None = None,
+    hf_discovery: ModelScopeDiscovery | None = None,
     modelhub_client: ModelHubClient | None = None,
     template_selector: TemplateSelector | None = None,
     outcome_tracker: OutcomeTracker | None = None,
@@ -351,8 +352,14 @@ def run_submission(
     if not target_gpus:
         raise RuntimeError("No auto-submittable GPUs are available for the selected task types")
 
-    hf_discovery = hf_discovery or HuggingFaceDiscovery(base_url=args.hf_base_url)
-    modelhub_client = modelhub_client or ModelHubClient(token=args.modelhub_token, base_url=args.modelhub_base_url)
+    discovery_base_url = getattr(args, "modelscope_base_url", None) or args.hf_base_url
+    hf_discovery = hf_discovery or ModelScopeDiscovery(base_url=discovery_base_url)
+    if modelhub_client is None:
+        modelhub_tokens = list(getattr(args, "modelhub_tokens", []) or [])
+        if len(modelhub_tokens) > 1:
+            modelhub_client = MultiModelHubClient(tokens=modelhub_tokens, base_url=args.modelhub_base_url)
+        else:
+            modelhub_client = ModelHubClient(token=args.modelhub_token, base_url=args.modelhub_base_url)
 
     runs_dir = Path(args.runs_dir)
     ledger_path = Path(args.ledger_path)

modelhub_submmit_api/modelhub_client.py

@@ -2,6 +2,8 @@ from __future__ import annotations
 
 import os
 from datetime import datetime, timedelta
+from itertools import cycle
+from threading import Lock
 from typing import Any
 
 from common import format_modelhub_datetime, parse_datetime
@@ -205,6 +207,73 @@ class ModelHubClient:
         return payload
 
 
+class MultiModelHubClient:
+    def __init__(
+        self,
+        *,
+        tokens: list[str],
+        base_url: str = "https://modelhub.org.cn",
+        timeout: int = 30,
+        retries: int = 2,
+    ) -> None:
+        deduped: list[str] = []
+        for token in tokens:
+            clean = token.strip()
+            if clean and clean not in deduped:
+                deduped.append(clean)
+        if not deduped:
+            raise ValueError("At least one ModelHub token is required")
+
+        self.clients = [
+            ModelHubClient(token=token, base_url=base_url, timeout=timeout, retries=retries)
+            for token in deduped
+        ]
+        self._cycle = cycle(self.clients)
+        self._lock = Lock()
+
+    def _next_client(self) -> ModelHubClient:
+        with self._lock:
+            return next(self._cycle)
+
+    def add_task(self, payload: dict[str, Any]) -> dict[str, Any]:
+        return self._next_client().add_task(payload)
+
+    def search_by_model_id(self, model_id: str) -> dict[str, Any]:
+        return self.clients[0].search_by_model_id(model_id)
+
+    def is_model_processed_for_gpu(self, model_id: str, target_gpu: str) -> bool:
+        return self.clients[0].is_model_processed_for_gpu(model_id, target_gpu)
+
+    def get_verify_result_map(self, model_id: str) -> dict[str, Any]:
+        return self.clients[0].get_verify_result_map(model_id)
+
+    def processed_gpus_for_model(self, model_id: str) -> set[str]:
+        return self.clients[0].processed_gpus_for_model(model_id)
+
+    def list_tasks_page(self, **kwargs: Any) -> dict[str, Any]:
+        return self.clients[0].list_tasks_page(**kwargs)
+
+    def list_tasks(self, **kwargs: Any) -> list[dict[str, Any]]:
+        return self.clients[0].list_tasks(**kwargs)
+
+    def find_recent_task_id(self, model_id: str, gpu_type: str, submitted_after: datetime) -> str | None:
+        return self.clients[0].find_recent_task_id(model_id, gpu_type, submitted_after)
+
+    def count_active_tasks(self, **kwargs: Any) -> int:
+        return self.clients[0].count_active_tasks(**kwargs)
+
+    def active_task_counts(self) -> list[int]:
+        counts: list[int] = []
+        now = datetime.utcnow()
+        begin_time = now - timedelta(days=1)
+        for client in self.clients:
+            counts.append(client.count_active_tasks(begin_time=begin_time, end_time=now, max_count=100))
+        return counts
+
+    def available_submit_slots(self, max_active_per_account: int = 5) -> int:
+        return sum(max(0, max_active_per_account - count) for count in self.active_task_counts())
+
+
 ACTIVE_TASK_STATUSES = {
     "waiting",
     "running",

modelhub_submmit_api/models.py

@@ -23,9 +23,12 @@ class HFModelSummary:
     last_modified: datetime | None
     pipeline_tag: str | None
     created_at: datetime | None = None
+    source: str = "huggingface"
 
     @property
     def model_address(self) -> str:
+        if self.source == "modelscope":
+            return f"https://modelscope.cn/models/{self.repo_id}"
         return f"https://huggingface.co/{self.repo_id}"
 
 

modelhub_submmit_api/modelscope_discovery.py

@@ -0,0 +1,243 @@
+from __future__ import annotations
+
+import os
+import threading
+from typing import Any
+from urllib.parse import quote
+
+from common import parse_datetime
+from defaults import EMBEDDED_MODELSCOPE_TOKEN
+from hf_discovery import inspect_repo_tree
+from http_json import HttpJsonError, JsonHttpClient
+from models import HFModelSummary, ModelInspection
+
+
+MODELSCOPE_TASK_TAGS = {
+    "text-generation": "text-generation",
+    "image-text-to-text": "image-text-to-text",
+    "visual-question-answering": "visual-question-answering",
+    "document-question-answering": "document-question-answering",
+    "video-text-to-text": "video-text-to-text",
+    "text-to-image": "text-to-image-synthesis",
+    "image-to-image": "image-to-image",
+    "automatic-speech-recognition": "auto-speech-recognition",
+    "question-answering": "question-answering",
+    "feature-extraction": "feature-extraction",
+    "sentence-similarity": "sentence-similarity",
+    "image-classification": "image-classification",
+    "zero-shot-image-classification": "zero-shot-image-classification",
+    "text-classification": "text-classification",
+    "zero-shot-classification": "zero-shot-classification",
+    "reinforcement-learning": "reinforcement-learning",
+}
+
+
+class ModelScopeDiscovery:
+    def __init__(
+        self,
+        base_url: str = "https://modelscope.cn",
+        http_client: JsonHttpClient | None = None,
+        legacy_http_client: JsonHttpClient | None = None,
+        timeout: int = 30,
+        retries: int = 2,
+    ) -> None:
+        token = os.getenv("MODELSCOPE_API_TOKEN") or os.getenv("MODELSCOPE_TOKEN") or EMBEDDED_MODELSCOPE_TOKEN
+        headers = {"User-Agent": "modelhub-submmit-cli/0.1"}
+        if token:
+            headers["Authorization"] = f"Bearer {token}"
+            headers["Cookie"] = f"m_session_id={token}"
+        self.http_client = http_client or JsonHttpClient(
+            base_url=f"{base_url.rstrip('/')}/openapi/v1",
+            default_headers=headers,
+            timeout=timeout,
+            retries=retries,
+        )
+        self.legacy_http_client = legacy_http_client or JsonHttpClient(
+            base_url=base_url,
+            default_headers=headers,
+            timeout=timeout,
+            retries=retries,
+        )
+        self._repo_tree_cache: dict[str, list[dict[str, Any]]] = {}
+        self._repo_tree_lock = threading.Lock()
+
+    def list_recent_models(
+        self,
+        *,
+        pipeline_tags: list[str],
+        limit: int,
+        min_downloads: int,
+        updated_after=None,
+        read_concurrency: int = 1,
+    ) -> list[HFModelSummary]:
+        del read_concurrency
+        if not pipeline_tags:
+            return []
+
+        deduped: dict[str, HFModelSummary] = {}
+        for pipeline_tag in pipeline_tags:
+            for model in self._query_recent_models(
+                pipeline_tag=pipeline_tag,
+                limit=limit,
+                min_downloads=min_downloads,
+                updated_after=updated_after,
+            ):
+                current = deduped.get(model.repo_id)
+                if current is None or (model.last_modified or parse_datetime("1970-01-01")) > (
+                    current.last_modified or parse_datetime("1970-01-01")
+                ):
+                    deduped[model.repo_id] = model
+
+        models = list(deduped.values())
+        models.sort(key=lambda item: item.last_modified or parse_datetime("1970-01-01"), reverse=True)
+        return models
+
+    def _query_recent_models(
+        self,
+        *,
+        pipeline_tag: str,
+        limit: int,
+        min_downloads: int,
+        updated_after=None,
+    ) -> list[HFModelSummary]:
+        page_size = min(max(1, limit), 100)
+        max_items = min(max(1, limit), 3000)
+        task_tag = MODELSCOPE_TASK_TAGS.get(pipeline_tag, pipeline_tag)
+        models: list[HFModelSummary] = []
+
+        for page_number in range(1, (max_items + page_size - 1) // page_size + 1):
+            try:
+                payload = self.http_client.request_json(
+                    "GET",
+                    "/models",
+                    query={
+                        "page_number": page_number,
+                        "page_size": page_size,
+                        "sort": "last_modified",
+                        "filter.task": task_tag,
+                    },
+                )
+            except HttpJsonError as exc:
+                print(f"[modelscope] list_models_error task={task_tag} page={page_number} error={exc}", flush=True)
+                break
+
+            items = self._extract_models(payload)
+            if not items:
+                break
+            for item in items:
+                model = self._parse_model(
+                    item,
+                    fallback_pipeline_tag=pipeline_tag,
+                    min_downloads=min_downloads,
+                    updated_after=updated_after,
+                )
+                if model is not None:
+                    models.append(model)
+                if len(models) >= max_items:
+                    return models
+            if len(items) < page_size:
+                break
+
+        return models
+
+    def list_recent_text_generation_models(
+        self,
+        *,
+        limit: int,
+        min_downloads: int,
+        updated_after=None,
+    ) -> list[HFModelSummary]:
+        return self.list_recent_models(
+            pipeline_tags=["text-generation"],
+            limit=limit,
+            min_downloads=min_downloads,
+            updated_after=updated_after,
+        )
+
+    def inspect_model(self, model: HFModelSummary) -> ModelInspection:
+        entries = self.list_repo_tree(model.repo_id)
+        return inspect_repo_tree(model.repo_id, entries)
+
+    def list_repo_tree(self, repo_id: str) -> list[dict[str, Any]]:
+        with self._repo_tree_lock:
+            cached = self._repo_tree_cache.get(repo_id)
+        if cached is not None:
+            return list(cached)
+
+        encoded_repo_id = "/".join(quote(part, safe="") for part in repo_id.split("/"))
+        try:
+            payload = self.legacy_http_client.request_json(
+                "GET",
+                f"/api/v1/models/{encoded_repo_id}/repo/files",
+                query={"Revision": "master", "Recursive": "true"},
+            )
+        except HttpJsonError as exc:
+            print(f"[modelscope] repo_tree_error repo={repo_id} error={exc}", flush=True)
+            payload = None
+
+        entries = self._extract_files(payload)
+        with self._repo_tree_lock:
+            self._repo_tree_cache[repo_id] = list(entries)
+        return list(entries)
+
+    @staticmethod
+    def _extract_models(payload: Any) -> list[dict[str, Any]]:
+        data = payload.get("data") if isinstance(payload, dict) else payload
+        if isinstance(data, dict):
+            for key in ("models", "Models", "items", "list", "data", "results"):
+                value = data.get(key)
+                if isinstance(value, list):
+                    return [item for item in value if isinstance(item, dict)]
+        if isinstance(data, list):
+            return [item for item in data if isinstance(item, dict)]
+        return []
+
+    @staticmethod
+    def _extract_files(payload: Any) -> list[dict[str, Any]]:
+        data = payload.get("Data") if isinstance(payload, dict) else payload
+        if isinstance(data, dict):
+            for key in ("Files", "files", "items", "tree"):
+                value = data.get(key)
+                if isinstance(value, list):
+                    return [item for item in value if isinstance(item, dict)]
+        if isinstance(data, list):
+            return [item for item in data if isinstance(item, dict)]
+        return []
+
+    @staticmethod
+    def _parse_model(
+        item: dict[str, Any],
+        *,
+        fallback_pipeline_tag: str,
+        min_downloads: int,
+        updated_after=None,
+    ) -> HFModelSummary | None:
+        repo_id = item.get("id") or item.get("model_id") or item.get("modelId")
+        if not repo_id:
+            owner = item.get("owner") or item.get("Owner") or item.get("Path")
+            name = item.get("name") or item.get("Name")
+            repo_id = f"{owner}/{name}" if owner and name else None
+        if not repo_id:
+            return None
+
+        downloads = int(item.get("downloads") or item.get("Downloads") or 0)
+        if downloads < min_downloads:
+            return None
+
+        tasks = item.get("tasks") or item.get("Tasks") or []
+        if isinstance(tasks, list) and tasks:
+            pipeline_tag = str(tasks[0])
+        else:
+            pipeline_tag = fallback_pipeline_tag
+        last_modified = parse_datetime(item.get("last_modified") or item.get("UpdatedAt") or item.get("LastUpdatedTime"))
+        if updated_after and last_modified and last_modified < updated_after:
+            return None
+
+        return HFModelSummary(
+            repo_id=repo_id,
+            downloads=downloads,
+            last_modified=last_modified,
+            pipeline_tag=pipeline_tag,
+            created_at=parse_datetime(item.get("created_at") or item.get("CreatedAt")),
+            source="modelscope",
+        )

modelhub_submmit_api/poll_runner.py

@@ -9,9 +9,9 @@ from typing import Any, Callable
 
 from common import utc_now, write_json
 from daily_runner import DEFAULT_DAILY_RUNS_DIR, log, run_daily_batches
-from hf_discovery import HuggingFaceDiscovery
 from main import DEFAULT_LEDGER_PATH, DEFAULT_RUNS_DIR
-from modelhub_client import ModelHubClient
+from modelhub_client import ModelHubClient, MultiModelHubClient
+from modelscope_discovery import ModelScopeDiscovery
 from outcome_tracker import DEFAULT_OUTCOMES_PATH, OutcomeTracker
 from runner_common import DEFAULT_KEY_PATH, ensure_tokens
 from template_selector import TemplateSelector
@@ -25,7 +25,7 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument("--daily-target", type=int, default=0, help="Total submissions to aim for per UTC day; 0 means unlimited")
     parser.add_argument("--gpu", help="Single GPU alias or platform name, for example: k100")
     parser.add_argument("--gpus", help="Comma-separated GPU aliases/platform names. Omit to auto-use all safe GPUs.")
-    parser.add_argument("--min-downloads", type=int, default=50, help="Minimum Hugging Face download threshold")
+    parser.add_argument("--min-downloads", type=int, default=50, help="Minimum ModelScope download threshold")
     parser.add_argument(
         "--history-stats-threshold",
         type=int,
@@ -55,7 +55,7 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument("--skip-outcome-sync", action="store_true", help="Skip outcome sync from ModelHub before scanning")
     parser.add_argument("--skip-history-archive", action="store_true", help="Skip historical task archive download for this run")
     parser.add_argument("--dry-run", action="store_true", help="Plan the day without creating tasks")
-    parser.add_argument("--key-path", default=str(DEFAULT_KEY_PATH), help="Path to KEY.md containing HF_TOKEN/XC_TOKEN")
+    parser.add_argument("--key-path", default=str(DEFAULT_KEY_PATH), help="Path to KEY.md containing MODELSCOPE_TOKEN/XC_TOKEN")
     parser.add_argument("--runs-dir", default=str(DEFAULT_RUNS_DIR), help=argparse.SUPPRESS)
     parser.add_argument("--ledger-path", default=str(DEFAULT_LEDGER_PATH), help=argparse.SUPPRESS)
     parser.add_argument("--history-archive-path", default="history/platform_tasks.jsonl", help=argparse.SUPPRESS)
@@ -63,10 +63,12 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument("--daily-runs-dir", default=str(DEFAULT_DAILY_RUNS_DIR), help=argparse.SUPPRESS)
     parser.add_argument("--poll-runs-dir", default=str(DEFAULT_POLL_RUNS_DIR), help=argparse.SUPPRESS)
     parser.add_argument("--outcomes-path", default=str(DEFAULT_OUTCOMES_PATH), help=argparse.SUPPRESS)
-    parser.add_argument("--hf-base-url", default="https://huggingface.co", help=argparse.SUPPRESS)
+    parser.add_argument("--hf-base-url", default="https://modelscope.cn", help=argparse.SUPPRESS)
+    parser.add_argument("--modelscope-base-url", default="https://modelscope.cn", help=argparse.SUPPRESS)
     parser.add_argument("--modelhub-base-url", default="https://modelhub.org.cn", help=argparse.SUPPRESS)
     parser.add_argument("--modelhub-token", default=None, help=argparse.SUPPRESS)
     parser.add_argument("--hf-token", default=None, help=argparse.SUPPRESS)
+    parser.add_argument("--modelscope-token", default=None, help=argparse.SUPPRESS)
     parser.add_argument("--poll-interval-seconds", type=int, default=60, help="Sleep between polling cycles when no slots are available")
     parser.add_argument("--idle-interval-seconds", type=int, default=30, help="Sleep between cycles when a scan submits nothing")
     parser.add_argument("--post-cycle-cooldown-seconds", type=int, default=0, help="Short sleep after a successful cycle")
@@ -85,6 +87,9 @@ def _make_cycle_args(base_args: argparse.Namespace) -> argparse.Namespace:
 
 
 def _build_modelhub_client(base_args: argparse.Namespace) -> ModelHubClient:
+    modelhub_tokens = list(getattr(base_args, "modelhub_tokens", []) or [])
+    if len(modelhub_tokens) > 1:
+        return MultiModelHubClient(tokens=modelhub_tokens, base_url=base_args.modelhub_base_url)
     return ModelHubClient(token=base_args.modelhub_token, base_url=base_args.modelhub_base_url)
 
 
@@ -93,13 +98,14 @@ def run_poll_loop(
     base_args: argparse.Namespace,
     now=None,
     run_fn: Callable[..., dict[str, Any]] = run_daily_batches,
-    hf_discovery: HuggingFaceDiscovery | None = None,
+    hf_discovery: ModelScopeDiscovery | None = None,
     modelhub_client: ModelHubClient | None = None,
     template_selector: TemplateSelector | None = None,
     outcome_tracker: OutcomeTracker | None = None,
 ) -> dict[str, Any]:
     now = now or utc_now()
-    hf_discovery = hf_discovery or HuggingFaceDiscovery(base_url=base_args.hf_base_url)
+    discovery_base_url = getattr(base_args, "modelscope_base_url", None) or base_args.hf_base_url
+    hf_discovery = hf_discovery or ModelScopeDiscovery(base_url=discovery_base_url)
     modelhub_client = modelhub_client or _build_modelhub_client(base_args)
     template_selector = template_selector or TemplateSelector()
 
@@ -244,8 +250,8 @@ def main(argv: list[str] | None = None) -> int:
         return 0
 
     log(
-        f"[poll] hf_token={'set' if bool(args.hf_token) else 'missing'} "
-        f"xc_token={'set' if bool(args.modelhub_token) else 'missing'}"
+        f"[poll] modelscope_token={'set' if bool(args.modelscope_token) else 'missing'} "
+        f"xc_tokens={len(getattr(args, 'modelhub_tokens', []) or [])}"
     )
     summary = run_poll_loop(base_args=args)
     print(f"poll_run_dir={summary['pollRunDir']}")

modelhub_submmit_api/runner_common.py

@@ -4,12 +4,15 @@ import argparse
 import os
 from pathlib import Path
 
-from defaults import EMBEDDED_HF_TOKEN, EMBEDDED_MODELHUB_XC_TOKEN
+from defaults import EMBEDDED_HF_TOKEN, EMBEDDED_MODELHUB_XC_TOKENS, EMBEDDED_MODELSCOPE_TOKEN
 
 
 DEFAULT_KEY_PATH = Path("KEY.md")
+MODULE_DIR = Path(__file__).resolve().parent
 XC_TOKEN_ENV_NAMES = ("MODELHUB_XC_TOKEN", "XC_TOKEN", "MODELHUB_TOKEN")
+XC_TOKEN_LIST_ENV_NAMES = ("MODELHUB_XC_TOKENS", "XC_TOKENS", "MODELHUB_TOKENS")
 JWT_TOKEN_ENV_NAMES = ("MODELHUB_JWT_TOKEN", "JWT_TOKEN")
+MODELSCOPE_TOKEN_ENV_NAMES = ("MODELSCOPE_API_TOKEN", "MODELSCOPE_TOKEN")
 
 
 def load_key_file(path: Path) -> dict[str, str]:
@@ -33,33 +36,100 @@ def first_value(values: dict[str, str], names: tuple[str, ...]) -> str | None:
     return None
 
 
+def split_token_list(value: str | None) -> list[str]:
+    if not value:
+        return []
+    tokens: list[str] = []
+    for normalized in value.replace(",", "\n").replace(";", "\n").splitlines():
+        token = normalized.strip()
+        if token:
+            tokens.append(token)
+    return tokens
+
+
+def discover_key_paths(primary_path: Path) -> list[Path]:
+    candidates: list[Path] = []
+    search_dirs = [primary_path.parent, primary_path.parent.parent]
+    for path in [primary_path, primary_path.parent / "KEYS.md"]:
+        if path not in candidates:
+            candidates.append(path)
+    for directory in search_dirs:
+        if not directory.exists():
+            continue
+        for path in sorted(directory.glob("KEYS*")):
+            if path.is_file() and path not in candidates:
+                candidates.append(path)
+    return candidates
+
+
+def collect_modelhub_tokens(values_by_path: list[dict[str, str]], explicit_token: str | None) -> list[str]:
+    tokens: list[str] = []
+
+    def add(value: str | None) -> None:
+        for token in split_token_list(value):
+            if token and token not in tokens:
+                tokens.append(token)
+
+    add(explicit_token)
+    for env_name in XC_TOKEN_LIST_ENV_NAMES:
+        add(os.getenv(env_name))
+    for env_name in XC_TOKEN_ENV_NAMES:
+        add(os.getenv(env_name))
+
+    for values in values_by_path:
+        for key, value in values.items():
+            normalized = key.strip().upper()
+            if normalized in XC_TOKEN_ENV_NAMES or normalized.startswith("XC_TOKEN") or normalized.startswith("MODELHUB_XC_TOKEN"):
+                add(value)
+
+    for token in EMBEDDED_MODELHUB_XC_TOKENS:
+        add(token)
+    return tokens
+
+
 def ensure_tokens(args: argparse.Namespace) -> None:
     primary_key_path = Path(getattr(args, "key_path", DEFAULT_KEY_PATH))
     if not primary_key_path.exists():
-        parent_key = primary_key_path.parent.parent / primary_key_path.name
-        if parent_key.exists():
-            primary_key_path = parent_key
-    values = load_key_file(primary_key_path)
+        fallback_paths = [
+            MODULE_DIR / primary_key_path.name,
+            primary_key_path.parent.parent / primary_key_path.name,
+            MODULE_DIR.parent / primary_key_path.name,
+        ]
+        for fallback_path in fallback_paths:
+            if fallback_path.exists():
+                primary_key_path = fallback_path
+                break
+    values_by_path = [load_key_file(path) for path in discover_key_paths(primary_key_path) if path.exists()]
+    values: dict[str, str] = {}
+    for loaded in values_by_path:
+        values.update(loaded)
 
     if not getattr(args, "hf_token", None):
         args.hf_token = os.getenv("HF_TOKEN") or values.get("HF_TOKEN")
     if not getattr(args, "hf_token", None):
         args.hf_token = EMBEDDED_HF_TOKEN
+    if not getattr(args, "modelscope_token", None):
+        args.modelscope_token = first_value(values, MODELSCOPE_TOKEN_ENV_NAMES)
+    if not getattr(args, "modelscope_token", None):
+        args.modelscope_token = EMBEDDED_MODELSCOPE_TOKEN
 
-    modelhub_token = getattr(args, "modelhub_token", None)
-    if not modelhub_token:
-        modelhub_token = first_value(values, XC_TOKEN_ENV_NAMES)
-    if not modelhub_token:
-        modelhub_token = EMBEDDED_MODELHUB_XC_TOKEN
+    modelhub_tokens = collect_modelhub_tokens(values_by_path, getattr(args, "modelhub_token", None))
+    modelhub_token = modelhub_tokens[0] if modelhub_tokens else None
     jwt_token = first_value(values, JWT_TOKEN_ENV_NAMES)
 
     args.modelhub_token = modelhub_token
+    args.modelhub_tokens = modelhub_tokens
 
     if args.hf_token:
         os.environ["HF_TOKEN"] = args.hf_token
+    if args.modelscope_token:
+        os.environ["MODELSCOPE_API_TOKEN"] = args.modelscope_token
+        os.environ["MODELSCOPE_TOKEN"] = args.modelscope_token
     if args.modelhub_token:
         os.environ["MODELHUB_XC_TOKEN"] = args.modelhub_token
         os.environ["XC_TOKEN"] = args.modelhub_token
+    if args.modelhub_tokens:
+        os.environ["MODELHUB_XC_TOKENS"] = ",".join(args.modelhub_tokens)
     if jwt_token:
         os.environ["MODELHUB_JWT_TOKEN"] = jwt_token
     if not args.modelhub_token and not jwt_token: