From 3a2fa86e1e73ce511bd9ba60eef671156e355b6d Mon Sep 17 00:00:00 2001 From: CoolBoy Date: Fri, 10 Jul 2026 01:44:16 +0800 Subject: [PATCH] Switch agent discovery to ModelScope --- .dockerignore | 6 + .gitignore | 6 + README.md | 8 +- main.py | 4 +- modelhub_submmit_api/README.md | 12 +- modelhub_submmit_api/daily_runner.py | 28 ++- modelhub_submmit_api/defaults.py | 17 +- modelhub_submmit_api/hf_discovery.py | 4 +- modelhub_submmit_api/main.py | 25 +- modelhub_submmit_api/modelhub_client.py | 69 ++++++ modelhub_submmit_api/models.py | 3 + modelhub_submmit_api/modelscope_discovery.py | 243 +++++++++++++++++++ modelhub_submmit_api/poll_runner.py | 24 +- modelhub_submmit_api/runner_common.py | 90 ++++++- 14 files changed, 486 insertions(+), 53 deletions(-) create mode 100644 modelhub_submmit_api/modelscope_discovery.py diff --git a/.dockerignore b/.dockerignore index 861494b..39f942a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -15,6 +15,12 @@ modelhub_submmit_api/.ipynb_checkpoints/KEY-checkpoint.md modelhub_submmit_api/.ipynb_checkpoints/KEYS-checkpoint.md # Runtime artifacts are regenerated by the deployed strategy. +runs/ +daily_runs/ +poll_runs/ +ledger/ +outcomes/ +history/ modelhub_submmit_api/runs/ modelhub_submmit_api/daily_runs/ modelhub_submmit_api/poll_runs/ diff --git a/.gitignore b/.gitignore index b092a4a..aa4525d 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,12 @@ modelhub_submmit_api/.ipynb_checkpoints/KEY-checkpoint.md modelhub_submmit_api/.ipynb_checkpoints/KEYS-checkpoint.md # Runtime artifacts. +runs/ +daily_runs/ +poll_runs/ +ledger/ +outcomes/ +history/ modelhub_submmit_api/runs/ modelhub_submmit_api/daily_runs/ modelhub_submmit_api/poll_runs/ diff --git a/README.md b/README.md index 2c36b0b..63df6af 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,13 @@ submission poller in a child process. ## Runtime Environment -The image includes single-account ModelHub and Hugging Face token fallbacks for +The image includes single-account ModelHub and ModelScope token fallbacks for the agent platform. Environment variables can override them without rebuilding the image. - `MODELHUB_XC_TOKEN`, `XC_TOKEN`, or `MODELHUB_TOKEN` for ModelHub API authentication - `MODELHUB_JWT_TOKEN` or `JWT_TOKEN` can be used instead when the platform provides a JWT -- `HF_TOKEN` optional override for the embedded Hugging Face fallback token +- `MODELSCOPE_API_TOKEN` or `MODELSCOPE_TOKEN` optional override for the embedded ModelScope fallback token - `STRATEGY_ID` is expected to be injected by the ModelHub agent platform and is attached to submissions for strategy attribution; it is not an API authentication token Optional tuning: @@ -40,6 +40,6 @@ Optional tuning: Create a tag and submit the repository URL plus tag in "我的适配智能体". ```bash -git tag agent-v1 -git push origin agent-v1 +git tag agent-v4 +git push origin agent-v4 ``` diff --git a/main.py b/main.py index c6c7b4a..0df8148 100644 --- a/main.py +++ b/main.py @@ -9,7 +9,7 @@ import time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path -from modelhub_submmit_api.defaults import EMBEDDED_HF_TOKEN, EMBEDDED_MODELHUB_XC_TOKEN +from modelhub_submmit_api.defaults import EMBEDDED_MODELHUB_XC_TOKEN, EMBEDDED_MODELSCOPE_TOKEN HOST = "0.0.0.0" @@ -75,7 +75,7 @@ def _worker_command() -> list[str]: def _config() -> dict[str, object]: return { "strategy_id_present": bool(os.getenv("STRATEGY_ID")), - "hf_token_present": bool(os.getenv("HF_TOKEN") or EMBEDDED_HF_TOKEN), + "modelscope_token_present": bool(os.getenv("MODELSCOPE_API_TOKEN") or os.getenv("MODELSCOPE_TOKEN") or EMBEDDED_MODELSCOPE_TOKEN), "modelhub_auth_present": _has_modelhub_auth(), "config_error": config_error, "worker_running": worker is not None and worker.poll() is None, diff --git a/modelhub_submmit_api/README.md b/modelhub_submmit_api/README.md index e6333ec..28730a8 100644 --- a/modelhub_submmit_api/README.md +++ b/modelhub_submmit_api/README.md @@ -1,6 +1,6 @@ # ModelHub Submission Runner -This package automates Hugging Face model discovery and ModelHub submission. +This package automates ModelScope model discovery and ModelHub submission. It currently supports: - one-shot submission planning via `main.py` @@ -15,7 +15,7 @@ It currently supports: - `daily_runner.py`: daily wave orchestration - `poll_runner.py`: long-running queue refiller - `runner_common.py`: shared token / key file loading -- `hf_discovery.py`: Hugging Face model discovery and inspection +- `modelscope_discovery.py`: ModelScope model discovery and inspection - `modelhub_client.py`: ModelHub API client - `history_stats.py`: online history aggregation, ranking, and warnings - `template_selector.py`: template lookup and GPU normalization @@ -24,7 +24,7 @@ It currently supports: ## Key Files -- `KEY.md`: optional local Hugging Face and ModelHub tokens +- `KEY.md`: optional local ModelScope and ModelHub tokens - `templates/public_submit/adapt_task_templates.jsonl`: public submit templates The agent platform deployment should use environment variables instead of key @@ -81,9 +81,9 @@ bash run_poll.sh --dry-run Common flags: - `--daily-target`: total target submissions for the day; `0` means unlimited -- `--min-downloads`: Hugging Face download floor +- `--min-downloads`: ModelScope download floor - `--history-stats-threshold`: local ledger threshold before using online history stats -- `--max-scan-models`: hard cap on scanned HF models for a run (0 = auto) +- `--max-scan-models`: hard cap on scanned ModelScope models for a run (0 = auto) - `--scan-multiplier`: multiplier used for auto scan cap derivation from quota/queue capacity - `--read-concurrency`: concurrent HTTP reads while scanning model candidates (default 4) - `--max-submits-per-run`: max tasks to submit per run cycle (0 = unlimited) @@ -97,7 +97,7 @@ Common flags: - `--poll-interval-seconds`: sleep when the account has no available slots - `--idle-interval-seconds`: sleep when a cycle submits nothing -- `--max-scan-models`: hard cap on scanned HF models for this cycle (0 = auto) +- `--max-scan-models`: hard cap on scanned ModelScope models for this cycle (0 = auto) - `--scan-multiplier`: multiplier used for auto scan cap derivation from quota/queue capacity - `--max-submits-per-run`: max tasks to submit per poll cycle (0 = unlimited) - `--skip-outcome-sync`: skip outcome sync before scanning diff --git a/modelhub_submmit_api/daily_runner.py b/modelhub_submmit_api/daily_runner.py index c837707..8b75cd7 100644 --- a/modelhub_submmit_api/daily_runner.py +++ b/modelhub_submmit_api/daily_runner.py @@ -7,9 +7,9 @@ from pathlib import Path from typing import Any, Callable from common import utc_now, write_json -from hf_discovery import HuggingFaceDiscovery from main import DEFAULT_LEDGER_PATH, DEFAULT_RUNS_DIR, run_submission -from modelhub_client import ModelHubClient +from modelhub_client import ModelHubClient, MultiModelHubClient +from modelscope_discovery import ModelScopeDiscovery from outcome_tracker import OutcomeTracker from runner_common import DEFAULT_KEY_PATH, ensure_tokens from template_selector import TemplateSelector @@ -55,7 +55,7 @@ def build_parser() -> argparse.ArgumentParser: default=4, help="Multiplier used when auto-deriving scan limit from quota/queue capacity", ) - parser.add_argument("--min-downloads", type=int, default=50, help="Minimum Hugging Face download threshold") + parser.add_argument("--min-downloads", type=int, default=50, help="Minimum ModelScope download threshold") parser.add_argument( "--history-stats-threshold", type=int, @@ -78,16 +78,18 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--skip-outcome-sync", action="store_true", help="Skip outcome sync from ModelHub before scanning") parser.add_argument("--skip-history-archive", action="store_true", help="Skip historical task archive download for this run") parser.add_argument("--dry-run", action="store_true", help="Plan the day without creating tasks") - parser.add_argument("--key-path", default=str(DEFAULT_KEY_PATH), help="Path to KEY.md containing HF_TOKEN/XC_TOKEN") + parser.add_argument("--key-path", default=str(DEFAULT_KEY_PATH), help="Path to KEY.md containing MODELSCOPE_TOKEN/XC_TOKEN") parser.add_argument("--runs-dir", default=str(DEFAULT_RUNS_DIR), help=argparse.SUPPRESS) parser.add_argument("--ledger-path", default=str(DEFAULT_LEDGER_PATH), help=argparse.SUPPRESS) parser.add_argument("--history-archive-path", default="history/platform_tasks.jsonl", help=argparse.SUPPRESS) parser.add_argument("--history-archive-limit", type=int, default=5000, help=argparse.SUPPRESS) parser.add_argument("--daily-runs-dir", default=str(DEFAULT_DAILY_RUNS_DIR), help=argparse.SUPPRESS) - parser.add_argument("--hf-base-url", default=os.getenv("HF_BASE_URL", "https://huggingface.co"), help=argparse.SUPPRESS) + parser.add_argument("--hf-base-url", default=os.getenv("MODELSCOPE_BASE_URL", "https://modelscope.cn"), help=argparse.SUPPRESS) + parser.add_argument("--modelscope-base-url", default=os.getenv("MODELSCOPE_BASE_URL", "https://modelscope.cn"), help=argparse.SUPPRESS) parser.add_argument("--modelhub-base-url", default=os.getenv("MODELHUB_BASE_URL", "https://modelhub.org.cn"), help=argparse.SUPPRESS) parser.add_argument("--modelhub-token", default=os.getenv("MODELHUB_XC_TOKEN") or os.getenv("XC_TOKEN"), help=argparse.SUPPRESS) parser.add_argument("--hf-token", default=os.getenv("HF_TOKEN"), help=argparse.SUPPRESS) + parser.add_argument("--modelscope-token", default=os.getenv("MODELSCOPE_API_TOKEN") or os.getenv("MODELSCOPE_TOKEN"), help=argparse.SUPPRESS) return parser @@ -119,6 +121,7 @@ def make_wave_namespace(base_args: argparse.Namespace, wave: WaveSpec) -> argpar history_archive_path=base_args.history_archive_path, history_archive_limit=base_args.history_archive_limit, hf_base_url=base_args.hf_base_url, + modelscope_base_url=getattr(base_args, "modelscope_base_url", base_args.hf_base_url), modelhub_base_url=base_args.modelhub_base_url, modelhub_token=base_args.modelhub_token, ) @@ -130,15 +133,20 @@ def run_daily_batches( waves: tuple[WaveSpec, ...] = DEFAULT_WAVES, now=None, run_fn: Callable[..., dict[str, Any]] = run_submission, - hf_discovery: HuggingFaceDiscovery | None = None, + hf_discovery: ModelScopeDiscovery | None = None, modelhub_client: ModelHubClient | None = None, template_selector: TemplateSelector | None = None, outcome_tracker: OutcomeTracker | None = None, ) -> dict[str, Any]: now = now or utc_now() - hf_discovery = hf_discovery or HuggingFaceDiscovery(base_url=base_args.hf_base_url) + discovery_base_url = getattr(base_args, "modelscope_base_url", None) or base_args.hf_base_url + hf_discovery = hf_discovery or ModelScopeDiscovery(base_url=discovery_base_url) if modelhub_client is None: - modelhub_client = ModelHubClient(token=base_args.modelhub_token, base_url=base_args.modelhub_base_url) + modelhub_tokens = list(getattr(base_args, "modelhub_tokens", []) or []) + if len(modelhub_tokens) > 1: + modelhub_client = MultiModelHubClient(tokens=modelhub_tokens, base_url=base_args.modelhub_base_url) + else: + modelhub_client = ModelHubClient(token=base_args.modelhub_token, base_url=base_args.modelhub_base_url) template_selector = template_selector or TemplateSelector() daily_run_dir = Path(base_args.daily_runs_dir) / now.strftime("%Y%m%dT%H%M%SZ") @@ -284,8 +292,8 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) ensure_tokens(args) log( - f"[daily] hf_token={'set' if bool(args.hf_token) else 'missing'} " - f"xc_token={'set' if bool(args.modelhub_token) else 'missing'}" + f"[daily] modelscope_token={'set' if bool(args.modelscope_token) else 'missing'} " + f"xc_tokens={len(getattr(args, 'modelhub_tokens', []) or [])}" ) summary = run_daily_batches(base_args=args) print(f"daily_run_dir={summary['dailyRunDir']}") diff --git a/modelhub_submmit_api/defaults.py b/modelhub_submmit_api/defaults.py index a0081d5..5ed817c 100644 --- a/modelhub_submmit_api/defaults.py +++ b/modelhub_submmit_api/defaults.py @@ -1,2 +1,17 @@ -EMBEDDED_MODELHUB_XC_TOKEN = "a14776f6e7ad4c04a1710260613c294c" +EMBEDDED_MODELHUB_XC_TOKENS = [ + "a14776f6e7ad4c04a1710260613c294c", + "8408eb2f73034fa09e1919406774d111", + "b0b3add66b4641d5afdc7487589905f9", + "edff358739594aa4b9f26beed6ab8799", + "d260bdd83c4b4c6faa063a16685b8908", + "fc72d989bfa84d529fff6ea33e073945", + "30d137aa08944ded8b86aac942904cc0", + "fd60e08b313048efa413f627badfe0b3", + "9bd1431410ba4558bc730481dff74d49", + "a96aea9071d84576a963c706dc71dfac", + "be09a04434fb4e379344b0554aaf6e6a", + "6032b66717e041d3adb0217e86103c6d", +] +EMBEDDED_MODELHUB_XC_TOKEN = EMBEDDED_MODELHUB_XC_TOKENS[0] EMBEDDED_HF_TOKEN = "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" +EMBEDDED_MODELSCOPE_TOKEN = "ms-b4918c83-7eb3-4034-8635-f154938ed3f0" diff --git a/modelhub_submmit_api/hf_discovery.py b/modelhub_submmit_api/hf_discovery.py index 3f1432a..e06a628 100644 --- a/modelhub_submmit_api/hf_discovery.py +++ b/modelhub_submmit_api/hf_discovery.py @@ -192,10 +192,10 @@ def inspect_repo_tree(repo_id: str, entries: list[dict[str, Any]]) -> ModelInspe onnx_files: list[str] = [] for entry in entries: - path = entry.get("path") or entry.get("rfilename") or entry.get("name") + path = entry.get("path") or entry.get("Path") or entry.get("rfilename") or entry.get("name") or entry.get("Name") if not path: continue - entry_type = (entry.get("type") or "").lower() + entry_type = (entry.get("type") or entry.get("Type") or "").lower() if entry_type in {"directory", "dir", "folder"}: continue file_paths.append(path) diff --git a/modelhub_submmit_api/main.py b/modelhub_submmit_api/main.py index e1a1fef..3f7463a 100644 --- a/modelhub_submmit_api/main.py +++ b/modelhub_submmit_api/main.py @@ -8,7 +8,6 @@ from pathlib import Path from typing import Any from common import parse_datetime, utc_now, write_json, write_jsonl -from hf_discovery import HuggingFaceDiscovery from history_stats import ( append_ledger_entry, build_empty_pre_submit_report, @@ -16,8 +15,9 @@ from history_stats import ( load_ledger, update_history_archive, ) -from modelhub_client import ModelHubAPIError, ModelHubClient +from modelhub_client import ModelHubAPIError, ModelHubClient, MultiModelHubClient from models import CandidateModel, HFModelSummary, ModelInspection +from modelscope_discovery import ModelScopeDiscovery from outcome_tracker import DEFAULT_OUTCOMES_PATH, OutcomeTracker from task_registry import TASK_SPEC_BY_TYPE, all_task_types, choose_framework_for_task, choose_text_generation_framework, pipeline_tags_for_task_types, task_specs_for_model from template_selector import TemplateSelector @@ -28,11 +28,11 @@ DEFAULT_LEDGER_PATH = Path("ledger/submissions.jsonl") def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Auto discover and submit public Hugging Face models to ModelHub.") + parser = argparse.ArgumentParser(description="Auto discover and submit public ModelScope models to ModelHub.") parser.add_argument("--gpu", help="Single GPU alias or platform name, for example: k100") parser.add_argument("--gpus", help="Comma-separated GPU aliases/platform names. Omit to auto-use all safe GPUs from templates.") parser.add_argument("--task-types", help="Comma-separated ModelHub task types. Omit to auto-enable all supported task types.") - parser.add_argument("--limit", type=int, default=300, help="Maximum number of Hugging Face models to scan per pipeline tag") + parser.add_argument("--limit", type=int, default=300, help="Maximum number of ModelScope models to scan per pipeline tag") parser.add_argument("--max-scan-models", type=int, default=0, help="Hard cap on total scanned models (0 means auto)") parser.add_argument( "--scan-multiplier", @@ -73,7 +73,8 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--outcomes-path", default=str(DEFAULT_OUTCOMES_PATH), help=argparse.SUPPRESS) parser.add_argument("--history-archive-path", default="history/platform_tasks.jsonl", help=argparse.SUPPRESS) parser.add_argument("--history-archive-limit", type=int, default=5000, help=argparse.SUPPRESS) - parser.add_argument("--hf-base-url", default=os.getenv("HF_BASE_URL", "https://huggingface.co"), help=argparse.SUPPRESS) + parser.add_argument("--hf-base-url", default=os.getenv("MODELSCOPE_BASE_URL", "https://modelscope.cn"), help=argparse.SUPPRESS) + parser.add_argument("--modelscope-base-url", default=os.getenv("MODELSCOPE_BASE_URL", "https://modelscope.cn"), help=argparse.SUPPRESS) parser.add_argument("--modelhub-base-url", default=os.getenv("MODELHUB_BASE_URL", "https://modelhub.org.cn"), help=argparse.SUPPRESS) parser.add_argument("--modelhub-token", default=os.getenv("MODELHUB_XC_TOKEN") or os.getenv("XC_TOKEN"), help=argparse.SUPPRESS) return parser @@ -227,7 +228,7 @@ def resolve_max_submit_count( def process_model_for_candidates( *, model: HFModelSummary, - hf_discovery: HuggingFaceDiscovery, + hf_discovery: ModelScopeDiscovery, modelhub_client: ModelHubClient, template_selector: TemplateSelector, target_gpus: list[str], @@ -339,7 +340,7 @@ def run_submission( args: argparse.Namespace, *, now=None, - hf_discovery: HuggingFaceDiscovery | None = None, + hf_discovery: ModelScopeDiscovery | None = None, modelhub_client: ModelHubClient | None = None, template_selector: TemplateSelector | None = None, outcome_tracker: OutcomeTracker | None = None, @@ -351,8 +352,14 @@ def run_submission( if not target_gpus: raise RuntimeError("No auto-submittable GPUs are available for the selected task types") - hf_discovery = hf_discovery or HuggingFaceDiscovery(base_url=args.hf_base_url) - modelhub_client = modelhub_client or ModelHubClient(token=args.modelhub_token, base_url=args.modelhub_base_url) + discovery_base_url = getattr(args, "modelscope_base_url", None) or args.hf_base_url + hf_discovery = hf_discovery or ModelScopeDiscovery(base_url=discovery_base_url) + if modelhub_client is None: + modelhub_tokens = list(getattr(args, "modelhub_tokens", []) or []) + if len(modelhub_tokens) > 1: + modelhub_client = MultiModelHubClient(tokens=modelhub_tokens, base_url=args.modelhub_base_url) + else: + modelhub_client = ModelHubClient(token=args.modelhub_token, base_url=args.modelhub_base_url) runs_dir = Path(args.runs_dir) ledger_path = Path(args.ledger_path) diff --git a/modelhub_submmit_api/modelhub_client.py b/modelhub_submmit_api/modelhub_client.py index 7001a16..dd3aa39 100644 --- a/modelhub_submmit_api/modelhub_client.py +++ b/modelhub_submmit_api/modelhub_client.py @@ -2,6 +2,8 @@ from __future__ import annotations import os from datetime import datetime, timedelta +from itertools import cycle +from threading import Lock from typing import Any from common import format_modelhub_datetime, parse_datetime @@ -205,6 +207,73 @@ class ModelHubClient: return payload +class MultiModelHubClient: + def __init__( + self, + *, + tokens: list[str], + base_url: str = "https://modelhub.org.cn", + timeout: int = 30, + retries: int = 2, + ) -> None: + deduped: list[str] = [] + for token in tokens: + clean = token.strip() + if clean and clean not in deduped: + deduped.append(clean) + if not deduped: + raise ValueError("At least one ModelHub token is required") + + self.clients = [ + ModelHubClient(token=token, base_url=base_url, timeout=timeout, retries=retries) + for token in deduped + ] + self._cycle = cycle(self.clients) + self._lock = Lock() + + def _next_client(self) -> ModelHubClient: + with self._lock: + return next(self._cycle) + + def add_task(self, payload: dict[str, Any]) -> dict[str, Any]: + return self._next_client().add_task(payload) + + def search_by_model_id(self, model_id: str) -> dict[str, Any]: + return self.clients[0].search_by_model_id(model_id) + + def is_model_processed_for_gpu(self, model_id: str, target_gpu: str) -> bool: + return self.clients[0].is_model_processed_for_gpu(model_id, target_gpu) + + def get_verify_result_map(self, model_id: str) -> dict[str, Any]: + return self.clients[0].get_verify_result_map(model_id) + + def processed_gpus_for_model(self, model_id: str) -> set[str]: + return self.clients[0].processed_gpus_for_model(model_id) + + def list_tasks_page(self, **kwargs: Any) -> dict[str, Any]: + return self.clients[0].list_tasks_page(**kwargs) + + def list_tasks(self, **kwargs: Any) -> list[dict[str, Any]]: + return self.clients[0].list_tasks(**kwargs) + + def find_recent_task_id(self, model_id: str, gpu_type: str, submitted_after: datetime) -> str | None: + return self.clients[0].find_recent_task_id(model_id, gpu_type, submitted_after) + + def count_active_tasks(self, **kwargs: Any) -> int: + return self.clients[0].count_active_tasks(**kwargs) + + def active_task_counts(self) -> list[int]: + counts: list[int] = [] + now = datetime.utcnow() + begin_time = now - timedelta(days=1) + for client in self.clients: + counts.append(client.count_active_tasks(begin_time=begin_time, end_time=now, max_count=100)) + return counts + + def available_submit_slots(self, max_active_per_account: int = 5) -> int: + return sum(max(0, max_active_per_account - count) for count in self.active_task_counts()) + + ACTIVE_TASK_STATUSES = { "waiting", "running", diff --git a/modelhub_submmit_api/models.py b/modelhub_submmit_api/models.py index 053592d..8e73592 100644 --- a/modelhub_submmit_api/models.py +++ b/modelhub_submmit_api/models.py @@ -23,9 +23,12 @@ class HFModelSummary: last_modified: datetime | None pipeline_tag: str | None created_at: datetime | None = None + source: str = "huggingface" @property def model_address(self) -> str: + if self.source == "modelscope": + return f"https://modelscope.cn/models/{self.repo_id}" return f"https://huggingface.co/{self.repo_id}" diff --git a/modelhub_submmit_api/modelscope_discovery.py b/modelhub_submmit_api/modelscope_discovery.py new file mode 100644 index 0000000..481998b --- /dev/null +++ b/modelhub_submmit_api/modelscope_discovery.py @@ -0,0 +1,243 @@ +from __future__ import annotations + +import os +import threading +from typing import Any +from urllib.parse import quote + +from common import parse_datetime +from defaults import EMBEDDED_MODELSCOPE_TOKEN +from hf_discovery import inspect_repo_tree +from http_json import HttpJsonError, JsonHttpClient +from models import HFModelSummary, ModelInspection + + +MODELSCOPE_TASK_TAGS = { + "text-generation": "text-generation", + "image-text-to-text": "image-text-to-text", + "visual-question-answering": "visual-question-answering", + "document-question-answering": "document-question-answering", + "video-text-to-text": "video-text-to-text", + "text-to-image": "text-to-image-synthesis", + "image-to-image": "image-to-image", + "automatic-speech-recognition": "auto-speech-recognition", + "question-answering": "question-answering", + "feature-extraction": "feature-extraction", + "sentence-similarity": "sentence-similarity", + "image-classification": "image-classification", + "zero-shot-image-classification": "zero-shot-image-classification", + "text-classification": "text-classification", + "zero-shot-classification": "zero-shot-classification", + "reinforcement-learning": "reinforcement-learning", +} + + +class ModelScopeDiscovery: + def __init__( + self, + base_url: str = "https://modelscope.cn", + http_client: JsonHttpClient | None = None, + legacy_http_client: JsonHttpClient | None = None, + timeout: int = 30, + retries: int = 2, + ) -> None: + token = os.getenv("MODELSCOPE_API_TOKEN") or os.getenv("MODELSCOPE_TOKEN") or EMBEDDED_MODELSCOPE_TOKEN + headers = {"User-Agent": "modelhub-submmit-cli/0.1"} + if token: + headers["Authorization"] = f"Bearer {token}" + headers["Cookie"] = f"m_session_id={token}" + self.http_client = http_client or JsonHttpClient( + base_url=f"{base_url.rstrip('/')}/openapi/v1", + default_headers=headers, + timeout=timeout, + retries=retries, + ) + self.legacy_http_client = legacy_http_client or JsonHttpClient( + base_url=base_url, + default_headers=headers, + timeout=timeout, + retries=retries, + ) + self._repo_tree_cache: dict[str, list[dict[str, Any]]] = {} + self._repo_tree_lock = threading.Lock() + + def list_recent_models( + self, + *, + pipeline_tags: list[str], + limit: int, + min_downloads: int, + updated_after=None, + read_concurrency: int = 1, + ) -> list[HFModelSummary]: + del read_concurrency + if not pipeline_tags: + return [] + + deduped: dict[str, HFModelSummary] = {} + for pipeline_tag in pipeline_tags: + for model in self._query_recent_models( + pipeline_tag=pipeline_tag, + limit=limit, + min_downloads=min_downloads, + updated_after=updated_after, + ): + current = deduped.get(model.repo_id) + if current is None or (model.last_modified or parse_datetime("1970-01-01")) > ( + current.last_modified or parse_datetime("1970-01-01") + ): + deduped[model.repo_id] = model + + models = list(deduped.values()) + models.sort(key=lambda item: item.last_modified or parse_datetime("1970-01-01"), reverse=True) + return models + + def _query_recent_models( + self, + *, + pipeline_tag: str, + limit: int, + min_downloads: int, + updated_after=None, + ) -> list[HFModelSummary]: + page_size = min(max(1, limit), 100) + max_items = min(max(1, limit), 3000) + task_tag = MODELSCOPE_TASK_TAGS.get(pipeline_tag, pipeline_tag) + models: list[HFModelSummary] = [] + + for page_number in range(1, (max_items + page_size - 1) // page_size + 1): + try: + payload = self.http_client.request_json( + "GET", + "/models", + query={ + "page_number": page_number, + "page_size": page_size, + "sort": "last_modified", + "filter.task": task_tag, + }, + ) + except HttpJsonError as exc: + print(f"[modelscope] list_models_error task={task_tag} page={page_number} error={exc}", flush=True) + break + + items = self._extract_models(payload) + if not items: + break + for item in items: + model = self._parse_model( + item, + fallback_pipeline_tag=pipeline_tag, + min_downloads=min_downloads, + updated_after=updated_after, + ) + if model is not None: + models.append(model) + if len(models) >= max_items: + return models + if len(items) < page_size: + break + + return models + + def list_recent_text_generation_models( + self, + *, + limit: int, + min_downloads: int, + updated_after=None, + ) -> list[HFModelSummary]: + return self.list_recent_models( + pipeline_tags=["text-generation"], + limit=limit, + min_downloads=min_downloads, + updated_after=updated_after, + ) + + def inspect_model(self, model: HFModelSummary) -> ModelInspection: + entries = self.list_repo_tree(model.repo_id) + return inspect_repo_tree(model.repo_id, entries) + + def list_repo_tree(self, repo_id: str) -> list[dict[str, Any]]: + with self._repo_tree_lock: + cached = self._repo_tree_cache.get(repo_id) + if cached is not None: + return list(cached) + + encoded_repo_id = "/".join(quote(part, safe="") for part in repo_id.split("/")) + try: + payload = self.legacy_http_client.request_json( + "GET", + f"/api/v1/models/{encoded_repo_id}/repo/files", + query={"Revision": "master", "Recursive": "true"}, + ) + except HttpJsonError as exc: + print(f"[modelscope] repo_tree_error repo={repo_id} error={exc}", flush=True) + payload = None + + entries = self._extract_files(payload) + with self._repo_tree_lock: + self._repo_tree_cache[repo_id] = list(entries) + return list(entries) + + @staticmethod + def _extract_models(payload: Any) -> list[dict[str, Any]]: + data = payload.get("data") if isinstance(payload, dict) else payload + if isinstance(data, dict): + for key in ("models", "Models", "items", "list", "data", "results"): + value = data.get(key) + if isinstance(value, list): + return [item for item in value if isinstance(item, dict)] + if isinstance(data, list): + return [item for item in data if isinstance(item, dict)] + return [] + + @staticmethod + def _extract_files(payload: Any) -> list[dict[str, Any]]: + data = payload.get("Data") if isinstance(payload, dict) else payload + if isinstance(data, dict): + for key in ("Files", "files", "items", "tree"): + value = data.get(key) + if isinstance(value, list): + return [item for item in value if isinstance(item, dict)] + if isinstance(data, list): + return [item for item in data if isinstance(item, dict)] + return [] + + @staticmethod + def _parse_model( + item: dict[str, Any], + *, + fallback_pipeline_tag: str, + min_downloads: int, + updated_after=None, + ) -> HFModelSummary | None: + repo_id = item.get("id") or item.get("model_id") or item.get("modelId") + if not repo_id: + owner = item.get("owner") or item.get("Owner") or item.get("Path") + name = item.get("name") or item.get("Name") + repo_id = f"{owner}/{name}" if owner and name else None + if not repo_id: + return None + + downloads = int(item.get("downloads") or item.get("Downloads") or 0) + if downloads < min_downloads: + return None + + tasks = item.get("tasks") or item.get("Tasks") or [] + if isinstance(tasks, list) and tasks: + pipeline_tag = str(tasks[0]) + else: + pipeline_tag = fallback_pipeline_tag + last_modified = parse_datetime(item.get("last_modified") or item.get("UpdatedAt") or item.get("LastUpdatedTime")) + if updated_after and last_modified and last_modified < updated_after: + return None + + return HFModelSummary( + repo_id=repo_id, + downloads=downloads, + last_modified=last_modified, + pipeline_tag=pipeline_tag, + created_at=parse_datetime(item.get("created_at") or item.get("CreatedAt")), + source="modelscope", + ) diff --git a/modelhub_submmit_api/poll_runner.py b/modelhub_submmit_api/poll_runner.py index b526b50..1e46b88 100644 --- a/modelhub_submmit_api/poll_runner.py +++ b/modelhub_submmit_api/poll_runner.py @@ -9,9 +9,9 @@ from typing import Any, Callable from common import utc_now, write_json from daily_runner import DEFAULT_DAILY_RUNS_DIR, log, run_daily_batches -from hf_discovery import HuggingFaceDiscovery from main import DEFAULT_LEDGER_PATH, DEFAULT_RUNS_DIR -from modelhub_client import ModelHubClient +from modelhub_client import ModelHubClient, MultiModelHubClient +from modelscope_discovery import ModelScopeDiscovery from outcome_tracker import DEFAULT_OUTCOMES_PATH, OutcomeTracker from runner_common import DEFAULT_KEY_PATH, ensure_tokens from template_selector import TemplateSelector @@ -25,7 +25,7 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--daily-target", type=int, default=0, help="Total submissions to aim for per UTC day; 0 means unlimited") parser.add_argument("--gpu", help="Single GPU alias or platform name, for example: k100") parser.add_argument("--gpus", help="Comma-separated GPU aliases/platform names. Omit to auto-use all safe GPUs.") - parser.add_argument("--min-downloads", type=int, default=50, help="Minimum Hugging Face download threshold") + parser.add_argument("--min-downloads", type=int, default=50, help="Minimum ModelScope download threshold") parser.add_argument( "--history-stats-threshold", type=int, @@ -55,7 +55,7 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--skip-outcome-sync", action="store_true", help="Skip outcome sync from ModelHub before scanning") parser.add_argument("--skip-history-archive", action="store_true", help="Skip historical task archive download for this run") parser.add_argument("--dry-run", action="store_true", help="Plan the day without creating tasks") - parser.add_argument("--key-path", default=str(DEFAULT_KEY_PATH), help="Path to KEY.md containing HF_TOKEN/XC_TOKEN") + parser.add_argument("--key-path", default=str(DEFAULT_KEY_PATH), help="Path to KEY.md containing MODELSCOPE_TOKEN/XC_TOKEN") parser.add_argument("--runs-dir", default=str(DEFAULT_RUNS_DIR), help=argparse.SUPPRESS) parser.add_argument("--ledger-path", default=str(DEFAULT_LEDGER_PATH), help=argparse.SUPPRESS) parser.add_argument("--history-archive-path", default="history/platform_tasks.jsonl", help=argparse.SUPPRESS) @@ -63,10 +63,12 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--daily-runs-dir", default=str(DEFAULT_DAILY_RUNS_DIR), help=argparse.SUPPRESS) parser.add_argument("--poll-runs-dir", default=str(DEFAULT_POLL_RUNS_DIR), help=argparse.SUPPRESS) parser.add_argument("--outcomes-path", default=str(DEFAULT_OUTCOMES_PATH), help=argparse.SUPPRESS) - parser.add_argument("--hf-base-url", default="https://huggingface.co", help=argparse.SUPPRESS) + parser.add_argument("--hf-base-url", default="https://modelscope.cn", help=argparse.SUPPRESS) + parser.add_argument("--modelscope-base-url", default="https://modelscope.cn", help=argparse.SUPPRESS) parser.add_argument("--modelhub-base-url", default="https://modelhub.org.cn", help=argparse.SUPPRESS) parser.add_argument("--modelhub-token", default=None, help=argparse.SUPPRESS) parser.add_argument("--hf-token", default=None, help=argparse.SUPPRESS) + parser.add_argument("--modelscope-token", default=None, help=argparse.SUPPRESS) parser.add_argument("--poll-interval-seconds", type=int, default=60, help="Sleep between polling cycles when no slots are available") parser.add_argument("--idle-interval-seconds", type=int, default=30, help="Sleep between cycles when a scan submits nothing") parser.add_argument("--post-cycle-cooldown-seconds", type=int, default=0, help="Short sleep after a successful cycle") @@ -85,6 +87,9 @@ def _make_cycle_args(base_args: argparse.Namespace) -> argparse.Namespace: def _build_modelhub_client(base_args: argparse.Namespace) -> ModelHubClient: + modelhub_tokens = list(getattr(base_args, "modelhub_tokens", []) or []) + if len(modelhub_tokens) > 1: + return MultiModelHubClient(tokens=modelhub_tokens, base_url=base_args.modelhub_base_url) return ModelHubClient(token=base_args.modelhub_token, base_url=base_args.modelhub_base_url) @@ -93,13 +98,14 @@ def run_poll_loop( base_args: argparse.Namespace, now=None, run_fn: Callable[..., dict[str, Any]] = run_daily_batches, - hf_discovery: HuggingFaceDiscovery | None = None, + hf_discovery: ModelScopeDiscovery | None = None, modelhub_client: ModelHubClient | None = None, template_selector: TemplateSelector | None = None, outcome_tracker: OutcomeTracker | None = None, ) -> dict[str, Any]: now = now or utc_now() - hf_discovery = hf_discovery or HuggingFaceDiscovery(base_url=base_args.hf_base_url) + discovery_base_url = getattr(base_args, "modelscope_base_url", None) or base_args.hf_base_url + hf_discovery = hf_discovery or ModelScopeDiscovery(base_url=discovery_base_url) modelhub_client = modelhub_client or _build_modelhub_client(base_args) template_selector = template_selector or TemplateSelector() @@ -244,8 +250,8 @@ def main(argv: list[str] | None = None) -> int: return 0 log( - f"[poll] hf_token={'set' if bool(args.hf_token) else 'missing'} " - f"xc_token={'set' if bool(args.modelhub_token) else 'missing'}" + f"[poll] modelscope_token={'set' if bool(args.modelscope_token) else 'missing'} " + f"xc_tokens={len(getattr(args, 'modelhub_tokens', []) or [])}" ) summary = run_poll_loop(base_args=args) print(f"poll_run_dir={summary['pollRunDir']}") diff --git a/modelhub_submmit_api/runner_common.py b/modelhub_submmit_api/runner_common.py index 1944833..ee3ab19 100644 --- a/modelhub_submmit_api/runner_common.py +++ b/modelhub_submmit_api/runner_common.py @@ -4,12 +4,15 @@ import argparse import os from pathlib import Path -from defaults import EMBEDDED_HF_TOKEN, EMBEDDED_MODELHUB_XC_TOKEN +from defaults import EMBEDDED_HF_TOKEN, EMBEDDED_MODELHUB_XC_TOKENS, EMBEDDED_MODELSCOPE_TOKEN DEFAULT_KEY_PATH = Path("KEY.md") +MODULE_DIR = Path(__file__).resolve().parent XC_TOKEN_ENV_NAMES = ("MODELHUB_XC_TOKEN", "XC_TOKEN", "MODELHUB_TOKEN") +XC_TOKEN_LIST_ENV_NAMES = ("MODELHUB_XC_TOKENS", "XC_TOKENS", "MODELHUB_TOKENS") JWT_TOKEN_ENV_NAMES = ("MODELHUB_JWT_TOKEN", "JWT_TOKEN") +MODELSCOPE_TOKEN_ENV_NAMES = ("MODELSCOPE_API_TOKEN", "MODELSCOPE_TOKEN") def load_key_file(path: Path) -> dict[str, str]: @@ -33,33 +36,100 @@ def first_value(values: dict[str, str], names: tuple[str, ...]) -> str | None: return None +def split_token_list(value: str | None) -> list[str]: + if not value: + return [] + tokens: list[str] = [] + for normalized in value.replace(",", "\n").replace(";", "\n").splitlines(): + token = normalized.strip() + if token: + tokens.append(token) + return tokens + + +def discover_key_paths(primary_path: Path) -> list[Path]: + candidates: list[Path] = [] + search_dirs = [primary_path.parent, primary_path.parent.parent] + for path in [primary_path, primary_path.parent / "KEYS.md"]: + if path not in candidates: + candidates.append(path) + for directory in search_dirs: + if not directory.exists(): + continue + for path in sorted(directory.glob("KEYS*")): + if path.is_file() and path not in candidates: + candidates.append(path) + return candidates + + +def collect_modelhub_tokens(values_by_path: list[dict[str, str]], explicit_token: str | None) -> list[str]: + tokens: list[str] = [] + + def add(value: str | None) -> None: + for token in split_token_list(value): + if token and token not in tokens: + tokens.append(token) + + add(explicit_token) + for env_name in XC_TOKEN_LIST_ENV_NAMES: + add(os.getenv(env_name)) + for env_name in XC_TOKEN_ENV_NAMES: + add(os.getenv(env_name)) + + for values in values_by_path: + for key, value in values.items(): + normalized = key.strip().upper() + if normalized in XC_TOKEN_ENV_NAMES or normalized.startswith("XC_TOKEN") or normalized.startswith("MODELHUB_XC_TOKEN"): + add(value) + + for token in EMBEDDED_MODELHUB_XC_TOKENS: + add(token) + return tokens + + def ensure_tokens(args: argparse.Namespace) -> None: primary_key_path = Path(getattr(args, "key_path", DEFAULT_KEY_PATH)) if not primary_key_path.exists(): - parent_key = primary_key_path.parent.parent / primary_key_path.name - if parent_key.exists(): - primary_key_path = parent_key - values = load_key_file(primary_key_path) + fallback_paths = [ + MODULE_DIR / primary_key_path.name, + primary_key_path.parent.parent / primary_key_path.name, + MODULE_DIR.parent / primary_key_path.name, + ] + for fallback_path in fallback_paths: + if fallback_path.exists(): + primary_key_path = fallback_path + break + values_by_path = [load_key_file(path) for path in discover_key_paths(primary_key_path) if path.exists()] + values: dict[str, str] = {} + for loaded in values_by_path: + values.update(loaded) if not getattr(args, "hf_token", None): args.hf_token = os.getenv("HF_TOKEN") or values.get("HF_TOKEN") if not getattr(args, "hf_token", None): args.hf_token = EMBEDDED_HF_TOKEN + if not getattr(args, "modelscope_token", None): + args.modelscope_token = first_value(values, MODELSCOPE_TOKEN_ENV_NAMES) + if not getattr(args, "modelscope_token", None): + args.modelscope_token = EMBEDDED_MODELSCOPE_TOKEN - modelhub_token = getattr(args, "modelhub_token", None) - if not modelhub_token: - modelhub_token = first_value(values, XC_TOKEN_ENV_NAMES) - if not modelhub_token: - modelhub_token = EMBEDDED_MODELHUB_XC_TOKEN + modelhub_tokens = collect_modelhub_tokens(values_by_path, getattr(args, "modelhub_token", None)) + modelhub_token = modelhub_tokens[0] if modelhub_tokens else None jwt_token = first_value(values, JWT_TOKEN_ENV_NAMES) args.modelhub_token = modelhub_token + args.modelhub_tokens = modelhub_tokens if args.hf_token: os.environ["HF_TOKEN"] = args.hf_token + if args.modelscope_token: + os.environ["MODELSCOPE_API_TOKEN"] = args.modelscope_token + os.environ["MODELSCOPE_TOKEN"] = args.modelscope_token if args.modelhub_token: os.environ["MODELHUB_XC_TOKEN"] = args.modelhub_token os.environ["XC_TOKEN"] = args.modelhub_token + if args.modelhub_tokens: + os.environ["MODELHUB_XC_TOKENS"] = ",".join(args.modelhub_tokens) if jwt_token: os.environ["MODELHUB_JWT_TOKEN"] = jwt_token if not args.modelhub_token and not jwt_token: