CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Switch agent discovery to ModelScope

Browse Source
This commit is contained in:
CoolBoy
2026-07-10 01:44:16 +08:00
parent 5460fbd108
commit 3a2fa86e1e
14 changed files with 486 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
modelhub_submmit_api/main.py
View File

@@ -8,7 +8,6 @@ from pathlib import Path
from typing import Any
from common import parse_datetime, utc_now, write_json, write_jsonl
from hf_discovery import HuggingFaceDiscovery
from history_stats import (
append_ledger_entry,
build_empty_pre_submit_report,
@@ -16,8 +15,9 @@ from history_stats import (
load_ledger,
update_history_archive,
)
from modelhub_client import ModelHubAPIError, ModelHubClient
from modelhub_client import ModelHubAPIError, ModelHubClient, MultiModelHubClient
from models import CandidateModel, HFModelSummary, ModelInspection
from modelscope_discovery import ModelScopeDiscovery
from outcome_tracker import DEFAULT_OUTCOMES_PATH, OutcomeTracker
from task_registry import TASK_SPEC_BY_TYPE, all_task_types, choose_framework_for_task, choose_text_generation_framework, pipeline_tags_for_task_types, task_specs_for_model
from template_selector import TemplateSelector
@@ -28,11 +28,11 @@ DEFAULT_LEDGER_PATH = Path("ledger/submissions.jsonl")
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Auto discover and submit public Hugging Face models to ModelHub.")
parser = argparse.ArgumentParser(description="Auto discover and submit public ModelScope models to ModelHub.")
parser.add_argument("--gpu", help="Single GPU alias or platform name, for example: k100")
parser.add_argument("--gpus", help="Comma-separated GPU aliases/platform names. Omit to auto-use all safe GPUs from templates.")
parser.add_argument("--task-types", help="Comma-separated ModelHub task types. Omit to auto-enable all supported task types.")
parser.add_argument("--limit", type=int, default=300, help="Maximum number of Hugging Face models to scan per pipeline tag")
parser.add_argument("--limit", type=int, default=300, help="Maximum number of ModelScope models to scan per pipeline tag")
parser.add_argument("--max-scan-models", type=int, default=0, help="Hard cap on total scanned models (0 means auto)")
parser.add_argument(
"--scan-multiplier",
@@ -73,7 +73,8 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument("--outcomes-path", default=str(DEFAULT_OUTCOMES_PATH), help=argparse.SUPPRESS)
parser.add_argument("--history-archive-path", default="history/platform_tasks.jsonl", help=argparse.SUPPRESS)
parser.add_argument("--history-archive-limit", type=int, default=5000, help=argparse.SUPPRESS)
parser.add_argument("--hf-base-url", default=os.getenv("HF_BASE_URL", "https://huggingface.co"), help=argparse.SUPPRESS)
parser.add_argument("--hf-base-url", default=os.getenv("MODELSCOPE_BASE_URL", "https://modelscope.cn"), help=argparse.SUPPRESS)
parser.add_argument("--modelscope-base-url", default=os.getenv("MODELSCOPE_BASE_URL", "https://modelscope.cn"), help=argparse.SUPPRESS)
parser.add_argument("--modelhub-base-url", default=os.getenv("MODELHUB_BASE_URL", "https://modelhub.org.cn"), help=argparse.SUPPRESS)
parser.add_argument("--modelhub-token", default=os.getenv("MODELHUB_XC_TOKEN") or os.getenv("XC_TOKEN"), help=argparse.SUPPRESS)
return parser
@@ -227,7 +228,7 @@ def resolve_max_submit_count(
def process_model_for_candidates(
*,
model: HFModelSummary,
hf_discovery: HuggingFaceDiscovery,
hf_discovery: ModelScopeDiscovery,
modelhub_client: ModelHubClient,
template_selector: TemplateSelector,
target_gpus: list[str],
@@ -339,7 +340,7 @@ def run_submission(
args: argparse.Namespace,
*,
now=None,
hf_discovery: HuggingFaceDiscovery | None = None,
hf_discovery: ModelScopeDiscovery | None = None,
modelhub_client: ModelHubClient | None = None,
template_selector: TemplateSelector | None = None,
outcome_tracker: OutcomeTracker | None = None,
@@ -351,8 +352,14 @@ def run_submission(
if not target_gpus:
raise RuntimeError("No auto-submittable GPUs are available for the selected task types")
hf_discovery = hf_discovery or HuggingFaceDiscovery(base_url=args.hf_base_url)
modelhub_client = modelhub_client or ModelHubClient(token=args.modelhub_token, base_url=args.modelhub_base_url)
discovery_base_url = getattr(args, "modelscope_base_url", None) or args.hf_base_url
hf_discovery = hf_discovery or ModelScopeDiscovery(base_url=discovery_base_url)
if modelhub_client is None:
modelhub_tokens = list(getattr(args, "modelhub_tokens", []) or [])
if len(modelhub_tokens) > 1:
modelhub_client = MultiModelHubClient(tokens=modelhub_tokens, base_url=args.modelhub_base_url)
else:
modelhub_client = ModelHubClient(token=args.modelhub_token, base_url=args.modelhub_base_url)
runs_dir = Path(args.runs_dir)
ledger_path = Path(args.ledger_path)