fix: coordinate concurrent account capacity filling

modelhub_submmit_api/main.py

@@ -7,7 +7,7 @@ from datetime import timedelta
 from pathlib import Path
 from typing import Any
 
-from common import parse_datetime, utc_now, write_json, write_jsonl
+from common import parse_datetime, runtime_instance_id, utc_now, write_json, write_jsonl
 from hf_discovery import HuggingFaceDiscovery
 from history_stats import (
     append_ledger_entry,
@@ -19,6 +19,7 @@ from history_stats import (
 from modelhub_client import ModelHubAPIError, ModelHubClient, ModelHubClientPool
 from models import CandidateModel, HFModelSummary, ModelInspection
 from outcome_tracker import DEFAULT_OUTCOMES_PATH, OutcomeTracker
+from submission_claims import DEFAULT_CLAIMS_PATH, SubmissionClaimStore, diversify_candidates
 from task_registry import TASK_SPEC_BY_TYPE, all_task_types, choose_framework_for_task, choose_text_generation_framework, pipeline_tags_for_task_types, task_specs_for_model
 from template_selector import TemplateSelector
 
@@ -71,6 +72,11 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument("--runs-dir", default=str(DEFAULT_RUNS_DIR), help=argparse.SUPPRESS)
     parser.add_argument("--ledger-path", default=str(DEFAULT_LEDGER_PATH), help=argparse.SUPPRESS)
     parser.add_argument("--outcomes-path", default=str(DEFAULT_OUTCOMES_PATH), help=argparse.SUPPRESS)
+    parser.add_argument(
+        "--claims-path",
+        default=os.getenv("MODELHUB_AGENT_CLAIMS_PATH", str(DEFAULT_CLAIMS_PATH)),
+        help=argparse.SUPPRESS,
+    )
     parser.add_argument("--history-archive-path", default="history/platform_tasks.jsonl", help=argparse.SUPPRESS)
     parser.add_argument("--history-archive-limit", type=int, default=5000, help=argparse.SUPPRESS)
     parser.add_argument("--hf-base-url", default=os.getenv("MODELSCOPE_BASE_URL", "https://modelscope.cn"), help=argparse.SUPPRESS)
@@ -216,9 +222,7 @@ def resolve_max_submit_count(
     remaining_daily_quota: int,
 ) -> int:
     explicit_limit = int(getattr(args, "max_submits_per_run", 0) or 0)
-    planned_submit_count = planned_count
-    if remaining_daily_quota > 0:
-        planned_submit_count = min(planned_submit_count, remaining_daily_quota)
+    planned_submit_count = min(planned_count, max(0, remaining_daily_quota))
     if explicit_limit > 0:
         planned_submit_count = min(planned_submit_count, explicit_limit)
     return planned_submit_count
@@ -328,16 +332,16 @@ def submit_candidate(
 
 
 def make_run_dir(runs_dir: Path, now) -> Path:
+    runs_dir.mkdir(parents=True, exist_ok=True)
     base_name = now.strftime("%Y%m%dT%H%M%SZ")
-    run_dir = runs_dir / base_name
-    if not run_dir.exists():
-        return run_dir
-
-    for suffix in range(1, 1000):
-        candidate = runs_dir / f"{base_name}.{suffix:03d}"
-        if not candidate.exists():
+    for suffix in range(0, 1000):
+        candidate = runs_dir / (base_name if suffix == 0 else f"{base_name}.{suffix:03d}")
+        try:
+            candidate.mkdir(exist_ok=False)
             return candidate
-    return candidate
+        except FileExistsError:
+            continue
+    raise RuntimeError(f"Unable to allocate a unique run directory under {runs_dir}")
 
 
 def run_submission(
@@ -359,17 +363,17 @@ def run_submission(
     hf_discovery = hf_discovery or HuggingFaceDiscovery(base_url=args.hf_base_url)
     if modelhub_client is None:
         modelhub_tokens = list(getattr(args, "modelhub_tokens", None) or ([] if not args.modelhub_token else [args.modelhub_token]))
-        if len(modelhub_tokens) > 1:
-            clients = [ModelHubClient(token=token, base_url=args.modelhub_base_url) for token in modelhub_tokens]
-            modelhub_client = ModelHubClientPool(clients)
-        else:
-            modelhub_client = ModelHubClient(token=args.modelhub_token, base_url=args.modelhub_base_url)
+        token_values: list[str | None] = modelhub_tokens or [args.modelhub_token]
+        clients = [ModelHubClient(token=token, base_url=args.modelhub_base_url) for token in token_values]
+        modelhub_client = ModelHubClientPool(clients)
+
+    if hasattr(modelhub_client, "begin_cycle"):
+        modelhub_client.begin_cycle()
 
     runs_dir = Path(args.runs_dir)
     ledger_path = Path(args.ledger_path)
     history_archive_path = Path(args.history_archive_path)
     run_dir = make_run_dir(runs_dir, now)
-    run_dir.mkdir(parents=True, exist_ok=True)
     ledger_path.parent.mkdir(parents=True, exist_ok=True)
     history_archive_path.parent.mkdir(parents=True, exist_ok=True)
 
@@ -387,8 +391,26 @@ def run_submission(
     day_start = now.replace(hour=0, minute=0, second=0, microsecond=0)
     # Count today's submissions from the local ledger (avoids expensive paginated API call)
     daily_snapshot = count_submissions_for_day(tasks=[], ledger_entries=ledger_entries, day_start=day_start, day_end=now)
-    # Fallback: if ledger has no entries yet, do a quick API check (limit to 1 page)
-    if daily_snapshot["totalCount"] <= 0:
+    # A configured daily target must include every pooled account and every
+    # concurrent worker, not only the first token's first page.
+    if args.daily_target > 0:
+        list_kwargs: dict[str, Any] = {
+            "page_size": 100,
+            "only_mine": True,
+            "begin_time": day_start,
+            "end_time": now,
+        }
+        if isinstance(modelhub_client, ModelHubClientPool):
+            list_kwargs["_fanout_all"] = True
+        today_tasks = modelhub_client.list_tasks(**list_kwargs)
+        daily_snapshot = count_submissions_for_day(
+            tasks=today_tasks,
+            ledger_entries=ledger_entries,
+            day_start=day_start,
+            day_end=now,
+        )
+    # In unlimited mode, keep the inexpensive one-page fallback for an empty ledger.
+    elif daily_snapshot["totalCount"] <= 0:
         today_tasks_page = modelhub_client.list_tasks_page(
             current=1,
             page_size=20,
@@ -533,7 +555,17 @@ def run_submission(
         planned_count=len(candidates),
         remaining_daily_quota=remaining_daily_quota,
     )
-    planned_candidates = candidates[:planned_submit_count]
+    instance_id = runtime_instance_id()
+    diversified_candidates = diversify_candidates(candidates, instance_id=instance_id)
+    claim_store: SubmissionClaimStore | None = None
+    if args.dry_run:
+        planned_candidates = diversified_candidates[:planned_submit_count]
+    else:
+        claim_store = SubmissionClaimStore(
+            Path(getattr(args, "claims_path", DEFAULT_CLAIMS_PATH)),
+            owner_id=instance_id,
+        )
+        planned_candidates = claim_store.claim(diversified_candidates, limit=planned_submit_count)
     submit_workers = 1
     if not args.dry_run:
         submit_workers = resolve_submit_concurrency(
@@ -609,6 +641,12 @@ def run_submission(
                 submit_time=result["submitTime"],
             )
 
+        if claim_store is not None:
+            submitted_candidates = [result["candidate"] for result in ordered_results.values() if result.get("outcome") == "submitted"]
+            failed_candidates = [result["candidate"] for result in ordered_results.values() if result.get("outcome") != "submitted"]
+            claim_store.mark_submitted(submitted_candidates)
+            claim_store.release(failed_candidates)
+
     write_jsonl(run_dir / "submitted.jsonl", submitted)
     write_jsonl(run_dir / "skipped.jsonl", skipped)
     write_jsonl(run_dir / "failed.jsonl", failed)