CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

fix: coordinate concurrent account capacity filling

Browse Source
This commit is contained in:
CoolBoy
2026-08-01 16:50:50 +08:00
parent c06169d906
commit 0cabc43abf
14 changed files with 794 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
modelhub_submmit_api/daily_runner.py
View File

@@ -8,10 +8,11 @@ from typing import Any, Callable
from common import utc_now, write_json
from hf_discovery import HuggingFaceDiscovery
from main import DEFAULT_LEDGER_PATH, DEFAULT_RUNS_DIR, run_submission
from main import DEFAULT_LEDGER_PATH, DEFAULT_RUNS_DIR, make_run_dir, run_submission
from modelhub_client import ModelHubClient, ModelHubClientPool
from outcome_tracker import OutcomeTracker
from runner_common import DEFAULT_KEY_PATH, ensure_tokens
from submission_claims import DEFAULT_CLAIMS_PATH
from template_selector import TemplateSelector
@@ -81,6 +82,11 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument("--key-path", default=str(DEFAULT_KEY_PATH), help="Path to KEY.md containing MODELSCOPE_TOKEN/XC_TOKEN")
parser.add_argument("--runs-dir", default=str(DEFAULT_RUNS_DIR), help=argparse.SUPPRESS)
parser.add_argument("--ledger-path", default=str(DEFAULT_LEDGER_PATH), help=argparse.SUPPRESS)
parser.add_argument(
"--claims-path",
default=os.getenv("MODELHUB_AGENT_CLAIMS_PATH", str(DEFAULT_CLAIMS_PATH)),
help=argparse.SUPPRESS,
)
parser.add_argument("--history-archive-path", default="history/platform_tasks.jsonl", help=argparse.SUPPRESS)
parser.add_argument("--history-archive-limit", type=int, default=5000, help=argparse.SUPPRESS)
parser.add_argument("--daily-runs-dir", default=str(DEFAULT_DAILY_RUNS_DIR), help=argparse.SUPPRESS)
@@ -117,6 +123,7 @@ def make_wave_namespace(base_args: argparse.Namespace, wave: WaveSpec) -> argpar
runs_dir=base_args.runs_dir,
ledger_path=base_args.ledger_path,
outcomes_path=getattr(base_args, "outcomes_path", "outcomes/submissions.jsonl"),
claims_path=getattr(base_args, "claims_path", str(DEFAULT_CLAIMS_PATH)),
history_archive_path=base_args.history_archive_path,
history_archive_limit=base_args.history_archive_limit,
hf_base_url=base_args.hf_base_url,
@@ -132,7 +139,7 @@ def run_daily_batches(
now=None,
run_fn: Callable[..., dict[str, Any]] = run_submission,
hf_discovery: HuggingFaceDiscovery | None = None,
modelhub_client: ModelHubClient | None = None,
modelhub_client: ModelHubClient | ModelHubClientPool | None = None,
template_selector: TemplateSelector | None = None,
outcome_tracker: OutcomeTracker | None = None,
) -> dict[str, Any]:
@@ -140,15 +147,12 @@ def run_daily_batches(
hf_discovery = hf_discovery or HuggingFaceDiscovery(base_url=base_args.hf_base_url)
if modelhub_client is None:
modelhub_tokens = list(getattr(base_args, "modelhub_tokens", None) or ([] if not base_args.modelhub_token else [base_args.modelhub_token]))
if len(modelhub_tokens) > 1:
clients = [ModelHubClient(token=token, base_url=base_args.modelhub_base_url) for token in modelhub_tokens]
modelhub_client = ModelHubClientPool(clients)
else:
modelhub_client = ModelHubClient(token=base_args.modelhub_token, base_url=base_args.modelhub_base_url)
token_values: list[str | None] = modelhub_tokens or [base_args.modelhub_token]
clients = [ModelHubClient(token=token, base_url=base_args.modelhub_base_url) for token in token_values]
modelhub_client = ModelHubClientPool(clients)
template_selector = template_selector or TemplateSelector()
daily_run_dir = Path(base_args.daily_runs_dir) / now.strftime("%Y%m%dT%H%M%SZ")
daily_run_dir.mkdir(parents=True, exist_ok=True)
daily_run_dir = make_run_dir(Path(base_args.daily_runs_dir), now)
log(f"[daily] daily_run_dir={daily_run_dir}")
log(
f"[daily] target={base_args.daily_target} rounds={base_args.rounds} "