CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

fix: replace duplicate submissions while refilling queues

Browse Source
This commit is contained in:
CoolBoy
2026-08-02 15:44:28 +08:00
parent 670c76fe4e
commit 0e42ef73dc
8 changed files with 309 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

215
modelhub_submmit_api/main.py
View File

@@ -16,10 +16,10 @@ from history_stats import (
load_ledger,
update_history_archive,
)
from modelhub_client import ModelHubAPIError, ModelHubClient, ModelHubClientPool
from modelhub_client import ModelHubAPIError, ModelHubClient, ModelHubClientPool, is_duplicate_submission_error
from models import CandidateModel, HFModelSummary, ModelInspection
from outcome_tracker import DEFAULT_OUTCOMES_PATH, OutcomeTracker
from submission_claims import DEFAULT_CLAIMS_PATH, SubmissionClaimStore, diversify_candidates
from submission_claims import DEFAULT_CLAIMS_PATH, SubmissionClaimStore, candidate_key, diversify_candidates
from task_registry import TASK_SPEC_BY_TYPE, all_task_types, choose_framework_for_task, choose_text_generation_framework, pipeline_tags_for_task_types, task_specs_for_model
from template_selector import TemplateSelector
@@ -319,6 +319,17 @@ def submit_candidate(
"responseData": response.get("data"),
}
except ModelHubAPIError as exc:
if is_duplicate_submission_error(exc):
print(
f"[submit] skipped repo={candidate['repoId']} gpu={candidate['targetGpu']} "
f"framework={candidate['framework']} reason=already_validating",
flush=True,
)
return {
"outcome": "duplicate",
"candidate": candidate,
"reason": str(exc),
}
print(
f"[submit] failed repo={candidate['repoId']} gpu={candidate['targetGpu']} "
f"framework={candidate['framework']} reason={exc}",
@@ -458,8 +469,11 @@ def run_submission(
"scanLimit": 0,
"scannedModels": 0,
"candidateCount": 0,
"targetSubmitCount": 0,
"maxSubmitAttempts": 0,
"plannedSubmitCount": 0,
"submittedCount": 0,
"duplicateCount": 0,
"skippedCount": 0,
"failedCount": 0,
"warnings": report.get("warnings", []),
@@ -550,7 +564,8 @@ def run_submission(
write_jsonl(run_dir / "candidates.jsonl", candidates)
submitted: list[dict[str, Any]] = []
planned_submit_count = resolve_max_submit_count(
duplicate_candidates: list[dict[str, Any]] = []
target_submit_count = resolve_max_submit_count(
args=args,
planned_count=len(candidates),
remaining_daily_quota=remaining_daily_quota,
@@ -558,94 +573,134 @@ def run_submission(
instance_id = runtime_instance_id()
diversified_candidates = diversify_candidates(candidates, instance_id=instance_id)
claim_store: SubmissionClaimStore | None = None
attempted_candidates: list[dict[str, Any]] = []
submit_workers = 1
if args.dry_run:
planned_candidates = diversified_candidates[:planned_submit_count]
attempted_candidates = diversified_candidates[:target_submit_count]
else:
claim_store = SubmissionClaimStore(
Path(getattr(args, "claims_path", DEFAULT_CLAIMS_PATH)),
owner_id=instance_id,
)
planned_candidates = claim_store.claim(diversified_candidates, limit=planned_submit_count)
submit_workers = 1
if not args.dry_run:
submit_workers = resolve_submit_concurrency(
args,
modelhub_client=modelhub_client,
planned_submit_count=len(planned_candidates),
attempted_keys: set[str] = set()
attempt_multiplier = max(1, int(getattr(args, "scan_multiplier", 4) or 1))
max_submit_attempts = min(
len(diversified_candidates),
max(target_submit_count, target_submit_count * attempt_multiplier),
)
with ThreadPoolExecutor(max_workers=submit_workers) as executor:
futures = {
executor.submit(
submit_candidate,
candidate,
modelhub_client,
): index
for index, candidate in enumerate(planned_candidates)
}
ordered_results: dict[int, dict[str, Any]] = {}
for future in as_completed(futures):
index = futures[future]
try:
ordered_results[index] = future.result()
except Exception as exc:
candidate = planned_candidates[index]
ordered_results[index] = {
"outcome": "failed",
"candidate": candidate,
"reason": str(exc),
}
for index in range(len(planned_candidates)):
result = ordered_results.get(index)
if result is None:
continue
candidate = result["candidate"]
if result["outcome"] == "failed":
failed.append(
# A batch can contain candidates another machine has already submitted.
# Keep those duplicate claims and immediately draw replacements from the
# already-scanned pool until the desired number of real submissions is
# reached or account capacity is genuinely exhausted.
while len(submitted) < target_submit_count and len(attempted_candidates) < max_submit_attempts:
remaining_candidates = [
candidate
for candidate in diversified_candidates
if candidate_key(candidate) not in attempted_keys
]
desired_count = min(
target_submit_count - len(submitted),
max_submit_attempts - len(attempted_candidates),
)
batch_candidates = claim_store.claim(remaining_candidates, limit=desired_count)
if not batch_candidates:
break
attempted_candidates.extend(batch_candidates)
attempted_keys.update(candidate_key(candidate) for candidate in batch_candidates)
batch_workers = resolve_submit_concurrency(
args,
modelhub_client=modelhub_client,
planned_submit_count=len(batch_candidates),
)
submit_workers = max(submit_workers, batch_workers)
with ThreadPoolExecutor(max_workers=batch_workers) as executor:
futures = {
executor.submit(submit_candidate, candidate, modelhub_client): index
for index, candidate in enumerate(batch_candidates)
}
ordered_results: dict[int, dict[str, Any]] = {}
for future in as_completed(futures):
index = futures[future]
try:
ordered_results[index] = future.result()
except Exception as exc:
candidate = batch_candidates[index]
ordered_results[index] = {
"outcome": "failed",
"candidate": candidate,
"reason": str(exc),
}
batch_submitted_candidates: list[dict[str, Any]] = []
batch_duplicate_candidates: list[dict[str, Any]] = []
batch_failed_candidates: list[dict[str, Any]] = []
for index in range(len(batch_candidates)):
result = ordered_results.get(index)
if result is None:
continue
candidate = result["candidate"]
if result["outcome"] == "duplicate":
duplicate_candidates.append(candidate)
batch_duplicate_candidates.append(candidate)
skipped.append(
{
"repoId": candidate["repoId"],
"targetGpu": candidate["targetGpu"],
"reason": "already_validating_on_platform",
}
)
continue
if result["outcome"] == "failed":
batch_failed_candidates.append(candidate)
failed.append(
{
"repoId": candidate["repoId"],
"targetGpu": candidate["targetGpu"],
"framework": candidate["framework"],
"taskType": candidate["taskType"],
"reason": result.get("reason", "submission_failed"),
}
)
continue
batch_submitted_candidates.append(candidate)
submitted_record = {
**candidate,
"submitTime": result["submitTime"],
"taskId": result["taskId"],
"responseData": result["responseData"],
}
submitted.append(submitted_record)
append_ledger_entry(
ledger_path,
{
"repoId": candidate["repoId"],
"modelId": candidate["repoId"],
"modelAddress": candidate["modelAddress"],
"targetGpu": candidate["targetGpu"],
"framework": candidate["framework"],
"templateId": candidate["templateId"],
"taskId": result["taskId"],
"taskType": candidate["taskType"],
"reason": result.get("reason", "submission_failed"),
}
"submitTime": result["submitTime"],
},
)
outcome_tracker.record_submission(
model_id=candidate["repoId"],
target_gpu=candidate["targetGpu"],
framework=candidate["framework"],
task_type=candidate["taskType"],
task_id=result["taskId"],
submit_time=result["submitTime"],
)
continue
submitted_record = {
**candidate,
"submitTime": result["submitTime"],
"taskId": result["taskId"],
"responseData": result["responseData"],
}
submitted.append(submitted_record)
append_ledger_entry(
ledger_path,
{
"modelId": candidate["repoId"],
"modelAddress": candidate["modelAddress"],
"targetGpu": candidate["targetGpu"],
"framework": candidate["framework"],
"templateId": candidate["templateId"],
"taskId": result["taskId"],
"taskType": candidate["taskType"],
"submitTime": result["submitTime"],
},
)
outcome_tracker.record_submission(
model_id=candidate["repoId"],
target_gpu=candidate["targetGpu"],
framework=candidate["framework"],
task_type=candidate["taskType"],
task_id=result["taskId"],
submit_time=result["submitTime"],
)
claim_store.mark_submitted([*batch_submitted_candidates, *batch_duplicate_candidates])
claim_store.release(batch_failed_candidates)
if claim_store is not None:
submitted_candidates = [result["candidate"] for result in ordered_results.values() if result.get("outcome") == "submitted"]
failed_candidates = [result["candidate"] for result in ordered_results.values() if result.get("outcome") != "submitted"]
claim_store.mark_submitted(submitted_candidates)
claim_store.release(failed_candidates)
if hasattr(modelhub_client, "available_submit_slots") and modelhub_client.available_submit_slots() <= 0:
break
write_jsonl(run_dir / "submitted.jsonl", submitted)
write_jsonl(run_dir / "skipped.jsonl", skipped)
@@ -671,7 +726,13 @@ def run_submission(
"scanLimit": scan_limit,
"scannedModels": len(models),
"candidateCount": len(candidates),
"plannedSubmitCount": len(planned_candidates),
"targetSubmitCount": target_submit_count,
"maxSubmitAttempts": 0 if args.dry_run else min(
len(diversified_candidates),
max(target_submit_count, target_submit_count * max(1, int(getattr(args, "scan_multiplier", 4) or 1))),
),
"plannedSubmitCount": len(attempted_candidates),
"duplicateCount": len(duplicate_candidates),
"submittedCount": len(submitted),
"skippedCount": len(skipped),
"failedCount": len(failed),