CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

fix: prevent repeated model GPU submissions

Browse Source
This commit is contained in:
CoolBoy
2026-08-02 18:48:47 +08:00
parent 8e68c6f611
commit ccae7ff8f3
9 changed files with 489 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
modelhub_submmit_api/main.py
View File

@@ -18,10 +18,18 @@ from history_stats import (
load_ledger,
update_history_archive,
)
from modelhub_client import DEFAULT_CAPACITY_STATE_PATH, ModelHubAPIError, ModelHubClient, ModelHubClientPool, is_duplicate_submission_error
from modelhub_client import (
DEFAULT_CAPACITY_STATE_PATH,
ModelHubAPIError,
ModelHubClient,
ModelHubClientPool,
is_duplicate_submission_error,
is_model_uniqueness_error,
)
from models import CandidateModel, HFModelSummary, ModelInspection
from outcome_tracker import DEFAULT_OUTCOMES_PATH, OutcomeTracker
from submission_claims import DEFAULT_CLAIMS_PATH, SubmissionClaimStore, candidate_key, diversify_candidates
from submission_exclusions import DEFAULT_SUBMISSION_EXCLUSIONS_PATH, SubmissionExclusionStore
from task_registry import TASK_SPEC_BY_TYPE, all_task_types, choose_framework_for_task, choose_text_generation_framework, pipeline_tags_for_task_types, task_specs_for_model
from template_selector import TemplateSelector
@@ -94,6 +102,11 @@ def build_parser() -> argparse.ArgumentParser:
)
parser.add_argument("--history-archive-path", default="history/platform_tasks.jsonl", help=argparse.SUPPRESS)
parser.add_argument("--history-archive-limit", type=int, default=5000, help=argparse.SUPPRESS)
parser.add_argument(
"--submission-exclusions-path",
default=os.getenv("MODELHUB_SUBMISSION_EXCLUSIONS_PATH", str(DEFAULT_SUBMISSION_EXCLUSIONS_PATH)),
help=argparse.SUPPRESS,
)
parser.add_argument(
"--gpu-strategy-state-path",
default=os.getenv("MODELHUB_GPU_STRATEGY_STATE_PATH", str(DEFAULT_GPU_STRATEGY_PATH)),
@@ -270,12 +283,24 @@ def process_model_for_candidates(
target_gpus: list[str],
allowed_task_types: list[str],
outcome_tracker: OutcomeTracker | None = None,
submission_exclusion_store: SubmissionExclusionStore | None = None,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]:
specs = [spec for spec in task_specs_for_model(model) if spec.task_type in allowed_task_types]
if not specs:
return [], [{"repoId": model.repo_id, "reason": f"unsupported_pipeline_tag:{model.pipeline_tag or 'unknown'}"}], []
processed_gpus = modelhub_client.processed_gpus_for_model(model.repo_id)
try:
if hasattr(modelhub_client, "model_submission_precheck"):
precheck = modelhub_client.model_submission_precheck(model.repo_id)
processed_gpus = set(precheck.get("processedGpus") or set())
else:
processed_gpus = modelhub_client.processed_gpus_for_model(model.repo_id)
except Exception as exc:
return (
[],
[{"repoId": model.repo_id, "reason": "community_precheck_unavailable_fail_closed"}],
[{"repoId": model.repo_id, "reason": f"community_precheck_error:{exc}"}],
)
candidates: list[dict[str, Any]] = []
skipped: list[dict[str, Any]] = []
failed: list[dict[str, Any]] = []
@@ -283,6 +308,11 @@ def process_model_for_candidates(
pending_task_types_by_gpu: list[tuple[str, list[str]]] = []
for target_gpu in target_gpus:
if submission_exclusion_store is not None and submission_exclusion_store.is_blocked(model.repo_id, target_gpu):
skipped.append(
{"repoId": model.repo_id, "targetGpu": target_gpu, "reason": "model_gpu_uniqueness_blocklist"}
)
continue
if target_gpu in processed_gpus:
skipped.append({"repoId": model.repo_id, "targetGpu": target_gpu, "reason": "already_processed_for_gpu"})
continue
@@ -391,6 +421,7 @@ def collect_candidates_from_models(
target_gpus: list[str],
selected_task_types: list[str],
outcome_tracker: OutcomeTracker | None,
submission_exclusion_store: SubmissionExclusionStore | None,
read_concurrency: int,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]], int]:
candidates: list[dict[str, Any]] = []
@@ -419,6 +450,7 @@ def collect_candidates_from_models(
target_gpus=target_gpus,
allowed_task_types=selected_task_types,
outcome_tracker=outcome_tracker,
submission_exclusion_store=submission_exclusion_store,
): index
for index, model in enumerate(chunk)
}
@@ -445,6 +477,27 @@ def submit_candidate(
candidate: dict[str, Any],
modelhub_client: ModelHubClient,
) -> dict[str, Any]:
if hasattr(modelhub_client, "model_submission_precheck"):
try:
precheck = modelhub_client.model_submission_precheck(candidate["repoId"], force_refresh=True)
except Exception as exc:
print(
f"[submit] deferred repo={candidate['repoId']} gpu={candidate['targetGpu']} "
f"framework={candidate['framework']} reason=community_precheck_unavailable",
flush=True,
)
return {
"outcome": "precheck_deferred",
"candidate": candidate,
"reason": f"community_precheck_error:{exc}",
}
if candidate["targetGpu"] in set(precheck.get("processedGpus") or set()):
return {
"outcome": "duplicate",
"candidate": candidate,
"reason": "already_processed_for_gpu",
}
payload = {
"modelAddress": candidate["modelAddress"],
"taskType": candidate["taskType"],
@@ -469,6 +522,17 @@ def submit_candidate(
"responseData": response.get("data"),
}
except ModelHubAPIError as exc:
if is_model_uniqueness_error(exc):
print(
f"[submit] skipped repo={candidate['repoId']} gpu={candidate['targetGpu']} "
f"framework={candidate['framework']} reason=model_uniqueness_rejected",
flush=True,
)
return {
"outcome": "uniqueness_rejected",
"candidate": candidate,
"reason": str(exc),
}
if is_duplicate_submission_error(exc):
print(
f"[submit] skipped repo={candidate['repoId']} gpu={candidate['targetGpu']} "
@@ -505,6 +569,18 @@ def make_run_dir(runs_dir: Path, now) -> Path:
raise RuntimeError(f"Unable to allocate a unique run directory under {runs_dir}")
def one_candidate_per_model(candidates: list[dict[str, Any]]) -> list[dict[str, Any]]:
selected: list[dict[str, Any]] = []
seen_models: set[str] = set()
for candidate in candidates:
model_id = str(candidate.get("repoId") or candidate.get("modelAddress") or "")
if not model_id or model_id in seen_models:
continue
seen_models.add(model_id)
selected.append(candidate)
return selected
def run_submission(
args: argparse.Namespace,
*,
@@ -543,6 +619,9 @@ def run_submission(
history_archive_path.parent.mkdir(parents=True, exist_ok=True)
outcome_tracker = outcome_tracker or OutcomeTracker(Path(args.outcomes_path))
submission_exclusion_store = SubmissionExclusionStore(
Path(getattr(args, "submission_exclusions_path", DEFAULT_SUBMISSION_EXCLUSIONS_PATH))
)
strategy_manager: GPUStrategyManager | None = None
strategy_summary: dict[str, Any] = {"enabled": False}
strategy_enabled = (
@@ -638,9 +717,12 @@ def run_submission(
"plannedSubmitCount": 0,
"submittedCount": 0,
"duplicateCount": 0,
"modelGpuUniquenessRejectedCount": 0,
"skippedCount": 0,
"skipReasonCounts": {},
"failedCount": 0,
"failureReasonCounts": {},
"submissionExclusionsPath": str(submission_exclusion_store.path),
"warnings": report.get("warnings", []),
"runDir": str(run_dir),
"outcomeSyncCount": synced_count,
@@ -740,6 +822,7 @@ def run_submission(
target_gpus=target_gpus,
selected_task_types=selected_task_types,
outcome_tracker=outcome_tracker,
submission_exclusion_store=submission_exclusion_store,
read_concurrency=max(1, args.read_concurrency),
)
candidates.extend(stage_candidates)
@@ -767,6 +850,7 @@ def run_submission(
submitted: list[dict[str, Any]] = []
duplicate_candidates: list[dict[str, Any]] = []
uniqueness_rejected_candidates: list[dict[str, Any]] = []
target_submit_count = resolve_max_submit_count(
args=args,
planned_count=len(candidates),
@@ -806,7 +890,9 @@ def run_submission(
candidate
for candidate in diversified_candidates
if candidate_key(candidate) not in attempted_keys
and not submission_exclusion_store.is_blocked(candidate["repoId"], candidate["targetGpu"])
]
remaining_candidates = one_candidate_per_model(remaining_candidates)
desired_count = min(
target_submit_count - len(submitted),
max_submit_attempts - len(attempted_candidates),
@@ -843,12 +929,30 @@ def run_submission(
batch_submitted_candidates: list[dict[str, Any]] = []
batch_duplicate_candidates: list[dict[str, Any]] = []
batch_uniqueness_rejected_candidates: list[dict[str, Any]] = []
batch_failed_candidates: list[dict[str, Any]] = []
for index in range(len(batch_candidates)):
result = ordered_results.get(index)
if result is None:
continue
candidate = result["candidate"]
if result["outcome"] == "uniqueness_rejected":
duplicate_candidates.append(candidate)
uniqueness_rejected_candidates.append(candidate)
batch_uniqueness_rejected_candidates.append(candidate)
submission_exclusion_store.block(
candidate["repoId"],
candidate["targetGpu"],
reason=result.get("reason", "model_uniqueness_rejected"),
)
skipped.append(
{
"repoId": candidate["repoId"],
"targetGpu": candidate["targetGpu"],
"reason": "model_uniqueness_rejected",
}
)
continue
if result["outcome"] == "duplicate":
duplicate_candidates.append(candidate)
batch_duplicate_candidates.append(candidate)
@@ -860,7 +964,7 @@ def run_submission(
}
)
continue
if result["outcome"] == "failed":
if result["outcome"] in {"failed", "precheck_deferred"}:
batch_failed_candidates.append(candidate)
failed.append(
{
@@ -903,7 +1007,9 @@ def run_submission(
submit_time=result["submitTime"],
)
claim_store.mark_submitted([*batch_submitted_candidates, *batch_duplicate_candidates])
claim_store.mark_submitted(
[*batch_submitted_candidates, *batch_duplicate_candidates, *batch_uniqueness_rejected_candidates]
)
claim_store.release(batch_failed_candidates)
if strategy_manager is not None:
strategy_manager.record_accepted(batch_submitted_candidates)
@@ -950,11 +1056,13 @@ def run_submission(
),
"plannedSubmitCount": len(attempted_candidates),
"duplicateCount": len(duplicate_candidates),
"modelGpuUniquenessRejectedCount": len(uniqueness_rejected_candidates),
"submittedCount": len(submitted),
"skippedCount": len(skipped),
"skipReasonCounts": dict(skip_reason_counts.most_common()),
"failedCount": len(failed),
"failureReasonCounts": dict(failure_reason_counts.most_common()),
"submissionExclusionsPath": str(submission_exclusion_store.path),
"submitConcurrencyUsed": submit_workers,
"warnings": report.get("warnings", []),
"runDir": str(run_dir),