CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

fix: protect running validation tasks from queue cleanup

Browse Source
This commit is contained in:
CoolBoy
2026-08-12 01:04:34 +08:00
parent 2065ad6abc
commit d908706f9a
8 changed files with 343 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
modelhub_submmit_api/README.md
View File

@@ -124,8 +124,10 @@ bash run_poll.sh --dry-run
positions. The account pool enforces this per-account boundary atomically and updates it when
capacity probing discovers a higher limit. Cleanup stops OOM tasks first, recalculates the
surviving queue order, and applies the same limit-minus-10 boundary on startup. Scheduled
cleanup relaxes to limit minus 5 to avoid excessive pruning. Recent overflow tasks stay, and
unknown ModelScope timestamps never authorize a cancellation.
cleanup relaxes to limit minus 5 to avoid excessive pruning. Age cleanup stops waiting tasks
only; running tasks are protected and rechecked after OOM cleanup, immediately before the
age-only stop batch. Recent
overflow tasks stay, and unknown ModelScope timestamps never authorize a cancellation.
## Important Flags
@@ -167,6 +169,11 @@ minus 5, while admission continues to reserve the final 10 slots for recent
models. Override these suffix sizes with `MODELHUB_RECENT_MODEL_RESERVE_SLOTS`
and `MODELHUB_DYNAMIC_OLD_MODEL_CLEANUP_RESERVE_SLOTS`.
Every successful worker-initiated stop is persisted as `policy_cancelled` in the
outcome store. It is excluded from GPU/framework success rates, local failure
cooldowns, and circuit breakers. If a task races to a real success before the
stop takes effect, that success remains authoritative.
Failure-informed preflight is enabled by default. It rejects deterministic
missing-file and predicted-OOM cases, clamps unsafe context-length arguments,
and records its decisions in `candidatePreflight` and each candidate's

72
modelhub_submmit_api/outcome_tracker.py
View File

@@ -88,6 +88,49 @@ class OutcomeTracker:
self._by_model_gpu[(model_id, target_gpu)].append(record)
append_jsonl(self.path, record)
def mark_policy_cancellations(self, decisions: list[dict[str, Any]]) -> int:
"""Persist intentional task stops so they never become failure evidence."""
marked = 0
marked_at = _now_iso()
for decision in decisions:
task_id_value = decision.get("taskId")
if task_id_value is None:
continue
task_id = str(task_id_value)
existing = self._by_task_id.get(task_id)
if existing is not None and existing.get("outcome") in {"success", "failed"}:
continue
reasons = list(decision.get("cleanupReasons") or [decision.get("reason")])
reasons = [str(reason) for reason in reasons if reason]
if existing is None:
existing = {
"modelId": decision.get("modelId") or "",
"targetGpu": decision.get("gpuType") or decision.get("targetGpu") or "",
"framework": decision.get("framework") or "",
"taskType": decision.get("taskType") or "",
"taskId": task_id,
"submitTime": decision.get("submitTime") or marked_at,
"verifyResult": None,
}
self._records.append(existing)
self._by_task_id[task_id] = existing
self._by_model_gpu[(existing["modelId"], existing["targetGpu"])].append(existing)
existing.update(
{
"lastSyncTime": marked_at,
"status": "cancellation_requested",
"outcome": "policy_cancelled",
"failReason": None,
"policyCancelled": True,
"policyCancellationReasons": reasons,
"policyCancelledAt": marked_at,
}
)
marked += 1
if marked:
self._rebuild_failed_index()
return marked
def sync_from_api(self, client: ModelHubClient | ModelHubClientPool) -> int:
try:
begin = self._last_sync_time
@@ -109,7 +152,7 @@ class OutcomeTracker:
existing = self._by_task_id.get(task_id)
if existing is not None:
if existing.get("outcome") == "pending":
if existing.get("outcome") in {"pending", "policy_cancelled"}:
self._update_record_from_task(existing, task)
if existing.get("outcome") == "failed" and existing.get("logCosUrl"):
enrichment_candidates.append(existing)
@@ -287,6 +330,9 @@ class OutcomeTracker:
warnings.append(f"组合 {key} 近期失败集中，建议降低该 GPU+框架的提交优先级。")
pending_count = sum(1 for r in self._records if r.get("outcome") == "pending")
policy_cancelled_count = sum(
1 for record in self._records if record.get("outcome") == "policy_cancelled"
)
observed_gpu_memory: dict[str, float] = {}
for record in self._records:
gpu = str(record.get("targetGpu") or "")
@@ -302,6 +348,7 @@ class OutcomeTracker:
"generatedAt": now,
"totalRecords": len(self._records),
"pendingRecords": pending_count,
"policyCancelledRecords": policy_cancelled_count,
"terminalRecords": len(terminal),
"gpuSummaries": gpu_summaries,
"frameworkSummaries": framework_summaries,
@@ -337,9 +384,10 @@ class OutcomeTracker:
self._failed_model_gpus.clear()
latest_by_combo: dict[tuple[str, str], tuple[datetime, dict[str, Any]]] = {}
for record in self._records:
# Infrastructure failures neither clear nor create a model/GPU
# cooldown. Look through them to the latest attributable outcome.
if _is_platform_failure(record):
# Infrastructure failures and our own policy cancellations neither
# clear nor create a model/GPU cooldown. Look through them to the
# latest attributable outcome.
if _is_platform_failure(record) or _is_policy_cancelled(record):
continue
model_id = record.get("modelId") or ""
target_gpu = record.get("targetGpu") or ""
@@ -362,9 +410,19 @@ class OutcomeTracker:
record["lastSyncTime"] = _now_iso()
if task.get("logCosUrl"):
record["logCosUrl"] = task.get("logCosUrl")
status = str(task.get("status") or "").strip().lower()
if is_success(task):
record["outcome"] = "success"
record["failReason"] = None
if record.get("policyCancelled"):
record["policyCancellationResolvedAsSuccess"] = True
record["policyCancelled"] = False
elif record.get("policyCancelled"):
# A successful stop may be observed as waiting/running briefly before
# the platform publishes its terminal cancellation state. None of
# those intermediate states should become failure evidence.
record["outcome"] = "policy_cancelled"
record["failReason"] = None
elif is_failure(task):
record["outcome"] = "failed"
record["failReason"] = classify_failure(task)
@@ -439,6 +497,10 @@ def _is_platform_failure(record: dict[str, Any]) -> bool:
return scope == "platform" or category.startswith("platform_")
def _is_policy_cancelled(record: dict[str, Any]) -> bool:
return bool(record.get("policyCancelled")) or record.get("outcome") == "policy_cancelled"
def _consecutive_attributable_failures(records: list[dict[str, Any]]) -> int:
count = 0
for record in records:
@@ -486,7 +548,7 @@ def _outcome_record_key(record: dict[str, Any]) -> str:
def _outcome_version(record: dict[str, Any]) -> tuple[int, float, float]:
last_sync = parse_datetime(record.get("lastSyncTime"))
submit_time = parse_datetime(record.get("submitTime"))
outcome_rank = 1 if record.get("outcome") in {"success", "failed"} else 0
outcome_rank = 1 if record.get("outcome") in {"success", "failed", "policy_cancelled"} else 0
return (
outcome_rank,
last_sync.timestamp() if last_sync else 0.0,

7
modelhub_submmit_api/poll_runner.py
View File

@@ -324,6 +324,12 @@ def run_poll_loop(
age_reserved_slots=age_cleanup_reserve_slots,
log=log,
)
policy_cancelled_recorded = outcome_tracker.mark_policy_cancellations(
cleanup_summary["cancelledTasks"]
)
if policy_cancelled_recorded:
outcome_tracker.save()
cleanup_summary["policyCancelledRecorded"] = policy_cancelled_recorded
write_json(
Path(
getattr(
@@ -344,6 +350,7 @@ def run_poll_loop(
"certainOomCount": cleanup_summary["certainOomCount"],
"oldOverflowCount": cleanup_summary["oldOverflowCount"],
"cancelledCount": cleanup_summary["cancelledCount"],
"policyCancelledRecorded": policy_cancelled_recorded,
"stopErrorCount": len(cleanup_summary["stopErrors"]),
}
)

89
modelhub_submmit_api/queue_cleanup.py
View File

@@ -261,6 +261,7 @@ def find_old_overflow_tasks(
"recentOverflowTasks": 0,
"modelAgeUnknown": 0,
"accountThresholdUnknown": 0,
"runningOverflowProtected": 0,
}
for account_index, account_tasks in sorted(grouped.items()):
if account_index in incomplete_accounts:
@@ -278,6 +279,9 @@ def find_old_overflow_tasks(
for position, task in enumerate(ordered, start=1):
if position <= threshold:
continue
if task.status != "waiting":
skipped["runningOverflowProtected"] += 1
continue
last_modified = model_last_modified.get(task.model_id)
if last_modified is None:
skipped["modelAgeUnknown"] += 1
@@ -390,6 +394,7 @@ def cleanup_certain_oom_tasks(
(item for item in age_rank_tasks if item.account_index == account_index),
key=lambda item: item.task_id,
)[queue_thresholds.get(account_index, len(age_rank_tasks)):]
if task.status == "waiting"
}
model_last_modified, age_errors = _load_model_last_modified(
overflow_model_ids,
@@ -410,7 +415,8 @@ def cleanup_certain_oom_tasks(
f"thresholds={','.join(str(queue_thresholds[index]) for index in sorted(queue_thresholds))} "
f"reserve_recent_slots={reserved_slots} recent_days={recent_model_days} "
f"recent_overflow={age_skipped['recentOverflowTasks']} "
f"age_unknown={age_skipped['modelAgeUnknown']}"
f"age_unknown={age_skipped['modelAgeUnknown']} "
f"running_protected={age_skipped['runningOverflowProtected']}"
)
decisions_by_key: dict[tuple[int, int], dict[str, Any]] = {}
@@ -444,8 +450,10 @@ def cleanup_certain_oom_tasks(
refreshed_tasks, refresh_errors = collect_active_tasks(clients, read_concurrency=read_concurrency)
active_ids_by_account: dict[int, set[int]] = {}
active_positions_by_account: dict[int, dict[int, int]] = {}
active_status_by_account: dict[int, dict[int, str]] = {}
for task in refreshed_tasks:
active_ids_by_account.setdefault(task.account_index, set()).add(task.task_id)
active_status_by_account.setdefault(task.account_index, {})[task.task_id] = task.status
for account_index in range(len(clients)):
planned_oom_ids = {
int(decision["taskId"])
@@ -481,14 +489,25 @@ def cleanup_certain_oom_tasks(
cleanup_reasons = set(decision.get("cleanupReasons") or [decision.get("reason")])
age_only = cleanup_reasons == {"old_model_beyond_account_queue_threshold"}
current_position = active_positions_by_account.get(account_index, {}).get(int(decision["taskId"]))
current_status = active_status_by_account.get(account_index, {}).get(int(decision["taskId"]))
account_queue_threshold = queue_thresholds.get(account_index)
if age_only and (
current_position is None
or account_queue_threshold is None
or current_position <= account_queue_threshold
or current_status != "waiting"
):
policy_no_longer_applies.append(
{**decision, "recheckedQueuePosition": current_position}
{
**decision,
"recheckedQueuePosition": current_position,
"recheckedStatus": current_status,
"policyChangeReason": (
"task_started_running"
if current_status == "running"
else "queue_position_or_status_changed"
),
}
)
continue
if current_position is not None:
@@ -501,16 +520,72 @@ def cleanup_certain_oom_tasks(
if stop_failed:
break
decisions_by_id = {int(item["taskId"]): item for item in by_account[account_index]}
phase_ids = [
sorted(
for is_oom_phase in (True, False):
task_ids = sorted(
task_id
for task_id, decision in decisions_by_id.items()
if ("certain_oom_repository_size_exceeds_gpu_capacity" in decision["cleanupReasons"])
== is_oom_phase
)
for is_oom_phase in (True, False)
]
for task_ids in phase_ids:
if not is_oom_phase and task_ids:
# OOM stops can change actual positions, and a waiting task
# can start running after the account-wide recheck above.
# Re-read this account immediately before its age-only stop.
try:
phase_tasks: dict[int, OwnedTask] = {}
for status in ACTIVE_FILTER_STATUSES:
for task in _fetch_status_tasks(
clients[account_index],
account_index=account_index,
status=status,
):
phase_tasks[task.task_id] = task
except Exception as exc:
stop_errors.append(
{
"accountIndex": account_index + 1,
"taskIds": task_ids,
"error": f"age_policy_final_recheck_failed: {type(exc).__name__}: {exc}",
}
)
stop_failed = True
break
phase_positions = {
task_id: position
for position, task_id in enumerate(sorted(phase_tasks), start=1)
}
eligible_task_ids: list[int] = []
account_queue_threshold = queue_thresholds.get(account_index)
for task_id in task_ids:
current_task = phase_tasks.get(task_id)
if current_task is None:
disappeared.append(decisions_by_id[task_id])
continue
current_position = phase_positions.get(task_id)
if (
account_queue_threshold is None
or current_position is None
or current_position <= account_queue_threshold
or current_task.status != "waiting"
):
policy_no_longer_applies.append(
{
**decisions_by_id[task_id],
"recheckedQueuePosition": current_position,
"recheckedStatus": current_task.status,
"policyChangeReason": (
"task_started_running"
if current_task.status == "running"
else "queue_position_or_status_changed"
),
}
)
continue
decisions_by_id[task_id]["recheckedQueuePosition"] = current_position
eligible_task_ids.append(task_id)
task_ids = eligible_task_ids
for batch in _chunks(task_ids, batch_size):
try:
clients[account_index].stop_tasks(batch)

2
modelhub_submmit_api/version.py
View File

@@ -1 +1 @@
AGENT_VERSION = "2026.08.11.4"
AGENT_VERSION = "2026.08.12.1"