fix: protect running validation tasks from queue cleanup

modelhub_submmit_api/outcome_tracker.py

@@ -88,6 +88,49 @@ class OutcomeTracker:
         self._by_model_gpu[(model_id, target_gpu)].append(record)
         append_jsonl(self.path, record)
 
+    def mark_policy_cancellations(self, decisions: list[dict[str, Any]]) -> int:
+        """Persist intentional task stops so they never become failure evidence."""
+        marked = 0
+        marked_at = _now_iso()
+        for decision in decisions:
+            task_id_value = decision.get("taskId")
+            if task_id_value is None:
+                continue
+            task_id = str(task_id_value)
+            existing = self._by_task_id.get(task_id)
+            if existing is not None and existing.get("outcome") in {"success", "failed"}:
+                continue
+            reasons = list(decision.get("cleanupReasons") or [decision.get("reason")])
+            reasons = [str(reason) for reason in reasons if reason]
+            if existing is None:
+                existing = {
+                    "modelId": decision.get("modelId") or "",
+                    "targetGpu": decision.get("gpuType") or decision.get("targetGpu") or "",
+                    "framework": decision.get("framework") or "",
+                    "taskType": decision.get("taskType") or "",
+                    "taskId": task_id,
+                    "submitTime": decision.get("submitTime") or marked_at,
+                    "verifyResult": None,
+                }
+                self._records.append(existing)
+                self._by_task_id[task_id] = existing
+                self._by_model_gpu[(existing["modelId"], existing["targetGpu"])].append(existing)
+            existing.update(
+                {
+                    "lastSyncTime": marked_at,
+                    "status": "cancellation_requested",
+                    "outcome": "policy_cancelled",
+                    "failReason": None,
+                    "policyCancelled": True,
+                    "policyCancellationReasons": reasons,
+                    "policyCancelledAt": marked_at,
+                }
+            )
+            marked += 1
+        if marked:
+            self._rebuild_failed_index()
+        return marked
+
     def sync_from_api(self, client: ModelHubClient | ModelHubClientPool) -> int:
         try:
             begin = self._last_sync_time
@@ -109,7 +152,7 @@ class OutcomeTracker:
 
             existing = self._by_task_id.get(task_id)
             if existing is not None:
-                if existing.get("outcome") == "pending":
+                if existing.get("outcome") in {"pending", "policy_cancelled"}:
                     self._update_record_from_task(existing, task)
                     if existing.get("outcome") == "failed" and existing.get("logCosUrl"):
                         enrichment_candidates.append(existing)
@@ -287,6 +330,9 @@ class OutcomeTracker:
                 warnings.append(f"组合 {key} 近期失败集中，建议降低该 GPU+框架的提交优先级。")
 
         pending_count = sum(1 for r in self._records if r.get("outcome") == "pending")
+        policy_cancelled_count = sum(
+            1 for record in self._records if record.get("outcome") == "policy_cancelled"
+        )
         observed_gpu_memory: dict[str, float] = {}
         for record in self._records:
             gpu = str(record.get("targetGpu") or "")
@@ -302,6 +348,7 @@ class OutcomeTracker:
             "generatedAt": now,
             "totalRecords": len(self._records),
             "pendingRecords": pending_count,
+            "policyCancelledRecords": policy_cancelled_count,
             "terminalRecords": len(terminal),
             "gpuSummaries": gpu_summaries,
             "frameworkSummaries": framework_summaries,
@@ -337,9 +384,10 @@ class OutcomeTracker:
         self._failed_model_gpus.clear()
         latest_by_combo: dict[tuple[str, str], tuple[datetime, dict[str, Any]]] = {}
         for record in self._records:
-            # Infrastructure failures neither clear nor create a model/GPU
-            # cooldown. Look through them to the latest attributable outcome.
-            if _is_platform_failure(record):
+            # Infrastructure failures and our own policy cancellations neither
+            # clear nor create a model/GPU cooldown. Look through them to the
+            # latest attributable outcome.
+            if _is_platform_failure(record) or _is_policy_cancelled(record):
                 continue
             model_id = record.get("modelId") or ""
             target_gpu = record.get("targetGpu") or ""
@@ -362,9 +410,19 @@ class OutcomeTracker:
         record["lastSyncTime"] = _now_iso()
         if task.get("logCosUrl"):
             record["logCosUrl"] = task.get("logCosUrl")
+        status = str(task.get("status") or "").strip().lower()
         if is_success(task):
             record["outcome"] = "success"
             record["failReason"] = None
+            if record.get("policyCancelled"):
+                record["policyCancellationResolvedAsSuccess"] = True
+                record["policyCancelled"] = False
+        elif record.get("policyCancelled"):
+            # A successful stop may be observed as waiting/running briefly before
+            # the platform publishes its terminal cancellation state. None of
+            # those intermediate states should become failure evidence.
+            record["outcome"] = "policy_cancelled"
+            record["failReason"] = None
         elif is_failure(task):
             record["outcome"] = "failed"
             record["failReason"] = classify_failure(task)
@@ -439,6 +497,10 @@ def _is_platform_failure(record: dict[str, Any]) -> bool:
     return scope == "platform" or category.startswith("platform_")
 
 
+def _is_policy_cancelled(record: dict[str, Any]) -> bool:
+    return bool(record.get("policyCancelled")) or record.get("outcome") == "policy_cancelled"
+
+
 def _consecutive_attributable_failures(records: list[dict[str, Any]]) -> int:
     count = 0
     for record in records:
@@ -486,7 +548,7 @@ def _outcome_record_key(record: dict[str, Any]) -> str:
 def _outcome_version(record: dict[str, Any]) -> tuple[int, float, float]:
     last_sync = parse_datetime(record.get("lastSyncTime"))
     submit_time = parse_datetime(record.get("submitTime"))
-    outcome_rank = 1 if record.get("outcome") in {"success", "failed"} else 0
+    outcome_rank = 1 if record.get("outcome") in {"success", "failed", "policy_cancelled"} else 0
     return (
         outcome_rank,
         last_sync.timestamp() if last_sync else 0.0,