fix: protect running validation tasks from queue cleanup

This commit is contained in:
CoolBoy
2026-08-12 01:04:34 +08:00
parent 2065ad6abc
commit d908706f9a
8 changed files with 343 additions and 20 deletions

View File

@@ -391,6 +391,78 @@ class CandidatePreflightTests(unittest.TestCase):
self.assertEqual(1, report["profileCombinationStats"][key]["failureCount"])
self.assertEqual(1, report["recentProfileCombinationStats"][key]["consecutiveFailures"])
def test_policy_cancellation_is_excluded_from_failure_feedback(self) -> None:
with tempfile.TemporaryDirectory() as temporary_dir:
path = Path(temporary_dir) / "outcomes.jsonl"
tracker = OutcomeTracker(path)
tracker.record_submission(
"owner/model",
"gpu",
"vllm",
"text-generation",
"task-policy",
datetime.now(timezone.utc).isoformat(),
)
marked = tracker.mark_policy_cancellations(
[
{
"taskId": "task-policy",
"modelId": "owner/model",
"gpuType": "gpu",
"cleanupReasons": ["old_model_beyond_account_queue_threshold"],
}
]
)
tracker.save()
tracker.sync_from_api(
TaskClient(
[
{
"taskId": "task-policy",
"status": "cancelled",
"verifyResult": None,
}
]
) # type: ignore[arg-type]
)
report = tracker.get_stats_report()
self.assertEqual(1, marked)
self.assertEqual(1, report["policyCancelledRecords"])
self.assertEqual(0, report["terminalRecords"])
self.assertEqual({}, report["combinationStats"])
self.assertFalse(tracker.is_model_gpu_failed("owner/model", "gpu"))
def test_policy_cancellation_does_not_hide_a_racing_success(self) -> None:
with tempfile.TemporaryDirectory() as temporary_dir:
tracker = OutcomeTracker(Path(temporary_dir) / "outcomes.jsonl")
tracker.record_submission(
"owner/model",
"gpu",
"vllm",
"text-generation",
"task-success",
datetime.now(timezone.utc).isoformat(),
)
tracker.mark_policy_cancellations(
[{"taskId": "task-success", "modelId": "owner/model", "gpuType": "gpu"}]
)
tracker.sync_from_api(
TaskClient(
[
{
"taskId": "task-success",
"status": "success",
"verifyResult": 1,
}
]
) # type: ignore[arg-type]
)
report = tracker.get_stats_report()
self.assertEqual(0, report["policyCancelledRecords"])
self.assertEqual(1, report["totals"]["successCount"])
def test_failure_taxonomy_separates_platform_faults_from_model_faults(self) -> None:
platform = classify_failure_report(
"EXECUTE_EMPTY_RESULT",