fix: protect running validation tasks from queue cleanup
This commit is contained in:
@@ -391,6 +391,78 @@ class CandidatePreflightTests(unittest.TestCase):
|
||||
self.assertEqual(1, report["profileCombinationStats"][key]["failureCount"])
|
||||
self.assertEqual(1, report["recentProfileCombinationStats"][key]["consecutiveFailures"])
|
||||
|
||||
def test_policy_cancellation_is_excluded_from_failure_feedback(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_dir:
|
||||
path = Path(temporary_dir) / "outcomes.jsonl"
|
||||
tracker = OutcomeTracker(path)
|
||||
tracker.record_submission(
|
||||
"owner/model",
|
||||
"gpu",
|
||||
"vllm",
|
||||
"text-generation",
|
||||
"task-policy",
|
||||
datetime.now(timezone.utc).isoformat(),
|
||||
)
|
||||
marked = tracker.mark_policy_cancellations(
|
||||
[
|
||||
{
|
||||
"taskId": "task-policy",
|
||||
"modelId": "owner/model",
|
||||
"gpuType": "gpu",
|
||||
"cleanupReasons": ["old_model_beyond_account_queue_threshold"],
|
||||
}
|
||||
]
|
||||
)
|
||||
tracker.save()
|
||||
tracker.sync_from_api(
|
||||
TaskClient(
|
||||
[
|
||||
{
|
||||
"taskId": "task-policy",
|
||||
"status": "cancelled",
|
||||
"verifyResult": None,
|
||||
}
|
||||
]
|
||||
) # type: ignore[arg-type]
|
||||
)
|
||||
report = tracker.get_stats_report()
|
||||
|
||||
self.assertEqual(1, marked)
|
||||
self.assertEqual(1, report["policyCancelledRecords"])
|
||||
self.assertEqual(0, report["terminalRecords"])
|
||||
self.assertEqual({}, report["combinationStats"])
|
||||
self.assertFalse(tracker.is_model_gpu_failed("owner/model", "gpu"))
|
||||
|
||||
def test_policy_cancellation_does_not_hide_a_racing_success(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_dir:
|
||||
tracker = OutcomeTracker(Path(temporary_dir) / "outcomes.jsonl")
|
||||
tracker.record_submission(
|
||||
"owner/model",
|
||||
"gpu",
|
||||
"vllm",
|
||||
"text-generation",
|
||||
"task-success",
|
||||
datetime.now(timezone.utc).isoformat(),
|
||||
)
|
||||
tracker.mark_policy_cancellations(
|
||||
[{"taskId": "task-success", "modelId": "owner/model", "gpuType": "gpu"}]
|
||||
)
|
||||
tracker.sync_from_api(
|
||||
TaskClient(
|
||||
[
|
||||
{
|
||||
"taskId": "task-success",
|
||||
"status": "success",
|
||||
"verifyResult": 1,
|
||||
}
|
||||
]
|
||||
) # type: ignore[arg-type]
|
||||
)
|
||||
report = tracker.get_stats_report()
|
||||
|
||||
self.assertEqual(0, report["policyCancelledRecords"])
|
||||
self.assertEqual(1, report["totals"]["successCount"])
|
||||
|
||||
def test_failure_taxonomy_separates_platform_faults_from_model_faults(self) -> None:
|
||||
platform = classify_failure_report(
|
||||
"EXECUTE_EMPTY_RESULT",
|
||||
|
||||
Reference in New Issue
Block a user