Bootstrap architecture rules from task history
This commit is contained in:
@@ -574,6 +574,60 @@ class CandidatePreflightTests(unittest.TestCase):
|
||||
self.assertFalse(result["failureNeedsLlm"])
|
||||
self.assertEqual(1, classifier.calls)
|
||||
|
||||
def test_failure_archive_recovers_framework_from_target_docker_image(self) -> None:
|
||||
result = classify_failure_archive(
|
||||
make_failure_archive(
|
||||
"MODEL_NOT_SUPPORTED",
|
||||
"\n".join(
|
||||
[
|
||||
"[submit]docker_image: registry/enginex-sunrise/enginex-s2-vllm:v1",
|
||||
"model type `qwen3_5` but Transformers does not recognize this architecture",
|
||||
]
|
||||
),
|
||||
"请换用支持的模型",
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual("vllm", result["failureDetectedFramework"])
|
||||
self.assertEqual("target_docker_image", result["failureDetectedFrameworkSource"])
|
||||
self.assertEqual(["qwen3_5"], result["failureUnsupportedModelTypes"])
|
||||
|
||||
def test_history_bootstrap_builds_block_without_api_framework_field(self) -> None:
|
||||
now = datetime.now(timezone.utc)
|
||||
task = {
|
||||
"taskId": "history-failure",
|
||||
"modelId": "owner/history-model",
|
||||
"gpuType": "Biren_166m",
|
||||
"modelTaskLevelId": 23,
|
||||
"status": "success",
|
||||
"verifyResult": -1,
|
||||
"updateTime": now.isoformat(),
|
||||
"logCosUrl": "https://logs.invalid/history-failure.zip",
|
||||
}
|
||||
classification = {
|
||||
"failureCategory": "framework_architecture_unsupported",
|
||||
"failureScope": "model_gpu_framework",
|
||||
"failureAction": "block_gpu_framework_architecture",
|
||||
"failureDeterministic": True,
|
||||
"failureClassificationReason": "explicit_framework_model_unsupported",
|
||||
"failureUnsupportedModelTypes": ["qwen3_5"],
|
||||
"failureDetectedFramework": "vllm",
|
||||
"failureDetectedFrameworkSource": "target_docker_image",
|
||||
}
|
||||
with tempfile.TemporaryDirectory() as temporary_dir:
|
||||
tracker = OutcomeTracker(Path(temporary_dir) / "outcomes.jsonl")
|
||||
with patch(
|
||||
"outcome_tracker.fetch_and_classify_failure_log",
|
||||
return_value=classification,
|
||||
):
|
||||
summary = tracker.bootstrap_from_history_tasks([task])
|
||||
report = tracker.get_stats_report()
|
||||
|
||||
key = "biren_166m|vllm|text-generation|model_type:qwen3_5"
|
||||
self.assertEqual(1, summary["enrichmentAttempts"])
|
||||
self.assertEqual(1, summary["recoveredFrameworks"])
|
||||
self.assertIn(key, report["architectureCompatibilityBlocks"])
|
||||
|
||||
def test_explicit_failure_learns_exact_gpu_framework_architecture_block(self) -> None:
|
||||
now = datetime.now(timezone.utc)
|
||||
with tempfile.TemporaryDirectory() as temporary_dir:
|
||||
|
||||
@@ -15,10 +15,61 @@ if str(PACKAGE_DIR) in sys.path:
|
||||
sys.path.insert(0, str(PACKAGE_DIR))
|
||||
|
||||
from outcome_tracker import OutcomeTracker # noqa: E402
|
||||
from poll_runner import _load_task_compatibility_contexts, resolve_age_cleanup_policy # noqa: E402
|
||||
from poll_runner import ( # noqa: E402
|
||||
_bootstrap_architecture_history,
|
||||
_load_task_compatibility_contexts,
|
||||
resolve_age_cleanup_policy,
|
||||
)
|
||||
|
||||
|
||||
class PollPolicyTests(unittest.TestCase):
|
||||
def test_architecture_bootstrap_falls_back_when_public_logs_are_hidden(self) -> None:
|
||||
class HistoryClient:
|
||||
@staticmethod
|
||||
def list_tasks_page(**_kwargs): # noqa: ANN003
|
||||
return {
|
||||
"data": {
|
||||
"records": [
|
||||
{
|
||||
"taskId": "public-failure",
|
||||
"modelId": "public/model",
|
||||
"gpuType": "gpu",
|
||||
"status": "success",
|
||||
"verifyResult": -1,
|
||||
"logCosUrl": None,
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def list_tasks(**_kwargs): # noqa: ANN003
|
||||
return [
|
||||
{
|
||||
"taskId": "owned-success",
|
||||
"modelId": "owner/model",
|
||||
"gpuType": "gpu",
|
||||
"modelTaskLevelId": 23,
|
||||
"status": "success",
|
||||
"verifyResult": 1,
|
||||
"updateTime": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
]
|
||||
|
||||
with tempfile.TemporaryDirectory() as temporary_dir:
|
||||
root = Path(temporary_dir)
|
||||
tracker = OutcomeTracker(root / "outcomes.jsonl")
|
||||
summary = _bootstrap_architecture_history(
|
||||
modelhub_client=HistoryClient(), # type: ignore[arg-type]
|
||||
outcome_tracker=tracker,
|
||||
ledger_path=root / "ledger.jsonl",
|
||||
now=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
self.assertEqual("owned_full_history", summary["source"])
|
||||
self.assertEqual(1, summary["terminalRecords"])
|
||||
self.assertEqual(0, summary["communityUsableFailureDetails"])
|
||||
|
||||
def test_cleanup_contexts_merge_outcomes_with_older_ledger_entries(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_dir:
|
||||
root = Path(temporary_dir)
|
||||
|
||||
Reference in New Issue
Block a user