feat: learn framework architecture incompatibilities
This commit is contained in:
@@ -6,7 +6,7 @@ import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -475,6 +475,41 @@ class CandidatePreflightTests(unittest.TestCase):
|
||||
self.assertTrue(model.needs_llm)
|
||||
self.assertTrue(oom.deterministic)
|
||||
|
||||
def test_explicit_framework_model_error_is_deterministic_architecture_feedback(self) -> None:
|
||||
classifier = FailureClassifier()
|
||||
result = classify_failure_archive(
|
||||
make_failure_archive(
|
||||
"MODEL_NOT_SUPPORTED",
|
||||
"",
|
||||
"该框架不支持该模型,请换用支持的模型",
|
||||
),
|
||||
llm_classifier=classifier, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
self.assertEqual("framework_architecture_unsupported", result["failureCategory"])
|
||||
self.assertEqual("block_gpu_framework_architecture", result["failureAction"])
|
||||
self.assertTrue(result["failureDeterministic"])
|
||||
self.assertFalse(result["failureNeedsLlm"])
|
||||
self.assertEqual(0, classifier.calls)
|
||||
|
||||
def test_generic_unsupported_backend_does_not_create_architecture_feedback(self) -> None:
|
||||
classification = classify_failure_report(
|
||||
"ATTENTION_NOT_SUPPORTED",
|
||||
["Flash attention backend is not supported on this GPU"],
|
||||
)
|
||||
|
||||
self.assertEqual("attention_backend", classification.category)
|
||||
self.assertFalse(classification.deterministic)
|
||||
|
||||
def test_structured_oom_takes_priority_over_architecture_wording(self) -> None:
|
||||
classification = classify_failure_report(
|
||||
"PREFLIGHT_OOM",
|
||||
["该框架不支持该模型,请换用支持的模型"],
|
||||
)
|
||||
|
||||
self.assertEqual("memory_capacity", classification.category)
|
||||
self.assertEqual("structured_oom", classification.reason)
|
||||
|
||||
def test_failure_archive_uses_deterministic_platform_signature_without_llm(self) -> None:
|
||||
classifier = FailureClassifier()
|
||||
result = classify_failure_archive(
|
||||
@@ -511,6 +546,123 @@ class CandidatePreflightTests(unittest.TestCase):
|
||||
self.assertFalse(result["failureNeedsLlm"])
|
||||
self.assertEqual(1, classifier.calls)
|
||||
|
||||
def test_explicit_failure_learns_exact_gpu_framework_architecture_block(self) -> None:
|
||||
now = datetime.now(timezone.utc)
|
||||
with tempfile.TemporaryDirectory() as temporary_dir:
|
||||
tracker = OutcomeTracker(Path(temporary_dir) / "outcomes.jsonl")
|
||||
tracker.record_submission(
|
||||
"owner/source-model",
|
||||
"Biren_166m",
|
||||
"vllm",
|
||||
"text-generation",
|
||||
"task-architecture-failure",
|
||||
(now - timedelta(minutes=5)).isoformat(),
|
||||
model_profile={
|
||||
"modelType": "qwen2",
|
||||
"architectures": ["Qwen2ForCausalLM"],
|
||||
},
|
||||
)
|
||||
tracker._records[0].update( # noqa: SLF001
|
||||
{
|
||||
"outcome": "failed",
|
||||
"failureCategory": "framework_architecture_unsupported",
|
||||
"failureAction": "block_gpu_framework_architecture",
|
||||
"failureDeterministic": True,
|
||||
"failureClassificationReason": "explicit_framework_model_unsupported",
|
||||
}
|
||||
)
|
||||
report = tracker.get_stats_report()
|
||||
|
||||
key = "biren_166m|vllm|text-generation|architectures:qwen2forcausallm"
|
||||
self.assertIn(key, report["architectureCompatibilityBlocks"])
|
||||
self.assertEqual(
|
||||
1,
|
||||
report["architectureCompatibilitySummary"]["activeBlockCount"],
|
||||
)
|
||||
|
||||
advisor = CandidatePreflightAdvisor(gpu_memory_gib={})
|
||||
advisor.set_feedback_stats(report)
|
||||
exact_architecture = ModelInspection(
|
||||
repo_id="different-name/no-string-match-needed",
|
||||
model_config={
|
||||
"model_type": "qwen2",
|
||||
"architectures": ["Qwen2ForCausalLM"],
|
||||
},
|
||||
)
|
||||
blocked = advisor.assess(
|
||||
inspection=exact_architecture,
|
||||
task_type="text-generation",
|
||||
target_gpu="Biren_166m",
|
||||
framework="vllm",
|
||||
config_params="",
|
||||
)
|
||||
other_framework = advisor.assess(
|
||||
inspection=exact_architecture,
|
||||
task_type="text-generation",
|
||||
target_gpu="Biren_166m",
|
||||
framework="mindie",
|
||||
config_params="",
|
||||
)
|
||||
other_architecture = advisor.assess(
|
||||
inspection=ModelInspection(
|
||||
repo_id="owner/other",
|
||||
model_config={
|
||||
"model_type": "qwen2",
|
||||
"architectures": ["Qwen2ForSequenceClassification"],
|
||||
},
|
||||
),
|
||||
task_type="text-generation",
|
||||
target_gpu="Biren_166m",
|
||||
framework="vllm",
|
||||
config_params="",
|
||||
)
|
||||
|
||||
self.assertFalse(blocked.allowed)
|
||||
self.assertEqual("preflight_learned_architecture_incompatible", blocked.reason)
|
||||
self.assertTrue(other_framework.allowed)
|
||||
self.assertTrue(other_architecture.allowed)
|
||||
self.assertEqual(1, advisor.summary()["architectureCompatibilityBlocksApplied"])
|
||||
|
||||
def test_later_success_clears_learned_architecture_block(self) -> None:
|
||||
now = datetime.now(timezone.utc)
|
||||
profile = {
|
||||
"modelType": "qwen2",
|
||||
"architectures": ["Qwen2ForCausalLM"],
|
||||
}
|
||||
with tempfile.TemporaryDirectory() as temporary_dir:
|
||||
tracker = OutcomeTracker(Path(temporary_dir) / "outcomes.jsonl")
|
||||
tracker.record_submission(
|
||||
"owner/failed",
|
||||
"gpu",
|
||||
"vllm",
|
||||
"text-generation",
|
||||
"task-failed",
|
||||
(now - timedelta(hours=2)).isoformat(),
|
||||
model_profile=profile,
|
||||
)
|
||||
tracker._records[0].update( # noqa: SLF001
|
||||
{
|
||||
"outcome": "failed",
|
||||
"failureCategory": "framework_architecture_unsupported",
|
||||
"failureDeterministic": True,
|
||||
"failureClassificationReason": "explicit_framework_model_unsupported",
|
||||
}
|
||||
)
|
||||
tracker.record_submission(
|
||||
"owner/succeeded",
|
||||
"gpu",
|
||||
"vllm",
|
||||
"text-generation",
|
||||
"task-success",
|
||||
(now - timedelta(hours=1)).isoformat(),
|
||||
model_profile=profile,
|
||||
)
|
||||
tracker._records[1]["outcome"] = "success" # noqa: SLF001
|
||||
|
||||
report = tracker.get_stats_report()
|
||||
|
||||
self.assertEqual({}, report["architectureCompatibilityBlocks"])
|
||||
|
||||
def test_outcome_sync_enriches_failure_and_excludes_platform_fault_from_feedback(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_dir:
|
||||
path = Path(temporary_dir) / "outcomes.jsonl"
|
||||
|
||||
Reference in New Issue
Block a user