feat: learn framework architecture incompatibilities

This commit is contained in:
CoolBoy
2026-08-12 08:02:51 +08:00
parent d908706f9a
commit 615bcad124
10 changed files with 503 additions and 9 deletions

View File

@@ -6,7 +6,7 @@ import sys
import tempfile
import unittest
import zipfile
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
from pathlib import Path
from unittest.mock import patch
@@ -475,6 +475,41 @@ class CandidatePreflightTests(unittest.TestCase):
self.assertTrue(model.needs_llm)
self.assertTrue(oom.deterministic)
def test_explicit_framework_model_error_is_deterministic_architecture_feedback(self) -> None:
classifier = FailureClassifier()
result = classify_failure_archive(
make_failure_archive(
"MODEL_NOT_SUPPORTED",
"",
"该框架不支持该模型,请换用支持的模型",
),
llm_classifier=classifier, # type: ignore[arg-type]
)
self.assertEqual("framework_architecture_unsupported", result["failureCategory"])
self.assertEqual("block_gpu_framework_architecture", result["failureAction"])
self.assertTrue(result["failureDeterministic"])
self.assertFalse(result["failureNeedsLlm"])
self.assertEqual(0, classifier.calls)
def test_generic_unsupported_backend_does_not_create_architecture_feedback(self) -> None:
classification = classify_failure_report(
"ATTENTION_NOT_SUPPORTED",
["Flash attention backend is not supported on this GPU"],
)
self.assertEqual("attention_backend", classification.category)
self.assertFalse(classification.deterministic)
def test_structured_oom_takes_priority_over_architecture_wording(self) -> None:
classification = classify_failure_report(
"PREFLIGHT_OOM",
["该框架不支持该模型,请换用支持的模型"],
)
self.assertEqual("memory_capacity", classification.category)
self.assertEqual("structured_oom", classification.reason)
def test_failure_archive_uses_deterministic_platform_signature_without_llm(self) -> None:
classifier = FailureClassifier()
result = classify_failure_archive(
@@ -511,6 +546,123 @@ class CandidatePreflightTests(unittest.TestCase):
self.assertFalse(result["failureNeedsLlm"])
self.assertEqual(1, classifier.calls)
def test_explicit_failure_learns_exact_gpu_framework_architecture_block(self) -> None:
now = datetime.now(timezone.utc)
with tempfile.TemporaryDirectory() as temporary_dir:
tracker = OutcomeTracker(Path(temporary_dir) / "outcomes.jsonl")
tracker.record_submission(
"owner/source-model",
"Biren_166m",
"vllm",
"text-generation",
"task-architecture-failure",
(now - timedelta(minutes=5)).isoformat(),
model_profile={
"modelType": "qwen2",
"architectures": ["Qwen2ForCausalLM"],
},
)
tracker._records[0].update( # noqa: SLF001
{
"outcome": "failed",
"failureCategory": "framework_architecture_unsupported",
"failureAction": "block_gpu_framework_architecture",
"failureDeterministic": True,
"failureClassificationReason": "explicit_framework_model_unsupported",
}
)
report = tracker.get_stats_report()
key = "biren_166m|vllm|text-generation|architectures:qwen2forcausallm"
self.assertIn(key, report["architectureCompatibilityBlocks"])
self.assertEqual(
1,
report["architectureCompatibilitySummary"]["activeBlockCount"],
)
advisor = CandidatePreflightAdvisor(gpu_memory_gib={})
advisor.set_feedback_stats(report)
exact_architecture = ModelInspection(
repo_id="different-name/no-string-match-needed",
model_config={
"model_type": "qwen2",
"architectures": ["Qwen2ForCausalLM"],
},
)
blocked = advisor.assess(
inspection=exact_architecture,
task_type="text-generation",
target_gpu="Biren_166m",
framework="vllm",
config_params="",
)
other_framework = advisor.assess(
inspection=exact_architecture,
task_type="text-generation",
target_gpu="Biren_166m",
framework="mindie",
config_params="",
)
other_architecture = advisor.assess(
inspection=ModelInspection(
repo_id="owner/other",
model_config={
"model_type": "qwen2",
"architectures": ["Qwen2ForSequenceClassification"],
},
),
task_type="text-generation",
target_gpu="Biren_166m",
framework="vllm",
config_params="",
)
self.assertFalse(blocked.allowed)
self.assertEqual("preflight_learned_architecture_incompatible", blocked.reason)
self.assertTrue(other_framework.allowed)
self.assertTrue(other_architecture.allowed)
self.assertEqual(1, advisor.summary()["architectureCompatibilityBlocksApplied"])
def test_later_success_clears_learned_architecture_block(self) -> None:
now = datetime.now(timezone.utc)
profile = {
"modelType": "qwen2",
"architectures": ["Qwen2ForCausalLM"],
}
with tempfile.TemporaryDirectory() as temporary_dir:
tracker = OutcomeTracker(Path(temporary_dir) / "outcomes.jsonl")
tracker.record_submission(
"owner/failed",
"gpu",
"vllm",
"text-generation",
"task-failed",
(now - timedelta(hours=2)).isoformat(),
model_profile=profile,
)
tracker._records[0].update( # noqa: SLF001
{
"outcome": "failed",
"failureCategory": "framework_architecture_unsupported",
"failureDeterministic": True,
"failureClassificationReason": "explicit_framework_model_unsupported",
}
)
tracker.record_submission(
"owner/succeeded",
"gpu",
"vllm",
"text-generation",
"task-success",
(now - timedelta(hours=1)).isoformat(),
model_profile=profile,
)
tracker._records[1]["outcome"] = "success" # noqa: SLF001
report = tracker.get_stats_report()
self.assertEqual({}, report["architectureCompatibilityBlocks"])
def test_outcome_sync_enriches_failure_and_excludes_platform_fault_from_feedback(self) -> None:
with tempfile.TemporaryDirectory() as temporary_dir:
path = Path(temporary_dir) / "outcomes.jsonl"