fix: reject broken configs and improve accuracy feedback with isolated agent state

tests/test_accuracy_audit.py

@@ -0,0 +1,96 @@
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+from types import SimpleNamespace
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "modelhub_submmit_api"))
+from failure_taxonomy import classify_failure_report
+from market_intelligence import _validated_official_config, MarketIntelligenceManager
+from routing_engine import SuccessFirstRoutingEngine
+from state_sync import StateGitSync, DEFAULT_BRANCH
+from poll_runner import build_parser
+
+
+class AccuracyAuditTests(unittest.TestCase):
+    def config(self, command):
+        return f"framework: vllm_fix_tokenizer\nsut_config:\n  command: {command}\nref_config: {{}}\n"
+
+    def test_broken_official_and_cached_config_rejected(self):
+        bad = self.config('["/opt/entrypoint.sh    - /model", "--port", "80"]')
+        for gpu in ("Biren_166m", "Kunlunxin_p-800"):
+            self.assertEqual((False, "malformed_command_argv"), _validated_official_config(
+                bad, framework="vllm_fix_tokenizer", target_gpu=gpu))
+            with tempfile.TemporaryDirectory() as tmp:
+                manager = MarketIntelligenceManager(Path(tmp) / "market.json")
+                manager.state = {"frameworkStats": {"text-generation": {gpu: {
+                    "vllm_fix_tokenizer": {"officialConfigValid": True, "officialConfig": bad}}}}}
+                self.assertIsNone(manager.official_config(task_type="text-generation", target_gpu=gpu,
+                                                        framework="vllm_fix_tokenizer"))
+
+    def test_separate_argv_and_shell_command_remain_valid(self):
+        for command in ('["/opt/entrypoint.sh", "/model"]', '["/bin/sh", "-c", "echo hello"]'):
+            self.assertEqual((True, None), _validated_official_config(
+                self.config(command), framework="vllm_fix_tokenizer", target_gpu="Biren_166m"))
+
+    def test_specific_failure_survives_transient_resource_error(self):
+        result = classify_failure_report("MODEL_NOT_SUPPORTED", [
+            "找不到空闲卡", "当前框架版本不支持该模型架构"])
+        self.assertEqual("framework_architecture_unsupported", result.category)
+        self.assertEqual("tokenizer_compatibility", classify_failure_report(
+            "TOKENIZER_FAILED", ["no idle gpu", "Tokenizer class missing"]).category)
+        self.assertEqual("platform_infrastructure", classify_failure_report(
+            "EXECUTE_EMPTY_RESULT", ["no idle gpu"]).category)
+        self.assertEqual("framework_architecture_unsupported", classify_failure_report(
+            None, ["no idle gpu", "Model architectures ['NewForCausalLM'] are not supported for now"]).category)
+
+    def test_launch_and_quantization_have_specific_categories(self):
+        self.assertEqual("configuration_launch", classify_failure_report("EXECUTE_EMPTY_RESULT", [
+            'exec: "/opt/entrypoint.sh    - /model": no such file or directory']).category)
+        self.assertEqual("quantization_compatibility", classify_failure_report("EXECUTE_EMPTY_RESULT", [
+            "The quantization method compressed-tensors is not supported for the current GPU"]).category)
+
+    def test_sparse_failures_lower_score_even_without_classification(self):
+        candidate = {"targetGpu": "gpu", "framework": "vllm", "taskType": "text-generation",
+                     "frameworkMarketSamples": 1000, "frameworkMarketSuccessRate": 0.8,
+                     "preflightMetadata": {"modelType": "family", "estimatedLoadBytes": 1024}}
+        with tempfile.TemporaryDirectory() as tmp:
+            engine = SuccessFirstRoutingEngine(Path(tmp) / "route.json")
+            prior = engine._evidence(candidate)[3]
+            engine.outcome_stats = {"profileCombinationStats": {
+                "gpu|vllm|text-generation|family|none": {
+                    "successCount": 0, "failureCount": 5, "decisionTotal": 0}}}
+            evidence = engine._evidence(candidate)
+            self.assertEqual("local_profile_shrunk", evidence[0])
+            self.assertLess(evidence[3], prior)
+
+    def test_coexistence_defaults_and_submission_provenance(self):
+        with patch.dict(os.environ, {"MODELHUB_COEXISTENCE_MODE": "1", "STRATEGY_ID": "new-agent"}):
+            self.assertTrue(build_parser().parse_args([]).disable_queue_cleanup)
+            intent = StateGitSync._intent({"repoId": "owner/model"}, "batch")
+            self.assertEqual("new-agent", intent["strategyId"])
+            self.assertTrue(intent["agentVersion"])
+            self.assertTrue(intent["instanceId"])
+        self.assertNotEqual("agent-state", DEFAULT_BRANCH)
+
+    def test_model_specific_failures_change_framework_selection(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            manager = MarketIntelligenceManager(Path(tmp) / "market.json")
+            manager.state = {"frameworkStats": {"text-generation": {"gpu": {
+                "first": {"modelCount": 1000, "wilsonLowerBound": 0.8},
+                "second": {"modelCount": 1000, "wilsonLowerBound": 0.6},
+            }}}}
+            manager.set_local_outcome_stats({"profileCombinationStats": {
+                "gpu|first|text-generation|family|none": {"successCount": 0, "failureCount": 4}
+            }})
+            with patch.object(manager, "compatible_discovered_frameworks", return_value=[]):
+                ranked = manager.selectable_frameworks(
+                    task_type="text-generation", target_gpu="gpu", incumbent_frameworks=["first", "second"],
+                    inspection=SimpleNamespace(model_type="family", quantization_method=None))
+            self.assertEqual(["second", "first"], ranked)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_market_intelligence.py

@@ -283,6 +283,7 @@ class MarketIntelligenceTests(unittest.TestCase):
                         "successCount": 10,
                         "failureCount": 3,
                         "attributableFailureCount": 0,
+                        "platformFailureCount": 3,
                     }
                 },
                 "recentCombinationStats": {
@@ -290,6 +291,7 @@ class MarketIntelligenceTests(unittest.TestCase):
                         "successCount": 10,
                         "failureCount": 3,
                         "attributableFailureCount": 0,
+                        "platformFailureCount": 3,
                         "consecutiveFailures": 0,
                         "consecutivePlatformFailures": 3,
                         "lastPlatformFailureAt": datetime.now(timezone.utc).isoformat(),