fix: reject broken configs and improve accuracy feedback with isolated agent state

modelhub_submmit_api/market_intelligence.py

@@ -4,6 +4,7 @@ import math
 import re
 import statistics
 import threading
+import yaml
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime, timedelta
 from pathlib import Path
@@ -97,6 +98,31 @@ def _validated_official_config(config: str, *, framework: str, target_gpu: str)
         return False, "missing_sut_or_ref_config"
     if any(marker in config for marker in ("{{", "}}", "PLACEHOLDER")):
         return False, "unresolved_placeholder"
+    try:
+        document = yaml.safe_load(config)
+    except yaml.YAMLError:
+        return False, "invalid_yaml"
+    if not isinstance(document, dict):
+        return False, "invalid_config_structure"
+
+    def valid_commands(node: Any) -> bool:
+        if isinstance(node, dict):
+            for key, value in node.items():
+                if key in {"command", "entrypoint"} and isinstance(value, list):
+                    if not value or not all(isinstance(arg, str) for arg in value):
+                        return False
+                    # argv[0] is an executable, not a shell command. In
+                    # particular, a folded YAML list item must not eat argv[1].
+                    if not value[0].strip() or re.search(r"\s", value[0]):
+                        return False
+                if not valid_commands(value):
+                    return False
+        elif isinstance(node, list):
+            return all(valid_commands(value) for value in node)
+        return True
+
+    if not valid_commands(document):
+        return False, "malformed_command_argv"
 
     declared = re.search(r"(?m)^\s*framework:\s*['\"]?([^'\"\s]+)", config)
     if declared is None or declared.group(1).strip() != framework:
@@ -736,7 +762,7 @@ class MarketIntelligenceManager:
         local_success = max(0, int(local_item.get("successCount") or 0))
         local_failure = max(
             0,
-            int(local_item.get("attributableFailureCount", local_item.get("failureCount") or 0)),
+            int(local_item.get("failureCount") or 0) - int(local_item.get("platformFailureCount") or 0),
         )
         local_samples = local_success + local_failure
         local_score = _wilson_lower_bound(local_success, local_samples)
@@ -746,7 +772,7 @@ class MarketIntelligenceManager:
         recent_all_failures = max(0, int(recent_item.get("failureCount") or 0))
         recent_failure = max(
             0,
-            int(recent_item.get("attributableFailureCount", recent_item.get("failureCount") or 0)),
+            int(recent_item.get("failureCount") or 0) - int(recent_item.get("platformFailureCount") or 0),
         )
         recent_samples = recent_success + recent_failure
         recent_rate = recent_success / recent_samples if recent_samples else None
@@ -849,11 +875,31 @@ class MarketIntelligenceManager:
             if incumbent_best > 0 and float(evidence["combinedScore"]) < incumbent_best * NEW_FRAMEWORK_PROMOTION_MARGIN:
                 continue
             promoted.append(framework)
-        return self.rank_frameworks(
+        ranked = self.rank_frameworks(
             task_type=task_type,
             target_gpu=target_gpu,
             compatible_frameworks=promoted,
         )
+        model_type = str(getattr(inspection, "model_type", None) or "").strip()
+        quantization = str(getattr(inspection, "quantization_method", None) or "none").strip()
+        if not model_type:
+            return ranked
+
+        def profile_score(framework: str) -> float:
+            prior = float(self._framework_evidence(task_type, target_gpu, framework)["combinedScore"])
+            key = f"{target_gpu}|{framework}|{task_type}|{model_type}|{quantization}"
+            item = (self.local_outcome_stats.get("profileCombinationStats") or {}).get(key) or {}
+            success = max(0, int(item.get("successCount") or 0))
+            failure = max(0, int(item.get("failureCount") or 0) - int(item.get("platformFailureCount") or 0))
+            samples = success + failure
+            if not samples:
+                return prior
+            # Model-specific negative evidence must affect the framework
+            # choice, not only the ranking after the first framework wins.
+            lower = _wilson_lower_bound(success + 8 * prior, samples + 8)
+            return lower
+
+        return sorted(ranked, key=profile_score, reverse=True)
 
     def compatible_discovered_frameworks(
         self,
@@ -941,6 +987,11 @@ class MarketIntelligenceManager:
         config = item.get("officialConfig")
         if not isinstance(config, str) or not config:
             return None
+        valid, reason = _validated_official_config(config, framework=framework, target_gpu=target_gpu)
+        if not valid:
+            item["officialConfigValid"] = False
+            item["officialConfigError"] = reason
+            return None
         return _render_official_config(config, gguf_filename=gguf_filename)
 
     def framework_metadata(self, task_type: str, target_gpu: str, framework: str) -> dict[str, Any]: