From 484a10607c946fec2133de2b000685d54eeab056 Mon Sep 17 00:00:00 2001 From: CoolBoy Date: Sun, 18 Oct 2026 16:29:18 +0800 Subject: [PATCH] fix: reject broken configs and improve accuracy feedback with isolated agent state --- README.md | 29 ++++++- modelhub_submmit_api/failure_taxonomy.py | 36 +++++--- modelhub_submmit_api/market_intelligence.py | 57 +++++++++++- modelhub_submmit_api/outcome_tracker.py | 6 ++ modelhub_submmit_api/poll_runner.py | 9 +- modelhub_submmit_api/routing_engine.py | 22 ++++- modelhub_submmit_api/state_sync.py | 9 +- modelhub_submmit_api/version.py | 2 +- tests/test_accuracy_audit.py | 96 +++++++++++++++++++++ tests/test_market_intelligence.py | 2 + 10 files changed, 243 insertions(+), 25 deletions(-) create mode 100644 tests/test_accuracy_audit.py diff --git a/README.md b/README.md index 764606bafd..b13dbb9e9c 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,30 @@ This repository is packaged for the ModelHub XC agent platform. +## Accuracy release 2026.10.18.1 + +This release rejects malformed YAML command arrays, including the observed +`/opt/entrypoint.sh - /model` executable error, even in restored cached +official configurations. It preserves terminal compatibility errors when a +log also contains transient platform resource messages. Framework scoring +includes unresolved non-platform failures; sparse model-profile evidence is +blended with eight prior observations instead of discarded below a threshold. + +For operation alongside the baseline agent, the new default state branch is +`agent-state-accuracy-v2`. Do not override it with the baseline's `agent-state`. +The separate branch starts its own checkpoint on first boot; it does not +overwrite or migrate the baseline state. New submissions retain agent version, +instance ID and platform strategy ID. Historical tasks are not assigned an +invented version. + +`MODELHUB_COEXISTENCE_MODE=1` is the default: this agent does not run queue +cancellation, including deterministic cleanup, because the accounts are shared +with the baseline. Submission preflight remains enabled. Set this variable to +`0` only when account-wide cleanup is intended; `--disable-queue-cleanup` still +disables cleanup explicitly. The baseline agent is unchanged. Both agents share +account capacity and platform deduplication; separate state does not reserve +capacity or eliminate simultaneous submission races. + ## Platform Contract - Root-level `Dockerfile` @@ -67,7 +91,7 @@ Optional tuning: - `MODELHUB_RECENT_MODEL_RESERVE_SLOTS` default `5` per account - `MODELHUB_RECENT_MODEL_DAYS` default `7` - `MODELHUB_STATE_SYNC_REMOTE` default `https://dev.modelhub.org.cn/CoolBoy/submmit.git` -- `MODELHUB_STATE_SYNC_BRANCH` default `agent-state` +- `MODELHUB_STATE_SYNC_BRANCH` default `agent-state-accuracy-v2` - `MODELHUB_STATE_SYNC_BATCH_SIZE` default `20` Git state synchronization reads `modelhub_user_name`, `modelhub_user_email`, and @@ -140,7 +164,8 @@ unvetted GPU or framework. ## Durable State Branch The hosted worker synchronizes an allowlisted runtime snapshot to the orphan -`agent-state` branch of this repository. Before each batch of at most 20 API +`agent-state-accuracy-v2` branch of this repository (overridable via +`MODELHUB_STATE_SYNC_BRANCH`). Before each batch of at most 20 API submissions it pushes write-ahead intents containing model, GPU, task, framework, safe parameter vector, and a config fingerprint. It pushes task IDs and results after the batch. If either push fails, no further submission begins until the diff --git a/modelhub_submmit_api/failure_taxonomy.py b/modelhub_submmit_api/failure_taxonomy.py index bf8fa2f95f..ee4ba1c6fe 100644 --- a/modelhub_submmit_api/failure_taxonomy.py +++ b/modelhub_submmit_api/failure_taxonomy.py @@ -67,7 +67,7 @@ PLATFORM_PATTERNS = ( (re.compile(r"welcome\.sh: No such file|data: command not found", re.I), "broken_platform_launch_script"), (re.compile(r"id_rsa.*No such file", re.I), "missing_platform_credential"), (re.compile(r"找不到空闲卡|no idle (?:gpu|card)", re.I), "no_idle_device"), - (re.compile(r"storage|download.*timed? out|connection reset", re.I), "platform_io_transient"), + (re.compile(r"no space left on device|download.*timed? out|connection reset", re.I), "platform_io_transient"), ) DETERMINISTIC_LOG_PATTERNS = ( @@ -80,6 +80,8 @@ DETERMINISTIC_LOG_PATTERNS = ( # semantics. Generic messages such as "attention backend not supported" or # "GPU type not supported" must not create an architecture blacklist. EXPLICIT_FRAMEWORK_MODEL_UNSUPPORTED_PATTERNS = ( + re.compile(r"model type\s+[`'\"][^`'\"]+[`'\"].{0,100}does not recognize this architecture", re.I), + re.compile(r"Model architectures?\s*\[[^\]\n]+\]\s*(?:are|is) not supported", re.I), re.compile( r"(?:当前|该|此)(?:推理)?框架.{0,100}(?:不支持|暂不支持|无法支持|不兼容)" r".{0,100}(?:该|此|当前)?(?:模型架构|模型|架构)", @@ -117,19 +119,16 @@ def classify_failure_report(report_code: str | None, log_lines: Iterable[str] = code = str(report_code or "").strip().upper() text = "\n".join(str(line) for line in log_lines) - # Infrastructure signatures override broad report codes such as - # EXECUTE_EMPTY_RESULT so they do not poison model compatibility feedback. - for pattern, reason in PLATFORM_PATTERNS: - if pattern.search(text): - return FailureClassification( - "platform_infrastructure", - "gpu_framework", - "open_short_gpu_framework_circuit_and_retry_other_models", - True, - False, - reason, - ) - + if re.search(r'exec:.*entrypoint.*(?:no such file|not found)', text, re.I): + return FailureClassification( + "configuration_launch", "configuration", "reject_broken_config", True, False, + "invalid_executable_argv", + ) + if re.search(r'quantization method .*not supported for the current GPU', text, re.I): + return FailureClassification( + "quantization_compatibility", "model_gpu_framework", "reject_quantization_route", + True, False, "explicit_quantization_gpu_incompatibility", + ) if code in DETERMINISTIC_POLICIES: return DETERMINISTIC_POLICIES[code] for pattern, inferred_code in DETERMINISTIC_LOG_PATTERNS: @@ -149,6 +148,15 @@ def classify_failure_report(report_code: str | None, log_lines: Iterable[str] = ) if code in SEMANTIC_POLICIES: return SEMANTIC_POLICIES[code] + # Fall back to infrastructure signatures only after specific root causes. + # Logs may include a transient allocation failure before a model starts. + for pattern, reason in PLATFORM_PATTERNS: + if pattern.search(text): + return FailureClassification( + "platform_infrastructure", "gpu_framework", + "open_short_gpu_framework_circuit_and_retry_other_models", + True, False, reason, + ) return FailureClassification( "ambiguous_runtime", "unknown", diff --git a/modelhub_submmit_api/market_intelligence.py b/modelhub_submmit_api/market_intelligence.py index b9a86dde7f..58aae57ac2 100644 --- a/modelhub_submmit_api/market_intelligence.py +++ b/modelhub_submmit_api/market_intelligence.py @@ -4,6 +4,7 @@ import math import re import statistics import threading +import yaml from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timedelta from pathlib import Path @@ -97,6 +98,31 @@ def _validated_official_config(config: str, *, framework: str, target_gpu: str) return False, "missing_sut_or_ref_config" if any(marker in config for marker in ("{{", "}}", "PLACEHOLDER")): return False, "unresolved_placeholder" + try: + document = yaml.safe_load(config) + except yaml.YAMLError: + return False, "invalid_yaml" + if not isinstance(document, dict): + return False, "invalid_config_structure" + + def valid_commands(node: Any) -> bool: + if isinstance(node, dict): + for key, value in node.items(): + if key in {"command", "entrypoint"} and isinstance(value, list): + if not value or not all(isinstance(arg, str) for arg in value): + return False + # argv[0] is an executable, not a shell command. In + # particular, a folded YAML list item must not eat argv[1]. + if not value[0].strip() or re.search(r"\s", value[0]): + return False + if not valid_commands(value): + return False + elif isinstance(node, list): + return all(valid_commands(value) for value in node) + return True + + if not valid_commands(document): + return False, "malformed_command_argv" declared = re.search(r"(?m)^\s*framework:\s*['\"]?([^'\"\s]+)", config) if declared is None or declared.group(1).strip() != framework: @@ -736,7 +762,7 @@ class MarketIntelligenceManager: local_success = max(0, int(local_item.get("successCount") or 0)) local_failure = max( 0, - int(local_item.get("attributableFailureCount", local_item.get("failureCount") or 0)), + int(local_item.get("failureCount") or 0) - int(local_item.get("platformFailureCount") or 0), ) local_samples = local_success + local_failure local_score = _wilson_lower_bound(local_success, local_samples) @@ -746,7 +772,7 @@ class MarketIntelligenceManager: recent_all_failures = max(0, int(recent_item.get("failureCount") or 0)) recent_failure = max( 0, - int(recent_item.get("attributableFailureCount", recent_item.get("failureCount") or 0)), + int(recent_item.get("failureCount") or 0) - int(recent_item.get("platformFailureCount") or 0), ) recent_samples = recent_success + recent_failure recent_rate = recent_success / recent_samples if recent_samples else None @@ -849,11 +875,31 @@ class MarketIntelligenceManager: if incumbent_best > 0 and float(evidence["combinedScore"]) < incumbent_best * NEW_FRAMEWORK_PROMOTION_MARGIN: continue promoted.append(framework) - return self.rank_frameworks( + ranked = self.rank_frameworks( task_type=task_type, target_gpu=target_gpu, compatible_frameworks=promoted, ) + model_type = str(getattr(inspection, "model_type", None) or "").strip() + quantization = str(getattr(inspection, "quantization_method", None) or "none").strip() + if not model_type: + return ranked + + def profile_score(framework: str) -> float: + prior = float(self._framework_evidence(task_type, target_gpu, framework)["combinedScore"]) + key = f"{target_gpu}|{framework}|{task_type}|{model_type}|{quantization}" + item = (self.local_outcome_stats.get("profileCombinationStats") or {}).get(key) or {} + success = max(0, int(item.get("successCount") or 0)) + failure = max(0, int(item.get("failureCount") or 0) - int(item.get("platformFailureCount") or 0)) + samples = success + failure + if not samples: + return prior + # Model-specific negative evidence must affect the framework + # choice, not only the ranking after the first framework wins. + lower = _wilson_lower_bound(success + 8 * prior, samples + 8) + return lower + + return sorted(ranked, key=profile_score, reverse=True) def compatible_discovered_frameworks( self, @@ -941,6 +987,11 @@ class MarketIntelligenceManager: config = item.get("officialConfig") if not isinstance(config, str) or not config: return None + valid, reason = _validated_official_config(config, framework=framework, target_gpu=target_gpu) + if not valid: + item["officialConfigValid"] = False + item["officialConfigError"] = reason + return None return _render_official_config(config, gguf_filename=gguf_filename) def framework_metadata(self, task_type: str, target_gpu: str, framework: str) -> dict[str, Any]: diff --git a/modelhub_submmit_api/outcome_tracker.py b/modelhub_submmit_api/outcome_tracker.py index 9023dbe5f0..7ce4e65ee7 100644 --- a/modelhub_submmit_api/outcome_tracker.py +++ b/modelhub_submmit_api/outcome_tracker.py @@ -22,6 +22,8 @@ from history_stats import classify_failure, is_failure, is_success from llm_classifier import LLMAssistedClassifier from modelhub_client import ModelHubClient, ModelHubClientPool from task_registry import task_type_from_history_task +from common import runtime_instance_id +from version import AGENT_VERSION DEFAULT_OUTCOMES_PATH = Path("outcomes/submissions.jsonl") @@ -36,6 +38,7 @@ FAILURE_ENRICHMENT_MAX_ATTEMPTS = 3 TERMINAL_TASK_STATUSES = {"success", "failed", "error", "cancelled", "completed"} DECISION_RECORD_FIELDS = { + "agentVersion", "instanceId", "strategyId", "taskId", "modelId", "targetGpu", @@ -325,6 +328,9 @@ class OutcomeTracker: model_profile: dict[str, Any] | None = None, ) -> None: record: dict[str, Any] = { + "agentVersion": AGENT_VERSION, + "instanceId": runtime_instance_id(), + "strategyId": os.getenv("STRATEGY_ID"), "modelId": model_id, "targetGpu": target_gpu, "framework": framework, diff --git a/modelhub_submmit_api/poll_runner.py b/modelhub_submmit_api/poll_runner.py index 44744eabf1..41d6e44084 100644 --- a/modelhub_submmit_api/poll_runner.py +++ b/modelhub_submmit_api/poll_runner.py @@ -234,7 +234,9 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--post-cycle-cooldown-seconds", type=int, default=2, help="Short sleep after a successful cycle") parser.add_argument("--max-cycles", type=int, default=0, help="Optional hard stop after N cycles; 0 means run until quota is reached") parser.add_argument("--print-stats", action="store_true", help="Load outcomes, sync, print stats report, and exit") - parser.add_argument("--disable-queue-cleanup", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--disable-queue-cleanup", action="store_true", + default=os.getenv("MODELHUB_COEXISTENCE_MODE", "1") != "0", + help=argparse.SUPPRESS) parser.add_argument( "--queue-cleanup-interval-cycles", type=int, @@ -702,6 +704,11 @@ def run_poll_loop( STATS_PRINT_INTERVAL = 10 log(f"[poll] version={AGENT_VERSION} poll_run_dir={poll_run_dir}") + log( + f"[coexistence] state_branch={base_args.state_sync_branch} " + f"queue_cancellation={'disabled' if base_args.disable_queue_cleanup else 'enabled'} " + "submission_preflight=enabled" + ) log( "[state-retention] mode=decision_state_only full_archive=disabled " f"recent_outcomes={DEFAULT_RECENT_OUTCOME_LIMIT} " diff --git a/modelhub_submmit_api/routing_engine.py b/modelhub_submmit_api/routing_engine.py index ddc5c6c685..52130d56a8 100644 --- a/modelhub_submmit_api/routing_engine.py +++ b/modelhub_submmit_api/routing_engine.py @@ -46,6 +46,13 @@ class SuccessFirstRoutingEngine: return value def _evidence(self, candidate: dict[str, Any]) -> tuple[str, int, int, float]: + def observed_total(item: dict[str, Any]) -> int: + if "failureCount" in item: + return max(0, int(item.get("successCount") or 0) + int(item.get("failureCount") or 0) + - int(item.get("platformFailureCount") or 0)) + return max(0, int(item.get("decisionTotal") or 0)) + + sparse = None profile = candidate.get("preflightMetadata") or {} gpu = str(candidate.get("targetGpu") or "") framework = str(candidate.get("framework") or "") @@ -60,14 +67,18 @@ class SuccessFirstRoutingEngine: size_bucket = int(load_bytes).bit_length() - 1 key = f"{gpu}|{framework}|{task_type}|{model_type}|{quantization}|{size_bucket}" item = (self.outcome_stats.get("sizedProfileCombinationStats") or {}).get(key) or {} - total = int(item.get("decisionTotal") or 0) + total = observed_total(item) + if total: + sparse = ("local_profile_size_shrunk", item, total) if total >= 20: success = int(item.get("successCount") or 0) return "local_profile_size", success, total, wilson_lower_bound(success, total) if model_type: key = f"{gpu}|{framework}|{task_type}|{model_type}|{quantization}" item = (self.outcome_stats.get("profileCombinationStats") or {}).get(key) or {} - total = int(item.get("decisionTotal") or 0) + total = observed_total(item) + if total and sparse is None: + sparse = ("local_profile_shrunk", item, total) if total >= 40: success = int(item.get("successCount") or 0) return "local_profile", success, total, wilson_lower_bound(success, total) @@ -97,6 +108,13 @@ class SuccessFirstRoutingEngine: total = 2 success = 1 level = "global_conservative_prior" + if sparse is not None: + level, item, samples = sparse + # Eight prior observations stabilize small groups without hiding + # their failures behind an unreachable hard sample threshold. + prior_success = round(8 * success / max(1, total)) + success = int(item.get("successCount") or 0) + prior_success + total = samples + 8 return level, success, total, wilson_lower_bound(success, total) def annotate(self, candidate: dict[str, Any]) -> dict[str, Any]: diff --git a/modelhub_submmit_api/state_sync.py b/modelhub_submmit_api/state_sync.py index 839721e244..4d5c51a6a1 100644 --- a/modelhub_submmit_api/state_sync.py +++ b/modelhub_submmit_api/state_sync.py @@ -17,14 +17,14 @@ from typing import Any, Iterable from dulwich import porcelain from dulwich.repo import Repo -from common import read_json, read_jsonl, write_json, write_jsonl +from common import read_json, read_jsonl, write_json, write_jsonl, runtime_instance_id from runner_common import load_key_files from version import AGENT_VERSION STATE_SCHEMA_VERSION = 1 DEFAULT_REMOTE = "https://dev.modelhub.org.cn/CoolBoy/submmit.git" -DEFAULT_BRANCH = "agent-state" +DEFAULT_BRANCH = "agent-state-accuracy-v2" DEFAULT_BATCH_SIZE = 100 DEFAULT_HISTORY_DEPTH = 200 DEFAULT_RECENT_TERMINAL_INTENTS = 300 @@ -32,6 +32,7 @@ DEFAULT_LEDGER_RECORDS = 500 DEFAULT_CRASH_RECORDS = 50 STATE_OUTCOME_FIELDS = { + "agentVersion", "instanceId", "strategyId", "taskId", "modelId", "targetGpu", "framework", "taskType", "submitTime", "lastSyncTime", "status", "verifyResult", "outcome", "failReason", "modelProfile", "failureCode", "failureCategory", "failureScope", @@ -342,6 +343,7 @@ class StateGitSync: retained_terminal = terminal[:DEFAULT_RECENT_TERMINAL_INTENTS] discarded = terminal[DEFAULT_RECENT_TERMINAL_INTENTS:] compact_fields = { + "agentVersion", "instanceId", "strategyId", "intentId", "batchId", "status", "createdAt", "completedAt", "repoId", "targetGpu", "taskType", "framework", "configSource", "configFingerprint", "safeConfigVector", "taskId", @@ -372,6 +374,9 @@ class StateGitSync: config = str(candidate.get("configParams") or "") return { "intentId": uuid.uuid4().hex, + "agentVersion": AGENT_VERSION, + "instanceId": runtime_instance_id(), + "strategyId": os.getenv("STRATEGY_ID"), "batchId": batch_id, "status": "pending", "createdAt": _utc_now().isoformat(), diff --git a/modelhub_submmit_api/version.py b/modelhub_submmit_api/version.py index 669bf78e6e..0d4309eb82 100644 --- a/modelhub_submmit_api/version.py +++ b/modelhub_submmit_api/version.py @@ -1 +1 @@ -AGENT_VERSION = "2026.09.22.1" +AGENT_VERSION = "2026.10.18.1" diff --git a/tests/test_accuracy_audit.py b/tests/test_accuracy_audit.py new file mode 100644 index 0000000000..e31e021e41 --- /dev/null +++ b/tests/test_accuracy_audit.py @@ -0,0 +1,96 @@ +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch +from types import SimpleNamespace + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "modelhub_submmit_api")) +from failure_taxonomy import classify_failure_report +from market_intelligence import _validated_official_config, MarketIntelligenceManager +from routing_engine import SuccessFirstRoutingEngine +from state_sync import StateGitSync, DEFAULT_BRANCH +from poll_runner import build_parser + + +class AccuracyAuditTests(unittest.TestCase): + def config(self, command): + return f"framework: vllm_fix_tokenizer\nsut_config:\n command: {command}\nref_config: {{}}\n" + + def test_broken_official_and_cached_config_rejected(self): + bad = self.config('["/opt/entrypoint.sh - /model", "--port", "80"]') + for gpu in ("Biren_166m", "Kunlunxin_p-800"): + self.assertEqual((False, "malformed_command_argv"), _validated_official_config( + bad, framework="vllm_fix_tokenizer", target_gpu=gpu)) + with tempfile.TemporaryDirectory() as tmp: + manager = MarketIntelligenceManager(Path(tmp) / "market.json") + manager.state = {"frameworkStats": {"text-generation": {gpu: { + "vllm_fix_tokenizer": {"officialConfigValid": True, "officialConfig": bad}}}}} + self.assertIsNone(manager.official_config(task_type="text-generation", target_gpu=gpu, + framework="vllm_fix_tokenizer")) + + def test_separate_argv_and_shell_command_remain_valid(self): + for command in ('["/opt/entrypoint.sh", "/model"]', '["/bin/sh", "-c", "echo hello"]'): + self.assertEqual((True, None), _validated_official_config( + self.config(command), framework="vllm_fix_tokenizer", target_gpu="Biren_166m")) + + def test_specific_failure_survives_transient_resource_error(self): + result = classify_failure_report("MODEL_NOT_SUPPORTED", [ + "找不到空闲卡", "当前框架版本不支持该模型架构"]) + self.assertEqual("framework_architecture_unsupported", result.category) + self.assertEqual("tokenizer_compatibility", classify_failure_report( + "TOKENIZER_FAILED", ["no idle gpu", "Tokenizer class missing"]).category) + self.assertEqual("platform_infrastructure", classify_failure_report( + "EXECUTE_EMPTY_RESULT", ["no idle gpu"]).category) + self.assertEqual("framework_architecture_unsupported", classify_failure_report( + None, ["no idle gpu", "Model architectures ['NewForCausalLM'] are not supported for now"]).category) + + def test_launch_and_quantization_have_specific_categories(self): + self.assertEqual("configuration_launch", classify_failure_report("EXECUTE_EMPTY_RESULT", [ + 'exec: "/opt/entrypoint.sh - /model": no such file or directory']).category) + self.assertEqual("quantization_compatibility", classify_failure_report("EXECUTE_EMPTY_RESULT", [ + "The quantization method compressed-tensors is not supported for the current GPU"]).category) + + def test_sparse_failures_lower_score_even_without_classification(self): + candidate = {"targetGpu": "gpu", "framework": "vllm", "taskType": "text-generation", + "frameworkMarketSamples": 1000, "frameworkMarketSuccessRate": 0.8, + "preflightMetadata": {"modelType": "family", "estimatedLoadBytes": 1024}} + with tempfile.TemporaryDirectory() as tmp: + engine = SuccessFirstRoutingEngine(Path(tmp) / "route.json") + prior = engine._evidence(candidate)[3] + engine.outcome_stats = {"profileCombinationStats": { + "gpu|vllm|text-generation|family|none": { + "successCount": 0, "failureCount": 5, "decisionTotal": 0}}} + evidence = engine._evidence(candidate) + self.assertEqual("local_profile_shrunk", evidence[0]) + self.assertLess(evidence[3], prior) + + def test_coexistence_defaults_and_submission_provenance(self): + with patch.dict(os.environ, {"MODELHUB_COEXISTENCE_MODE": "1", "STRATEGY_ID": "new-agent"}): + self.assertTrue(build_parser().parse_args([]).disable_queue_cleanup) + intent = StateGitSync._intent({"repoId": "owner/model"}, "batch") + self.assertEqual("new-agent", intent["strategyId"]) + self.assertTrue(intent["agentVersion"]) + self.assertTrue(intent["instanceId"]) + self.assertNotEqual("agent-state", DEFAULT_BRANCH) + + def test_model_specific_failures_change_framework_selection(self): + with tempfile.TemporaryDirectory() as tmp: + manager = MarketIntelligenceManager(Path(tmp) / "market.json") + manager.state = {"frameworkStats": {"text-generation": {"gpu": { + "first": {"modelCount": 1000, "wilsonLowerBound": 0.8}, + "second": {"modelCount": 1000, "wilsonLowerBound": 0.6}, + }}}} + manager.set_local_outcome_stats({"profileCombinationStats": { + "gpu|first|text-generation|family|none": {"successCount": 0, "failureCount": 4} + }}) + with patch.object(manager, "compatible_discovered_frameworks", return_value=[]): + ranked = manager.selectable_frameworks( + task_type="text-generation", target_gpu="gpu", incumbent_frameworks=["first", "second"], + inspection=SimpleNamespace(model_type="family", quantization_method=None)) + self.assertEqual(["second", "first"], ranked) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_market_intelligence.py b/tests/test_market_intelligence.py index c4ab5e9f20..b20753a30d 100644 --- a/tests/test_market_intelligence.py +++ b/tests/test_market_intelligence.py @@ -283,6 +283,7 @@ class MarketIntelligenceTests(unittest.TestCase): "successCount": 10, "failureCount": 3, "attributableFailureCount": 0, + "platformFailureCount": 3, } }, "recentCombinationStats": { @@ -290,6 +291,7 @@ class MarketIntelligenceTests(unittest.TestCase): "successCount": 10, "failureCount": 3, "attributableFailureCount": 0, + "platformFailureCount": 3, "consecutiveFailures": 0, "consecutivePlatformFailures": 3, "lastPlatformFailureAt": datetime.now(timezone.utc).isoformat(),