fix: reject broken configs and improve accuracy feedback with isolated agent state
This commit is contained in:
29
README.md
29
README.md
@@ -2,6 +2,30 @@
|
||||
|
||||
This repository is packaged for the ModelHub XC agent platform.
|
||||
|
||||
## Accuracy release 2026.10.18.1
|
||||
|
||||
This release rejects malformed YAML command arrays, including the observed
|
||||
`/opt/entrypoint.sh - /model` executable error, even in restored cached
|
||||
official configurations. It preserves terminal compatibility errors when a
|
||||
log also contains transient platform resource messages. Framework scoring
|
||||
includes unresolved non-platform failures; sparse model-profile evidence is
|
||||
blended with eight prior observations instead of discarded below a threshold.
|
||||
|
||||
For operation alongside the baseline agent, the new default state branch is
|
||||
`agent-state-accuracy-v2`. Do not override it with the baseline's `agent-state`.
|
||||
The separate branch starts its own checkpoint on first boot; it does not
|
||||
overwrite or migrate the baseline state. New submissions retain agent version,
|
||||
instance ID and platform strategy ID. Historical tasks are not assigned an
|
||||
invented version.
|
||||
|
||||
`MODELHUB_COEXISTENCE_MODE=1` is the default: this agent does not run queue
|
||||
cancellation, including deterministic cleanup, because the accounts are shared
|
||||
with the baseline. Submission preflight remains enabled. Set this variable to
|
||||
`0` only when account-wide cleanup is intended; `--disable-queue-cleanup` still
|
||||
disables cleanup explicitly. The baseline agent is unchanged. Both agents share
|
||||
account capacity and platform deduplication; separate state does not reserve
|
||||
capacity or eliminate simultaneous submission races.
|
||||
|
||||
## Platform Contract
|
||||
|
||||
- Root-level `Dockerfile`
|
||||
@@ -67,7 +91,7 @@ Optional tuning:
|
||||
- `MODELHUB_RECENT_MODEL_RESERVE_SLOTS` default `5` per account
|
||||
- `MODELHUB_RECENT_MODEL_DAYS` default `7`
|
||||
- `MODELHUB_STATE_SYNC_REMOTE` default `https://dev.modelhub.org.cn/CoolBoy/submmit.git`
|
||||
- `MODELHUB_STATE_SYNC_BRANCH` default `agent-state`
|
||||
- `MODELHUB_STATE_SYNC_BRANCH` default `agent-state-accuracy-v2`
|
||||
- `MODELHUB_STATE_SYNC_BATCH_SIZE` default `20`
|
||||
|
||||
Git state synchronization reads `modelhub_user_name`, `modelhub_user_email`, and
|
||||
@@ -140,7 +164,8 @@ unvetted GPU or framework.
|
||||
## Durable State Branch
|
||||
|
||||
The hosted worker synchronizes an allowlisted runtime snapshot to the orphan
|
||||
`agent-state` branch of this repository. Before each batch of at most 20 API
|
||||
`agent-state-accuracy-v2` branch of this repository (overridable via
|
||||
`MODELHUB_STATE_SYNC_BRANCH`). Before each batch of at most 20 API
|
||||
submissions it pushes write-ahead intents containing model, GPU, task, framework,
|
||||
safe parameter vector, and a config fingerprint. It pushes task IDs and results
|
||||
after the batch. If either push fails, no further submission begins until the
|
||||
|
||||
@@ -67,7 +67,7 @@ PLATFORM_PATTERNS = (
|
||||
(re.compile(r"welcome\.sh: No such file|data: command not found", re.I), "broken_platform_launch_script"),
|
||||
(re.compile(r"id_rsa.*No such file", re.I), "missing_platform_credential"),
|
||||
(re.compile(r"找不到空闲卡|no idle (?:gpu|card)", re.I), "no_idle_device"),
|
||||
(re.compile(r"storage|download.*timed? out|connection reset", re.I), "platform_io_transient"),
|
||||
(re.compile(r"no space left on device|download.*timed? out|connection reset", re.I), "platform_io_transient"),
|
||||
)
|
||||
|
||||
DETERMINISTIC_LOG_PATTERNS = (
|
||||
@@ -80,6 +80,8 @@ DETERMINISTIC_LOG_PATTERNS = (
|
||||
# semantics. Generic messages such as "attention backend not supported" or
|
||||
# "GPU type not supported" must not create an architecture blacklist.
|
||||
EXPLICIT_FRAMEWORK_MODEL_UNSUPPORTED_PATTERNS = (
|
||||
re.compile(r"model type\s+[`'\"][^`'\"]+[`'\"].{0,100}does not recognize this architecture", re.I),
|
||||
re.compile(r"Model architectures?\s*\[[^\]\n]+\]\s*(?:are|is) not supported", re.I),
|
||||
re.compile(
|
||||
r"(?:当前|该|此)(?:推理)?框架.{0,100}(?:不支持|暂不支持|无法支持|不兼容)"
|
||||
r".{0,100}(?:该|此|当前)?(?:模型架构|模型|架构)",
|
||||
@@ -117,19 +119,16 @@ def classify_failure_report(report_code: str | None, log_lines: Iterable[str] =
|
||||
code = str(report_code or "").strip().upper()
|
||||
text = "\n".join(str(line) for line in log_lines)
|
||||
|
||||
# Infrastructure signatures override broad report codes such as
|
||||
# EXECUTE_EMPTY_RESULT so they do not poison model compatibility feedback.
|
||||
for pattern, reason in PLATFORM_PATTERNS:
|
||||
if pattern.search(text):
|
||||
return FailureClassification(
|
||||
"platform_infrastructure",
|
||||
"gpu_framework",
|
||||
"open_short_gpu_framework_circuit_and_retry_other_models",
|
||||
True,
|
||||
False,
|
||||
reason,
|
||||
)
|
||||
|
||||
if re.search(r'exec:.*entrypoint.*(?:no such file|not found)', text, re.I):
|
||||
return FailureClassification(
|
||||
"configuration_launch", "configuration", "reject_broken_config", True, False,
|
||||
"invalid_executable_argv",
|
||||
)
|
||||
if re.search(r'quantization method .*not supported for the current GPU', text, re.I):
|
||||
return FailureClassification(
|
||||
"quantization_compatibility", "model_gpu_framework", "reject_quantization_route",
|
||||
True, False, "explicit_quantization_gpu_incompatibility",
|
||||
)
|
||||
if code in DETERMINISTIC_POLICIES:
|
||||
return DETERMINISTIC_POLICIES[code]
|
||||
for pattern, inferred_code in DETERMINISTIC_LOG_PATTERNS:
|
||||
@@ -149,6 +148,15 @@ def classify_failure_report(report_code: str | None, log_lines: Iterable[str] =
|
||||
)
|
||||
if code in SEMANTIC_POLICIES:
|
||||
return SEMANTIC_POLICIES[code]
|
||||
# Fall back to infrastructure signatures only after specific root causes.
|
||||
# Logs may include a transient allocation failure before a model starts.
|
||||
for pattern, reason in PLATFORM_PATTERNS:
|
||||
if pattern.search(text):
|
||||
return FailureClassification(
|
||||
"platform_infrastructure", "gpu_framework",
|
||||
"open_short_gpu_framework_circuit_and_retry_other_models",
|
||||
True, False, reason,
|
||||
)
|
||||
return FailureClassification(
|
||||
"ambiguous_runtime",
|
||||
"unknown",
|
||||
|
||||
@@ -4,6 +4,7 @@ import math
|
||||
import re
|
||||
import statistics
|
||||
import threading
|
||||
import yaml
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
@@ -97,6 +98,31 @@ def _validated_official_config(config: str, *, framework: str, target_gpu: str)
|
||||
return False, "missing_sut_or_ref_config"
|
||||
if any(marker in config for marker in ("{{", "}}", "PLACEHOLDER")):
|
||||
return False, "unresolved_placeholder"
|
||||
try:
|
||||
document = yaml.safe_load(config)
|
||||
except yaml.YAMLError:
|
||||
return False, "invalid_yaml"
|
||||
if not isinstance(document, dict):
|
||||
return False, "invalid_config_structure"
|
||||
|
||||
def valid_commands(node: Any) -> bool:
|
||||
if isinstance(node, dict):
|
||||
for key, value in node.items():
|
||||
if key in {"command", "entrypoint"} and isinstance(value, list):
|
||||
if not value or not all(isinstance(arg, str) for arg in value):
|
||||
return False
|
||||
# argv[0] is an executable, not a shell command. In
|
||||
# particular, a folded YAML list item must not eat argv[1].
|
||||
if not value[0].strip() or re.search(r"\s", value[0]):
|
||||
return False
|
||||
if not valid_commands(value):
|
||||
return False
|
||||
elif isinstance(node, list):
|
||||
return all(valid_commands(value) for value in node)
|
||||
return True
|
||||
|
||||
if not valid_commands(document):
|
||||
return False, "malformed_command_argv"
|
||||
|
||||
declared = re.search(r"(?m)^\s*framework:\s*['\"]?([^'\"\s]+)", config)
|
||||
if declared is None or declared.group(1).strip() != framework:
|
||||
@@ -736,7 +762,7 @@ class MarketIntelligenceManager:
|
||||
local_success = max(0, int(local_item.get("successCount") or 0))
|
||||
local_failure = max(
|
||||
0,
|
||||
int(local_item.get("attributableFailureCount", local_item.get("failureCount") or 0)),
|
||||
int(local_item.get("failureCount") or 0) - int(local_item.get("platformFailureCount") or 0),
|
||||
)
|
||||
local_samples = local_success + local_failure
|
||||
local_score = _wilson_lower_bound(local_success, local_samples)
|
||||
@@ -746,7 +772,7 @@ class MarketIntelligenceManager:
|
||||
recent_all_failures = max(0, int(recent_item.get("failureCount") or 0))
|
||||
recent_failure = max(
|
||||
0,
|
||||
int(recent_item.get("attributableFailureCount", recent_item.get("failureCount") or 0)),
|
||||
int(recent_item.get("failureCount") or 0) - int(recent_item.get("platformFailureCount") or 0),
|
||||
)
|
||||
recent_samples = recent_success + recent_failure
|
||||
recent_rate = recent_success / recent_samples if recent_samples else None
|
||||
@@ -849,11 +875,31 @@ class MarketIntelligenceManager:
|
||||
if incumbent_best > 0 and float(evidence["combinedScore"]) < incumbent_best * NEW_FRAMEWORK_PROMOTION_MARGIN:
|
||||
continue
|
||||
promoted.append(framework)
|
||||
return self.rank_frameworks(
|
||||
ranked = self.rank_frameworks(
|
||||
task_type=task_type,
|
||||
target_gpu=target_gpu,
|
||||
compatible_frameworks=promoted,
|
||||
)
|
||||
model_type = str(getattr(inspection, "model_type", None) or "").strip()
|
||||
quantization = str(getattr(inspection, "quantization_method", None) or "none").strip()
|
||||
if not model_type:
|
||||
return ranked
|
||||
|
||||
def profile_score(framework: str) -> float:
|
||||
prior = float(self._framework_evidence(task_type, target_gpu, framework)["combinedScore"])
|
||||
key = f"{target_gpu}|{framework}|{task_type}|{model_type}|{quantization}"
|
||||
item = (self.local_outcome_stats.get("profileCombinationStats") or {}).get(key) or {}
|
||||
success = max(0, int(item.get("successCount") or 0))
|
||||
failure = max(0, int(item.get("failureCount") or 0) - int(item.get("platformFailureCount") or 0))
|
||||
samples = success + failure
|
||||
if not samples:
|
||||
return prior
|
||||
# Model-specific negative evidence must affect the framework
|
||||
# choice, not only the ranking after the first framework wins.
|
||||
lower = _wilson_lower_bound(success + 8 * prior, samples + 8)
|
||||
return lower
|
||||
|
||||
return sorted(ranked, key=profile_score, reverse=True)
|
||||
|
||||
def compatible_discovered_frameworks(
|
||||
self,
|
||||
@@ -941,6 +987,11 @@ class MarketIntelligenceManager:
|
||||
config = item.get("officialConfig")
|
||||
if not isinstance(config, str) or not config:
|
||||
return None
|
||||
valid, reason = _validated_official_config(config, framework=framework, target_gpu=target_gpu)
|
||||
if not valid:
|
||||
item["officialConfigValid"] = False
|
||||
item["officialConfigError"] = reason
|
||||
return None
|
||||
return _render_official_config(config, gguf_filename=gguf_filename)
|
||||
|
||||
def framework_metadata(self, task_type: str, target_gpu: str, framework: str) -> dict[str, Any]:
|
||||
|
||||
@@ -22,6 +22,8 @@ from history_stats import classify_failure, is_failure, is_success
|
||||
from llm_classifier import LLMAssistedClassifier
|
||||
from modelhub_client import ModelHubClient, ModelHubClientPool
|
||||
from task_registry import task_type_from_history_task
|
||||
from common import runtime_instance_id
|
||||
from version import AGENT_VERSION
|
||||
|
||||
|
||||
DEFAULT_OUTCOMES_PATH = Path("outcomes/submissions.jsonl")
|
||||
@@ -36,6 +38,7 @@ FAILURE_ENRICHMENT_MAX_ATTEMPTS = 3
|
||||
TERMINAL_TASK_STATUSES = {"success", "failed", "error", "cancelled", "completed"}
|
||||
|
||||
DECISION_RECORD_FIELDS = {
|
||||
"agentVersion", "instanceId", "strategyId",
|
||||
"taskId",
|
||||
"modelId",
|
||||
"targetGpu",
|
||||
@@ -325,6 +328,9 @@ class OutcomeTracker:
|
||||
model_profile: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
record: dict[str, Any] = {
|
||||
"agentVersion": AGENT_VERSION,
|
||||
"instanceId": runtime_instance_id(),
|
||||
"strategyId": os.getenv("STRATEGY_ID"),
|
||||
"modelId": model_id,
|
||||
"targetGpu": target_gpu,
|
||||
"framework": framework,
|
||||
|
||||
@@ -234,7 +234,9 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
parser.add_argument("--post-cycle-cooldown-seconds", type=int, default=2, help="Short sleep after a successful cycle")
|
||||
parser.add_argument("--max-cycles", type=int, default=0, help="Optional hard stop after N cycles; 0 means run until quota is reached")
|
||||
parser.add_argument("--print-stats", action="store_true", help="Load outcomes, sync, print stats report, and exit")
|
||||
parser.add_argument("--disable-queue-cleanup", action="store_true", help=argparse.SUPPRESS)
|
||||
parser.add_argument("--disable-queue-cleanup", action="store_true",
|
||||
default=os.getenv("MODELHUB_COEXISTENCE_MODE", "1") != "0",
|
||||
help=argparse.SUPPRESS)
|
||||
parser.add_argument(
|
||||
"--queue-cleanup-interval-cycles",
|
||||
type=int,
|
||||
@@ -702,6 +704,11 @@ def run_poll_loop(
|
||||
STATS_PRINT_INTERVAL = 10
|
||||
|
||||
log(f"[poll] version={AGENT_VERSION} poll_run_dir={poll_run_dir}")
|
||||
log(
|
||||
f"[coexistence] state_branch={base_args.state_sync_branch} "
|
||||
f"queue_cancellation={'disabled' if base_args.disable_queue_cleanup else 'enabled'} "
|
||||
"submission_preflight=enabled"
|
||||
)
|
||||
log(
|
||||
"[state-retention] mode=decision_state_only full_archive=disabled "
|
||||
f"recent_outcomes={DEFAULT_RECENT_OUTCOME_LIMIT} "
|
||||
|
||||
@@ -46,6 +46,13 @@ class SuccessFirstRoutingEngine:
|
||||
return value
|
||||
|
||||
def _evidence(self, candidate: dict[str, Any]) -> tuple[str, int, int, float]:
|
||||
def observed_total(item: dict[str, Any]) -> int:
|
||||
if "failureCount" in item:
|
||||
return max(0, int(item.get("successCount") or 0) + int(item.get("failureCount") or 0)
|
||||
- int(item.get("platformFailureCount") or 0))
|
||||
return max(0, int(item.get("decisionTotal") or 0))
|
||||
|
||||
sparse = None
|
||||
profile = candidate.get("preflightMetadata") or {}
|
||||
gpu = str(candidate.get("targetGpu") or "")
|
||||
framework = str(candidate.get("framework") or "")
|
||||
@@ -60,14 +67,18 @@ class SuccessFirstRoutingEngine:
|
||||
size_bucket = int(load_bytes).bit_length() - 1
|
||||
key = f"{gpu}|{framework}|{task_type}|{model_type}|{quantization}|{size_bucket}"
|
||||
item = (self.outcome_stats.get("sizedProfileCombinationStats") or {}).get(key) or {}
|
||||
total = int(item.get("decisionTotal") or 0)
|
||||
total = observed_total(item)
|
||||
if total:
|
||||
sparse = ("local_profile_size_shrunk", item, total)
|
||||
if total >= 20:
|
||||
success = int(item.get("successCount") or 0)
|
||||
return "local_profile_size", success, total, wilson_lower_bound(success, total)
|
||||
if model_type:
|
||||
key = f"{gpu}|{framework}|{task_type}|{model_type}|{quantization}"
|
||||
item = (self.outcome_stats.get("profileCombinationStats") or {}).get(key) or {}
|
||||
total = int(item.get("decisionTotal") or 0)
|
||||
total = observed_total(item)
|
||||
if total and sparse is None:
|
||||
sparse = ("local_profile_shrunk", item, total)
|
||||
if total >= 40:
|
||||
success = int(item.get("successCount") or 0)
|
||||
return "local_profile", success, total, wilson_lower_bound(success, total)
|
||||
@@ -97,6 +108,13 @@ class SuccessFirstRoutingEngine:
|
||||
total = 2
|
||||
success = 1
|
||||
level = "global_conservative_prior"
|
||||
if sparse is not None:
|
||||
level, item, samples = sparse
|
||||
# Eight prior observations stabilize small groups without hiding
|
||||
# their failures behind an unreachable hard sample threshold.
|
||||
prior_success = round(8 * success / max(1, total))
|
||||
success = int(item.get("successCount") or 0) + prior_success
|
||||
total = samples + 8
|
||||
return level, success, total, wilson_lower_bound(success, total)
|
||||
|
||||
def annotate(self, candidate: dict[str, Any]) -> dict[str, Any]:
|
||||
|
||||
@@ -17,14 +17,14 @@ from typing import Any, Iterable
|
||||
from dulwich import porcelain
|
||||
from dulwich.repo import Repo
|
||||
|
||||
from common import read_json, read_jsonl, write_json, write_jsonl
|
||||
from common import read_json, read_jsonl, write_json, write_jsonl, runtime_instance_id
|
||||
from runner_common import load_key_files
|
||||
from version import AGENT_VERSION
|
||||
|
||||
|
||||
STATE_SCHEMA_VERSION = 1
|
||||
DEFAULT_REMOTE = "https://dev.modelhub.org.cn/CoolBoy/submmit.git"
|
||||
DEFAULT_BRANCH = "agent-state"
|
||||
DEFAULT_BRANCH = "agent-state-accuracy-v2"
|
||||
DEFAULT_BATCH_SIZE = 100
|
||||
DEFAULT_HISTORY_DEPTH = 200
|
||||
DEFAULT_RECENT_TERMINAL_INTENTS = 300
|
||||
@@ -32,6 +32,7 @@ DEFAULT_LEDGER_RECORDS = 500
|
||||
DEFAULT_CRASH_RECORDS = 50
|
||||
|
||||
STATE_OUTCOME_FIELDS = {
|
||||
"agentVersion", "instanceId", "strategyId",
|
||||
"taskId", "modelId", "targetGpu", "framework", "taskType", "submitTime",
|
||||
"lastSyncTime", "status", "verifyResult", "outcome", "failReason",
|
||||
"modelProfile", "failureCode", "failureCategory", "failureScope",
|
||||
@@ -342,6 +343,7 @@ class StateGitSync:
|
||||
retained_terminal = terminal[:DEFAULT_RECENT_TERMINAL_INTENTS]
|
||||
discarded = terminal[DEFAULT_RECENT_TERMINAL_INTENTS:]
|
||||
compact_fields = {
|
||||
"agentVersion", "instanceId", "strategyId",
|
||||
"intentId", "batchId", "status", "createdAt", "completedAt",
|
||||
"repoId", "targetGpu", "taskType", "framework", "configSource",
|
||||
"configFingerprint", "safeConfigVector", "taskId",
|
||||
@@ -372,6 +374,9 @@ class StateGitSync:
|
||||
config = str(candidate.get("configParams") or "")
|
||||
return {
|
||||
"intentId": uuid.uuid4().hex,
|
||||
"agentVersion": AGENT_VERSION,
|
||||
"instanceId": runtime_instance_id(),
|
||||
"strategyId": os.getenv("STRATEGY_ID"),
|
||||
"batchId": batch_id,
|
||||
"status": "pending",
|
||||
"createdAt": _utc_now().isoformat(),
|
||||
|
||||
@@ -1 +1 @@
|
||||
AGENT_VERSION = "2026.09.22.1"
|
||||
AGENT_VERSION = "2026.10.18.1"
|
||||
|
||||
96
tests/test_accuracy_audit.py
Normal file
96
tests/test_accuracy_audit.py
Normal file
@@ -0,0 +1,96 @@
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
from types import SimpleNamespace
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "modelhub_submmit_api"))
|
||||
from failure_taxonomy import classify_failure_report
|
||||
from market_intelligence import _validated_official_config, MarketIntelligenceManager
|
||||
from routing_engine import SuccessFirstRoutingEngine
|
||||
from state_sync import StateGitSync, DEFAULT_BRANCH
|
||||
from poll_runner import build_parser
|
||||
|
||||
|
||||
class AccuracyAuditTests(unittest.TestCase):
|
||||
def config(self, command):
|
||||
return f"framework: vllm_fix_tokenizer\nsut_config:\n command: {command}\nref_config: {{}}\n"
|
||||
|
||||
def test_broken_official_and_cached_config_rejected(self):
|
||||
bad = self.config('["/opt/entrypoint.sh - /model", "--port", "80"]')
|
||||
for gpu in ("Biren_166m", "Kunlunxin_p-800"):
|
||||
self.assertEqual((False, "malformed_command_argv"), _validated_official_config(
|
||||
bad, framework="vllm_fix_tokenizer", target_gpu=gpu))
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
manager = MarketIntelligenceManager(Path(tmp) / "market.json")
|
||||
manager.state = {"frameworkStats": {"text-generation": {gpu: {
|
||||
"vllm_fix_tokenizer": {"officialConfigValid": True, "officialConfig": bad}}}}}
|
||||
self.assertIsNone(manager.official_config(task_type="text-generation", target_gpu=gpu,
|
||||
framework="vllm_fix_tokenizer"))
|
||||
|
||||
def test_separate_argv_and_shell_command_remain_valid(self):
|
||||
for command in ('["/opt/entrypoint.sh", "/model"]', '["/bin/sh", "-c", "echo hello"]'):
|
||||
self.assertEqual((True, None), _validated_official_config(
|
||||
self.config(command), framework="vllm_fix_tokenizer", target_gpu="Biren_166m"))
|
||||
|
||||
def test_specific_failure_survives_transient_resource_error(self):
|
||||
result = classify_failure_report("MODEL_NOT_SUPPORTED", [
|
||||
"找不到空闲卡", "当前框架版本不支持该模型架构"])
|
||||
self.assertEqual("framework_architecture_unsupported", result.category)
|
||||
self.assertEqual("tokenizer_compatibility", classify_failure_report(
|
||||
"TOKENIZER_FAILED", ["no idle gpu", "Tokenizer class missing"]).category)
|
||||
self.assertEqual("platform_infrastructure", classify_failure_report(
|
||||
"EXECUTE_EMPTY_RESULT", ["no idle gpu"]).category)
|
||||
self.assertEqual("framework_architecture_unsupported", classify_failure_report(
|
||||
None, ["no idle gpu", "Model architectures ['NewForCausalLM'] are not supported for now"]).category)
|
||||
|
||||
def test_launch_and_quantization_have_specific_categories(self):
|
||||
self.assertEqual("configuration_launch", classify_failure_report("EXECUTE_EMPTY_RESULT", [
|
||||
'exec: "/opt/entrypoint.sh - /model": no such file or directory']).category)
|
||||
self.assertEqual("quantization_compatibility", classify_failure_report("EXECUTE_EMPTY_RESULT", [
|
||||
"The quantization method compressed-tensors is not supported for the current GPU"]).category)
|
||||
|
||||
def test_sparse_failures_lower_score_even_without_classification(self):
|
||||
candidate = {"targetGpu": "gpu", "framework": "vllm", "taskType": "text-generation",
|
||||
"frameworkMarketSamples": 1000, "frameworkMarketSuccessRate": 0.8,
|
||||
"preflightMetadata": {"modelType": "family", "estimatedLoadBytes": 1024}}
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
engine = SuccessFirstRoutingEngine(Path(tmp) / "route.json")
|
||||
prior = engine._evidence(candidate)[3]
|
||||
engine.outcome_stats = {"profileCombinationStats": {
|
||||
"gpu|vllm|text-generation|family|none": {
|
||||
"successCount": 0, "failureCount": 5, "decisionTotal": 0}}}
|
||||
evidence = engine._evidence(candidate)
|
||||
self.assertEqual("local_profile_shrunk", evidence[0])
|
||||
self.assertLess(evidence[3], prior)
|
||||
|
||||
def test_coexistence_defaults_and_submission_provenance(self):
|
||||
with patch.dict(os.environ, {"MODELHUB_COEXISTENCE_MODE": "1", "STRATEGY_ID": "new-agent"}):
|
||||
self.assertTrue(build_parser().parse_args([]).disable_queue_cleanup)
|
||||
intent = StateGitSync._intent({"repoId": "owner/model"}, "batch")
|
||||
self.assertEqual("new-agent", intent["strategyId"])
|
||||
self.assertTrue(intent["agentVersion"])
|
||||
self.assertTrue(intent["instanceId"])
|
||||
self.assertNotEqual("agent-state", DEFAULT_BRANCH)
|
||||
|
||||
def test_model_specific_failures_change_framework_selection(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
manager = MarketIntelligenceManager(Path(tmp) / "market.json")
|
||||
manager.state = {"frameworkStats": {"text-generation": {"gpu": {
|
||||
"first": {"modelCount": 1000, "wilsonLowerBound": 0.8},
|
||||
"second": {"modelCount": 1000, "wilsonLowerBound": 0.6},
|
||||
}}}}
|
||||
manager.set_local_outcome_stats({"profileCombinationStats": {
|
||||
"gpu|first|text-generation|family|none": {"successCount": 0, "failureCount": 4}
|
||||
}})
|
||||
with patch.object(manager, "compatible_discovered_frameworks", return_value=[]):
|
||||
ranked = manager.selectable_frameworks(
|
||||
task_type="text-generation", target_gpu="gpu", incumbent_frameworks=["first", "second"],
|
||||
inspection=SimpleNamespace(model_type="family", quantization_method=None))
|
||||
self.assertEqual(["second", "first"], ranked)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -283,6 +283,7 @@ class MarketIntelligenceTests(unittest.TestCase):
|
||||
"successCount": 10,
|
||||
"failureCount": 3,
|
||||
"attributableFailureCount": 0,
|
||||
"platformFailureCount": 3,
|
||||
}
|
||||
},
|
||||
"recentCombinationStats": {
|
||||
@@ -290,6 +291,7 @@ class MarketIntelligenceTests(unittest.TestCase):
|
||||
"successCount": 10,
|
||||
"failureCount": 3,
|
||||
"attributableFailureCount": 0,
|
||||
"platformFailureCount": 3,
|
||||
"consecutiveFailures": 0,
|
||||
"consecutivePlatformFailures": 3,
|
||||
"lastPlatformFailureAt": datetime.now(timezone.utc).isoformat(),
|
||||
|
||||
Reference in New Issue
Block a user