1 Commits

Author SHA1 Message Date
CoolBoy
484a10607c fix: reject broken configs and improve accuracy feedback with isolated agent state 2026-10-18 16:29:18 +08:00
10 changed files with 243 additions and 25 deletions

View File

@@ -2,6 +2,30 @@
This repository is packaged for the ModelHub XC agent platform. This repository is packaged for the ModelHub XC agent platform.
## Accuracy release 2026.10.18.1
This release rejects malformed YAML command arrays, including the observed
`/opt/entrypoint.sh - /model` executable error, even in restored cached
official configurations. It preserves terminal compatibility errors when a
log also contains transient platform resource messages. Framework scoring
includes unresolved non-platform failures; sparse model-profile evidence is
blended with eight prior observations instead of discarded below a threshold.
For operation alongside the baseline agent, the new default state branch is
`agent-state-accuracy-v2`. Do not override it with the baseline's `agent-state`.
The separate branch starts its own checkpoint on first boot; it does not
overwrite or migrate the baseline state. New submissions retain agent version,
instance ID and platform strategy ID. Historical tasks are not assigned an
invented version.
`MODELHUB_COEXISTENCE_MODE=1` is the default: this agent does not run queue
cancellation, including deterministic cleanup, because the accounts are shared
with the baseline. Submission preflight remains enabled. Set this variable to
`0` only when account-wide cleanup is intended; `--disable-queue-cleanup` still
disables cleanup explicitly. The baseline agent is unchanged. Both agents share
account capacity and platform deduplication; separate state does not reserve
capacity or eliminate simultaneous submission races.
## Platform Contract ## Platform Contract
- Root-level `Dockerfile` - Root-level `Dockerfile`
@@ -67,7 +91,7 @@ Optional tuning:
- `MODELHUB_RECENT_MODEL_RESERVE_SLOTS` default `5` per account - `MODELHUB_RECENT_MODEL_RESERVE_SLOTS` default `5` per account
- `MODELHUB_RECENT_MODEL_DAYS` default `7` - `MODELHUB_RECENT_MODEL_DAYS` default `7`
- `MODELHUB_STATE_SYNC_REMOTE` default `https://dev.modelhub.org.cn/CoolBoy/submmit.git` - `MODELHUB_STATE_SYNC_REMOTE` default `https://dev.modelhub.org.cn/CoolBoy/submmit.git`
- `MODELHUB_STATE_SYNC_BRANCH` default `agent-state` - `MODELHUB_STATE_SYNC_BRANCH` default `agent-state-accuracy-v2`
- `MODELHUB_STATE_SYNC_BATCH_SIZE` default `20` - `MODELHUB_STATE_SYNC_BATCH_SIZE` default `20`
Git state synchronization reads `modelhub_user_name`, `modelhub_user_email`, and Git state synchronization reads `modelhub_user_name`, `modelhub_user_email`, and
@@ -140,7 +164,8 @@ unvetted GPU or framework.
## Durable State Branch ## Durable State Branch
The hosted worker synchronizes an allowlisted runtime snapshot to the orphan The hosted worker synchronizes an allowlisted runtime snapshot to the orphan
`agent-state` branch of this repository. Before each batch of at most 20 API `agent-state-accuracy-v2` branch of this repository (overridable via
`MODELHUB_STATE_SYNC_BRANCH`). Before each batch of at most 20 API
submissions it pushes write-ahead intents containing model, GPU, task, framework, submissions it pushes write-ahead intents containing model, GPU, task, framework,
safe parameter vector, and a config fingerprint. It pushes task IDs and results safe parameter vector, and a config fingerprint. It pushes task IDs and results
after the batch. If either push fails, no further submission begins until the after the batch. If either push fails, no further submission begins until the

View File

@@ -67,7 +67,7 @@ PLATFORM_PATTERNS = (
(re.compile(r"welcome\.sh: No such file|data: command not found", re.I), "broken_platform_launch_script"), (re.compile(r"welcome\.sh: No such file|data: command not found", re.I), "broken_platform_launch_script"),
(re.compile(r"id_rsa.*No such file", re.I), "missing_platform_credential"), (re.compile(r"id_rsa.*No such file", re.I), "missing_platform_credential"),
(re.compile(r"找不到空闲卡|no idle (?:gpu|card)", re.I), "no_idle_device"), (re.compile(r"找不到空闲卡|no idle (?:gpu|card)", re.I), "no_idle_device"),
(re.compile(r"storage|download.*timed? out|connection reset", re.I), "platform_io_transient"), (re.compile(r"no space left on device|download.*timed? out|connection reset", re.I), "platform_io_transient"),
) )
DETERMINISTIC_LOG_PATTERNS = ( DETERMINISTIC_LOG_PATTERNS = (
@@ -80,6 +80,8 @@ DETERMINISTIC_LOG_PATTERNS = (
# semantics. Generic messages such as "attention backend not supported" or # semantics. Generic messages such as "attention backend not supported" or
# "GPU type not supported" must not create an architecture blacklist. # "GPU type not supported" must not create an architecture blacklist.
EXPLICIT_FRAMEWORK_MODEL_UNSUPPORTED_PATTERNS = ( EXPLICIT_FRAMEWORK_MODEL_UNSUPPORTED_PATTERNS = (
re.compile(r"model type\s+[`'\"][^`'\"]+[`'\"].{0,100}does not recognize this architecture", re.I),
re.compile(r"Model architectures?\s*\[[^\]\n]+\]\s*(?:are|is) not supported", re.I),
re.compile( re.compile(
r"(?:当前|该|此)(?:推理)?框架.{0,100}(?:不支持|暂不支持|无法支持|不兼容)" r"(?:当前|该|此)(?:推理)?框架.{0,100}(?:不支持|暂不支持|无法支持|不兼容)"
r".{0,100}(?:该|此|当前)?(?:模型架构|模型|架构)", r".{0,100}(?:该|此|当前)?(?:模型架构|模型|架构)",
@@ -117,19 +119,16 @@ def classify_failure_report(report_code: str | None, log_lines: Iterable[str] =
code = str(report_code or "").strip().upper() code = str(report_code or "").strip().upper()
text = "\n".join(str(line) for line in log_lines) text = "\n".join(str(line) for line in log_lines)
# Infrastructure signatures override broad report codes such as if re.search(r'exec:.*entrypoint.*(?:no such file|not found)', text, re.I):
# EXECUTE_EMPTY_RESULT so they do not poison model compatibility feedback. return FailureClassification(
for pattern, reason in PLATFORM_PATTERNS: "configuration_launch", "configuration", "reject_broken_config", True, False,
if pattern.search(text): "invalid_executable_argv",
return FailureClassification( )
"platform_infrastructure", if re.search(r'quantization method .*not supported for the current GPU', text, re.I):
"gpu_framework", return FailureClassification(
"open_short_gpu_framework_circuit_and_retry_other_models", "quantization_compatibility", "model_gpu_framework", "reject_quantization_route",
True, True, False, "explicit_quantization_gpu_incompatibility",
False, )
reason,
)
if code in DETERMINISTIC_POLICIES: if code in DETERMINISTIC_POLICIES:
return DETERMINISTIC_POLICIES[code] return DETERMINISTIC_POLICIES[code]
for pattern, inferred_code in DETERMINISTIC_LOG_PATTERNS: for pattern, inferred_code in DETERMINISTIC_LOG_PATTERNS:
@@ -149,6 +148,15 @@ def classify_failure_report(report_code: str | None, log_lines: Iterable[str] =
) )
if code in SEMANTIC_POLICIES: if code in SEMANTIC_POLICIES:
return SEMANTIC_POLICIES[code] return SEMANTIC_POLICIES[code]
# Fall back to infrastructure signatures only after specific root causes.
# Logs may include a transient allocation failure before a model starts.
for pattern, reason in PLATFORM_PATTERNS:
if pattern.search(text):
return FailureClassification(
"platform_infrastructure", "gpu_framework",
"open_short_gpu_framework_circuit_and_retry_other_models",
True, False, reason,
)
return FailureClassification( return FailureClassification(
"ambiguous_runtime", "ambiguous_runtime",
"unknown", "unknown",

View File

@@ -4,6 +4,7 @@ import math
import re import re
import statistics import statistics
import threading import threading
import yaml
from concurrent.futures import ThreadPoolExecutor, as_completed from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timedelta from datetime import datetime, timedelta
from pathlib import Path from pathlib import Path
@@ -97,6 +98,31 @@ def _validated_official_config(config: str, *, framework: str, target_gpu: str)
return False, "missing_sut_or_ref_config" return False, "missing_sut_or_ref_config"
if any(marker in config for marker in ("{{", "}}", "PLACEHOLDER")): if any(marker in config for marker in ("{{", "}}", "PLACEHOLDER")):
return False, "unresolved_placeholder" return False, "unresolved_placeholder"
try:
document = yaml.safe_load(config)
except yaml.YAMLError:
return False, "invalid_yaml"
if not isinstance(document, dict):
return False, "invalid_config_structure"
def valid_commands(node: Any) -> bool:
if isinstance(node, dict):
for key, value in node.items():
if key in {"command", "entrypoint"} and isinstance(value, list):
if not value or not all(isinstance(arg, str) for arg in value):
return False
# argv[0] is an executable, not a shell command. In
# particular, a folded YAML list item must not eat argv[1].
if not value[0].strip() or re.search(r"\s", value[0]):
return False
if not valid_commands(value):
return False
elif isinstance(node, list):
return all(valid_commands(value) for value in node)
return True
if not valid_commands(document):
return False, "malformed_command_argv"
declared = re.search(r"(?m)^\s*framework:\s*['\"]?([^'\"\s]+)", config) declared = re.search(r"(?m)^\s*framework:\s*['\"]?([^'\"\s]+)", config)
if declared is None or declared.group(1).strip() != framework: if declared is None or declared.group(1).strip() != framework:
@@ -736,7 +762,7 @@ class MarketIntelligenceManager:
local_success = max(0, int(local_item.get("successCount") or 0)) local_success = max(0, int(local_item.get("successCount") or 0))
local_failure = max( local_failure = max(
0, 0,
int(local_item.get("attributableFailureCount", local_item.get("failureCount") or 0)), int(local_item.get("failureCount") or 0) - int(local_item.get("platformFailureCount") or 0),
) )
local_samples = local_success + local_failure local_samples = local_success + local_failure
local_score = _wilson_lower_bound(local_success, local_samples) local_score = _wilson_lower_bound(local_success, local_samples)
@@ -746,7 +772,7 @@ class MarketIntelligenceManager:
recent_all_failures = max(0, int(recent_item.get("failureCount") or 0)) recent_all_failures = max(0, int(recent_item.get("failureCount") or 0))
recent_failure = max( recent_failure = max(
0, 0,
int(recent_item.get("attributableFailureCount", recent_item.get("failureCount") or 0)), int(recent_item.get("failureCount") or 0) - int(recent_item.get("platformFailureCount") or 0),
) )
recent_samples = recent_success + recent_failure recent_samples = recent_success + recent_failure
recent_rate = recent_success / recent_samples if recent_samples else None recent_rate = recent_success / recent_samples if recent_samples else None
@@ -849,11 +875,31 @@ class MarketIntelligenceManager:
if incumbent_best > 0 and float(evidence["combinedScore"]) < incumbent_best * NEW_FRAMEWORK_PROMOTION_MARGIN: if incumbent_best > 0 and float(evidence["combinedScore"]) < incumbent_best * NEW_FRAMEWORK_PROMOTION_MARGIN:
continue continue
promoted.append(framework) promoted.append(framework)
return self.rank_frameworks( ranked = self.rank_frameworks(
task_type=task_type, task_type=task_type,
target_gpu=target_gpu, target_gpu=target_gpu,
compatible_frameworks=promoted, compatible_frameworks=promoted,
) )
model_type = str(getattr(inspection, "model_type", None) or "").strip()
quantization = str(getattr(inspection, "quantization_method", None) or "none").strip()
if not model_type:
return ranked
def profile_score(framework: str) -> float:
prior = float(self._framework_evidence(task_type, target_gpu, framework)["combinedScore"])
key = f"{target_gpu}|{framework}|{task_type}|{model_type}|{quantization}"
item = (self.local_outcome_stats.get("profileCombinationStats") or {}).get(key) or {}
success = max(0, int(item.get("successCount") or 0))
failure = max(0, int(item.get("failureCount") or 0) - int(item.get("platformFailureCount") or 0))
samples = success + failure
if not samples:
return prior
# Model-specific negative evidence must affect the framework
# choice, not only the ranking after the first framework wins.
lower = _wilson_lower_bound(success + 8 * prior, samples + 8)
return lower
return sorted(ranked, key=profile_score, reverse=True)
def compatible_discovered_frameworks( def compatible_discovered_frameworks(
self, self,
@@ -941,6 +987,11 @@ class MarketIntelligenceManager:
config = item.get("officialConfig") config = item.get("officialConfig")
if not isinstance(config, str) or not config: if not isinstance(config, str) or not config:
return None return None
valid, reason = _validated_official_config(config, framework=framework, target_gpu=target_gpu)
if not valid:
item["officialConfigValid"] = False
item["officialConfigError"] = reason
return None
return _render_official_config(config, gguf_filename=gguf_filename) return _render_official_config(config, gguf_filename=gguf_filename)
def framework_metadata(self, task_type: str, target_gpu: str, framework: str) -> dict[str, Any]: def framework_metadata(self, task_type: str, target_gpu: str, framework: str) -> dict[str, Any]:

View File

@@ -22,6 +22,8 @@ from history_stats import classify_failure, is_failure, is_success
from llm_classifier import LLMAssistedClassifier from llm_classifier import LLMAssistedClassifier
from modelhub_client import ModelHubClient, ModelHubClientPool from modelhub_client import ModelHubClient, ModelHubClientPool
from task_registry import task_type_from_history_task from task_registry import task_type_from_history_task
from common import runtime_instance_id
from version import AGENT_VERSION
DEFAULT_OUTCOMES_PATH = Path("outcomes/submissions.jsonl") DEFAULT_OUTCOMES_PATH = Path("outcomes/submissions.jsonl")
@@ -36,6 +38,7 @@ FAILURE_ENRICHMENT_MAX_ATTEMPTS = 3
TERMINAL_TASK_STATUSES = {"success", "failed", "error", "cancelled", "completed"} TERMINAL_TASK_STATUSES = {"success", "failed", "error", "cancelled", "completed"}
DECISION_RECORD_FIELDS = { DECISION_RECORD_FIELDS = {
"agentVersion", "instanceId", "strategyId",
"taskId", "taskId",
"modelId", "modelId",
"targetGpu", "targetGpu",
@@ -325,6 +328,9 @@ class OutcomeTracker:
model_profile: dict[str, Any] | None = None, model_profile: dict[str, Any] | None = None,
) -> None: ) -> None:
record: dict[str, Any] = { record: dict[str, Any] = {
"agentVersion": AGENT_VERSION,
"instanceId": runtime_instance_id(),
"strategyId": os.getenv("STRATEGY_ID"),
"modelId": model_id, "modelId": model_id,
"targetGpu": target_gpu, "targetGpu": target_gpu,
"framework": framework, "framework": framework,

View File

@@ -234,7 +234,9 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument("--post-cycle-cooldown-seconds", type=int, default=2, help="Short sleep after a successful cycle") parser.add_argument("--post-cycle-cooldown-seconds", type=int, default=2, help="Short sleep after a successful cycle")
parser.add_argument("--max-cycles", type=int, default=0, help="Optional hard stop after N cycles; 0 means run until quota is reached") parser.add_argument("--max-cycles", type=int, default=0, help="Optional hard stop after N cycles; 0 means run until quota is reached")
parser.add_argument("--print-stats", action="store_true", help="Load outcomes, sync, print stats report, and exit") parser.add_argument("--print-stats", action="store_true", help="Load outcomes, sync, print stats report, and exit")
parser.add_argument("--disable-queue-cleanup", action="store_true", help=argparse.SUPPRESS) parser.add_argument("--disable-queue-cleanup", action="store_true",
default=os.getenv("MODELHUB_COEXISTENCE_MODE", "1") != "0",
help=argparse.SUPPRESS)
parser.add_argument( parser.add_argument(
"--queue-cleanup-interval-cycles", "--queue-cleanup-interval-cycles",
type=int, type=int,
@@ -702,6 +704,11 @@ def run_poll_loop(
STATS_PRINT_INTERVAL = 10 STATS_PRINT_INTERVAL = 10
log(f"[poll] version={AGENT_VERSION} poll_run_dir={poll_run_dir}") log(f"[poll] version={AGENT_VERSION} poll_run_dir={poll_run_dir}")
log(
f"[coexistence] state_branch={base_args.state_sync_branch} "
f"queue_cancellation={'disabled' if base_args.disable_queue_cleanup else 'enabled'} "
"submission_preflight=enabled"
)
log( log(
"[state-retention] mode=decision_state_only full_archive=disabled " "[state-retention] mode=decision_state_only full_archive=disabled "
f"recent_outcomes={DEFAULT_RECENT_OUTCOME_LIMIT} " f"recent_outcomes={DEFAULT_RECENT_OUTCOME_LIMIT} "

View File

@@ -46,6 +46,13 @@ class SuccessFirstRoutingEngine:
return value return value
def _evidence(self, candidate: dict[str, Any]) -> tuple[str, int, int, float]: def _evidence(self, candidate: dict[str, Any]) -> tuple[str, int, int, float]:
def observed_total(item: dict[str, Any]) -> int:
if "failureCount" in item:
return max(0, int(item.get("successCount") or 0) + int(item.get("failureCount") or 0)
- int(item.get("platformFailureCount") or 0))
return max(0, int(item.get("decisionTotal") or 0))
sparse = None
profile = candidate.get("preflightMetadata") or {} profile = candidate.get("preflightMetadata") or {}
gpu = str(candidate.get("targetGpu") or "") gpu = str(candidate.get("targetGpu") or "")
framework = str(candidate.get("framework") or "") framework = str(candidate.get("framework") or "")
@@ -60,14 +67,18 @@ class SuccessFirstRoutingEngine:
size_bucket = int(load_bytes).bit_length() - 1 size_bucket = int(load_bytes).bit_length() - 1
key = f"{gpu}|{framework}|{task_type}|{model_type}|{quantization}|{size_bucket}" key = f"{gpu}|{framework}|{task_type}|{model_type}|{quantization}|{size_bucket}"
item = (self.outcome_stats.get("sizedProfileCombinationStats") or {}).get(key) or {} item = (self.outcome_stats.get("sizedProfileCombinationStats") or {}).get(key) or {}
total = int(item.get("decisionTotal") or 0) total = observed_total(item)
if total:
sparse = ("local_profile_size_shrunk", item, total)
if total >= 20: if total >= 20:
success = int(item.get("successCount") or 0) success = int(item.get("successCount") or 0)
return "local_profile_size", success, total, wilson_lower_bound(success, total) return "local_profile_size", success, total, wilson_lower_bound(success, total)
if model_type: if model_type:
key = f"{gpu}|{framework}|{task_type}|{model_type}|{quantization}" key = f"{gpu}|{framework}|{task_type}|{model_type}|{quantization}"
item = (self.outcome_stats.get("profileCombinationStats") or {}).get(key) or {} item = (self.outcome_stats.get("profileCombinationStats") or {}).get(key) or {}
total = int(item.get("decisionTotal") or 0) total = observed_total(item)
if total and sparse is None:
sparse = ("local_profile_shrunk", item, total)
if total >= 40: if total >= 40:
success = int(item.get("successCount") or 0) success = int(item.get("successCount") or 0)
return "local_profile", success, total, wilson_lower_bound(success, total) return "local_profile", success, total, wilson_lower_bound(success, total)
@@ -97,6 +108,13 @@ class SuccessFirstRoutingEngine:
total = 2 total = 2
success = 1 success = 1
level = "global_conservative_prior" level = "global_conservative_prior"
if sparse is not None:
level, item, samples = sparse
# Eight prior observations stabilize small groups without hiding
# their failures behind an unreachable hard sample threshold.
prior_success = round(8 * success / max(1, total))
success = int(item.get("successCount") or 0) + prior_success
total = samples + 8
return level, success, total, wilson_lower_bound(success, total) return level, success, total, wilson_lower_bound(success, total)
def annotate(self, candidate: dict[str, Any]) -> dict[str, Any]: def annotate(self, candidate: dict[str, Any]) -> dict[str, Any]:

View File

@@ -17,14 +17,14 @@ from typing import Any, Iterable
from dulwich import porcelain from dulwich import porcelain
from dulwich.repo import Repo from dulwich.repo import Repo
from common import read_json, read_jsonl, write_json, write_jsonl from common import read_json, read_jsonl, write_json, write_jsonl, runtime_instance_id
from runner_common import load_key_files from runner_common import load_key_files
from version import AGENT_VERSION from version import AGENT_VERSION
STATE_SCHEMA_VERSION = 1 STATE_SCHEMA_VERSION = 1
DEFAULT_REMOTE = "https://dev.modelhub.org.cn/CoolBoy/submmit.git" DEFAULT_REMOTE = "https://dev.modelhub.org.cn/CoolBoy/submmit.git"
DEFAULT_BRANCH = "agent-state" DEFAULT_BRANCH = "agent-state-accuracy-v2"
DEFAULT_BATCH_SIZE = 100 DEFAULT_BATCH_SIZE = 100
DEFAULT_HISTORY_DEPTH = 200 DEFAULT_HISTORY_DEPTH = 200
DEFAULT_RECENT_TERMINAL_INTENTS = 300 DEFAULT_RECENT_TERMINAL_INTENTS = 300
@@ -32,6 +32,7 @@ DEFAULT_LEDGER_RECORDS = 500
DEFAULT_CRASH_RECORDS = 50 DEFAULT_CRASH_RECORDS = 50
STATE_OUTCOME_FIELDS = { STATE_OUTCOME_FIELDS = {
"agentVersion", "instanceId", "strategyId",
"taskId", "modelId", "targetGpu", "framework", "taskType", "submitTime", "taskId", "modelId", "targetGpu", "framework", "taskType", "submitTime",
"lastSyncTime", "status", "verifyResult", "outcome", "failReason", "lastSyncTime", "status", "verifyResult", "outcome", "failReason",
"modelProfile", "failureCode", "failureCategory", "failureScope", "modelProfile", "failureCode", "failureCategory", "failureScope",
@@ -342,6 +343,7 @@ class StateGitSync:
retained_terminal = terminal[:DEFAULT_RECENT_TERMINAL_INTENTS] retained_terminal = terminal[:DEFAULT_RECENT_TERMINAL_INTENTS]
discarded = terminal[DEFAULT_RECENT_TERMINAL_INTENTS:] discarded = terminal[DEFAULT_RECENT_TERMINAL_INTENTS:]
compact_fields = { compact_fields = {
"agentVersion", "instanceId", "strategyId",
"intentId", "batchId", "status", "createdAt", "completedAt", "intentId", "batchId", "status", "createdAt", "completedAt",
"repoId", "targetGpu", "taskType", "framework", "configSource", "repoId", "targetGpu", "taskType", "framework", "configSource",
"configFingerprint", "safeConfigVector", "taskId", "configFingerprint", "safeConfigVector", "taskId",
@@ -372,6 +374,9 @@ class StateGitSync:
config = str(candidate.get("configParams") or "") config = str(candidate.get("configParams") or "")
return { return {
"intentId": uuid.uuid4().hex, "intentId": uuid.uuid4().hex,
"agentVersion": AGENT_VERSION,
"instanceId": runtime_instance_id(),
"strategyId": os.getenv("STRATEGY_ID"),
"batchId": batch_id, "batchId": batch_id,
"status": "pending", "status": "pending",
"createdAt": _utc_now().isoformat(), "createdAt": _utc_now().isoformat(),

View File

@@ -1 +1 @@
AGENT_VERSION = "2026.09.22.1" AGENT_VERSION = "2026.10.18.1"

View File

@@ -0,0 +1,96 @@
import os
import sys
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from types import SimpleNamespace
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "modelhub_submmit_api"))
from failure_taxonomy import classify_failure_report
from market_intelligence import _validated_official_config, MarketIntelligenceManager
from routing_engine import SuccessFirstRoutingEngine
from state_sync import StateGitSync, DEFAULT_BRANCH
from poll_runner import build_parser
class AccuracyAuditTests(unittest.TestCase):
def config(self, command):
return f"framework: vllm_fix_tokenizer\nsut_config:\n command: {command}\nref_config: {{}}\n"
def test_broken_official_and_cached_config_rejected(self):
bad = self.config('["/opt/entrypoint.sh - /model", "--port", "80"]')
for gpu in ("Biren_166m", "Kunlunxin_p-800"):
self.assertEqual((False, "malformed_command_argv"), _validated_official_config(
bad, framework="vllm_fix_tokenizer", target_gpu=gpu))
with tempfile.TemporaryDirectory() as tmp:
manager = MarketIntelligenceManager(Path(tmp) / "market.json")
manager.state = {"frameworkStats": {"text-generation": {gpu: {
"vllm_fix_tokenizer": {"officialConfigValid": True, "officialConfig": bad}}}}}
self.assertIsNone(manager.official_config(task_type="text-generation", target_gpu=gpu,
framework="vllm_fix_tokenizer"))
def test_separate_argv_and_shell_command_remain_valid(self):
for command in ('["/opt/entrypoint.sh", "/model"]', '["/bin/sh", "-c", "echo hello"]'):
self.assertEqual((True, None), _validated_official_config(
self.config(command), framework="vllm_fix_tokenizer", target_gpu="Biren_166m"))
def test_specific_failure_survives_transient_resource_error(self):
result = classify_failure_report("MODEL_NOT_SUPPORTED", [
"找不到空闲卡", "当前框架版本不支持该模型架构"])
self.assertEqual("framework_architecture_unsupported", result.category)
self.assertEqual("tokenizer_compatibility", classify_failure_report(
"TOKENIZER_FAILED", ["no idle gpu", "Tokenizer class missing"]).category)
self.assertEqual("platform_infrastructure", classify_failure_report(
"EXECUTE_EMPTY_RESULT", ["no idle gpu"]).category)
self.assertEqual("framework_architecture_unsupported", classify_failure_report(
None, ["no idle gpu", "Model architectures ['NewForCausalLM'] are not supported for now"]).category)
def test_launch_and_quantization_have_specific_categories(self):
self.assertEqual("configuration_launch", classify_failure_report("EXECUTE_EMPTY_RESULT", [
'exec: "/opt/entrypoint.sh - /model": no such file or directory']).category)
self.assertEqual("quantization_compatibility", classify_failure_report("EXECUTE_EMPTY_RESULT", [
"The quantization method compressed-tensors is not supported for the current GPU"]).category)
def test_sparse_failures_lower_score_even_without_classification(self):
candidate = {"targetGpu": "gpu", "framework": "vllm", "taskType": "text-generation",
"frameworkMarketSamples": 1000, "frameworkMarketSuccessRate": 0.8,
"preflightMetadata": {"modelType": "family", "estimatedLoadBytes": 1024}}
with tempfile.TemporaryDirectory() as tmp:
engine = SuccessFirstRoutingEngine(Path(tmp) / "route.json")
prior = engine._evidence(candidate)[3]
engine.outcome_stats = {"profileCombinationStats": {
"gpu|vllm|text-generation|family|none": {
"successCount": 0, "failureCount": 5, "decisionTotal": 0}}}
evidence = engine._evidence(candidate)
self.assertEqual("local_profile_shrunk", evidence[0])
self.assertLess(evidence[3], prior)
def test_coexistence_defaults_and_submission_provenance(self):
with patch.dict(os.environ, {"MODELHUB_COEXISTENCE_MODE": "1", "STRATEGY_ID": "new-agent"}):
self.assertTrue(build_parser().parse_args([]).disable_queue_cleanup)
intent = StateGitSync._intent({"repoId": "owner/model"}, "batch")
self.assertEqual("new-agent", intent["strategyId"])
self.assertTrue(intent["agentVersion"])
self.assertTrue(intent["instanceId"])
self.assertNotEqual("agent-state", DEFAULT_BRANCH)
def test_model_specific_failures_change_framework_selection(self):
with tempfile.TemporaryDirectory() as tmp:
manager = MarketIntelligenceManager(Path(tmp) / "market.json")
manager.state = {"frameworkStats": {"text-generation": {"gpu": {
"first": {"modelCount": 1000, "wilsonLowerBound": 0.8},
"second": {"modelCount": 1000, "wilsonLowerBound": 0.6},
}}}}
manager.set_local_outcome_stats({"profileCombinationStats": {
"gpu|first|text-generation|family|none": {"successCount": 0, "failureCount": 4}
}})
with patch.object(manager, "compatible_discovered_frameworks", return_value=[]):
ranked = manager.selectable_frameworks(
task_type="text-generation", target_gpu="gpu", incumbent_frameworks=["first", "second"],
inspection=SimpleNamespace(model_type="family", quantization_method=None))
self.assertEqual(["second", "first"], ranked)
if __name__ == "__main__":
unittest.main()

View File

@@ -283,6 +283,7 @@ class MarketIntelligenceTests(unittest.TestCase):
"successCount": 10, "successCount": 10,
"failureCount": 3, "failureCount": 3,
"attributableFailureCount": 0, "attributableFailureCount": 0,
"platformFailureCount": 3,
} }
}, },
"recentCombinationStats": { "recentCombinationStats": {
@@ -290,6 +291,7 @@ class MarketIntelligenceTests(unittest.TestCase):
"successCount": 10, "successCount": 10,
"failureCount": 3, "failureCount": 3,
"attributableFailureCount": 0, "attributableFailureCount": 0,
"platformFailureCount": 3,
"consecutiveFailures": 0, "consecutiveFailures": 0,
"consecutivePlatformFailures": 3, "consecutivePlatformFailures": 3,
"lastPlatformFailureAt": datetime.now(timezone.utc).isoformat(), "lastPlatformFailureAt": datetime.now(timezone.utc).isoformat(),