fix: reject broken configs and improve accuracy feedback with isolated agent state
This commit is contained in:
29
README.md
29
README.md
@@ -2,6 +2,30 @@
|
|||||||
|
|
||||||
This repository is packaged for the ModelHub XC agent platform.
|
This repository is packaged for the ModelHub XC agent platform.
|
||||||
|
|
||||||
|
## Accuracy release 2026.10.18.1
|
||||||
|
|
||||||
|
This release rejects malformed YAML command arrays, including the observed
|
||||||
|
`/opt/entrypoint.sh - /model` executable error, even in restored cached
|
||||||
|
official configurations. It preserves terminal compatibility errors when a
|
||||||
|
log also contains transient platform resource messages. Framework scoring
|
||||||
|
includes unresolved non-platform failures; sparse model-profile evidence is
|
||||||
|
blended with eight prior observations instead of discarded below a threshold.
|
||||||
|
|
||||||
|
For operation alongside the baseline agent, the new default state branch is
|
||||||
|
`agent-state-accuracy-v2`. Do not override it with the baseline's `agent-state`.
|
||||||
|
The separate branch starts its own checkpoint on first boot; it does not
|
||||||
|
overwrite or migrate the baseline state. New submissions retain agent version,
|
||||||
|
instance ID and platform strategy ID. Historical tasks are not assigned an
|
||||||
|
invented version.
|
||||||
|
|
||||||
|
`MODELHUB_COEXISTENCE_MODE=1` is the default: this agent does not run queue
|
||||||
|
cancellation, including deterministic cleanup, because the accounts are shared
|
||||||
|
with the baseline. Submission preflight remains enabled. Set this variable to
|
||||||
|
`0` only when account-wide cleanup is intended; `--disable-queue-cleanup` still
|
||||||
|
disables cleanup explicitly. The baseline agent is unchanged. Both agents share
|
||||||
|
account capacity and platform deduplication; separate state does not reserve
|
||||||
|
capacity or eliminate simultaneous submission races.
|
||||||
|
|
||||||
## Platform Contract
|
## Platform Contract
|
||||||
|
|
||||||
- Root-level `Dockerfile`
|
- Root-level `Dockerfile`
|
||||||
@@ -67,7 +91,7 @@ Optional tuning:
|
|||||||
- `MODELHUB_RECENT_MODEL_RESERVE_SLOTS` default `5` per account
|
- `MODELHUB_RECENT_MODEL_RESERVE_SLOTS` default `5` per account
|
||||||
- `MODELHUB_RECENT_MODEL_DAYS` default `7`
|
- `MODELHUB_RECENT_MODEL_DAYS` default `7`
|
||||||
- `MODELHUB_STATE_SYNC_REMOTE` default `https://dev.modelhub.org.cn/CoolBoy/submmit.git`
|
- `MODELHUB_STATE_SYNC_REMOTE` default `https://dev.modelhub.org.cn/CoolBoy/submmit.git`
|
||||||
- `MODELHUB_STATE_SYNC_BRANCH` default `agent-state`
|
- `MODELHUB_STATE_SYNC_BRANCH` default `agent-state-accuracy-v2`
|
||||||
- `MODELHUB_STATE_SYNC_BATCH_SIZE` default `20`
|
- `MODELHUB_STATE_SYNC_BATCH_SIZE` default `20`
|
||||||
|
|
||||||
Git state synchronization reads `modelhub_user_name`, `modelhub_user_email`, and
|
Git state synchronization reads `modelhub_user_name`, `modelhub_user_email`, and
|
||||||
@@ -140,7 +164,8 @@ unvetted GPU or framework.
|
|||||||
## Durable State Branch
|
## Durable State Branch
|
||||||
|
|
||||||
The hosted worker synchronizes an allowlisted runtime snapshot to the orphan
|
The hosted worker synchronizes an allowlisted runtime snapshot to the orphan
|
||||||
`agent-state` branch of this repository. Before each batch of at most 20 API
|
`agent-state-accuracy-v2` branch of this repository (overridable via
|
||||||
|
`MODELHUB_STATE_SYNC_BRANCH`). Before each batch of at most 20 API
|
||||||
submissions it pushes write-ahead intents containing model, GPU, task, framework,
|
submissions it pushes write-ahead intents containing model, GPU, task, framework,
|
||||||
safe parameter vector, and a config fingerprint. It pushes task IDs and results
|
safe parameter vector, and a config fingerprint. It pushes task IDs and results
|
||||||
after the batch. If either push fails, no further submission begins until the
|
after the batch. If either push fails, no further submission begins until the
|
||||||
|
|||||||
@@ -67,7 +67,7 @@ PLATFORM_PATTERNS = (
|
|||||||
(re.compile(r"welcome\.sh: No such file|data: command not found", re.I), "broken_platform_launch_script"),
|
(re.compile(r"welcome\.sh: No such file|data: command not found", re.I), "broken_platform_launch_script"),
|
||||||
(re.compile(r"id_rsa.*No such file", re.I), "missing_platform_credential"),
|
(re.compile(r"id_rsa.*No such file", re.I), "missing_platform_credential"),
|
||||||
(re.compile(r"找不到空闲卡|no idle (?:gpu|card)", re.I), "no_idle_device"),
|
(re.compile(r"找不到空闲卡|no idle (?:gpu|card)", re.I), "no_idle_device"),
|
||||||
(re.compile(r"storage|download.*timed? out|connection reset", re.I), "platform_io_transient"),
|
(re.compile(r"no space left on device|download.*timed? out|connection reset", re.I), "platform_io_transient"),
|
||||||
)
|
)
|
||||||
|
|
||||||
DETERMINISTIC_LOG_PATTERNS = (
|
DETERMINISTIC_LOG_PATTERNS = (
|
||||||
@@ -80,6 +80,8 @@ DETERMINISTIC_LOG_PATTERNS = (
|
|||||||
# semantics. Generic messages such as "attention backend not supported" or
|
# semantics. Generic messages such as "attention backend not supported" or
|
||||||
# "GPU type not supported" must not create an architecture blacklist.
|
# "GPU type not supported" must not create an architecture blacklist.
|
||||||
EXPLICIT_FRAMEWORK_MODEL_UNSUPPORTED_PATTERNS = (
|
EXPLICIT_FRAMEWORK_MODEL_UNSUPPORTED_PATTERNS = (
|
||||||
|
re.compile(r"model type\s+[`'\"][^`'\"]+[`'\"].{0,100}does not recognize this architecture", re.I),
|
||||||
|
re.compile(r"Model architectures?\s*\[[^\]\n]+\]\s*(?:are|is) not supported", re.I),
|
||||||
re.compile(
|
re.compile(
|
||||||
r"(?:当前|该|此)(?:推理)?框架.{0,100}(?:不支持|暂不支持|无法支持|不兼容)"
|
r"(?:当前|该|此)(?:推理)?框架.{0,100}(?:不支持|暂不支持|无法支持|不兼容)"
|
||||||
r".{0,100}(?:该|此|当前)?(?:模型架构|模型|架构)",
|
r".{0,100}(?:该|此|当前)?(?:模型架构|模型|架构)",
|
||||||
@@ -117,19 +119,16 @@ def classify_failure_report(report_code: str | None, log_lines: Iterable[str] =
|
|||||||
code = str(report_code or "").strip().upper()
|
code = str(report_code or "").strip().upper()
|
||||||
text = "\n".join(str(line) for line in log_lines)
|
text = "\n".join(str(line) for line in log_lines)
|
||||||
|
|
||||||
# Infrastructure signatures override broad report codes such as
|
if re.search(r'exec:.*entrypoint.*(?:no such file|not found)', text, re.I):
|
||||||
# EXECUTE_EMPTY_RESULT so they do not poison model compatibility feedback.
|
return FailureClassification(
|
||||||
for pattern, reason in PLATFORM_PATTERNS:
|
"configuration_launch", "configuration", "reject_broken_config", True, False,
|
||||||
if pattern.search(text):
|
"invalid_executable_argv",
|
||||||
return FailureClassification(
|
)
|
||||||
"platform_infrastructure",
|
if re.search(r'quantization method .*not supported for the current GPU', text, re.I):
|
||||||
"gpu_framework",
|
return FailureClassification(
|
||||||
"open_short_gpu_framework_circuit_and_retry_other_models",
|
"quantization_compatibility", "model_gpu_framework", "reject_quantization_route",
|
||||||
True,
|
True, False, "explicit_quantization_gpu_incompatibility",
|
||||||
False,
|
)
|
||||||
reason,
|
|
||||||
)
|
|
||||||
|
|
||||||
if code in DETERMINISTIC_POLICIES:
|
if code in DETERMINISTIC_POLICIES:
|
||||||
return DETERMINISTIC_POLICIES[code]
|
return DETERMINISTIC_POLICIES[code]
|
||||||
for pattern, inferred_code in DETERMINISTIC_LOG_PATTERNS:
|
for pattern, inferred_code in DETERMINISTIC_LOG_PATTERNS:
|
||||||
@@ -149,6 +148,15 @@ def classify_failure_report(report_code: str | None, log_lines: Iterable[str] =
|
|||||||
)
|
)
|
||||||
if code in SEMANTIC_POLICIES:
|
if code in SEMANTIC_POLICIES:
|
||||||
return SEMANTIC_POLICIES[code]
|
return SEMANTIC_POLICIES[code]
|
||||||
|
# Fall back to infrastructure signatures only after specific root causes.
|
||||||
|
# Logs may include a transient allocation failure before a model starts.
|
||||||
|
for pattern, reason in PLATFORM_PATTERNS:
|
||||||
|
if pattern.search(text):
|
||||||
|
return FailureClassification(
|
||||||
|
"platform_infrastructure", "gpu_framework",
|
||||||
|
"open_short_gpu_framework_circuit_and_retry_other_models",
|
||||||
|
True, False, reason,
|
||||||
|
)
|
||||||
return FailureClassification(
|
return FailureClassification(
|
||||||
"ambiguous_runtime",
|
"ambiguous_runtime",
|
||||||
"unknown",
|
"unknown",
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import math
|
|||||||
import re
|
import re
|
||||||
import statistics
|
import statistics
|
||||||
import threading
|
import threading
|
||||||
|
import yaml
|
||||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -97,6 +98,31 @@ def _validated_official_config(config: str, *, framework: str, target_gpu: str)
|
|||||||
return False, "missing_sut_or_ref_config"
|
return False, "missing_sut_or_ref_config"
|
||||||
if any(marker in config for marker in ("{{", "}}", "PLACEHOLDER")):
|
if any(marker in config for marker in ("{{", "}}", "PLACEHOLDER")):
|
||||||
return False, "unresolved_placeholder"
|
return False, "unresolved_placeholder"
|
||||||
|
try:
|
||||||
|
document = yaml.safe_load(config)
|
||||||
|
except yaml.YAMLError:
|
||||||
|
return False, "invalid_yaml"
|
||||||
|
if not isinstance(document, dict):
|
||||||
|
return False, "invalid_config_structure"
|
||||||
|
|
||||||
|
def valid_commands(node: Any) -> bool:
|
||||||
|
if isinstance(node, dict):
|
||||||
|
for key, value in node.items():
|
||||||
|
if key in {"command", "entrypoint"} and isinstance(value, list):
|
||||||
|
if not value or not all(isinstance(arg, str) for arg in value):
|
||||||
|
return False
|
||||||
|
# argv[0] is an executable, not a shell command. In
|
||||||
|
# particular, a folded YAML list item must not eat argv[1].
|
||||||
|
if not value[0].strip() or re.search(r"\s", value[0]):
|
||||||
|
return False
|
||||||
|
if not valid_commands(value):
|
||||||
|
return False
|
||||||
|
elif isinstance(node, list):
|
||||||
|
return all(valid_commands(value) for value in node)
|
||||||
|
return True
|
||||||
|
|
||||||
|
if not valid_commands(document):
|
||||||
|
return False, "malformed_command_argv"
|
||||||
|
|
||||||
declared = re.search(r"(?m)^\s*framework:\s*['\"]?([^'\"\s]+)", config)
|
declared = re.search(r"(?m)^\s*framework:\s*['\"]?([^'\"\s]+)", config)
|
||||||
if declared is None or declared.group(1).strip() != framework:
|
if declared is None or declared.group(1).strip() != framework:
|
||||||
@@ -736,7 +762,7 @@ class MarketIntelligenceManager:
|
|||||||
local_success = max(0, int(local_item.get("successCount") or 0))
|
local_success = max(0, int(local_item.get("successCount") or 0))
|
||||||
local_failure = max(
|
local_failure = max(
|
||||||
0,
|
0,
|
||||||
int(local_item.get("attributableFailureCount", local_item.get("failureCount") or 0)),
|
int(local_item.get("failureCount") or 0) - int(local_item.get("platformFailureCount") or 0),
|
||||||
)
|
)
|
||||||
local_samples = local_success + local_failure
|
local_samples = local_success + local_failure
|
||||||
local_score = _wilson_lower_bound(local_success, local_samples)
|
local_score = _wilson_lower_bound(local_success, local_samples)
|
||||||
@@ -746,7 +772,7 @@ class MarketIntelligenceManager:
|
|||||||
recent_all_failures = max(0, int(recent_item.get("failureCount") or 0))
|
recent_all_failures = max(0, int(recent_item.get("failureCount") or 0))
|
||||||
recent_failure = max(
|
recent_failure = max(
|
||||||
0,
|
0,
|
||||||
int(recent_item.get("attributableFailureCount", recent_item.get("failureCount") or 0)),
|
int(recent_item.get("failureCount") or 0) - int(recent_item.get("platformFailureCount") or 0),
|
||||||
)
|
)
|
||||||
recent_samples = recent_success + recent_failure
|
recent_samples = recent_success + recent_failure
|
||||||
recent_rate = recent_success / recent_samples if recent_samples else None
|
recent_rate = recent_success / recent_samples if recent_samples else None
|
||||||
@@ -849,11 +875,31 @@ class MarketIntelligenceManager:
|
|||||||
if incumbent_best > 0 and float(evidence["combinedScore"]) < incumbent_best * NEW_FRAMEWORK_PROMOTION_MARGIN:
|
if incumbent_best > 0 and float(evidence["combinedScore"]) < incumbent_best * NEW_FRAMEWORK_PROMOTION_MARGIN:
|
||||||
continue
|
continue
|
||||||
promoted.append(framework)
|
promoted.append(framework)
|
||||||
return self.rank_frameworks(
|
ranked = self.rank_frameworks(
|
||||||
task_type=task_type,
|
task_type=task_type,
|
||||||
target_gpu=target_gpu,
|
target_gpu=target_gpu,
|
||||||
compatible_frameworks=promoted,
|
compatible_frameworks=promoted,
|
||||||
)
|
)
|
||||||
|
model_type = str(getattr(inspection, "model_type", None) or "").strip()
|
||||||
|
quantization = str(getattr(inspection, "quantization_method", None) or "none").strip()
|
||||||
|
if not model_type:
|
||||||
|
return ranked
|
||||||
|
|
||||||
|
def profile_score(framework: str) -> float:
|
||||||
|
prior = float(self._framework_evidence(task_type, target_gpu, framework)["combinedScore"])
|
||||||
|
key = f"{target_gpu}|{framework}|{task_type}|{model_type}|{quantization}"
|
||||||
|
item = (self.local_outcome_stats.get("profileCombinationStats") or {}).get(key) or {}
|
||||||
|
success = max(0, int(item.get("successCount") or 0))
|
||||||
|
failure = max(0, int(item.get("failureCount") or 0) - int(item.get("platformFailureCount") or 0))
|
||||||
|
samples = success + failure
|
||||||
|
if not samples:
|
||||||
|
return prior
|
||||||
|
# Model-specific negative evidence must affect the framework
|
||||||
|
# choice, not only the ranking after the first framework wins.
|
||||||
|
lower = _wilson_lower_bound(success + 8 * prior, samples + 8)
|
||||||
|
return lower
|
||||||
|
|
||||||
|
return sorted(ranked, key=profile_score, reverse=True)
|
||||||
|
|
||||||
def compatible_discovered_frameworks(
|
def compatible_discovered_frameworks(
|
||||||
self,
|
self,
|
||||||
@@ -941,6 +987,11 @@ class MarketIntelligenceManager:
|
|||||||
config = item.get("officialConfig")
|
config = item.get("officialConfig")
|
||||||
if not isinstance(config, str) or not config:
|
if not isinstance(config, str) or not config:
|
||||||
return None
|
return None
|
||||||
|
valid, reason = _validated_official_config(config, framework=framework, target_gpu=target_gpu)
|
||||||
|
if not valid:
|
||||||
|
item["officialConfigValid"] = False
|
||||||
|
item["officialConfigError"] = reason
|
||||||
|
return None
|
||||||
return _render_official_config(config, gguf_filename=gguf_filename)
|
return _render_official_config(config, gguf_filename=gguf_filename)
|
||||||
|
|
||||||
def framework_metadata(self, task_type: str, target_gpu: str, framework: str) -> dict[str, Any]:
|
def framework_metadata(self, task_type: str, target_gpu: str, framework: str) -> dict[str, Any]:
|
||||||
|
|||||||
@@ -22,6 +22,8 @@ from history_stats import classify_failure, is_failure, is_success
|
|||||||
from llm_classifier import LLMAssistedClassifier
|
from llm_classifier import LLMAssistedClassifier
|
||||||
from modelhub_client import ModelHubClient, ModelHubClientPool
|
from modelhub_client import ModelHubClient, ModelHubClientPool
|
||||||
from task_registry import task_type_from_history_task
|
from task_registry import task_type_from_history_task
|
||||||
|
from common import runtime_instance_id
|
||||||
|
from version import AGENT_VERSION
|
||||||
|
|
||||||
|
|
||||||
DEFAULT_OUTCOMES_PATH = Path("outcomes/submissions.jsonl")
|
DEFAULT_OUTCOMES_PATH = Path("outcomes/submissions.jsonl")
|
||||||
@@ -36,6 +38,7 @@ FAILURE_ENRICHMENT_MAX_ATTEMPTS = 3
|
|||||||
TERMINAL_TASK_STATUSES = {"success", "failed", "error", "cancelled", "completed"}
|
TERMINAL_TASK_STATUSES = {"success", "failed", "error", "cancelled", "completed"}
|
||||||
|
|
||||||
DECISION_RECORD_FIELDS = {
|
DECISION_RECORD_FIELDS = {
|
||||||
|
"agentVersion", "instanceId", "strategyId",
|
||||||
"taskId",
|
"taskId",
|
||||||
"modelId",
|
"modelId",
|
||||||
"targetGpu",
|
"targetGpu",
|
||||||
@@ -325,6 +328,9 @@ class OutcomeTracker:
|
|||||||
model_profile: dict[str, Any] | None = None,
|
model_profile: dict[str, Any] | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
record: dict[str, Any] = {
|
record: dict[str, Any] = {
|
||||||
|
"agentVersion": AGENT_VERSION,
|
||||||
|
"instanceId": runtime_instance_id(),
|
||||||
|
"strategyId": os.getenv("STRATEGY_ID"),
|
||||||
"modelId": model_id,
|
"modelId": model_id,
|
||||||
"targetGpu": target_gpu,
|
"targetGpu": target_gpu,
|
||||||
"framework": framework,
|
"framework": framework,
|
||||||
|
|||||||
@@ -234,7 +234,9 @@ def build_parser() -> argparse.ArgumentParser:
|
|||||||
parser.add_argument("--post-cycle-cooldown-seconds", type=int, default=2, help="Short sleep after a successful cycle")
|
parser.add_argument("--post-cycle-cooldown-seconds", type=int, default=2, help="Short sleep after a successful cycle")
|
||||||
parser.add_argument("--max-cycles", type=int, default=0, help="Optional hard stop after N cycles; 0 means run until quota is reached")
|
parser.add_argument("--max-cycles", type=int, default=0, help="Optional hard stop after N cycles; 0 means run until quota is reached")
|
||||||
parser.add_argument("--print-stats", action="store_true", help="Load outcomes, sync, print stats report, and exit")
|
parser.add_argument("--print-stats", action="store_true", help="Load outcomes, sync, print stats report, and exit")
|
||||||
parser.add_argument("--disable-queue-cleanup", action="store_true", help=argparse.SUPPRESS)
|
parser.add_argument("--disable-queue-cleanup", action="store_true",
|
||||||
|
default=os.getenv("MODELHUB_COEXISTENCE_MODE", "1") != "0",
|
||||||
|
help=argparse.SUPPRESS)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--queue-cleanup-interval-cycles",
|
"--queue-cleanup-interval-cycles",
|
||||||
type=int,
|
type=int,
|
||||||
@@ -702,6 +704,11 @@ def run_poll_loop(
|
|||||||
STATS_PRINT_INTERVAL = 10
|
STATS_PRINT_INTERVAL = 10
|
||||||
|
|
||||||
log(f"[poll] version={AGENT_VERSION} poll_run_dir={poll_run_dir}")
|
log(f"[poll] version={AGENT_VERSION} poll_run_dir={poll_run_dir}")
|
||||||
|
log(
|
||||||
|
f"[coexistence] state_branch={base_args.state_sync_branch} "
|
||||||
|
f"queue_cancellation={'disabled' if base_args.disable_queue_cleanup else 'enabled'} "
|
||||||
|
"submission_preflight=enabled"
|
||||||
|
)
|
||||||
log(
|
log(
|
||||||
"[state-retention] mode=decision_state_only full_archive=disabled "
|
"[state-retention] mode=decision_state_only full_archive=disabled "
|
||||||
f"recent_outcomes={DEFAULT_RECENT_OUTCOME_LIMIT} "
|
f"recent_outcomes={DEFAULT_RECENT_OUTCOME_LIMIT} "
|
||||||
|
|||||||
@@ -46,6 +46,13 @@ class SuccessFirstRoutingEngine:
|
|||||||
return value
|
return value
|
||||||
|
|
||||||
def _evidence(self, candidate: dict[str, Any]) -> tuple[str, int, int, float]:
|
def _evidence(self, candidate: dict[str, Any]) -> tuple[str, int, int, float]:
|
||||||
|
def observed_total(item: dict[str, Any]) -> int:
|
||||||
|
if "failureCount" in item:
|
||||||
|
return max(0, int(item.get("successCount") or 0) + int(item.get("failureCount") or 0)
|
||||||
|
- int(item.get("platformFailureCount") or 0))
|
||||||
|
return max(0, int(item.get("decisionTotal") or 0))
|
||||||
|
|
||||||
|
sparse = None
|
||||||
profile = candidate.get("preflightMetadata") or {}
|
profile = candidate.get("preflightMetadata") or {}
|
||||||
gpu = str(candidate.get("targetGpu") or "")
|
gpu = str(candidate.get("targetGpu") or "")
|
||||||
framework = str(candidate.get("framework") or "")
|
framework = str(candidate.get("framework") or "")
|
||||||
@@ -60,14 +67,18 @@ class SuccessFirstRoutingEngine:
|
|||||||
size_bucket = int(load_bytes).bit_length() - 1
|
size_bucket = int(load_bytes).bit_length() - 1
|
||||||
key = f"{gpu}|{framework}|{task_type}|{model_type}|{quantization}|{size_bucket}"
|
key = f"{gpu}|{framework}|{task_type}|{model_type}|{quantization}|{size_bucket}"
|
||||||
item = (self.outcome_stats.get("sizedProfileCombinationStats") or {}).get(key) or {}
|
item = (self.outcome_stats.get("sizedProfileCombinationStats") or {}).get(key) or {}
|
||||||
total = int(item.get("decisionTotal") or 0)
|
total = observed_total(item)
|
||||||
|
if total:
|
||||||
|
sparse = ("local_profile_size_shrunk", item, total)
|
||||||
if total >= 20:
|
if total >= 20:
|
||||||
success = int(item.get("successCount") or 0)
|
success = int(item.get("successCount") or 0)
|
||||||
return "local_profile_size", success, total, wilson_lower_bound(success, total)
|
return "local_profile_size", success, total, wilson_lower_bound(success, total)
|
||||||
if model_type:
|
if model_type:
|
||||||
key = f"{gpu}|{framework}|{task_type}|{model_type}|{quantization}"
|
key = f"{gpu}|{framework}|{task_type}|{model_type}|{quantization}"
|
||||||
item = (self.outcome_stats.get("profileCombinationStats") or {}).get(key) or {}
|
item = (self.outcome_stats.get("profileCombinationStats") or {}).get(key) or {}
|
||||||
total = int(item.get("decisionTotal") or 0)
|
total = observed_total(item)
|
||||||
|
if total and sparse is None:
|
||||||
|
sparse = ("local_profile_shrunk", item, total)
|
||||||
if total >= 40:
|
if total >= 40:
|
||||||
success = int(item.get("successCount") or 0)
|
success = int(item.get("successCount") or 0)
|
||||||
return "local_profile", success, total, wilson_lower_bound(success, total)
|
return "local_profile", success, total, wilson_lower_bound(success, total)
|
||||||
@@ -97,6 +108,13 @@ class SuccessFirstRoutingEngine:
|
|||||||
total = 2
|
total = 2
|
||||||
success = 1
|
success = 1
|
||||||
level = "global_conservative_prior"
|
level = "global_conservative_prior"
|
||||||
|
if sparse is not None:
|
||||||
|
level, item, samples = sparse
|
||||||
|
# Eight prior observations stabilize small groups without hiding
|
||||||
|
# their failures behind an unreachable hard sample threshold.
|
||||||
|
prior_success = round(8 * success / max(1, total))
|
||||||
|
success = int(item.get("successCount") or 0) + prior_success
|
||||||
|
total = samples + 8
|
||||||
return level, success, total, wilson_lower_bound(success, total)
|
return level, success, total, wilson_lower_bound(success, total)
|
||||||
|
|
||||||
def annotate(self, candidate: dict[str, Any]) -> dict[str, Any]:
|
def annotate(self, candidate: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
|||||||
@@ -17,14 +17,14 @@ from typing import Any, Iterable
|
|||||||
from dulwich import porcelain
|
from dulwich import porcelain
|
||||||
from dulwich.repo import Repo
|
from dulwich.repo import Repo
|
||||||
|
|
||||||
from common import read_json, read_jsonl, write_json, write_jsonl
|
from common import read_json, read_jsonl, write_json, write_jsonl, runtime_instance_id
|
||||||
from runner_common import load_key_files
|
from runner_common import load_key_files
|
||||||
from version import AGENT_VERSION
|
from version import AGENT_VERSION
|
||||||
|
|
||||||
|
|
||||||
STATE_SCHEMA_VERSION = 1
|
STATE_SCHEMA_VERSION = 1
|
||||||
DEFAULT_REMOTE = "https://dev.modelhub.org.cn/CoolBoy/submmit.git"
|
DEFAULT_REMOTE = "https://dev.modelhub.org.cn/CoolBoy/submmit.git"
|
||||||
DEFAULT_BRANCH = "agent-state"
|
DEFAULT_BRANCH = "agent-state-accuracy-v2"
|
||||||
DEFAULT_BATCH_SIZE = 100
|
DEFAULT_BATCH_SIZE = 100
|
||||||
DEFAULT_HISTORY_DEPTH = 200
|
DEFAULT_HISTORY_DEPTH = 200
|
||||||
DEFAULT_RECENT_TERMINAL_INTENTS = 300
|
DEFAULT_RECENT_TERMINAL_INTENTS = 300
|
||||||
@@ -32,6 +32,7 @@ DEFAULT_LEDGER_RECORDS = 500
|
|||||||
DEFAULT_CRASH_RECORDS = 50
|
DEFAULT_CRASH_RECORDS = 50
|
||||||
|
|
||||||
STATE_OUTCOME_FIELDS = {
|
STATE_OUTCOME_FIELDS = {
|
||||||
|
"agentVersion", "instanceId", "strategyId",
|
||||||
"taskId", "modelId", "targetGpu", "framework", "taskType", "submitTime",
|
"taskId", "modelId", "targetGpu", "framework", "taskType", "submitTime",
|
||||||
"lastSyncTime", "status", "verifyResult", "outcome", "failReason",
|
"lastSyncTime", "status", "verifyResult", "outcome", "failReason",
|
||||||
"modelProfile", "failureCode", "failureCategory", "failureScope",
|
"modelProfile", "failureCode", "failureCategory", "failureScope",
|
||||||
@@ -342,6 +343,7 @@ class StateGitSync:
|
|||||||
retained_terminal = terminal[:DEFAULT_RECENT_TERMINAL_INTENTS]
|
retained_terminal = terminal[:DEFAULT_RECENT_TERMINAL_INTENTS]
|
||||||
discarded = terminal[DEFAULT_RECENT_TERMINAL_INTENTS:]
|
discarded = terminal[DEFAULT_RECENT_TERMINAL_INTENTS:]
|
||||||
compact_fields = {
|
compact_fields = {
|
||||||
|
"agentVersion", "instanceId", "strategyId",
|
||||||
"intentId", "batchId", "status", "createdAt", "completedAt",
|
"intentId", "batchId", "status", "createdAt", "completedAt",
|
||||||
"repoId", "targetGpu", "taskType", "framework", "configSource",
|
"repoId", "targetGpu", "taskType", "framework", "configSource",
|
||||||
"configFingerprint", "safeConfigVector", "taskId",
|
"configFingerprint", "safeConfigVector", "taskId",
|
||||||
@@ -372,6 +374,9 @@ class StateGitSync:
|
|||||||
config = str(candidate.get("configParams") or "")
|
config = str(candidate.get("configParams") or "")
|
||||||
return {
|
return {
|
||||||
"intentId": uuid.uuid4().hex,
|
"intentId": uuid.uuid4().hex,
|
||||||
|
"agentVersion": AGENT_VERSION,
|
||||||
|
"instanceId": runtime_instance_id(),
|
||||||
|
"strategyId": os.getenv("STRATEGY_ID"),
|
||||||
"batchId": batch_id,
|
"batchId": batch_id,
|
||||||
"status": "pending",
|
"status": "pending",
|
||||||
"createdAt": _utc_now().isoformat(),
|
"createdAt": _utc_now().isoformat(),
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
AGENT_VERSION = "2026.09.22.1"
|
AGENT_VERSION = "2026.10.18.1"
|
||||||
|
|||||||
96
tests/test_accuracy_audit.py
Normal file
96
tests/test_accuracy_audit.py
Normal file
@@ -0,0 +1,96 @@
|
|||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import patch
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "modelhub_submmit_api"))
|
||||||
|
from failure_taxonomy import classify_failure_report
|
||||||
|
from market_intelligence import _validated_official_config, MarketIntelligenceManager
|
||||||
|
from routing_engine import SuccessFirstRoutingEngine
|
||||||
|
from state_sync import StateGitSync, DEFAULT_BRANCH
|
||||||
|
from poll_runner import build_parser
|
||||||
|
|
||||||
|
|
||||||
|
class AccuracyAuditTests(unittest.TestCase):
|
||||||
|
def config(self, command):
|
||||||
|
return f"framework: vllm_fix_tokenizer\nsut_config:\n command: {command}\nref_config: {{}}\n"
|
||||||
|
|
||||||
|
def test_broken_official_and_cached_config_rejected(self):
|
||||||
|
bad = self.config('["/opt/entrypoint.sh - /model", "--port", "80"]')
|
||||||
|
for gpu in ("Biren_166m", "Kunlunxin_p-800"):
|
||||||
|
self.assertEqual((False, "malformed_command_argv"), _validated_official_config(
|
||||||
|
bad, framework="vllm_fix_tokenizer", target_gpu=gpu))
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
manager = MarketIntelligenceManager(Path(tmp) / "market.json")
|
||||||
|
manager.state = {"frameworkStats": {"text-generation": {gpu: {
|
||||||
|
"vllm_fix_tokenizer": {"officialConfigValid": True, "officialConfig": bad}}}}}
|
||||||
|
self.assertIsNone(manager.official_config(task_type="text-generation", target_gpu=gpu,
|
||||||
|
framework="vllm_fix_tokenizer"))
|
||||||
|
|
||||||
|
def test_separate_argv_and_shell_command_remain_valid(self):
|
||||||
|
for command in ('["/opt/entrypoint.sh", "/model"]', '["/bin/sh", "-c", "echo hello"]'):
|
||||||
|
self.assertEqual((True, None), _validated_official_config(
|
||||||
|
self.config(command), framework="vllm_fix_tokenizer", target_gpu="Biren_166m"))
|
||||||
|
|
||||||
|
def test_specific_failure_survives_transient_resource_error(self):
|
||||||
|
result = classify_failure_report("MODEL_NOT_SUPPORTED", [
|
||||||
|
"找不到空闲卡", "当前框架版本不支持该模型架构"])
|
||||||
|
self.assertEqual("framework_architecture_unsupported", result.category)
|
||||||
|
self.assertEqual("tokenizer_compatibility", classify_failure_report(
|
||||||
|
"TOKENIZER_FAILED", ["no idle gpu", "Tokenizer class missing"]).category)
|
||||||
|
self.assertEqual("platform_infrastructure", classify_failure_report(
|
||||||
|
"EXECUTE_EMPTY_RESULT", ["no idle gpu"]).category)
|
||||||
|
self.assertEqual("framework_architecture_unsupported", classify_failure_report(
|
||||||
|
None, ["no idle gpu", "Model architectures ['NewForCausalLM'] are not supported for now"]).category)
|
||||||
|
|
||||||
|
def test_launch_and_quantization_have_specific_categories(self):
|
||||||
|
self.assertEqual("configuration_launch", classify_failure_report("EXECUTE_EMPTY_RESULT", [
|
||||||
|
'exec: "/opt/entrypoint.sh - /model": no such file or directory']).category)
|
||||||
|
self.assertEqual("quantization_compatibility", classify_failure_report("EXECUTE_EMPTY_RESULT", [
|
||||||
|
"The quantization method compressed-tensors is not supported for the current GPU"]).category)
|
||||||
|
|
||||||
|
def test_sparse_failures_lower_score_even_without_classification(self):
|
||||||
|
candidate = {"targetGpu": "gpu", "framework": "vllm", "taskType": "text-generation",
|
||||||
|
"frameworkMarketSamples": 1000, "frameworkMarketSuccessRate": 0.8,
|
||||||
|
"preflightMetadata": {"modelType": "family", "estimatedLoadBytes": 1024}}
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
engine = SuccessFirstRoutingEngine(Path(tmp) / "route.json")
|
||||||
|
prior = engine._evidence(candidate)[3]
|
||||||
|
engine.outcome_stats = {"profileCombinationStats": {
|
||||||
|
"gpu|vllm|text-generation|family|none": {
|
||||||
|
"successCount": 0, "failureCount": 5, "decisionTotal": 0}}}
|
||||||
|
evidence = engine._evidence(candidate)
|
||||||
|
self.assertEqual("local_profile_shrunk", evidence[0])
|
||||||
|
self.assertLess(evidence[3], prior)
|
||||||
|
|
||||||
|
def test_coexistence_defaults_and_submission_provenance(self):
|
||||||
|
with patch.dict(os.environ, {"MODELHUB_COEXISTENCE_MODE": "1", "STRATEGY_ID": "new-agent"}):
|
||||||
|
self.assertTrue(build_parser().parse_args([]).disable_queue_cleanup)
|
||||||
|
intent = StateGitSync._intent({"repoId": "owner/model"}, "batch")
|
||||||
|
self.assertEqual("new-agent", intent["strategyId"])
|
||||||
|
self.assertTrue(intent["agentVersion"])
|
||||||
|
self.assertTrue(intent["instanceId"])
|
||||||
|
self.assertNotEqual("agent-state", DEFAULT_BRANCH)
|
||||||
|
|
||||||
|
def test_model_specific_failures_change_framework_selection(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
manager = MarketIntelligenceManager(Path(tmp) / "market.json")
|
||||||
|
manager.state = {"frameworkStats": {"text-generation": {"gpu": {
|
||||||
|
"first": {"modelCount": 1000, "wilsonLowerBound": 0.8},
|
||||||
|
"second": {"modelCount": 1000, "wilsonLowerBound": 0.6},
|
||||||
|
}}}}
|
||||||
|
manager.set_local_outcome_stats({"profileCombinationStats": {
|
||||||
|
"gpu|first|text-generation|family|none": {"successCount": 0, "failureCount": 4}
|
||||||
|
}})
|
||||||
|
with patch.object(manager, "compatible_discovered_frameworks", return_value=[]):
|
||||||
|
ranked = manager.selectable_frameworks(
|
||||||
|
task_type="text-generation", target_gpu="gpu", incumbent_frameworks=["first", "second"],
|
||||||
|
inspection=SimpleNamespace(model_type="family", quantization_method=None))
|
||||||
|
self.assertEqual(["second", "first"], ranked)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -283,6 +283,7 @@ class MarketIntelligenceTests(unittest.TestCase):
|
|||||||
"successCount": 10,
|
"successCount": 10,
|
||||||
"failureCount": 3,
|
"failureCount": 3,
|
||||||
"attributableFailureCount": 0,
|
"attributableFailureCount": 0,
|
||||||
|
"platformFailureCount": 3,
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"recentCombinationStats": {
|
"recentCombinationStats": {
|
||||||
@@ -290,6 +291,7 @@ class MarketIntelligenceTests(unittest.TestCase):
|
|||||||
"successCount": 10,
|
"successCount": 10,
|
||||||
"failureCount": 3,
|
"failureCount": 3,
|
||||||
"attributableFailureCount": 0,
|
"attributableFailureCount": 0,
|
||||||
|
"platformFailureCount": 3,
|
||||||
"consecutiveFailures": 0,
|
"consecutiveFailures": 0,
|
||||||
"consecutivePlatformFailures": 3,
|
"consecutivePlatformFailures": 3,
|
||||||
"lastPlatformFailureAt": datetime.now(timezone.utc).isoformat(),
|
"lastPlatformFailureAt": datetime.now(timezone.utc).isoformat(),
|
||||||
|
|||||||
Reference in New Issue
Block a user