CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

fix: reject broken configs and improve accuracy feedback with isolated agent state

Browse Source
This commit is contained in:
CoolBoy
2026-10-18 16:29:18 +08:00
parent 964362818e
commit 484a10607c
10 changed files with 243 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
modelhub_submmit_api/failure_taxonomy.py
View File

@@ -67,7 +67,7 @@ PLATFORM_PATTERNS = (
(re.compile(r"welcome\.sh: No such file|data: command not found", re.I), "broken_platform_launch_script"),
(re.compile(r"id_rsa.*No such file", re.I), "missing_platform_credential"),
(re.compile(r"找不到空闲卡|no idle (?:gpu|card)", re.I), "no_idle_device"),
(re.compile(r"storage|download.*timed? out|connection reset", re.I), "platform_io_transient"),
(re.compile(r"no space left on device|download.*timed? out|connection reset", re.I), "platform_io_transient"),
)
DETERMINISTIC_LOG_PATTERNS = (
@@ -80,6 +80,8 @@ DETERMINISTIC_LOG_PATTERNS = (
# semantics. Generic messages such as "attention backend not supported" or
# "GPU type not supported" must not create an architecture blacklist.
EXPLICIT_FRAMEWORK_MODEL_UNSUPPORTED_PATTERNS = (
re.compile(r"model type\s+[`'\"][^`'\"]+[`'\"].{0,100}does not recognize this architecture", re.I),
re.compile(r"Model architectures?\s*\[[^\]\n]+\]\s*(?:are|is) not supported", re.I),
re.compile(
r"(?:当前|该|此)(?:推理)?框架.{0,100}(?:不支持|暂不支持|无法支持|不兼容)"
r".{0,100}(?:该|此|当前)?(?:模型架构|模型|架构)",
@@ -117,19 +119,16 @@ def classify_failure_report(report_code: str | None, log_lines: Iterable[str] =
code = str(report_code or "").strip().upper()
text = "\n".join(str(line) for line in log_lines)
# Infrastructure signatures override broad report codes such as
# EXECUTE_EMPTY_RESULT so they do not poison model compatibility feedback.
for pattern, reason in PLATFORM_PATTERNS:
if pattern.search(text):
return FailureClassification(
"platform_infrastructure",
"gpu_framework",
"open_short_gpu_framework_circuit_and_retry_other_models",
True,
False,
reason,
)
if re.search(r'exec:.*entrypoint.*(?:no such file|not found)', text, re.I):
return FailureClassification(
"configuration_launch", "configuration", "reject_broken_config", True, False,
"invalid_executable_argv",
)
if re.search(r'quantization method .*not supported for the current GPU', text, re.I):
return FailureClassification(
"quantization_compatibility", "model_gpu_framework", "reject_quantization_route",
True, False, "explicit_quantization_gpu_incompatibility",
)
if code in DETERMINISTIC_POLICIES:
return DETERMINISTIC_POLICIES[code]
for pattern, inferred_code in DETERMINISTIC_LOG_PATTERNS:
@@ -149,6 +148,15 @@ def classify_failure_report(report_code: str | None, log_lines: Iterable[str] =
)
if code in SEMANTIC_POLICIES:
return SEMANTIC_POLICIES[code]
# Fall back to infrastructure signatures only after specific root causes.
# Logs may include a transient allocation failure before a model starts.
for pattern, reason in PLATFORM_PATTERNS:
if pattern.search(text):
return FailureClassification(
"platform_infrastructure", "gpu_framework",
"open_short_gpu_framework_circuit_and_retry_other_models",
True, False, reason,
)
return FailureClassification(
"ambiguous_runtime",
"unknown",

57
modelhub_submmit_api/market_intelligence.py
View File

@@ -4,6 +4,7 @@ import math
import re
import statistics
import threading
import yaml
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timedelta
from pathlib import Path
@@ -97,6 +98,31 @@ def _validated_official_config(config: str, *, framework: str, target_gpu: str)
return False, "missing_sut_or_ref_config"
if any(marker in config for marker in ("{{", "}}", "PLACEHOLDER")):
return False, "unresolved_placeholder"
try:
document = yaml.safe_load(config)
except yaml.YAMLError:
return False, "invalid_yaml"
if not isinstance(document, dict):
return False, "invalid_config_structure"
def valid_commands(node: Any) -> bool:
if isinstance(node, dict):
for key, value in node.items():
if key in {"command", "entrypoint"} and isinstance(value, list):
if not value or not all(isinstance(arg, str) for arg in value):
return False
# argv[0] is an executable, not a shell command. In
# particular, a folded YAML list item must not eat argv[1].
if not value[0].strip() or re.search(r"\s", value[0]):
return False
if not valid_commands(value):
return False
elif isinstance(node, list):
return all(valid_commands(value) for value in node)
return True
if not valid_commands(document):
return False, "malformed_command_argv"
declared = re.search(r"(?m)^\s*framework:\s*['\"]?([^'\"\s]+)", config)
if declared is None or declared.group(1).strip() != framework:
@@ -736,7 +762,7 @@ class MarketIntelligenceManager:
local_success = max(0, int(local_item.get("successCount") or 0))
local_failure = max(
0,
int(local_item.get("attributableFailureCount", local_item.get("failureCount") or 0)),
int(local_item.get("failureCount") or 0) - int(local_item.get("platformFailureCount") or 0),
)
local_samples = local_success + local_failure
local_score = _wilson_lower_bound(local_success, local_samples)
@@ -746,7 +772,7 @@ class MarketIntelligenceManager:
recent_all_failures = max(0, int(recent_item.get("failureCount") or 0))
recent_failure = max(
0,
int(recent_item.get("attributableFailureCount", recent_item.get("failureCount") or 0)),
int(recent_item.get("failureCount") or 0) - int(recent_item.get("platformFailureCount") or 0),
)
recent_samples = recent_success + recent_failure
recent_rate = recent_success / recent_samples if recent_samples else None
@@ -849,11 +875,31 @@ class MarketIntelligenceManager:
if incumbent_best > 0 and float(evidence["combinedScore"]) < incumbent_best * NEW_FRAMEWORK_PROMOTION_MARGIN:
continue
promoted.append(framework)
return self.rank_frameworks(
ranked = self.rank_frameworks(
task_type=task_type,
target_gpu=target_gpu,
compatible_frameworks=promoted,
)
model_type = str(getattr(inspection, "model_type", None) or "").strip()
quantization = str(getattr(inspection, "quantization_method", None) or "none").strip()
if not model_type:
return ranked
def profile_score(framework: str) -> float:
prior = float(self._framework_evidence(task_type, target_gpu, framework)["combinedScore"])
key = f"{target_gpu}|{framework}|{task_type}|{model_type}|{quantization}"
item = (self.local_outcome_stats.get("profileCombinationStats") or {}).get(key) or {}
success = max(0, int(item.get("successCount") or 0))
failure = max(0, int(item.get("failureCount") or 0) - int(item.get("platformFailureCount") or 0))
samples = success + failure
if not samples:
return prior
# Model-specific negative evidence must affect the framework
# choice, not only the ranking after the first framework wins.
lower = _wilson_lower_bound(success + 8 * prior, samples + 8)
return lower
return sorted(ranked, key=profile_score, reverse=True)
def compatible_discovered_frameworks(
self,
@@ -941,6 +987,11 @@ class MarketIntelligenceManager:
config = item.get("officialConfig")
if not isinstance(config, str) or not config:
return None
valid, reason = _validated_official_config(config, framework=framework, target_gpu=target_gpu)
if not valid:
item["officialConfigValid"] = False
item["officialConfigError"] = reason
return None
return _render_official_config(config, gguf_filename=gguf_filename)
def framework_metadata(self, task_type: str, target_gpu: str, framework: str) -> dict[str, Any]:

6
modelhub_submmit_api/outcome_tracker.py
View File

@@ -22,6 +22,8 @@ from history_stats import classify_failure, is_failure, is_success
from llm_classifier import LLMAssistedClassifier
from modelhub_client import ModelHubClient, ModelHubClientPool
from task_registry import task_type_from_history_task
from common import runtime_instance_id
from version import AGENT_VERSION
DEFAULT_OUTCOMES_PATH = Path("outcomes/submissions.jsonl")
@@ -36,6 +38,7 @@ FAILURE_ENRICHMENT_MAX_ATTEMPTS = 3
TERMINAL_TASK_STATUSES = {"success", "failed", "error", "cancelled", "completed"}
DECISION_RECORD_FIELDS = {
"agentVersion", "instanceId", "strategyId",
"taskId",
"modelId",
"targetGpu",
@@ -325,6 +328,9 @@ class OutcomeTracker:
model_profile: dict[str, Any] | None = None,
) -> None:
record: dict[str, Any] = {
"agentVersion": AGENT_VERSION,
"instanceId": runtime_instance_id(),
"strategyId": os.getenv("STRATEGY_ID"),
"modelId": model_id,
"targetGpu": target_gpu,
"framework": framework,

9
modelhub_submmit_api/poll_runner.py
View File

@@ -234,7 +234,9 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument("--post-cycle-cooldown-seconds", type=int, default=2, help="Short sleep after a successful cycle")
parser.add_argument("--max-cycles", type=int, default=0, help="Optional hard stop after N cycles; 0 means run until quota is reached")
parser.add_argument("--print-stats", action="store_true", help="Load outcomes, sync, print stats report, and exit")
parser.add_argument("--disable-queue-cleanup", action="store_true", help=argparse.SUPPRESS)
parser.add_argument("--disable-queue-cleanup", action="store_true",
default=os.getenv("MODELHUB_COEXISTENCE_MODE", "1") != "0",
help=argparse.SUPPRESS)
parser.add_argument(
"--queue-cleanup-interval-cycles",
type=int,
@@ -702,6 +704,11 @@ def run_poll_loop(
STATS_PRINT_INTERVAL = 10
log(f"[poll] version={AGENT_VERSION} poll_run_dir={poll_run_dir}")
log(
f"[coexistence] state_branch={base_args.state_sync_branch} "
f"queue_cancellation={'disabled' if base_args.disable_queue_cleanup else 'enabled'} "
"submission_preflight=enabled"
)
log(
"[state-retention] mode=decision_state_only full_archive=disabled "
f"recent_outcomes={DEFAULT_RECENT_OUTCOME_LIMIT} "

22
modelhub_submmit_api/routing_engine.py
View File

@@ -46,6 +46,13 @@ class SuccessFirstRoutingEngine:
return value
def _evidence(self, candidate: dict[str, Any]) -> tuple[str, int, int, float]:
def observed_total(item: dict[str, Any]) -> int:
if "failureCount" in item:
return max(0, int(item.get("successCount") or 0) + int(item.get("failureCount") or 0)
- int(item.get("platformFailureCount") or 0))
return max(0, int(item.get("decisionTotal") or 0))
sparse = None
profile = candidate.get("preflightMetadata") or {}
gpu = str(candidate.get("targetGpu") or "")
framework = str(candidate.get("framework") or "")
@@ -60,14 +67,18 @@ class SuccessFirstRoutingEngine:
size_bucket = int(load_bytes).bit_length() - 1
key = f"{gpu}|{framework}|{task_type}|{model_type}|{quantization}|{size_bucket}"
item = (self.outcome_stats.get("sizedProfileCombinationStats") or {}).get(key) or {}
total = int(item.get("decisionTotal") or 0)
total = observed_total(item)
if total:
sparse = ("local_profile_size_shrunk", item, total)
if total >= 20:
success = int(item.get("successCount") or 0)
return "local_profile_size", success, total, wilson_lower_bound(success, total)
if model_type:
key = f"{gpu}|{framework}|{task_type}|{model_type}|{quantization}"
item = (self.outcome_stats.get("profileCombinationStats") or {}).get(key) or {}
total = int(item.get("decisionTotal") or 0)
total = observed_total(item)
if total and sparse is None:
sparse = ("local_profile_shrunk", item, total)
if total >= 40:
success = int(item.get("successCount") or 0)
return "local_profile", success, total, wilson_lower_bound(success, total)
@@ -97,6 +108,13 @@ class SuccessFirstRoutingEngine:
total = 2
success = 1
level = "global_conservative_prior"
if sparse is not None:
level, item, samples = sparse
# Eight prior observations stabilize small groups without hiding
# their failures behind an unreachable hard sample threshold.
prior_success = round(8 * success / max(1, total))
success = int(item.get("successCount") or 0) + prior_success
total = samples + 8
return level, success, total, wilson_lower_bound(success, total)
def annotate(self, candidate: dict[str, Any]) -> dict[str, Any]:

9
modelhub_submmit_api/state_sync.py
View File

@@ -17,14 +17,14 @@ from typing import Any, Iterable
from dulwich import porcelain
from dulwich.repo import Repo
from common import read_json, read_jsonl, write_json, write_jsonl
from common import read_json, read_jsonl, write_json, write_jsonl, runtime_instance_id
from runner_common import load_key_files
from version import AGENT_VERSION
STATE_SCHEMA_VERSION = 1
DEFAULT_REMOTE = "https://dev.modelhub.org.cn/CoolBoy/submmit.git"
DEFAULT_BRANCH = "agent-state"
DEFAULT_BRANCH = "agent-state-accuracy-v2"
DEFAULT_BATCH_SIZE = 100
DEFAULT_HISTORY_DEPTH = 200
DEFAULT_RECENT_TERMINAL_INTENTS = 300
@@ -32,6 +32,7 @@ DEFAULT_LEDGER_RECORDS = 500
DEFAULT_CRASH_RECORDS = 50
STATE_OUTCOME_FIELDS = {
"agentVersion", "instanceId", "strategyId",
"taskId", "modelId", "targetGpu", "framework", "taskType", "submitTime",
"lastSyncTime", "status", "verifyResult", "outcome", "failReason",
"modelProfile", "failureCode", "failureCategory", "failureScope",
@@ -342,6 +343,7 @@ class StateGitSync:
retained_terminal = terminal[:DEFAULT_RECENT_TERMINAL_INTENTS]
discarded = terminal[DEFAULT_RECENT_TERMINAL_INTENTS:]
compact_fields = {
"agentVersion", "instanceId", "strategyId",
"intentId", "batchId", "status", "createdAt", "completedAt",
"repoId", "targetGpu", "taskType", "framework", "configSource",
"configFingerprint", "safeConfigVector", "taskId",
@@ -372,6 +374,9 @@ class StateGitSync:
config = str(candidate.get("configParams") or "")
return {
"intentId": uuid.uuid4().hex,
"agentVersion": AGENT_VERSION,
"instanceId": runtime_instance_id(),
"strategyId": os.getenv("STRATEGY_ID"),
"batchId": batch_id,
"status": "pending",
"createdAt": _utc_now().isoformat(),

2
modelhub_submmit_api/version.py
View File

@@ -1 +1 @@
AGENT_VERSION = "2026.09.22.1"
AGENT_VERSION = "2026.10.18.1"