CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: add failure-aware preflight and Qwen review

Browse Source
This commit is contained in:
CoolBoy
2026-08-10 21:44:42 +08:00
parent 5e47d9e695
commit 3d15f60284
21 changed files with 2672 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
modelhub_submmit_api/main.py
View File

@@ -9,6 +9,7 @@ from pathlib import Path
from typing import Any
from common import parse_datetime, runtime_instance_id, utc_now, write_json, write_jsonl
from candidate_preflight import CandidatePreflightAdvisor
from gpu_strategy import DEFAULT_GPU_STRATEGY_PATH, GPUStrategyManager
from hf_discovery import HuggingFaceDiscovery
from history_stats import (
@@ -27,6 +28,7 @@ from market_intelligence import (
DEFAULT_THROUGHPUT_WINDOW_HOURS,
MarketIntelligenceManager,
)
from llm_classifier import DEFAULT_LLM_CACHE_PATH, LLMAssistedClassifier
from modelhub_client import (
DEFAULT_CAPACITY_STATE_PATH,
ModelHubAPIError,
@@ -106,6 +108,43 @@ def build_parser() -> argparse.ArgumentParser:
default=0,
help="Maximum tasks to submit in one run (0 means unlimited)",
)
parser.add_argument(
"--disable-candidate-preflight",
action="store_true",
help="Disable repository structure, memory, and context-length preflight checks",
)
parser.add_argument(
"--llm-classifier-endpoint",
default=os.getenv("MODELHUB_LLM_CLASSIFIER_ENDPOINT"),
help=argparse.SUPPRESS,
)
parser.add_argument(
"--llm-classifier-model",
default=os.getenv("MODELHUB_LLM_CLASSIFIER_MODEL"),
help=argparse.SUPPRESS,
)
parser.add_argument(
"--llm-classifier-api-key",
default=os.getenv("MODELHUB_LLM_CLASSIFIER_API_KEY"),
help=argparse.SUPPRESS,
)
parser.add_argument(
"--llm-classifier-timeout-seconds",
type=int,
default=int(os.getenv("MODELHUB_LLM_CLASSIFIER_TIMEOUT_SECONDS", "20")),
help=argparse.SUPPRESS,
)
parser.add_argument(
"--llm-classifier-min-deny-confidence",
type=float,
default=float(os.getenv("MODELHUB_LLM_CLASSIFIER_MIN_DENY_CONFIDENCE", "0.85")),
help=argparse.SUPPRESS,
)
parser.add_argument(
"--llm-classifier-cache-path",
default=os.getenv("MODELHUB_LLM_CLASSIFIER_CACHE_PATH", str(DEFAULT_LLM_CACHE_PATH)),
help=argparse.SUPPRESS,
)
parser.add_argument("--runs-dir", default=str(DEFAULT_RUNS_DIR), help=argparse.SUPPRESS)
parser.add_argument("--ledger-path", default=str(DEFAULT_LEDGER_PATH), help=argparse.SUPPRESS)
parser.add_argument("--outcomes-path", default=str(DEFAULT_OUTCOMES_PATH), help=argparse.SUPPRESS)
@@ -259,7 +298,31 @@ def choose_candidate_for_gpu(
task_types: list[str],
target_gpu: str,
market_intelligence: MarketIntelligenceManager | None = None,
preflight_advisor: CandidatePreflightAdvisor | None = None,
) -> CandidateModel | None:
candidate, _reason = choose_candidate_for_gpu_detailed(
model=model,
inspection=inspection,
template_selector=template_selector,
task_types=task_types,
target_gpu=target_gpu,
market_intelligence=market_intelligence,
preflight_advisor=preflight_advisor,
)
return candidate
def choose_candidate_for_gpu_detailed(
*,
model: HFModelSummary,
inspection: ModelInspection,
template_selector: TemplateSelector,
task_types: list[str],
target_gpu: str,
market_intelligence: MarketIntelligenceManager | None = None,
preflight_advisor: CandidatePreflightAdvisor | None = None,
) -> tuple[CandidateModel | None, str | None]:
last_rejection_reason: str | None = None
for task_type in task_types:
supported_frameworks = template_selector.supported_frameworks_for_auto(task_type, target_gpu)
compatible_frameworks: list[str] = []
@@ -317,6 +380,21 @@ def choose_candidate_for_gpu(
warnings.append("framework_selected_from_success_evidence")
if config_params is not None and bool(framework_metadata.get("frameworkOfficialConfigValid", False)):
warnings.append("official_build_config_synced")
preflight_metadata: dict[str, Any] = {}
if preflight_advisor is not None:
assessment = preflight_advisor.assess(
inspection=inspection,
task_type=task_type,
target_gpu=target_gpu,
framework=framework,
config_params=config_params,
)
if not assessment.allowed:
last_rejection_reason = assessment.reason
continue
config_params = assessment.config_params
warnings.extend(assessment.warnings)
preflight_metadata = assessment.metadata
spec = TASK_SPEC_BY_TYPE[task_type]
return CandidateModel(
repo_id=model.repo_id,
@@ -333,8 +411,9 @@ def choose_candidate_for_gpu(
gguf_filename=inspection.selected_gguf,
score=score,
warnings=warnings,
)
return None
preflight_metadata=preflight_metadata,
), None
return None, last_rejection_reason
def resolve_submit_concurrency(
@@ -379,6 +458,7 @@ def process_model_for_candidates(
outcome_tracker: OutcomeTracker | None = None,
submission_exclusion_store: SubmissionExclusionStore | None = None,
market_intelligence: MarketIntelligenceManager | None = None,
preflight_advisor: CandidatePreflightAdvisor | None = None,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]:
specs = [spec for spec in task_specs_for_model(model) if spec.task_type in allowed_task_types]
if not specs:
@@ -434,19 +514,23 @@ def process_model_for_candidates(
return [], skipped, [{"repoId": model.repo_id, "reason": str(exc)}]
for target_gpu, task_types in pending_task_types_by_gpu:
best = choose_candidate_for_gpu(
best, preflight_reason = choose_candidate_for_gpu_detailed(
model=model,
inspection=inspection,
template_selector=template_selector,
task_types=task_types,
target_gpu=target_gpu,
market_intelligence=market_intelligence,
preflight_advisor=preflight_advisor,
)
if best is None:
reason = (
"no_publicly_vetted_compatible_framework"
if market_intelligence is not None
else "no_compatible_auto_template_or_framework"
preflight_reason
or (
"no_publicly_vetted_compatible_framework"
if market_intelligence is not None
else "no_compatible_auto_template_or_framework"
)
)
skipped.append({"repoId": model.repo_id, "targetGpu": target_gpu, "reason": reason})
continue
@@ -529,6 +613,7 @@ def collect_candidates_from_models(
submission_exclusion_store: SubmissionExclusionStore | None,
read_concurrency: int,
market_intelligence: MarketIntelligenceManager | None = None,
preflight_advisor: CandidatePreflightAdvisor | None = None,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]], int]:
candidates: list[dict[str, Any]] = []
skipped: list[dict[str, Any]] = []
@@ -558,6 +643,7 @@ def collect_candidates_from_models(
outcome_tracker=outcome_tracker,
submission_exclusion_store=submission_exclusion_store,
market_intelligence=market_intelligence,
preflight_advisor=preflight_advisor,
): index
for index, model in enumerate(chunk)
}
@@ -730,6 +816,39 @@ def run_submission(
submission_exclusion_store = SubmissionExclusionStore(
Path(getattr(args, "submission_exclusions_path", DEFAULT_SUBMISSION_EXCLUSIONS_PATH))
)
preflight_advisor: CandidatePreflightAdvisor | None = None
disable_preflight = bool(getattr(args, "disable_candidate_preflight", False)) or os.getenv(
"MODELHUB_DISABLE_CANDIDATE_PREFLIGHT", ""
).strip().lower() in {"1", "true", "yes"}
if not disable_preflight:
llm_classifier = LLMAssistedClassifier(
endpoint=getattr(args, "llm_classifier_endpoint", None)
or os.getenv("MODELHUB_LLM_CLASSIFIER_ENDPOINT"),
model=getattr(args, "llm_classifier_model", None)
or os.getenv("MODELHUB_LLM_CLASSIFIER_MODEL"),
api_key=getattr(args, "llm_classifier_api_key", None)
or os.getenv("MODELHUB_LLM_CLASSIFIER_API_KEY"),
timeout_seconds=max(
1,
int(
getattr(args, "llm_classifier_timeout_seconds", 0)
or os.getenv("MODELHUB_LLM_CLASSIFIER_TIMEOUT_SECONDS", "20")
),
),
min_deny_confidence=float(
getattr(args, "llm_classifier_min_deny_confidence", 0.0)
or os.getenv("MODELHUB_LLM_CLASSIFIER_MIN_DENY_CONFIDENCE", "0.85")
),
cache_path=Path(
getattr(args, "llm_classifier_cache_path", None)
or os.getenv("MODELHUB_LLM_CLASSIFIER_CACHE_PATH", str(DEFAULT_LLM_CACHE_PATH))
),
)
outcome_tracker.set_failure_llm_classifier(llm_classifier)
preflight_advisor = CandidatePreflightAdvisor(llm_classifier=llm_classifier)
preflight_summary: dict[str, Any] = (
preflight_advisor.summary() if preflight_advisor is not None else {"enabled": False}
)
strategy_manager: GPUStrategyManager | None = None
strategy_summary: dict[str, Any] = {"enabled": False}
market_intelligence: MarketIntelligenceManager | None = None
@@ -745,6 +864,11 @@ def run_submission(
synced_count = outcome_tracker.sync_from_api(modelhub_client)
except Exception:
pass
if preflight_advisor is not None:
try:
preflight_advisor.set_feedback_stats(outcome_tracker.get_stats_report())
except Exception:
preflight_advisor.set_feedback_stats(None)
updated_after = determine_updated_after(args, now)
history_begin = now - timedelta(days=args.stats_window_days)
@@ -819,6 +943,7 @@ def run_submission(
"historyArchiveRecordCount": len(archived_history),
"gpuStrategy": strategy_summary,
"marketIntelligence": market_summary,
"candidatePreflight": preflight_summary,
"scanLimit": 0,
"candidateGoal": 0,
"scanStages": [],
@@ -999,6 +1124,7 @@ def run_submission(
submission_exclusion_store=submission_exclusion_store,
read_concurrency=max(1, args.read_concurrency),
market_intelligence=market_intelligence,
preflight_advisor=preflight_advisor,
)
candidates.extend(stage_candidates)
skipped.extend(stage_skipped)
@@ -1180,6 +1306,7 @@ def run_submission(
task_type=candidate["taskType"],
task_id=result["taskId"],
submit_time=result["submitTime"],
model_profile=candidate.get("preflightMetadata") or {},
)
claim_store.mark_submitted(
@@ -1198,6 +1325,8 @@ def run_submission(
if strategy_manager is not None:
strategy_summary = strategy_manager.summary()
if preflight_advisor is not None:
preflight_summary = preflight_advisor.summary()
outcome_tracker.save()
skip_reason_counts = Counter(str(item.get("reason") or "unknown") for item in skipped)
@@ -1224,6 +1353,7 @@ def run_submission(
"historyArchiveRecordCount": len(archived_history),
"gpuStrategy": strategy_summary,
"marketIntelligence": market_summary,
"candidatePreflight": preflight_summary,
"scanLimit": scan_limit,
"candidateGoal": candidate_goal,
"scanStages": scan_stages,
@@ -1272,6 +1402,7 @@ def candidate_to_record(candidate: CandidateModel) -> dict[str, Any]:
"ggufFilename": candidate.gguf_filename,
"score": candidate.score,
"warnings": candidate.warnings,
"preflightMetadata": candidate.preflight_metadata,
}