CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

fix: refill from expanded candidate history

Browse Source
This commit is contained in:
CoolBoy
2026-08-02 17:45:21 +08:00
parent eab5ab6dce
commit 8e68c6f611
10 changed files with 418 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
modelhub_submmit_api/README.md
View File

@@ -79,6 +79,10 @@ bash run_poll.sh --dry-run
duplicates do not advance it. The next cycle refreshes platform history before submitting again.
- Strategy state is stored in `.modelhub_state/gpu_strategy.json`; a generation never recalculates
during candidate submission.
- Candidate discovery starts with the configured recent window, then automatically expands to
7 days, 30 days, and older history (up to 3,000 models) when the recent pool is exhausted.
- Model verification responses are cached across poll cycles for 15 minutes. Local model/GPU
failures cool down after 24 hours instead of remaining permanently blocked.
- Each model can be submitted at most once per GPU.
- Multiple ModelHub tokens are pooled and used to route submissions to the account with available async capacity.
- Concurrent submissions reserve account slots locally, and an account-capacity race automatically falls through to another account.
@@ -88,6 +92,8 @@ bash run_poll.sh --dry-run
on the next cycle.
- Every third poll cycle, a full account gets one controlled capacity probe. A successful probe
raises that account's persisted known limit; a capacity rejection enters cooldown.
- Each `[scan]` log records the discovery stage and candidate yield. The final `[daily] wave_done`
log includes `skip_reasons`, making empty candidate pools distinguishable from API failures.
## Important Flags

6
modelhub_submmit_api/daily_runner.py
View File

@@ -236,11 +236,17 @@ def run_daily_batches(
wave_results.append(wave_result)
submitted_total += summary["submittedCount"]
round_submitted += summary["submittedCount"]
skip_reasons = summary.get("skipReasonCounts") or {}
skip_reason_text = ",".join(
f"{reason}:{count}"
for reason, count in list(skip_reasons.items())[:4]
) or "none"
log(
f"[daily] wave_done name={wave.name} "
f"candidates={summary['candidateCount']} planned={summary['plannedSubmitCount']} "
f"submitted={summary['submittedCount']} skipped={summary['skippedCount']} "
f"duplicates={summary.get('duplicateCount', 0)} failed={summary['failedCount']} "
f"skip_reasons={skip_reason_text} "
f"remaining_before_run={summary['remainingDailyQuotaBeforeRun']}"
)

248
modelhub_submmit_api/main.py
View File

@@ -2,6 +2,7 @@ from __future__ import annotations
import argparse
import os
from collections import Counter
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import timedelta
from pathlib import Path
@@ -27,6 +28,8 @@ from template_selector import TemplateSelector
DEFAULT_RUNS_DIR = Path("runs")
DEFAULT_LEDGER_PATH = Path("ledger/submissions.jsonl")
ADAPTIVE_SCAN_MAX_MODELS = 3000
ADAPTIVE_SCAN_MIN_FALLBACK_MODELS = 500
def build_parser() -> argparse.ArgumentParser:
@@ -284,7 +287,7 @@ def process_model_for_candidates(
skipped.append({"repoId": model.repo_id, "targetGpu": target_gpu, "reason": "already_processed_for_gpu"})
continue
if outcome_tracker and outcome_tracker.is_model_gpu_failed(model.repo_id, target_gpu):
skipped.append({"repoId": model.repo_id, "targetGpu": target_gpu, "reason": "previously_failed_locally"})
skipped.append({"repoId": model.repo_id, "targetGpu": target_gpu, "reason": "local_failure_cooldown_24h"})
continue
compatible_task_types = [
@@ -321,6 +324,123 @@ def process_model_for_candidates(
return candidates, skipped, failed
def build_adaptive_scan_stages(
*,
now,
initial_updated_after,
initial_limit: int,
max_models: int = ADAPTIVE_SCAN_MAX_MODELS,
) -> list[dict[str, Any]]:
max_models = max(1, min(ADAPTIVE_SCAN_MAX_MODELS, int(max_models)))
initial_limit = min(max_models, max(1, int(initial_limit)))
stages: list[dict[str, Any]] = [
{
"name": "configured_window",
"updatedAfter": initial_updated_after,
"limit": initial_limit,
}
]
seven_days_ago = now - timedelta(days=7)
thirty_days_ago = now - timedelta(days=30)
if initial_updated_after is not None and initial_updated_after > seven_days_ago:
stages.append(
{
"name": "last_7_days",
"updatedAfter": seven_days_ago,
"limit": min(max_models, max(initial_limit, ADAPTIVE_SCAN_MIN_FALLBACK_MODELS)),
}
)
if initial_updated_after is not None and initial_updated_after > thirty_days_ago:
stages.append(
{
"name": "last_30_days",
"updatedAfter": thirty_days_ago,
"limit": min(max_models, max(initial_limit, 1500)),
}
)
if initial_updated_after is not None or initial_limit < max_models:
stages.append(
{
"name": "all_history",
"updatedAfter": None,
"limit": max_models,
}
)
deduped: list[dict[str, Any]] = []
seen: set[tuple[str | None, int]] = set()
for stage in stages:
updated_after = stage["updatedAfter"]
signature = (updated_after.isoformat() if updated_after is not None else None, int(stage["limit"]))
if signature in seen:
continue
seen.add(signature)
deduped.append(stage)
return deduped
def collect_candidates_from_models(
*,
models: list[HFModelSummary],
seen_model_ids: set[str],
candidate_goal: int,
hf_discovery: HuggingFaceDiscovery,
modelhub_client: ModelHubClient,
template_selector: TemplateSelector,
target_gpus: list[str],
selected_task_types: list[str],
outcome_tracker: OutcomeTracker | None,
read_concurrency: int,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]], int]:
candidates: list[dict[str, Any]] = []
skipped: list[dict[str, Any]] = []
failed: list[dict[str, Any]] = []
processed_count = 0
workers = max(1, int(read_concurrency))
chunk_size = max(16, workers * 8)
new_models = [model for model in models if model.repo_id not in seen_model_ids]
for offset in range(0, len(new_models), chunk_size):
if len(candidates) >= candidate_goal:
break
chunk = new_models[offset : offset + chunk_size]
for model in chunk:
seen_model_ids.add(model.repo_id)
with ThreadPoolExecutor(max_workers=workers) as executor:
futures = {
executor.submit(
process_model_for_candidates,
model=model,
hf_discovery=hf_discovery,
modelhub_client=modelhub_client,
template_selector=template_selector,
target_gpus=target_gpus,
allowed_task_types=selected_task_types,
outcome_tracker=outcome_tracker,
): index
for index, model in enumerate(chunk)
}
ordered_results: dict[int, tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]] = {}
for future in as_completed(futures):
index = futures[future]
model = chunk[index]
try:
ordered_results[index] = future.result()
except Exception as exc:
ordered_results[index] = ([], [], [{"repoId": model.repo_id, "reason": str(exc)}])
for index in range(len(chunk)):
model_candidates, model_skipped, model_failed = ordered_results.get(index, ([], [], []))
candidates.extend(model_candidates)
skipped.extend(model_skipped)
failed.extend(model_failed)
processed_count += len(chunk)
return candidates, skipped, failed, processed_count
def submit_candidate(
candidate: dict[str, Any],
modelhub_client: ModelHubClient,
@@ -509,6 +629,8 @@ def run_submission(
"historyArchiveRecordCount": len(archived_history),
"gpuStrategy": strategy_summary,
"scanLimit": 0,
"candidateGoal": 0,
"scanStages": [],
"scannedModels": 0,
"candidateCount": 0,
"targetSubmitCount": 0,
@@ -517,6 +639,7 @@ def run_submission(
"submittedCount": 0,
"duplicateCount": 0,
"skippedCount": 0,
"skipReasonCounts": {},
"failedCount": 0,
"warnings": report.get("warnings", []),
"runDir": str(run_dir),
@@ -563,55 +686,84 @@ def run_submission(
platform_available_slots=platform_available_slots,
)
model_query_kwargs = {
"pipeline_tags": pipeline_tags_for_task_types(selected_task_types),
"limit": scan_limit,
"min_downloads": args.min_downloads,
"updated_after": updated_after,
}
try:
model_query_kwargs["read_concurrency"] = max(1, args.read_concurrency)
models = hf_discovery.list_recent_models(**model_query_kwargs)
except TypeError:
models = hf_discovery.list_recent_models(
pipeline_tags=model_query_kwargs["pipeline_tags"],
limit=model_query_kwargs["limit"],
min_downloads=model_query_kwargs["min_downloads"],
updated_after=model_query_kwargs["updated_after"],
)
candidates: list[dict[str, Any]] = []
skipped: list[dict[str, Any]] = []
failed: list[dict[str, Any]] = []
scan_stages: list[dict[str, Any]] = []
seen_model_ids: set[str] = set()
scanned_model_count = 0
explicit_submit_limit = max(0, int(getattr(args, "max_submits_per_run", 0) or 0))
if platform_available_slots is None and unlimited_daily_target:
submission_goal = scan_limit
else:
submission_goal = max(0, int(remaining_daily_quota))
if explicit_submit_limit > 0:
submission_goal = min(submission_goal, explicit_submit_limit)
if strategy_manager is not None:
submission_goal = min(submission_goal, strategy_manager.submissions_until_refresh)
attempt_multiplier = max(1, int(getattr(args, "scan_multiplier", 4) or 1))
candidate_goal = max(submission_goal, submission_goal * attempt_multiplier)
with ThreadPoolExecutor(max_workers=max(1, args.read_concurrency)) as executor:
futures = {
executor.submit(
process_model_for_candidates,
model=model,
hf_discovery=hf_discovery,
modelhub_client=modelhub_client,
template_selector=template_selector,
target_gpus=target_gpus,
allowed_task_types=selected_task_types,
outcome_tracker=outcome_tracker,
): index
for index, model in enumerate(models)
pipeline_tags = pipeline_tags_for_task_types(selected_task_types)
explicit_scan_cap = max(0, int(getattr(args, "max_scan_models", 0) or 0))
for stage in build_adaptive_scan_stages(
now=now,
initial_updated_after=updated_after,
initial_limit=scan_limit,
max_models=explicit_scan_cap or ADAPTIVE_SCAN_MAX_MODELS,
):
if candidate_goal <= 0 or len(candidates) >= candidate_goal:
break
stage_updated_after = stage["updatedAfter"]
stage_limit = int(stage["limit"])
query_kwargs = {
"pipeline_tags": pipeline_tags,
"limit": stage_limit,
"min_downloads": args.min_downloads,
"updated_after": stage_updated_after,
}
ordered_results: dict[int, tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]] = {}
for future in as_completed(futures):
index = futures[future]
model = models[index]
try:
ordered_results[index] = future.result()
except Exception as exc:
ordered_results[index] = ([], [], [{"repoId": model.repo_id, "reason": str(exc)}])
try:
models = hf_discovery.list_recent_models(
**query_kwargs,
read_concurrency=max(1, args.read_concurrency),
)
except TypeError:
models = hf_discovery.list_recent_models(**query_kwargs)
for index in range(len(models)):
model_candidates, model_skipped, model_failed = ordered_results.get(index, ([], [], []))
candidates.extend(model_candidates)
skipped.extend(model_skipped)
failed.extend(model_failed)
stage_candidates, stage_skipped, stage_failed, processed_count = collect_candidates_from_models(
models=models,
seen_model_ids=seen_model_ids,
candidate_goal=max(1, candidate_goal - len(candidates)),
hf_discovery=hf_discovery,
modelhub_client=modelhub_client,
template_selector=template_selector,
target_gpus=target_gpus,
selected_task_types=selected_task_types,
outcome_tracker=outcome_tracker,
read_concurrency=max(1, args.read_concurrency),
)
candidates.extend(stage_candidates)
skipped.extend(stage_skipped)
failed.extend(stage_failed)
scanned_model_count += processed_count
stage_summary = {
"name": stage["name"],
"updatedAfter": stage_updated_after.isoformat() if stage_updated_after is not None else None,
"requestedLimit": stage_limit,
"discoveredModels": len(models),
"newModelsProcessed": processed_count,
"candidatesAdded": len(stage_candidates),
"candidateCountAfterStage": len(candidates),
"skippedAdded": len(stage_skipped),
"failedAdded": len(stage_failed),
}
scan_stages.append(stage_summary)
print(
f"[scan] stage={stage['name']} discovered={len(models)} new_processed={processed_count} "
f"candidates_added={len(stage_candidates)} candidates_total={len(candidates)}/{candidate_goal} "
f"skipped_added={len(stage_skipped)}",
flush=True,
)
submitted: list[dict[str, Any]] = []
duplicate_candidates: list[dict[str, Any]] = []
@@ -767,6 +919,8 @@ def run_submission(
strategy_summary = strategy_manager.summary()
outcome_tracker.save()
skip_reason_counts = Counter(str(item.get("reason") or "unknown") for item in skipped)
failure_reason_counts = Counter(str(item.get("reason") or "unknown") for item in failed)
summary = {
"generatedAt": now.isoformat(),
@@ -785,7 +939,9 @@ def run_submission(
"historyArchiveRecordCount": len(archived_history),
"gpuStrategy": strategy_summary,
"scanLimit": scan_limit,
"scannedModels": len(models),
"candidateGoal": candidate_goal,
"scanStages": scan_stages,
"scannedModels": scanned_model_count,
"candidateCount": len(candidates),
"targetSubmitCount": target_submit_count,
"maxSubmitAttempts": 0 if args.dry_run else min(
@@ -796,7 +952,9 @@ def run_submission(
"duplicateCount": len(duplicate_candidates),
"submittedCount": len(submitted),
"skippedCount": len(skipped),
"skipReasonCounts": dict(skip_reason_counts.most_common()),
"failedCount": len(failed),
"failureReasonCounts": dict(failure_reason_counts.most_common()),
"submitConcurrencyUsed": submit_workers,
"warnings": report.get("warnings", []),
"runDir": str(run_dir),

11
modelhub_submmit_api/modelhub_client.py
View File

@@ -321,7 +321,7 @@ class ModelHubClientPool:
identity_hash = hashlib.sha256(identity.encode("utf-8")).hexdigest()
self._selection_cursor = int(identity_hash[:12], 16) % len(clients)
self._verify_cache: dict[str, tuple[float, dict[str, Any]]] = {}
self._verify_cache_ttl = max(1.0, float(os.getenv("MODELHUB_AGENT_VERIFY_CACHE_TTL_SECONDS", "30")))
self._verify_cache_ttl = max(1.0, float(os.getenv("MODELHUB_AGENT_VERIFY_CACHE_TTL_SECONDS", "900")))
# Single reader client to avoid fanout on read operations
self._reader = clients[0]
@@ -539,12 +539,15 @@ class ModelHubClientPool:
self._verify_cache[model_id] = (time.monotonic(), payload)
def begin_cycle(self) -> None:
"""Drop model verification cache entries from the previous scan cycle."""
"""Keep recent verification results across cycles and prune expired entries."""
with self._state_lock:
self._verify_cache.clear()
now = time.monotonic()
for model_id, (cached_at, _payload) in list(self._verify_cache.items()):
if now - cached_at >= self._verify_cache_ttl:
self._verify_cache.pop(model_id, None)
def search_by_model_id(self, model_id: str) -> dict[str, Any]:
# Check cache first (per-cycle cache to avoid repeated API calls for the same model)
# Reuse recent model verification results across short poll cycles.
with self._state_lock:
cached = self._read_from_cache(model_id)
if cached is not None:

34
modelhub_submmit_api/outcome_tracker.py
View File

@@ -23,7 +23,7 @@ class OutcomeTracker:
self._records: list[dict[str, Any]] = []
self._by_task_id: dict[str, dict[str, Any]] = {}
self._by_model_gpu: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
self._failed_model_gpus: set[tuple[str, str]] = set()
self._failed_model_gpus: dict[tuple[str, str], datetime] = {}
self._records = read_jsonl(self.path)
self._rebuild_indexes()
@@ -117,8 +117,19 @@ class OutcomeTracker:
return updated_count
def is_model_gpu_failed(self, model_id: str, target_gpu: str) -> bool:
return (model_id, target_gpu) in self._failed_model_gpus
def is_model_gpu_failed(
self,
model_id: str,
target_gpu: str,
*,
cooldown_hours: int = 24,
now: datetime | None = None,
) -> bool:
failed_at = self._failed_model_gpus.get((model_id, target_gpu))
if failed_at is None:
return False
now = now or utc_now()
return failed_at >= now - timedelta(hours=max(0, int(cooldown_hours)))
def get_stats_report(self) -> dict[str, Any]:
now = _now_iso()
@@ -186,12 +197,21 @@ class OutcomeTracker:
def _rebuild_failed_index(self) -> None:
self._failed_model_gpus.clear()
latest_by_combo: dict[tuple[str, str], tuple[datetime, dict[str, Any]]] = {}
for record in self._records:
model_id = record.get("modelId") or ""
target_gpu = record.get("targetGpu") or ""
event_time = parse_datetime(record.get("lastSyncTime")) or parse_datetime(record.get("submitTime"))
if not model_id or not target_gpu or event_time is None:
continue
key = (model_id, target_gpu)
current = latest_by_combo.get(key)
if current is None or event_time >= current[0]:
latest_by_combo[key] = (event_time, record)
for key, (event_time, record) in latest_by_combo.items():
if record.get("outcome") == "failed":
model_id = record.get("modelId") or ""
target_gpu = record.get("targetGpu") or ""
if model_id and target_gpu:
self._failed_model_gpus.add((model_id, target_gpu))
self._failed_model_gpus[key] = event_time
@staticmethod
def _update_record_from_task(record: dict[str, Any], task: dict[str, Any]) -> None:

2
modelhub_submmit_api/version.py
View File

@@ -1 +1 @@
AGENT_VERSION = "2026.08.02.3"
AGENT_VERSION = "2026.08.02.4"