CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: dynamically clean incompatible architectures

Browse Source
This commit is contained in:
CoolBoy
2026-08-12 08:19:53 +08:00
parent 615bcad124
commit 7ec875563e
12 changed files with 979 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
modelhub_submmit_api/poll_runner.py
View File

@@ -8,7 +8,7 @@ import time
from pathlib import Path
from typing import Any, Callable
from common import utc_now, write_json
from common import read_jsonl, utc_now, write_json
from daily_runner import DEFAULT_DAILY_RUNS_DIR, log, run_daily_batches
from gpu_strategy import DEFAULT_GPU_STRATEGY_PATH
from hf_discovery import HuggingFaceDiscovery
@@ -31,6 +31,9 @@ from version import AGENT_VERSION
DEFAULT_POLL_RUNS_DIR = Path("poll_runs")
DEFAULT_ARCHITECTURE_BLACKLIST_PATH = Path(
".modelhub_state/architecture_compatibility_blacklist.json"
)
def build_parser() -> argparse.ArgumentParser:
@@ -204,6 +207,14 @@ def build_parser() -> argparse.ArgumentParser:
default=os.getenv("MODELHUB_QUEUE_CLEANUP_REPORT_PATH", ".modelhub_state/queue_cleanup_latest.json"),
help=argparse.SUPPRESS,
)
parser.add_argument(
"--architecture-blacklist-path",
default=os.getenv(
"MODELHUB_ARCHITECTURE_BLACKLIST_PATH",
str(DEFAULT_ARCHITECTURE_BLACKLIST_PATH),
),
help=argparse.SUPPRESS,
)
return parser
@@ -251,6 +262,53 @@ def resolve_age_cleanup_policy(
return "dynamic", dynamic_reserve_slots
def _persist_architecture_blacklist(
report: dict[str, Any],
*,
path: Path,
) -> set[str]:
blocks = report.get("architectureCompatibilityBlocks") or {}
blocks = blocks if isinstance(blocks, dict) else {}
write_json(
path,
{
"generatedAt": report.get("generatedAt"),
"summary": report.get("architectureCompatibilitySummary") or {},
"blocks": blocks,
},
)
return set(str(key) for key in blocks)
def _load_task_compatibility_contexts(
outcome_tracker: OutcomeTracker,
*,
ledger_path: Path,
) -> dict[str, dict[str, Any]]:
contexts = outcome_tracker.get_task_compatibility_contexts()
for record in read_jsonl(ledger_path):
task_id_value = record.get("taskId")
if task_id_value is None:
continue
task_id = str(task_id_value)
existing = contexts.get(task_id)
if existing is None:
contexts[task_id] = {
"taskId": task_id,
"modelId": str(record.get("modelId") or ""),
"targetGpu": str(record.get("targetGpu") or ""),
"framework": str(record.get("framework") or ""),
"taskType": str(record.get("taskType") or ""),
"modelProfile": {},
"submitTime": record.get("submitTime"),
}
continue
for field in ("modelId", "targetGpu", "framework", "taskType", "submitTime"):
if not existing.get(field) and record.get(field):
existing[field] = record.get(field)
return contexts
def run_poll_loop(
*,
base_args: argparse.Namespace,
@@ -286,6 +344,8 @@ def run_poll_loop(
cycles = 0
stopped_reason = "max_cycles_reached"
initial_age_cleanup_pending = True
pending_architecture_cleanup = False
last_cleaned_architecture_blocks: set[str] = set()
while True:
if base_args.max_cycles and cycles >= base_args.max_cycles:
@@ -296,22 +356,90 @@ def run_poll_loop(
if hasattr(modelhub_client, "configure_capacity_probe"):
modelhub_client.configure_capacity_probe(cycles)
outcome_synced_this_cycle = False
if (
cycles % OUTCOME_SYNC_INTERVAL == 0
and not getattr(base_args, "skip_outcome_sync", False)
):
try:
synced = outcome_tracker.sync_from_api(modelhub_client)
outcome_synced_this_cycle = True
if synced > 0:
log(f"[poll] cycle={cycles} outcome_sync_updated={synced}")
sync_feedback = outcome_tracker.get_stats_report()
active_block_keys = _persist_architecture_blacklist(
sync_feedback,
path=Path(
getattr(
base_args,
"architecture_blacklist_path",
DEFAULT_ARCHITECTURE_BLACKLIST_PATH,
)
),
)
new_block_keys = active_block_keys - last_cleaned_architecture_blocks
if (
new_block_keys
and not bool(getattr(base_args, "disable_queue_cleanup", False))
and isinstance(modelhub_client, ModelHubClientPool)
):
pending_architecture_cleanup = True
log(
f"[queue-cleanup] dynamic_architecture_blocks_added={len(new_block_keys)} "
f"cleanup_next=immediate"
)
except Exception as exc:
log(f"[poll] cycle={cycles} outcome_sync_error={exc}")
cleanup_interval = max(0, int(getattr(base_args, "queue_cleanup_interval_cycles", 120) or 0))
scheduled_queue_cleanup = bool(
cycles == 1 or (cleanup_interval > 0 and cycles % cleanup_interval == 0)
)
architecture_only_cleanup = bool(
pending_architecture_cleanup and not scheduled_queue_cleanup
)
should_cleanup_queue = (
not bool(getattr(base_args, "disable_queue_cleanup", False))
and isinstance(modelhub_client, ModelHubClientPool)
and (cycles == 1 or (cleanup_interval > 0 and cycles % cleanup_interval == 0))
and (
scheduled_queue_cleanup
or pending_architecture_cleanup
)
)
if should_cleanup_queue:
try:
if (
not outcome_synced_this_cycle
and not getattr(base_args, "skip_outcome_sync", False)
):
synced_before_cleanup = outcome_tracker.sync_from_api(modelhub_client)
if synced_before_cleanup:
log(
f"[queue-cleanup] outcome_sync_updated={synced_before_cleanup}"
)
cleanup_feedback = outcome_tracker.get_stats_report()
cleanup_gpu_memory = cleanup_feedback.get("observedGpuMemoryGiB") or {}
cleanup_architecture_blocks = (
cleanup_feedback.get("architectureCompatibilityBlocks") or {}
)
active_architecture_block_keys = _persist_architecture_blacklist(
cleanup_feedback,
path=Path(
getattr(
base_args,
"architecture_blacklist_path",
DEFAULT_ARCHITECTURE_BLACKLIST_PATH,
)
),
)
if active_architecture_block_keys - last_cleaned_architecture_blocks:
pending_architecture_cleanup = True
age_cleanup_mode, age_cleanup_reserve_slots = resolve_age_cleanup_policy(
base_args,
initial_cleanup_pending=initial_age_cleanup_pending,
)
log(
f"[queue-cleanup] mode={age_cleanup_mode} "
f"[queue-cleanup] mode={'architecture_dynamic' if architecture_only_cleanup else age_cleanup_mode} "
f"reserve_recent_slots={age_cleanup_reserve_slots} "
f"recent_days={max(1, int(getattr(base_args, 'recent_model_days', 7) or 7))}"
)
@@ -321,6 +449,18 @@ def run_poll_loop(
dry_run=bool(base_args.dry_run),
read_concurrency=max(1, int(getattr(base_args, "queue_cleanup_read_concurrency", 6) or 6)),
gpu_memory_gib=cleanup_gpu_memory if isinstance(cleanup_gpu_memory, dict) else None,
architecture_compatibility_blocks=(
cleanup_architecture_blocks
if isinstance(cleanup_architecture_blocks, dict)
else None
),
task_compatibility_contexts=(
_load_task_compatibility_contexts(
outcome_tracker,
ledger_path=Path(base_args.ledger_path),
)
),
architecture_only=architecture_only_cleanup,
age_reserved_slots=age_cleanup_reserve_slots,
log=log,
)
@@ -343,18 +483,31 @@ def run_poll_loop(
queue_cleanup_runs.append(
{
"cycle": cycles,
"mode": age_cleanup_mode,
"mode": (
"architecture_dynamic"
if architecture_only_cleanup
else age_cleanup_mode
),
"ageReservedSlots": age_cleanup_reserve_slots,
"ageQueueThresholds": cleanup_summary["oldModelQueueThresholds"],
"activeScanned": cleanup_summary["activeScanned"],
"certainOomCount": cleanup_summary["certainOomCount"],
"architectureBlockCount": cleanup_summary[
"architectureBlockCount"
],
"architectureIncompatibleCount": cleanup_summary[
"architectureIncompatibleCount"
],
"oldOverflowCount": cleanup_summary["oldOverflowCount"],
"cancelledCount": cleanup_summary["cancelledCount"],
"policyCancelledRecorded": policy_cancelled_recorded,
"stopErrorCount": len(cleanup_summary["stopErrors"]),
}
)
initial_age_cleanup_pending = False
if not architecture_only_cleanup:
initial_age_cleanup_pending = False
pending_architecture_cleanup = False
last_cleaned_architecture_blocks = active_architecture_block_keys
except Exception as exc:
log(f"[queue-cleanup] error={type(exc).__name__}: {exc} continue_polling=true")
@@ -410,14 +563,6 @@ def run_poll_loop(
time.sleep(base_args.idle_interval_seconds)
continue
if cycles % OUTCOME_SYNC_INTERVAL == 0:
try:
synced = outcome_tracker.sync_from_api(modelhub_client)
if synced > 0:
log(f"[poll] cycle={cycles} outcome_sync_updated={synced}")
except Exception as exc:
log(f"[poll] cycle={cycles} outcome_sync_error={exc}")
if cycles % STATS_PRINT_INTERVAL == 0:
try:
stats = outcome_tracker.get_stats_report()