CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: reserve queue capacity for recent models

Browse Source
This commit is contained in:
CoolBoy
2026-08-11 01:29:54 +08:00
parent 38ab25fc3c
commit f12d96b138
13 changed files with 927 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
modelhub_submmit_api/poll_runner.py
View File

@@ -58,6 +58,24 @@ def build_parser() -> argparse.ArgumentParser:
default=0,
help="Maximum tasks to submit in one cycle (0 means unlimited)",
)
parser.add_argument(
"--old-model-queue-threshold",
type=int,
default=int(os.getenv("MODELHUB_OLD_MODEL_QUEUE_THRESHOLD", "80")),
help=argparse.SUPPRESS,
)
parser.add_argument(
"--recent-model-days",
type=int,
default=int(os.getenv("MODELHUB_RECENT_MODEL_DAYS", "7")),
help=argparse.SUPPRESS,
)
parser.add_argument(
"--dynamic-old-model-cleanup-threshold",
type=int,
default=int(os.getenv("MODELHUB_DYNAMIC_OLD_MODEL_CLEANUP_THRESHOLD", "95")),
help=argparse.SUPPRESS,
)
parser.add_argument("--disable-candidate-preflight", action="store_true", help=argparse.SUPPRESS)
parser.add_argument("--llm-classifier-endpoint", default=os.getenv("MODELHUB_LLM_CLASSIFIER_ENDPOINT"), help=argparse.SUPPRESS)
parser.add_argument("--llm-classifier-model", default=os.getenv("MODELHUB_LLM_CLASSIFIER_MODEL"), help=argparse.SUPPRESS)
@@ -206,9 +224,30 @@ def _build_modelhub_client(base_args: argparse.Namespace) -> ModelHubClientPool:
clients,
capacity_probe_interval_cycles=max(0, int(getattr(base_args, "capacity_probe_interval_cycles", 3) or 0)),
capacity_state_path=Path(getattr(base_args, "capacity_state_path", DEFAULT_CAPACITY_STATE_PATH)),
old_model_queue_threshold=max(1, int(getattr(base_args, "old_model_queue_threshold", 80) or 80)),
recent_model_days=max(1, int(getattr(base_args, "recent_model_days", 7) or 7)),
)
def resolve_age_cleanup_policy(
base_args: argparse.Namespace,
*,
initial_cleanup_pending: bool,
) -> tuple[str, int]:
"""Use the strict admission boundary once, then retain a 15-slot buffer."""
initial_threshold = max(
1,
int(getattr(base_args, "old_model_queue_threshold", 80) or 80),
)
dynamic_threshold = max(
initial_threshold,
int(getattr(base_args, "dynamic_old_model_cleanup_threshold", 95) or 95),
)
if initial_cleanup_pending:
return "initial", initial_threshold
return "dynamic", dynamic_threshold
def run_poll_loop(
*,
base_args: argparse.Namespace,
@@ -243,6 +282,7 @@ def run_poll_loop(
submitted_total = 0
cycles = 0
stopped_reason = "max_cycles_reached"
initial_age_cleanup_pending = True
while True:
if base_args.max_cycles and cycles >= base_args.max_cycles:
@@ -263,12 +303,22 @@ def run_poll_loop(
try:
cleanup_feedback = outcome_tracker.get_stats_report()
cleanup_gpu_memory = cleanup_feedback.get("observedGpuMemoryGiB") or {}
age_cleanup_mode, age_cleanup_threshold = resolve_age_cleanup_policy(
base_args,
initial_cleanup_pending=initial_age_cleanup_pending,
)
log(
f"[queue-cleanup] mode={age_cleanup_mode} "
f"old_model_threshold={age_cleanup_threshold} "
f"recent_days={max(1, int(getattr(base_args, 'recent_model_days', 7) or 7))}"
)
cleanup_summary = cleanup_certain_oom_tasks(
modelhub_client,
hf_discovery,
dry_run=bool(base_args.dry_run),
read_concurrency=max(1, int(getattr(base_args, "queue_cleanup_read_concurrency", 6) or 6)),
gpu_memory_gib=cleanup_gpu_memory if isinstance(cleanup_gpu_memory, dict) else None,
age_queue_threshold=age_cleanup_threshold,
log=log,
)
write_json(
@@ -284,12 +334,16 @@ def run_poll_loop(
queue_cleanup_runs.append(
{
"cycle": cycles,
"mode": age_cleanup_mode,
"ageQueueThreshold": age_cleanup_threshold,
"activeScanned": cleanup_summary["activeScanned"],
"certainOomCount": cleanup_summary["certainOomCount"],
"oldOverflowCount": cleanup_summary["oldOverflowCount"],
"cancelledCount": cleanup_summary["cancelledCount"],
"stopErrorCount": len(cleanup_summary["stopErrors"]),
}
)
initial_age_cleanup_pending = False
except Exception as exc:
log(f"[queue-cleanup] error={type(exc).__name__}: {exc} continue_polling=true")