CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: derive recent queue reserve from account capacity

Browse Source
This commit is contained in:
CoolBoy
2026-08-11 01:41:26 +08:00
parent f12d96b138
commit 2065ad6abc
11 changed files with 266 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
modelhub_submmit_api/poll_runner.py
View File

@@ -59,9 +59,9 @@ def build_parser() -> argparse.ArgumentParser:
help="Maximum tasks to submit in one cycle (0 means unlimited)",
)
parser.add_argument(
"--old-model-queue-threshold",
"--recent-model-reserve-slots",
type=int,
default=int(os.getenv("MODELHUB_OLD_MODEL_QUEUE_THRESHOLD", "80")),
default=int(os.getenv("MODELHUB_RECENT_MODEL_RESERVE_SLOTS", "10")),
help=argparse.SUPPRESS,
)
parser.add_argument(
@@ -71,9 +71,9 @@ def build_parser() -> argparse.ArgumentParser:
help=argparse.SUPPRESS,
)
parser.add_argument(
"--dynamic-old-model-cleanup-threshold",
"--dynamic-old-model-cleanup-reserve-slots",
type=int,
default=int(os.getenv("MODELHUB_DYNAMIC_OLD_MODEL_CLEANUP_THRESHOLD", "95")),
default=int(os.getenv("MODELHUB_DYNAMIC_OLD_MODEL_CLEANUP_RESERVE_SLOTS", "5")),
help=argparse.SUPPRESS,
)
parser.add_argument("--disable-candidate-preflight", action="store_true", help=argparse.SUPPRESS)
@@ -224,7 +224,7 @@ def _build_modelhub_client(base_args: argparse.Namespace) -> ModelHubClientPool:
clients,
capacity_probe_interval_cycles=max(0, int(getattr(base_args, "capacity_probe_interval_cycles", 3) or 0)),
capacity_state_path=Path(getattr(base_args, "capacity_state_path", DEFAULT_CAPACITY_STATE_PATH)),
old_model_queue_threshold=max(1, int(getattr(base_args, "old_model_queue_threshold", 80) or 80)),
recent_model_reserve_slots=max(0, int(getattr(base_args, "recent_model_reserve_slots", 10) or 0)),
recent_model_days=max(1, int(getattr(base_args, "recent_model_days", 7) or 7)),
)
@@ -234,18 +234,21 @@ def resolve_age_cleanup_policy(
*,
initial_cleanup_pending: bool,
) -> tuple[str, int]:
"""Use the strict admission boundary once, then retain a 15-slot buffer."""
initial_threshold = max(
1,
int(getattr(base_args, "old_model_queue_threshold", 80) or 80),
"""Reserve ten slots initially, then use a five-slot cleanup hysteresis."""
initial_reserve_slots = max(
0,
int(getattr(base_args, "recent_model_reserve_slots", 10) or 0),
)
dynamic_threshold = max(
initial_threshold,
int(getattr(base_args, "dynamic_old_model_cleanup_threshold", 95) or 95),
dynamic_reserve_slots = min(
initial_reserve_slots,
max(
0,
int(getattr(base_args, "dynamic_old_model_cleanup_reserve_slots", 5) or 0),
),
)
if initial_cleanup_pending:
return "initial", initial_threshold
return "dynamic", dynamic_threshold
return "initial", initial_reserve_slots
return "dynamic", dynamic_reserve_slots
def run_poll_loop(
@@ -303,13 +306,13 @@ def run_poll_loop(
try:
cleanup_feedback = outcome_tracker.get_stats_report()
cleanup_gpu_memory = cleanup_feedback.get("observedGpuMemoryGiB") or {}
age_cleanup_mode, age_cleanup_threshold = resolve_age_cleanup_policy(
age_cleanup_mode, age_cleanup_reserve_slots = resolve_age_cleanup_policy(
base_args,
initial_cleanup_pending=initial_age_cleanup_pending,
)
log(
f"[queue-cleanup] mode={age_cleanup_mode} "
f"old_model_threshold={age_cleanup_threshold} "
f"reserve_recent_slots={age_cleanup_reserve_slots} "
f"recent_days={max(1, int(getattr(base_args, 'recent_model_days', 7) or 7))}"
)
cleanup_summary = cleanup_certain_oom_tasks(
@@ -318,7 +321,7 @@ def run_poll_loop(
dry_run=bool(base_args.dry_run),
read_concurrency=max(1, int(getattr(base_args, "queue_cleanup_read_concurrency", 6) or 6)),
gpu_memory_gib=cleanup_gpu_memory if isinstance(cleanup_gpu_memory, dict) else None,
age_queue_threshold=age_cleanup_threshold,
age_reserved_slots=age_cleanup_reserve_slots,
log=log,
)
write_json(
@@ -335,7 +338,8 @@ def run_poll_loop(
{
"cycle": cycles,
"mode": age_cleanup_mode,
"ageQueueThreshold": age_cleanup_threshold,
"ageReservedSlots": age_cleanup_reserve_slots,
"ageQueueThresholds": cleanup_summary["oldModelQueueThresholds"],
"activeScanned": cleanup_summary["activeScanned"],
"certainOomCount": cleanup_summary["certainOomCount"],
"oldOverflowCount": cleanup_summary["oldOverflowCount"],