feat: add adaptive GPU scheduling

This commit is contained in:
CoolBoy
2026-08-02 16:59:44 +08:00
parent 80a1b8518d
commit eab5ab6dce
14 changed files with 1216 additions and 52 deletions

View File

@@ -14,7 +14,7 @@ if str(PACKAGE_DIR) in sys.path:
sys.path.remove(str(PACKAGE_DIR))
sys.path.insert(0, str(PACKAGE_DIR))
from common import read_jsonl # noqa: E402
from common import read_json, read_jsonl # noqa: E402
from main import build_parser, make_run_dir, run_submission, submit_candidate # noqa: E402
from modelhub_client import ModelHubAPIError, ModelHubClientPool, is_duplicate_submission_error # noqa: E402
from models import HFModelSummary, ModelInspection # noqa: E402
@@ -106,6 +106,54 @@ class DuplicateThenSuccessClient:
return {"code": 0, "data": {"id": f"task-{self.calls}"}}
class DynamicCapacityClient:
def __init__(self, *, active: int, limit: int, token: str = "dynamic-test") -> None:
self.active = active
self.limit = limit
self.token = token
def count_active_tasks(self, *, max_count: int, **_kwargs) -> int: # noqa: ANN003
return min(self.active, max_count)
def add_task(self, _payload: dict) -> dict:
if self.active >= self.limit:
raise ModelHubAPIError("当前等待中或运行中的异步模型验证任务数量已达上限")
self.active += 1
return {"code": 0, "data": {"id": self.active}}
class AutoStrategyClient:
def __init__(self, available: int) -> None:
self.available = available
self.calls = 0
@staticmethod
def begin_cycle() -> None:
return None
def available_submit_slots(self) -> int:
return self.available
@staticmethod
def list_tasks(**_kwargs) -> list[dict]: # noqa: ANN003
return []
@staticmethod
def list_tasks_page(**_kwargs) -> dict: # noqa: ANN003
return {"code": 0, "data": {"records": [], "pages": 0}}
@staticmethod
def processed_gpus_for_model(_model_id: str) -> set[str]:
return set()
def add_task(self, _payload: dict) -> dict:
self.calls += 1
if self.calls == 1:
raise ModelHubAPIError("模型正在验证中,请勿重复提交")
self.available -= 1
return {"code": 0, "data": {"id": f"strategy-{self.calls}"}}
def make_candidate(index: int) -> dict:
return {
"repoId": f"owner/model-{index}",
@@ -115,6 +163,42 @@ def make_candidate(index: int) -> dict:
class ClientPoolConcurrencyTests(unittest.TestCase):
def test_adaptive_strategy_counts_only_platform_accepted_tasks(self) -> None:
with tempfile.TemporaryDirectory() as temporary_dir:
root = Path(temporary_dir)
args = build_parser().parse_args(
[
"--task-types",
"text-generation",
"--limit",
"4",
"--max-scan-models",
"4",
"--skip-outcome-sync",
"--skip-history-archive",
]
)
args.runs_dir = str(root / "runs")
args.ledger_path = str(root / "ledger.jsonl")
args.outcomes_path = str(root / "outcomes.jsonl")
args.claims_path = str(root / "claims.jsonl")
args.history_archive_path = str(root / "history.jsonl")
args.gpu_strategy_state_path = str(root / "strategy.json")
client = AutoStrategyClient(available=3)
summary = run_submission(
args,
now=datetime(2026, 1, 1, 12, tzinfo=timezone.utc),
hf_discovery=FakeDiscovery(4), # type: ignore[arg-type]
modelhub_client=client, # type: ignore[arg-type]
)
state = read_json(root / "strategy.json")
self.assertEqual(3, summary["submittedCount"])
self.assertEqual(1, summary["duplicateCount"])
self.assertEqual(3, state["acceptedSinceRefresh"])
self.assertEqual(3, sum(state["acceptedByCategory"].values()))
def test_duplicate_batch_is_replaced_until_available_slots_are_filled(self) -> None:
with tempfile.TemporaryDirectory() as temporary_dir:
root = Path(temporary_dir)
@@ -231,6 +315,45 @@ class ClientPoolConcurrencyTests(unittest.TestCase):
pool.search_by_model_id("owner/model")
self.assertEqual(2, client.search_calls)
def test_capacity_probe_discovers_a_higher_dynamic_account_limit(self) -> None:
client = DynamicCapacityClient(active=2, limit=3)
pool = ModelHubClientPool(
[client], # type: ignore[arg-type]
active_task_cap=2,
active_counts_ttl=60,
capacity_probe_interval_cycles=3,
instance_id="capacity-growth-test",
)
pool.configure_capacity_probe(2)
self.assertEqual(0, pool.available_submit_slots())
pool.configure_capacity_probe(3)
self.assertEqual(1, pool.available_submit_slots())
pool.add_task({"model": "probe"})
self.assertEqual([3], pool.account_capacity_limits())
self.assertEqual([3], pool.active_task_counts())
self.assertEqual(0, pool.available_submit_slots())
def test_rejected_capacity_probe_keeps_known_limit_and_enters_cooldown(self) -> None:
client = DynamicCapacityClient(active=2, limit=2)
pool = ModelHubClientPool(
[client], # type: ignore[arg-type]
active_task_cap=2,
active_counts_ttl=60,
capacity_probe_interval_cycles=3,
capacity_probe_cooldown_cycles=3,
instance_id="capacity-rejection-test",
)
pool.configure_capacity_probe(3)
self.assertEqual(1, pool.available_submit_slots())
with self.assertRaises(ModelHubAPIError):
pool.add_task({"model": "probe"})
self.assertEqual([2], pool.account_capacity_limits())
self.assertEqual(0, pool.available_submit_slots())
class ProcessCoordinationTests(unittest.TestCase):
def test_duplicate_claims_are_retained_so_next_batch_moves_forward(self) -> None: