feat: add adaptive GPU scheduling
This commit is contained in:
@@ -14,7 +14,7 @@ if str(PACKAGE_DIR) in sys.path:
|
||||
sys.path.remove(str(PACKAGE_DIR))
|
||||
sys.path.insert(0, str(PACKAGE_DIR))
|
||||
|
||||
from common import read_jsonl # noqa: E402
|
||||
from common import read_json, read_jsonl # noqa: E402
|
||||
from main import build_parser, make_run_dir, run_submission, submit_candidate # noqa: E402
|
||||
from modelhub_client import ModelHubAPIError, ModelHubClientPool, is_duplicate_submission_error # noqa: E402
|
||||
from models import HFModelSummary, ModelInspection # noqa: E402
|
||||
@@ -106,6 +106,54 @@ class DuplicateThenSuccessClient:
|
||||
return {"code": 0, "data": {"id": f"task-{self.calls}"}}
|
||||
|
||||
|
||||
class DynamicCapacityClient:
|
||||
def __init__(self, *, active: int, limit: int, token: str = "dynamic-test") -> None:
|
||||
self.active = active
|
||||
self.limit = limit
|
||||
self.token = token
|
||||
|
||||
def count_active_tasks(self, *, max_count: int, **_kwargs) -> int: # noqa: ANN003
|
||||
return min(self.active, max_count)
|
||||
|
||||
def add_task(self, _payload: dict) -> dict:
|
||||
if self.active >= self.limit:
|
||||
raise ModelHubAPIError("当前等待中或运行中的异步模型验证任务数量已达上限")
|
||||
self.active += 1
|
||||
return {"code": 0, "data": {"id": self.active}}
|
||||
|
||||
|
||||
class AutoStrategyClient:
|
||||
def __init__(self, available: int) -> None:
|
||||
self.available = available
|
||||
self.calls = 0
|
||||
|
||||
@staticmethod
|
||||
def begin_cycle() -> None:
|
||||
return None
|
||||
|
||||
def available_submit_slots(self) -> int:
|
||||
return self.available
|
||||
|
||||
@staticmethod
|
||||
def list_tasks(**_kwargs) -> list[dict]: # noqa: ANN003
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def list_tasks_page(**_kwargs) -> dict: # noqa: ANN003
|
||||
return {"code": 0, "data": {"records": [], "pages": 0}}
|
||||
|
||||
@staticmethod
|
||||
def processed_gpus_for_model(_model_id: str) -> set[str]:
|
||||
return set()
|
||||
|
||||
def add_task(self, _payload: dict) -> dict:
|
||||
self.calls += 1
|
||||
if self.calls == 1:
|
||||
raise ModelHubAPIError("模型正在验证中,请勿重复提交")
|
||||
self.available -= 1
|
||||
return {"code": 0, "data": {"id": f"strategy-{self.calls}"}}
|
||||
|
||||
|
||||
def make_candidate(index: int) -> dict:
|
||||
return {
|
||||
"repoId": f"owner/model-{index}",
|
||||
@@ -115,6 +163,42 @@ def make_candidate(index: int) -> dict:
|
||||
|
||||
|
||||
class ClientPoolConcurrencyTests(unittest.TestCase):
|
||||
def test_adaptive_strategy_counts_only_platform_accepted_tasks(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_dir:
|
||||
root = Path(temporary_dir)
|
||||
args = build_parser().parse_args(
|
||||
[
|
||||
"--task-types",
|
||||
"text-generation",
|
||||
"--limit",
|
||||
"4",
|
||||
"--max-scan-models",
|
||||
"4",
|
||||
"--skip-outcome-sync",
|
||||
"--skip-history-archive",
|
||||
]
|
||||
)
|
||||
args.runs_dir = str(root / "runs")
|
||||
args.ledger_path = str(root / "ledger.jsonl")
|
||||
args.outcomes_path = str(root / "outcomes.jsonl")
|
||||
args.claims_path = str(root / "claims.jsonl")
|
||||
args.history_archive_path = str(root / "history.jsonl")
|
||||
args.gpu_strategy_state_path = str(root / "strategy.json")
|
||||
client = AutoStrategyClient(available=3)
|
||||
|
||||
summary = run_submission(
|
||||
args,
|
||||
now=datetime(2026, 1, 1, 12, tzinfo=timezone.utc),
|
||||
hf_discovery=FakeDiscovery(4), # type: ignore[arg-type]
|
||||
modelhub_client=client, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
state = read_json(root / "strategy.json")
|
||||
self.assertEqual(3, summary["submittedCount"])
|
||||
self.assertEqual(1, summary["duplicateCount"])
|
||||
self.assertEqual(3, state["acceptedSinceRefresh"])
|
||||
self.assertEqual(3, sum(state["acceptedByCategory"].values()))
|
||||
|
||||
def test_duplicate_batch_is_replaced_until_available_slots_are_filled(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_dir:
|
||||
root = Path(temporary_dir)
|
||||
@@ -231,6 +315,45 @@ class ClientPoolConcurrencyTests(unittest.TestCase):
|
||||
pool.search_by_model_id("owner/model")
|
||||
self.assertEqual(2, client.search_calls)
|
||||
|
||||
def test_capacity_probe_discovers_a_higher_dynamic_account_limit(self) -> None:
|
||||
client = DynamicCapacityClient(active=2, limit=3)
|
||||
pool = ModelHubClientPool(
|
||||
[client], # type: ignore[arg-type]
|
||||
active_task_cap=2,
|
||||
active_counts_ttl=60,
|
||||
capacity_probe_interval_cycles=3,
|
||||
instance_id="capacity-growth-test",
|
||||
)
|
||||
|
||||
pool.configure_capacity_probe(2)
|
||||
self.assertEqual(0, pool.available_submit_slots())
|
||||
pool.configure_capacity_probe(3)
|
||||
self.assertEqual(1, pool.available_submit_slots())
|
||||
pool.add_task({"model": "probe"})
|
||||
|
||||
self.assertEqual([3], pool.account_capacity_limits())
|
||||
self.assertEqual([3], pool.active_task_counts())
|
||||
self.assertEqual(0, pool.available_submit_slots())
|
||||
|
||||
def test_rejected_capacity_probe_keeps_known_limit_and_enters_cooldown(self) -> None:
|
||||
client = DynamicCapacityClient(active=2, limit=2)
|
||||
pool = ModelHubClientPool(
|
||||
[client], # type: ignore[arg-type]
|
||||
active_task_cap=2,
|
||||
active_counts_ttl=60,
|
||||
capacity_probe_interval_cycles=3,
|
||||
capacity_probe_cooldown_cycles=3,
|
||||
instance_id="capacity-rejection-test",
|
||||
)
|
||||
|
||||
pool.configure_capacity_probe(3)
|
||||
self.assertEqual(1, pool.available_submit_slots())
|
||||
with self.assertRaises(ModelHubAPIError):
|
||||
pool.add_task({"model": "probe"})
|
||||
|
||||
self.assertEqual([2], pool.account_capacity_limits())
|
||||
self.assertEqual(0, pool.available_submit_slots())
|
||||
|
||||
|
||||
class ProcessCoordinationTests(unittest.TestCase):
|
||||
def test_duplicate_claims_are_retained_so_next_batch_moves_forward(self) -> None:
|
||||
|
||||
Reference in New Issue
Block a user