feat: prioritize proven GPU framework combinations
This commit is contained in:
@@ -68,7 +68,7 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
parser.add_argument(
|
||||
"--disable-gpu-strategy",
|
||||
action="store_true",
|
||||
help="Disable adaptive 50/30/20 GPU scheduling and keep the legacy candidate order",
|
||||
help="Disable adaptive 70/30 proven-GPU scheduling and keep the legacy candidate order",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--disable-market-intelligence",
|
||||
@@ -274,22 +274,14 @@ def choose_candidate_for_gpu(
|
||||
except ValueError:
|
||||
pass
|
||||
if market_intelligence is not None:
|
||||
discovered = market_intelligence.compatible_discovered_frameworks(
|
||||
compatible_frameworks = market_intelligence.selectable_frameworks(
|
||||
task_type=task_type,
|
||||
target_gpu=target_gpu,
|
||||
incumbent_frameworks=compatible_frameworks,
|
||||
inspection=inspection,
|
||||
)
|
||||
compatible_frameworks.extend(
|
||||
framework for framework in discovered if framework not in compatible_frameworks
|
||||
)
|
||||
if not compatible_frameworks:
|
||||
continue
|
||||
if market_intelligence is not None:
|
||||
compatible_frameworks = market_intelligence.rank_frameworks(
|
||||
task_type=task_type,
|
||||
target_gpu=target_gpu,
|
||||
compatible_frameworks=compatible_frameworks,
|
||||
)
|
||||
|
||||
for framework in compatible_frameworks:
|
||||
config_params = (
|
||||
@@ -451,7 +443,12 @@ def process_model_for_candidates(
|
||||
market_intelligence=market_intelligence,
|
||||
)
|
||||
if best is None:
|
||||
skipped.append({"repoId": model.repo_id, "targetGpu": target_gpu, "reason": "no_compatible_auto_template_or_framework"})
|
||||
reason = (
|
||||
"no_publicly_vetted_compatible_framework"
|
||||
if market_intelligence is not None
|
||||
else "no_compatible_auto_template_or_framework"
|
||||
)
|
||||
skipped.append({"repoId": model.repo_id, "targetGpu": target_gpu, "reason": reason})
|
||||
continue
|
||||
record = candidate_to_record(best)
|
||||
if market_intelligence is not None:
|
||||
@@ -706,6 +703,7 @@ def run_submission(
|
||||
target_gpus = resolve_target_gpus(args, template_selector, selected_task_types)
|
||||
if not target_gpus:
|
||||
raise RuntimeError("No auto-submittable GPUs are available for the selected task types")
|
||||
submission_target_gpus = list(target_gpus)
|
||||
|
||||
hf_discovery = hf_discovery or HuggingFaceDiscovery(base_url=args.hf_base_url)
|
||||
if modelhub_client is None:
|
||||
@@ -809,6 +807,7 @@ def run_submission(
|
||||
"dryRun": bool(args.dry_run),
|
||||
"selectedTaskTypes": selected_task_types,
|
||||
"targetGpus": target_gpus,
|
||||
"submissionEligibleGpus": submission_target_gpus,
|
||||
"dailyTarget": args.daily_target,
|
||||
"unlimitedDailyTarget": unlimited_daily_target,
|
||||
"submittedTodayBeforeRun": daily_snapshot["totalCount"],
|
||||
@@ -893,6 +892,14 @@ def run_submission(
|
||||
except Exception:
|
||||
market_intelligence.set_local_outcome_stats(None)
|
||||
market_summary = market_intelligence.summary()
|
||||
submission_target_gpus = market_intelligence.eligible_gpus(target_gpus)
|
||||
market_summary["eligibleGpus"] = submission_target_gpus
|
||||
market_summary["shadowOnlyGpus"] = [gpu for gpu in target_gpus if gpu not in submission_target_gpus]
|
||||
print(
|
||||
f"[market] eligible_gpus={','.join(submission_target_gpus) or 'none'} "
|
||||
f"shadow_only={','.join(market_summary['shadowOnlyGpus']) or 'none'}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if strategy_enabled:
|
||||
strategy_manager = GPUStrategyManager(
|
||||
@@ -930,7 +937,7 @@ def run_submission(
|
||||
|
||||
scan_limit = resolve_scan_limit(
|
||||
args,
|
||||
target_gpu_count=len(target_gpus),
|
||||
target_gpu_count=len(submission_target_gpus),
|
||||
remaining_daily_quota=remaining_daily_quota,
|
||||
platform_available_slots=platform_available_slots,
|
||||
)
|
||||
@@ -986,7 +993,7 @@ def run_submission(
|
||||
hf_discovery=hf_discovery,
|
||||
modelhub_client=modelhub_client,
|
||||
template_selector=template_selector,
|
||||
target_gpus=target_gpus,
|
||||
target_gpus=submission_target_gpus,
|
||||
selected_task_types=selected_task_types,
|
||||
outcome_tracker=outcome_tracker,
|
||||
submission_exclusion_store=submission_exclusion_store,
|
||||
@@ -1204,6 +1211,7 @@ def run_submission(
|
||||
"dryRun": bool(args.dry_run),
|
||||
"selectedTaskTypes": selected_task_types,
|
||||
"targetGpus": target_gpus,
|
||||
"submissionEligibleGpus": submission_target_gpus,
|
||||
"maxSubmitsPerRun": int(getattr(args, "max_submits_per_run", 0) or 0),
|
||||
"dailyTarget": args.daily_target,
|
||||
"unlimitedDailyTarget": unlimited_daily_target,
|
||||
|
||||
Reference in New Issue
Block a user