feat: compact durable logs and refine framework routing
This commit is contained in:
CoolBoy
@@ -36,7 +36,9 @@ from modelhub_client import (
|
||||
ModelHubClient,
|
||||
ModelHubClientPool,
|
||||
OldModelQueuePolicyError,
|
||||
is_capacity_error,
|
||||
is_duplicate_submission_error,
|
||||
is_framework_prerequisite_error,
|
||||
is_model_uniqueness_error,
|
||||
)
|
||||
from models import CandidateModel, HFModelSummary, ModelInspection
|
||||
@@ -651,6 +653,10 @@ def process_model_for_candidates(
|
||||
skipped.append({"repoId": model.repo_id, "targetGpu": target_gpu, "reason": reason})
|
||||
continue
|
||||
record = candidate_to_record(best)
|
||||
if outcome_tracker is not None:
|
||||
record["transformersPrerequisiteSatisfied"] = (
|
||||
outcome_tracker.has_non_transformers_failure(model.repo_id)
|
||||
)
|
||||
if market_intelligence is not None:
|
||||
record.update(market_intelligence.gpu_metadata(target_gpu))
|
||||
record.update(market_intelligence.framework_metadata(best.task_type, target_gpu, best.framework))
|
||||
@@ -879,6 +885,17 @@ def submit_candidate(
|
||||
"candidate": candidate,
|
||||
"reason": "age_policy_skipped",
|
||||
}
|
||||
if is_framework_prerequisite_error(exc):
|
||||
print(
|
||||
f"[submit] deferred repo={candidate['repoId']} gpu={candidate['targetGpu']} "
|
||||
f"framework={candidate['framework']} reason=framework_prerequisite",
|
||||
flush=True,
|
||||
)
|
||||
return {
|
||||
"outcome": "framework_prerequisite_deferred",
|
||||
"candidate": candidate,
|
||||
"reason": "framework_prerequisite_deferred",
|
||||
}
|
||||
if is_model_uniqueness_error(exc):
|
||||
print(
|
||||
f"[submit] skipped repo={candidate['repoId']} gpu={candidate['targetGpu']} "
|
||||
@@ -901,6 +918,17 @@ def submit_candidate(
|
||||
"candidate": candidate,
|
||||
"reason": str(exc),
|
||||
}
|
||||
if is_capacity_error(exc):
|
||||
print(
|
||||
f"[submit] deferred repo={candidate['repoId']} gpu={candidate['targetGpu']} "
|
||||
f"framework={candidate['framework']} reason=account_capacity_saturated",
|
||||
flush=True,
|
||||
)
|
||||
return {
|
||||
"outcome": "capacity_deferred",
|
||||
"candidate": candidate,
|
||||
"reason": "account_capacity_saturated",
|
||||
}
|
||||
print(
|
||||
f"[submit] failed repo={candidate['repoId']} gpu={candidate['targetGpu']} "
|
||||
f"framework={candidate['framework']} reason={exc}",
|
||||
@@ -1052,6 +1080,7 @@ def run_submission(
|
||||
run_dir = make_run_dir(runs_dir, now)
|
||||
ledger_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
history_archive_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
ledger_entries = load_ledger(ledger_path)
|
||||
|
||||
outcome_tracker = outcome_tracker or OutcomeTracker(Path(args.outcomes_path))
|
||||
# Online decisions are deliberately deterministic. The optional classifier
|
||||
@@ -1085,7 +1114,25 @@ def run_submission(
|
||||
synced_count = 0
|
||||
if not getattr(args, "skip_outcome_sync", False):
|
||||
try:
|
||||
synced_count = outcome_tracker.sync_from_api(modelhub_client)
|
||||
task_contexts = outcome_tracker.get_task_compatibility_contexts()
|
||||
for entry in ledger_entries:
|
||||
task_id = str(entry.get("taskId") or "")
|
||||
if not task_id:
|
||||
continue
|
||||
context = task_contexts.setdefault(task_id, {})
|
||||
for field, source in (
|
||||
("modelId", "modelId"),
|
||||
("targetGpu", "targetGpu"),
|
||||
("framework", "framework"),
|
||||
("taskType", "taskType"),
|
||||
("submitTime", "submitTime"),
|
||||
):
|
||||
if not context.get(field) and entry.get(source):
|
||||
context[field] = entry.get(source)
|
||||
synced_count = outcome_tracker.sync_from_api(
|
||||
modelhub_client,
|
||||
task_contexts=task_contexts,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
if preflight_advisor is not None:
|
||||
@@ -1096,7 +1143,6 @@ def run_submission(
|
||||
|
||||
updated_after = determine_updated_after(args, now)
|
||||
history_begin = now - timedelta(days=args.stats_window_days)
|
||||
ledger_entries = load_ledger(ledger_path)
|
||||
day_start = now.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
# Count today's submissions from the local ledger (avoids expensive paginated API call)
|
||||
daily_snapshot = count_submissions_for_day(tasks=[], ledger_entries=ledger_entries, day_start=day_start, day_end=now)
|
||||
@@ -1473,6 +1519,7 @@ def run_submission(
|
||||
submit_workers = 1
|
||||
state_sync = getattr(args, "_state_sync_manager", None)
|
||||
state_sync_paused = False
|
||||
capacity_saturated = False
|
||||
|
||||
if args.dry_run:
|
||||
attempted_candidates = diversified_candidates[:target_submit_count]
|
||||
@@ -1549,6 +1596,7 @@ def run_submission(
|
||||
batch_duplicate_candidates: list[dict[str, Any]] = []
|
||||
batch_uniqueness_rejected_candidates: list[dict[str, Any]] = []
|
||||
batch_policy_skipped_candidates: list[dict[str, Any]] = []
|
||||
batch_capacity_deferred_candidates: list[dict[str, Any]] = []
|
||||
batch_failed_candidates: list[dict[str, Any]] = []
|
||||
for index in range(len(batch_candidates)):
|
||||
result = ordered_results.get(index)
|
||||
@@ -1593,6 +1641,30 @@ def run_submission(
|
||||
}
|
||||
)
|
||||
continue
|
||||
if result["outcome"] == "capacity_deferred":
|
||||
capacity_saturated = True
|
||||
batch_capacity_deferred_candidates.append(candidate)
|
||||
skipped.append(
|
||||
{
|
||||
"repoId": candidate["repoId"],
|
||||
"targetGpu": candidate["targetGpu"],
|
||||
"reason": "account_capacity_saturated",
|
||||
"deferred": True,
|
||||
}
|
||||
)
|
||||
continue
|
||||
if result["outcome"] == "framework_prerequisite_deferred":
|
||||
batch_policy_skipped_candidates.append(candidate)
|
||||
skipped.append(
|
||||
{
|
||||
"repoId": candidate["repoId"],
|
||||
"targetGpu": candidate["targetGpu"],
|
||||
"framework": candidate["framework"],
|
||||
"reason": "framework_prerequisite_deferred",
|
||||
"deferred": True,
|
||||
}
|
||||
)
|
||||
continue
|
||||
if result["outcome"] in {"failed", "precheck_deferred"}:
|
||||
batch_failed_candidates.append(candidate)
|
||||
failed.append(
|
||||
@@ -1646,7 +1718,13 @@ def run_submission(
|
||||
claim_store.mark_submitted(
|
||||
[*batch_submitted_candidates, *batch_duplicate_candidates, *batch_uniqueness_rejected_candidates]
|
||||
)
|
||||
claim_store.release([*batch_failed_candidates, *batch_policy_skipped_candidates])
|
||||
claim_store.release(
|
||||
[
|
||||
*batch_failed_candidates,
|
||||
*batch_policy_skipped_candidates,
|
||||
*batch_capacity_deferred_candidates,
|
||||
]
|
||||
)
|
||||
if strategy_manager is not None:
|
||||
strategy_manager.record_accepted(batch_submitted_candidates)
|
||||
outcome_tracker.save()
|
||||
@@ -1663,6 +1741,11 @@ def run_submission(
|
||||
|
||||
if hasattr(modelhub_client, "available_submit_slots") and modelhub_client.available_submit_slots() <= 0:
|
||||
break
|
||||
if capacity_saturated:
|
||||
# Every account was tried by the pool. More attempts in this
|
||||
# cycle only duplicate the same capacity response and pollute
|
||||
# durable logs; refresh capacity on the next poll instead.
|
||||
break
|
||||
|
||||
write_jsonl(run_dir / "submitted.jsonl", submitted)
|
||||
write_jsonl(run_dir / "skipped.jsonl", skipped)
|
||||
|
||||
Reference in New Issue
Block a user