perf: complete history bootstrap in one streaming phase

This commit is contained in:
CoolBoy
2026-09-04 11:35:43 +08:00
parent 54ed4351a0
commit 2c748dcec2
6 changed files with 183 additions and 298 deletions

View File

@@ -17,60 +17,13 @@ sys.path.insert(0, str(PACKAGE_DIR))
from outcome_tracker import OutcomeTracker # noqa: E402
from poll_runner import ( # noqa: E402
_advance_architecture_history_backfill,
_bootstrap_architecture_history,
_complete_architecture_history_backfill,
_load_task_compatibility_contexts,
build_parser,
)
class PollPolicyTests(unittest.TestCase):
def test_architecture_bootstrap_falls_back_when_public_logs_are_hidden(self) -> None:
class HistoryClient:
@staticmethod
def list_tasks_page(**_kwargs): # noqa: ANN003
return {
"data": {
"records": [
{
"taskId": "public-failure",
"modelId": "public/model",
"gpuType": "gpu",
"status": "success",
"verifyResult": -1,
"logCosUrl": None,
}
]
}
}
@staticmethod
def list_tasks(**_kwargs): # noqa: ANN003
return [
{
"taskId": "owned-success",
"modelId": "owner/model",
"gpuType": "gpu",
"modelTaskLevelId": 23,
"status": "success",
"verifyResult": 1,
"updateTime": datetime.now(timezone.utc).isoformat(),
}
]
with tempfile.TemporaryDirectory() as temporary_dir:
root = Path(temporary_dir)
tracker = OutcomeTracker(root / "outcomes.jsonl")
summary = _bootstrap_architecture_history(
modelhub_client=HistoryClient(), # type: ignore[arg-type]
outcome_tracker=tracker,
ledger_path=root / "ledger.jsonl",
now=datetime.now(timezone.utc),
)
self.assertEqual("owned_recent_bounded", summary["source"])
self.assertEqual(1, summary["terminalRecords"])
self.assertEqual(0, summary["communityUsableFailureDetails"])
def test_architecture_history_backfill_resumes_and_keeps_only_decision_state(self) -> None:
class PagedHistoryClient:
calls: list[int] = []
@@ -109,7 +62,7 @@ class PollPolicyTests(unittest.TestCase):
with tempfile.TemporaryDirectory() as temporary_dir, patch.dict(
"os.environ",
{"MODELHUB_ARCHITECTURE_BACKFILL_PAGES_PER_CYCLE": "1"},
{"MODELHUB_ARCHITECTURE_BACKFILL_PAGES_PER_BATCH": "1"},
):
root = Path(temporary_dir)
outcomes = root / "outcomes.jsonl"
@@ -154,6 +107,53 @@ class PollPolicyTests(unittest.TestCase):
self.assertNotIn("owner/model", persisted_progress)
self.assertNotIn("logs.invalid", persisted_progress)
def test_cold_start_backfill_finishes_all_pages_in_one_phase(self) -> None:
class ThreePageClient:
calls: list[int] = []
def list_tasks_page(self, **kwargs): # noqa: ANN003, ANN201
current = int(kwargs["current"])
self.calls.append(current)
return {
"data": {
"records": [
{
"taskId": f"task-{current}",
"modelId": f"owner/model-{current}",
"gpuType": "gpu-a",
"status": "success",
"verifyResult": 1,
}
],
"pages": 3,
}
}
with tempfile.TemporaryDirectory() as temporary_dir, patch.dict(
"os.environ",
{"MODELHUB_ARCHITECTURE_BACKFILL_PAGES_PER_BATCH": "1"},
):
root = Path(temporary_dir)
tracker = OutcomeTracker(
root / "outcomes.jsonl",
checkpoint_path=root / "checkpoint.json",
recent_path=root / "recent.jsonl",
)
client = ThreePageClient()
durable_phases: list[str] = []
progress = _complete_architecture_history_backfill(
modelhub_client=client, # type: ignore[arg-type]
outcome_tracker=tracker,
ledger_path=root / "ledger.jsonl",
progress_path=root / "backfill.json",
sync_callback=lambda phase: durable_phases.append(phase) or True,
)
self.assertTrue(progress["complete"])
self.assertEqual([1, 2, 3], client.calls)
self.assertEqual(["history_backfill"], durable_phases)
self.assertEqual(3, tracker.get_stats_report()["terminalRecords"])
def test_cleanup_contexts_merge_outcomes_with_older_ledger_entries(self) -> None:
with tempfile.TemporaryDirectory() as temporary_dir:
root = Path(temporary_dir)