fix: stream cold-start history into decision state

This commit is contained in:
CoolBoy
2026-09-04 11:20:51 +08:00
parent d60551e130
commit 54ed4351a0
7 changed files with 538 additions and 58 deletions

View File

@@ -6,6 +6,7 @@ import tempfile
import unittest
from datetime import datetime, timezone
from pathlib import Path
from unittest.mock import patch
PACKAGE_DIR = Path(__file__).resolve().parents[1] / "modelhub_submmit_api"
@@ -15,6 +16,7 @@ sys.path.insert(0, str(PACKAGE_DIR))
from outcome_tracker import OutcomeTracker # noqa: E402
from poll_runner import ( # noqa: E402
_advance_architecture_history_backfill,
_bootstrap_architecture_history,
_load_task_compatibility_contexts,
build_parser,
@@ -65,10 +67,93 @@ class PollPolicyTests(unittest.TestCase):
now=datetime.now(timezone.utc),
)
self.assertEqual("owned_full_history", summary["source"])
self.assertEqual("owned_recent_bounded", summary["source"])
self.assertEqual(1, summary["terminalRecords"])
self.assertEqual(0, summary["communityUsableFailureDetails"])
def test_architecture_history_backfill_resumes_and_keeps_only_decision_state(self) -> None:
class PagedHistoryClient:
calls: list[int] = []
def list_tasks_page(self, **kwargs): # noqa: ANN003, ANN201
current = int(kwargs["current"])
self.calls.append(current)
records = {
1: [
{
"taskId": "history-1",
"modelId": "owner/model-1",
"gpuType": "gpu-a",
"status": "success",
"verifyResult": 1,
},
{
"taskId": "history-2",
"modelId": "owner/model-2",
"gpuType": "gpu-a",
"status": "failed",
"verifyResult": -1,
},
],
2: [
{
"taskId": "history-3",
"modelId": "owner/model-3",
"gpuType": "gpu-b",
"status": "success",
"verifyResult": 1,
}
],
}[current]
return {"data": {"records": records, "pages": 2}}
with tempfile.TemporaryDirectory() as temporary_dir, patch.dict(
"os.environ",
{"MODELHUB_ARCHITECTURE_BACKFILL_PAGES_PER_CYCLE": "1"},
):
root = Path(temporary_dir)
outcomes = root / "outcomes.jsonl"
checkpoint = root / "checkpoint.json"
recent = root / "recent.jsonl"
progress_path = root / "backfill.json"
tracker = OutcomeTracker(
outcomes,
checkpoint_path=checkpoint,
recent_path=recent,
)
client = PagedHistoryClient()
first = _advance_architecture_history_backfill(
modelhub_client=client, # type: ignore[arg-type]
outcome_tracker=tracker,
ledger_path=root / "ledger.jsonl",
progress_path=progress_path,
now=datetime.now(timezone.utc),
)
self.assertFalse(first["complete"])
self.assertTrue(tracker.has_durable_checkpoint)
restored = OutcomeTracker(
outcomes,
checkpoint_path=checkpoint,
recent_path=recent,
)
second = _advance_architecture_history_backfill(
modelhub_client=client, # type: ignore[arg-type]
outcome_tracker=restored,
ledger_path=root / "ledger.jsonl",
progress_path=progress_path,
now=datetime.now(timezone.utc),
)
self.assertTrue(second["complete"])
self.assertNotIn("seenTaskIds", second)
self.assertEqual([1, 2], client.calls)
self.assertEqual(3, restored.get_stats_report()["terminalRecords"])
self.assertEqual([], json.loads(outcomes.read_text(encoding="utf-8") or "[]"))
persisted_progress = progress_path.read_text(encoding="utf-8")
self.assertNotIn("owner/model", persisted_progress)
self.assertNotIn("logs.invalid", persisted_progress)
def test_cleanup_contexts_merge_outcomes_with_older_ledger_entries(self) -> None:
with tempfile.TemporaryDirectory() as temporary_dir:
root = Path(temporary_dir)
@@ -106,6 +191,47 @@ class PollPolicyTests(unittest.TestCase):
self.assertEqual("mindie", contexts["task-old"]["framework"])
self.assertEqual({}, contexts["task-old"]["modelProfile"])
def test_history_batch_is_classified_before_checkpoint_compaction(self) -> None:
tasks = [
{
"taskId": f"failed-{index}",
"modelId": f"owner/model-{index}",
"gpuType": "gpu-a",
"status": "failed",
"verifyResult": -1,
"logCosUrl": f"https://logs.invalid/{index}",
}
for index in range(600)
]
def classify(records, **kwargs): # noqa: ANN001, ANN003, ANN202
for record in records:
record["failureCategory"] = "model_runtime"
record["failureScope"] = "model"
record["failureDeterministic"] = True
progress = kwargs.get("progress")
if progress is not None:
progress(len(records), len(records), len(records), 0)
return len(records)
with tempfile.TemporaryDirectory() as temporary_dir:
root = Path(temporary_dir)
tracker = OutcomeTracker(
root / "outcomes.jsonl",
checkpoint_path=root / "checkpoint.json",
recent_path=root / "recent.jsonl",
)
with patch.object(tracker, "_enrich_failure_records", side_effect=classify):
summary = tracker.bootstrap_from_history_tasks(
tasks,
enrichment_limit=600,
)
report = tracker.get_stats_report()
self.assertEqual(600, summary["enrichmentAttempts"])
self.assertEqual(600, report["totals"]["attributableFailureCount"])
self.assertEqual(600, report["totals"]["failureBreakdown"]["model_runtime"])
def test_age_policy_defaults_to_admission_only_reserve_five(self) -> None:
args = build_parser().parse_args([])