fix: coordinate concurrent account capacity filling

modelhub_submmit_api/outcome_tracker.py

@@ -5,7 +5,7 @@ from datetime import datetime, timedelta
 from pathlib import Path
 from typing import Any
 
-from common import append_jsonl, parse_datetime, read_jsonl, utc_now, write_jsonl
+from common import append_jsonl, parse_datetime, read_jsonl, update_jsonl, utc_now
 from history_stats import classify_failure, is_failure, is_success
 from modelhub_client import ModelHubClient, ModelHubClientPool
 
@@ -25,9 +25,20 @@ class OutcomeTracker:
         self._by_model_gpu: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
         self._failed_model_gpus: set[tuple[str, str]] = set()
 
-        loaded = read_jsonl(self.path)
-        for record in loaded:
-            self._records.append(record)
+        self._records = read_jsonl(self.path)
+        self._rebuild_indexes()
+
+        last_sync_times = [
+            parse_datetime(record.get("lastSyncTime"))
+            for record in self._records
+            if record.get("lastSyncTime")
+        ]
+        self._last_sync_time: datetime = max(last_sync_times) if last_sync_times else utc_now() - timedelta(days=7)
+
+    def _rebuild_indexes(self) -> None:
+        self._by_task_id.clear()
+        self._by_model_gpu.clear()
+        for record in self._records:
             task_id = record.get("taskId")
             if task_id:
                 self._by_task_id[str(task_id)] = record
@@ -37,13 +48,6 @@ class OutcomeTracker:
 
         self._rebuild_failed_index()
 
-        last_sync_times = [
-            parse_datetime(record.get("lastSyncTime"))
-            for record in self._records
-            if record.get("lastSyncTime")
-        ]
-        self._last_sync_time: datetime = max(last_sync_times) if last_sync_times else utc_now() - timedelta(days=7)
-
     def record_submission(
         self,
         model_id: str,
@@ -162,8 +166,23 @@ class OutcomeTracker:
         }
 
     def save(self) -> None:
-        self.path.parent.mkdir(parents=True, exist_ok=True)
-        write_jsonl(self.path, self._records)
+        local_records = list(self._records)
+
+        def merge(existing: list[dict[str, Any]]) -> list[dict[str, Any]]:
+            merged = list(existing)
+            index_by_key = {_outcome_record_key(record): index for index, record in enumerate(merged)}
+            for record in local_records:
+                key = _outcome_record_key(record)
+                existing_index = index_by_key.get(key)
+                if existing_index is None:
+                    index_by_key[key] = len(merged)
+                    merged.append(record)
+                    continue
+                merged[existing_index] = _prefer_newer_outcome(merged[existing_index], record)
+            return merged
+
+        self._records = update_jsonl(self.path, merge)
+        self._rebuild_indexes()
 
     def _rebuild_failed_index(self) -> None:
         self._failed_model_gpus.clear()
@@ -234,3 +253,31 @@ def _summarize(records: list[dict[str, Any]]) -> dict[str, Any]:
         "pendingRate": round(pending_count / total, 4) if total > 0 else 0.0,
         "failureBreakdown": dict(failure_breakdown),
     }
+
+
+def _outcome_record_key(record: dict[str, Any]) -> str:
+    task_id = record.get("taskId")
+    if task_id is not None:
+        return f"task:{task_id}"
+    return "fallback:{model}|{gpu}|{time}".format(
+        model=record.get("modelId") or "",
+        gpu=record.get("targetGpu") or "",
+        time=record.get("submitTime") or "",
+    )
+
+
+def _outcome_version(record: dict[str, Any]) -> tuple[int, float, float]:
+    last_sync = parse_datetime(record.get("lastSyncTime"))
+    submit_time = parse_datetime(record.get("submitTime"))
+    outcome_rank = 1 if record.get("outcome") in {"success", "failed"} else 0
+    return (
+        outcome_rank,
+        last_sync.timestamp() if last_sync else 0.0,
+        submit_time.timestamp() if submit_time else 0.0,
+    )
+
+
+def _prefer_newer_outcome(existing: dict[str, Any], candidate: dict[str, Any]) -> dict[str, Any]:
+    if _outcome_version(candidate) >= _outcome_version(existing):
+        return candidate
+    return existing