feat: add adaptive GPU scheduling

modelhub_submmit_api/http_json.py

@@ -4,6 +4,8 @@ import json
 import socket
 import time
 from dataclasses import dataclass
+from datetime import datetime, timezone
+from email.utils import parsedate_to_datetime
 from http.client import IncompleteRead, RemoteDisconnected
 from typing import Any
 from urllib.error import HTTPError, URLError
@@ -76,17 +78,34 @@ class JsonHttpClient:
             except HTTPError as exc:
                 payload = self._decode_error_payload(exc)
                 if exc.code in retryable_statuses and attempt < self.retries:
-                    time.sleep(self.backoff_seconds * (attempt + 1))
+                    retry_after = exc.headers.get("Retry-After") if exc.headers is not None else None
+                    time.sleep(self._retry_delay(attempt, retry_after=retry_after))
                     continue
                 raise HttpJsonError(f"HTTP request failed for {url}", status_code=exc.code, payload=payload) from exc
             except (URLError, TimeoutError, socket.timeout, RemoteDisconnected, IncompleteRead) as exc:
                 if attempt < self.retries:
-                    time.sleep(self.backoff_seconds * (attempt + 1))
+                    time.sleep(self._retry_delay(attempt))
                     continue
                 raise HttpJsonError(f"Network request failed for {url}") from exc
 
         raise HttpJsonError(f"Request exhausted retries for {url}")
 
+    def _retry_delay(self, attempt: int, *, retry_after: str | None = None) -> float:
+        exponential_delay = self.backoff_seconds * (2**attempt)
+        retry_after_delay = 0.0
+        if retry_after:
+            try:
+                retry_after_delay = max(0.0, float(retry_after))
+            except ValueError:
+                try:
+                    parsed = parsedate_to_datetime(retry_after)
+                    if parsed.tzinfo is None:
+                        parsed = parsed.replace(tzinfo=timezone.utc)
+                    retry_after_delay = max(0.0, (parsed - datetime.now(timezone.utc)).total_seconds())
+                except (TypeError, ValueError, OverflowError):
+                    retry_after_delay = 0.0
+        return min(60.0, max(exponential_delay, retry_after_delay))
+
     def _build_url(self, path: str, query: dict[str, Any] | None) -> str:
         if path.startswith("http://") or path.startswith("https://"):
             url = path