135 lines
5.1 KiB
Python
135 lines
5.1 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import socket
|
|
import time
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from email.utils import parsedate_to_datetime
|
|
from http.client import IncompleteRead, RemoteDisconnected
|
|
from typing import Any
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.parse import urlencode, urljoin
|
|
from urllib.request import OpenerDirector, Request, build_opener
|
|
|
|
|
|
@dataclass
|
|
class HttpJsonError(RuntimeError):
|
|
message: str
|
|
status_code: int | None = None
|
|
payload: Any = None
|
|
|
|
def __str__(self) -> str:
|
|
if self.status_code is None:
|
|
return self.message
|
|
return f"{self.message} (status={self.status_code})"
|
|
|
|
|
|
class JsonHttpClient:
|
|
def __init__(
|
|
self,
|
|
base_url: str = "",
|
|
default_headers: dict[str, str] | None = None,
|
|
timeout: int = 30,
|
|
retries: int = 2,
|
|
backoff_seconds: float = 1.0,
|
|
opener: OpenerDirector | None = None,
|
|
) -> None:
|
|
self.base_url = base_url.rstrip("/")
|
|
self.default_headers = default_headers or {}
|
|
self.timeout = timeout
|
|
self.retries = retries
|
|
self.backoff_seconds = backoff_seconds
|
|
self.opener = opener or build_opener()
|
|
|
|
def request_json(
|
|
self,
|
|
method: str,
|
|
path: str,
|
|
*,
|
|
query: dict[str, Any] | None = None,
|
|
data: dict[str, Any] | None = None,
|
|
headers: dict[str, str] | None = None,
|
|
retryable_statuses: set[int] | None = None,
|
|
) -> Any:
|
|
retryable_statuses = retryable_statuses or {429, 500, 502, 503, 504}
|
|
url = self._build_url(path, query)
|
|
merged_headers = {
|
|
"Accept": "application/json",
|
|
**self.default_headers,
|
|
**(headers or {}),
|
|
}
|
|
body = None
|
|
if data is not None:
|
|
body = json.dumps(data).encode("utf-8")
|
|
merged_headers.setdefault("Content-Type", "application/json")
|
|
|
|
for attempt in range(self.retries + 1):
|
|
request = Request(url=url, data=body, headers=merged_headers, method=method.upper())
|
|
try:
|
|
with self.opener.open(request, timeout=self.timeout) as response:
|
|
raw = response.read().decode("utf-8")
|
|
if not raw:
|
|
return None
|
|
try:
|
|
return json.loads(raw)
|
|
except json.JSONDecodeError as exc:
|
|
raise HttpJsonError(f"Invalid JSON response from {url}", payload=raw) from exc
|
|
except HTTPError as exc:
|
|
payload = self._decode_error_payload(exc)
|
|
if exc.code in retryable_statuses and attempt < self.retries:
|
|
retry_after = exc.headers.get("Retry-After") if exc.headers is not None else None
|
|
time.sleep(self._retry_delay(attempt, retry_after=retry_after))
|
|
continue
|
|
raise HttpJsonError(f"HTTP request failed for {url}", status_code=exc.code, payload=payload) from exc
|
|
except (URLError, TimeoutError, socket.timeout, RemoteDisconnected, IncompleteRead) as exc:
|
|
if attempt < self.retries:
|
|
time.sleep(self._retry_delay(attempt))
|
|
continue
|
|
raise HttpJsonError(f"Network request failed for {url}") from exc
|
|
|
|
raise HttpJsonError(f"Request exhausted retries for {url}")
|
|
|
|
def _retry_delay(self, attempt: int, *, retry_after: str | None = None) -> float:
|
|
exponential_delay = self.backoff_seconds * (2**attempt)
|
|
retry_after_delay = 0.0
|
|
if retry_after:
|
|
try:
|
|
retry_after_delay = max(0.0, float(retry_after))
|
|
except ValueError:
|
|
try:
|
|
parsed = parsedate_to_datetime(retry_after)
|
|
if parsed.tzinfo is None:
|
|
parsed = parsed.replace(tzinfo=timezone.utc)
|
|
retry_after_delay = max(0.0, (parsed - datetime.now(timezone.utc)).total_seconds())
|
|
except (TypeError, ValueError, OverflowError):
|
|
retry_after_delay = 0.0
|
|
return min(60.0, max(exponential_delay, retry_after_delay))
|
|
|
|
def _build_url(self, path: str, query: dict[str, Any] | None) -> str:
|
|
if path.startswith("http://") or path.startswith("https://"):
|
|
url = path
|
|
else:
|
|
base = self.base_url + "/"
|
|
url = urljoin(base, path.lstrip("/"))
|
|
if query:
|
|
pairs = [(key, value) for key, value in query.items() if value is not None]
|
|
encoded = urlencode(pairs)
|
|
if encoded:
|
|
separator = "&" if "?" in url else "?"
|
|
url = f"{url}{separator}{encoded}"
|
|
return url
|
|
|
|
@staticmethod
|
|
def _decode_error_payload(exc: HTTPError) -> Any:
|
|
try:
|
|
raw = exc.read().decode("utf-8")
|
|
except Exception:
|
|
return None
|
|
if not raw:
|
|
return None
|
|
try:
|
|
return json.loads(raw)
|
|
except json.JSONDecodeError:
|
|
return raw
|