#!/usr/bin/env python3
"""
Shared HTTP GET helper with exponential-backoff retry for rate limiting (429)
and transient server errors (5xx). Stdlib only.

Every search_*.py / verify_citation.py network call should go through this
instead of calling urllib.request.urlopen directly, so a single source being
temporarily rate-limited doesn't quietly degrade a whole batch of candidates
to "unverified" the way Semantic Scholar's 429s did before this existed.

Importable:
    from http_utils import fetch
"""
import time
import urllib.request
import urllib.error

RETRYABLE_CODES = {429, 500, 502, 503, 504}


def _wait_seconds(http_error, attempt, base=1.0, cap=20.0):
    retry_after = http_error.headers.get("Retry-After") if http_error.headers else None
    if retry_after:
        try:
            return min(float(retry_after), cap)
        except ValueError:
            pass
    return min(base * (2 ** attempt), cap)


def fetch(url, headers=None, timeout=20, max_retries=4):
    """GET url, retrying on 429/5xx with exponential backoff (honoring a
    Retry-After header when the server sends one). Returns the raw response
    bytes. Raises the underlying urllib error if retries are exhausted or the
    error isn't retryable (e.g. 404)."""
    req = urllib.request.Request(url, headers=headers or {})
    attempt = 0
    while True:
        try:
            with urllib.request.urlopen(req, timeout=timeout) as resp:
                return resp.read()
        except urllib.error.HTTPError as e:
            if e.code not in RETRYABLE_CODES or attempt >= max_retries:
                raise
            time.sleep(_wait_seconds(e, attempt))
            attempt += 1