politics-tracking/backend/politrack/core/adapter.py

"""Adapter / Enricher base classes and the request/response/context types they work with.

An Adapter turns one publisher's data into Records in two steps:

    start(ctx)          -> Request...             entry points (lists, feeds, index pages)
    parse(ctx, resp)    -> Record | Request...    PURE: no DB access, no network

The runner fetches requests (policy-checked, archived), calls parse() and hands records to the
Loader. See docs/contributing/adapters.md for a walkthrough.
"""

from __future__ import annotations

import hashlib
import json
import logging
from collections import Counter
from collections.abc import Iterable, Iterator
from dataclasses import dataclass, field
from datetime import datetime
from functools import cached_property
from typing import TYPE_CHECKING, Any, ClassVar

from bs4 import BeautifulSoup

from politrack.core.spec import SourceSpec

if TYPE_CHECKING:
    from psycopg import Connection

    from politrack.core.records import Record


SECRET_PARAMS = {"key", "api_key", "apikey", "token", "access_token"}


def redact_url(url: str) -> str:
    """Remove credentials from a URL before it is logged or stored."""
    from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit

    parts = urlsplit(url)
    query = [(k, "<redacted>" if k.lower() in SECRET_PARAMS else v) for k, v in parse_qsl(parts.query, keep_blank_values=True)]
    return urlunsplit(parts._replace(query=urlencode(query)))


@dataclass(frozen=True)
class Request:
    url: str
    kind: str = "default"                      # parse() dispatch key
    method: str = "GET"
    params: dict[str, Any] | None = None
    json: Any = None
    headers: dict[str, str] | None = None
    meta: dict[str, Any] = field(default_factory=dict, compare=False, hash=False)
    conditional: bool = True                   # send If-None-Match / If-Modified-Since
    skip_unchanged: bool = True                # body + parser_version unchanged: follow-ups only

    @property
    def fingerprint(self) -> str:
        raw = json.dumps([self.method, self.url, self.params or {}, self.json], sort_keys=True, default=str)
        return hashlib.sha256(raw.encode()).hexdigest()[:32]

    def describe(self) -> dict[str, Any]:
        """What gets stored in the archive index (secrets removed)."""
        params = {k: ("<redacted>" if k.lower() in SECRET_PARAMS else v) for k, v in (self.params or {}).items()}
        return {"kind": self.kind, "params": params or None, "json": self.json, "meta": self.meta}


@dataclass
class Response:
    request: Request
    status: int
    url: str
    headers: dict[str, str]
    body: bytes
    fetched_at: datetime
    sha256: str
    fetch_id: int | None = None
    tdm_reserved: bool = False
    from_archive: bool = False

    @cached_property
    def text(self) -> str:
        encoding = "utf-8"
        content_type = self.headers.get("content-type", "")
        if "charset=" in content_type:
            encoding = content_type.split("charset=", 1)[1].split(";")[0].strip() or "utf-8"
        try:
            return self.body.decode(encoding)
        except (LookupError, UnicodeDecodeError):
            return self.body.decode("utf-8", errors="replace")

    def json(self) -> Any:
        return json.loads(self.text, strict=False)

    @cached_property
    def soup(self) -> BeautifulSoup:
        return BeautifulSoup(self.text, "html.parser")

    @property
    def meta(self) -> dict[str, Any]:
        return self.request.meta


class StateStore:
    """Cursors of incremental adapters (persisted in the `state` table)."""

    def __init__(self, conn: Connection | None, adapter: str) -> None:
        self.conn = conn
        self.adapter = adapter
        self._local: dict[str, Any] = {}

    def get(self, key: str, default: Any = None) -> Any:
        if self.conn is None:
            return self._local.get(key, default)
        row = self.conn.execute("SELECT value FROM state WHERE adapter = %s AND key = %s",
                                (self.adapter, key)).fetchone()
        return row["value"] if row else default

    def set(self, key: str, value: Any) -> None:
        if self.conn is None:
            self._local[key] = value
            return
        from psycopg.types.json import Jsonb

        self.conn.execute(
            """INSERT INTO state (adapter, key, value) VALUES (%s, %s, %s)
               ON CONFLICT (adapter, key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()""",
            (self.adapter, key, Jsonb(value)),
        )


@dataclass
class RunContext:
    adapter: str
    spec: SourceSpec
    state: StateStore
    stats: Counter = field(default_factory=Counter)
    since: datetime | None = None
    params: dict[str, Any] = field(default_factory=dict)   # CLI/worker parameters, e.g. {"gp": "XXVIII"}
    mode: str = "fetch"                                    # fetch | reparse | test
    conn: Connection | None = None                         # only for enrichers / the loader, never in parse()
    response: Response | None = None                       # set by the runner while parsing
    creates_persons: bool = False
    log: logging.Logger = field(default_factory=lambda: logging.getLogger("politrack"))


class Adapter:
    """Base class for source adapters. Subclass, set the class attributes, implement start()/parse()."""

    spec: ClassVar[SourceSpec]
    name: ClassVar[str]
    description: ClassVar[str] = ""
    schedule: ClassVar[str | None] = None      # cron, UTC; None = manual only
    queue: ClassVar[str] = "scrape"            # scrape | bulk | social | enrich
    parser_version: ClassVar[int] = 1          # bump when parse() output changes -> `politrack reparse`
    creates_persons: ClassVar[bool] = False    # may this adapter create new person entities?

    def start(self, ctx: RunContext) -> Iterable[Request]:
        raise NotImplementedError

    def parse(self, ctx: RunContext, response: Response) -> Iterator[Record | Request]:
        raise NotImplementedError


class Enricher:
    """Derives records from data already in the database (linking, events, corroboration...)."""

    spec: ClassVar[SourceSpec]
    name: ClassVar[str]
    description: ClassVar[str] = ""
    schedule: ClassVar[str | None] = None
    queue: ClassVar[str] = "enrich"
    creates_persons: ClassVar[bool] = False

    def run(self, ctx: RunContext) -> Iterable[Record] | None:
        raise NotImplementedError