"""Adapter / Enricher base classes and the request/response/context types they work with. An Adapter turns one publisher's data into Records in two steps: start(ctx) -> Request... entry points (lists, feeds, index pages) parse(ctx, resp) -> Record | Request... PURE: no DB access, no network The runner fetches requests (policy-checked, archived), calls parse() and hands records to the Loader. See docs/contributing/adapters.md for a walkthrough. """ from __future__ import annotations import hashlib import json import logging from collections import Counter from collections.abc import Iterable, Iterator from dataclasses import dataclass, field from datetime import datetime from functools import cached_property from typing import TYPE_CHECKING, Any, ClassVar from bs4 import BeautifulSoup from politrack.core.spec import SourceSpec if TYPE_CHECKING: from psycopg import Connection from politrack.core.records import Record SECRET_PARAMS = {"key", "api_key", "apikey", "token", "access_token"} def redact_url(url: str) -> str: """Remove credentials from a URL before it is logged or stored.""" from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit parts = urlsplit(url) query = [(k, "" if k.lower() in SECRET_PARAMS else v) for k, v in parse_qsl(parts.query, keep_blank_values=True)] return urlunsplit(parts._replace(query=urlencode(query))) @dataclass(frozen=True) class Request: url: str kind: str = "default" # parse() dispatch key method: str = "GET" params: dict[str, Any] | None = None json: Any = None headers: dict[str, str] | None = None meta: dict[str, Any] = field(default_factory=dict, compare=False, hash=False) conditional: bool = True # send If-None-Match / If-Modified-Since skip_unchanged: bool = True # body + parser_version unchanged: follow-ups only @property def fingerprint(self) -> str: raw = json.dumps([self.method, self.url, self.params or {}, self.json], sort_keys=True, default=str) return hashlib.sha256(raw.encode()).hexdigest()[:32] def describe(self) -> dict[str, Any]: """What gets stored in the archive index (secrets removed).""" params = {k: ("" if k.lower() in SECRET_PARAMS else v) for k, v in (self.params or {}).items()} return {"kind": self.kind, "params": params or None, "json": self.json, "meta": self.meta} @dataclass class Response: request: Request status: int url: str headers: dict[str, str] body: bytes fetched_at: datetime sha256: str fetch_id: int | None = None tdm_reserved: bool = False from_archive: bool = False @cached_property def text(self) -> str: encoding = "utf-8" content_type = self.headers.get("content-type", "") if "charset=" in content_type: encoding = content_type.split("charset=", 1)[1].split(";")[0].strip() or "utf-8" try: return self.body.decode(encoding) except (LookupError, UnicodeDecodeError): return self.body.decode("utf-8", errors="replace") def json(self) -> Any: return json.loads(self.text, strict=False) @cached_property def soup(self) -> BeautifulSoup: return BeautifulSoup(self.text, "html.parser") @property def meta(self) -> dict[str, Any]: return self.request.meta class StateStore: """Cursors of incremental adapters (persisted in the `state` table).""" def __init__(self, conn: Connection | None, adapter: str) -> None: self.conn = conn self.adapter = adapter self._local: dict[str, Any] = {} def get(self, key: str, default: Any = None) -> Any: if self.conn is None: return self._local.get(key, default) row = self.conn.execute("SELECT value FROM state WHERE adapter = %s AND key = %s", (self.adapter, key)).fetchone() return row["value"] if row else default def set(self, key: str, value: Any) -> None: if self.conn is None: self._local[key] = value return from psycopg.types.json import Jsonb self.conn.execute( """INSERT INTO state (adapter, key, value) VALUES (%s, %s, %s) ON CONFLICT (adapter, key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()""", (self.adapter, key, Jsonb(value)), ) @dataclass class RunContext: adapter: str spec: SourceSpec state: StateStore stats: Counter = field(default_factory=Counter) since: datetime | None = None params: dict[str, Any] = field(default_factory=dict) # CLI/worker parameters, e.g. {"gp": "XXVIII"} mode: str = "fetch" # fetch | reparse | test conn: Connection | None = None # only for enrichers / the loader, never in parse() response: Response | None = None # set by the runner while parsing creates_persons: bool = False log: logging.Logger = field(default_factory=lambda: logging.getLogger("politrack")) class Adapter: """Base class for source adapters. Subclass, set the class attributes, implement start()/parse().""" spec: ClassVar[SourceSpec] name: ClassVar[str] description: ClassVar[str] = "" schedule: ClassVar[str | None] = None # cron, UTC; None = manual only queue: ClassVar[str] = "scrape" # scrape | bulk | social | enrich parser_version: ClassVar[int] = 1 # bump when parse() output changes -> `politrack reparse` creates_persons: ClassVar[bool] = False # may this adapter create new person entities? def start(self, ctx: RunContext) -> Iterable[Request]: raise NotImplementedError def parse(self, ctx: RunContext, response: Response) -> Iterator[Record | Request]: raise NotImplementedError class Enricher: """Derives records from data already in the database (linking, events, corroboration...).""" spec: ClassVar[SourceSpec] name: ClassVar[str] description: ClassVar[str] = "" schedule: ClassVar[str | None] = None queue: ClassVar[str] = "enrich" creates_persons: ClassVar[bool] = False def run(self, ctx: RunContext) -> Iterable[Record] | None: raise NotImplementedError