Politrack: track Austrian politicians from public sources
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
583ccba31c
187 files changed
+39890
No files matched your search
@@ -0,0 +1,177 @@
|
||||
"""Adapter / Enricher base classes and the request/response/context types they work with.
|
||||
|
||||
An Adapter turns one publisher's data into Records in two steps:
|
||||
|
||||
start(ctx) -> Request... entry points (lists, feeds, index pages)
|
||||
parse(ctx, resp) -> Record | Request... PURE: no DB access, no network
|
||||
|
||||
The runner fetches requests (policy-checked, archived), calls parse() and hands records to the
|
||||
Loader. See docs/contributing/adapters.md for a walkthrough.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable, Iterator
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from functools import cached_property
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from politrack.core.spec import SourceSpec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from psycopg import Connection
|
||||
|
||||
from politrack.core.records import Record
|
||||
|
||||
|
||||
SECRET_PARAMS = {"key", "api_key", "apikey", "token", "access_token"}
|
||||
|
||||
|
||||
def redact_url(url: str) -> str:
|
||||
"""Remove credentials from a URL before it is logged or stored."""
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
|
||||
parts = urlsplit(url)
|
||||
query = [(k, "<redacted>" if k.lower() in SECRET_PARAMS else v) for k, v in parse_qsl(parts.query, keep_blank_values=True)]
|
||||
return urlunsplit(parts._replace(query=urlencode(query)))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Request:
|
||||
url: str
|
||||
kind: str = "default" # parse() dispatch key
|
||||
method: str = "GET"
|
||||
params: dict[str, Any] | None = None
|
||||
json: Any = None
|
||||
headers: dict[str, str] | None = None
|
||||
meta: dict[str, Any] = field(default_factory=dict, compare=False, hash=False)
|
||||
conditional: bool = True # send If-None-Match / If-Modified-Since
|
||||
skip_unchanged: bool = True # body + parser_version unchanged: follow-ups only
|
||||
|
||||
@property
|
||||
def fingerprint(self) -> str:
|
||||
raw = json.dumps([self.method, self.url, self.params or {}, self.json], sort_keys=True, default=str)
|
||||
return hashlib.sha256(raw.encode()).hexdigest()[:32]
|
||||
|
||||
def describe(self) -> dict[str, Any]:
|
||||
"""What gets stored in the archive index (secrets removed)."""
|
||||
params = {k: ("<redacted>" if k.lower() in SECRET_PARAMS else v) for k, v in (self.params or {}).items()}
|
||||
return {"kind": self.kind, "params": params or None, "json": self.json, "meta": self.meta}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Response:
|
||||
request: Request
|
||||
status: int
|
||||
url: str
|
||||
headers: dict[str, str]
|
||||
body: bytes
|
||||
fetched_at: datetime
|
||||
sha256: str
|
||||
fetch_id: int | None = None
|
||||
tdm_reserved: bool = False
|
||||
from_archive: bool = False
|
||||
|
||||
@cached_property
|
||||
def text(self) -> str:
|
||||
encoding = "utf-8"
|
||||
content_type = self.headers.get("content-type", "")
|
||||
if "charset=" in content_type:
|
||||
encoding = content_type.split("charset=", 1)[1].split(";")[0].strip() or "utf-8"
|
||||
try:
|
||||
return self.body.decode(encoding)
|
||||
except (LookupError, UnicodeDecodeError):
|
||||
return self.body.decode("utf-8", errors="replace")
|
||||
|
||||
def json(self) -> Any:
|
||||
return json.loads(self.text, strict=False)
|
||||
|
||||
@cached_property
|
||||
def soup(self) -> BeautifulSoup:
|
||||
return BeautifulSoup(self.text, "html.parser")
|
||||
|
||||
@property
|
||||
def meta(self) -> dict[str, Any]:
|
||||
return self.request.meta
|
||||
|
||||
|
||||
class StateStore:
|
||||
"""Cursors of incremental adapters (persisted in the `state` table)."""
|
||||
|
||||
def __init__(self, conn: Connection | None, adapter: str) -> None:
|
||||
self.conn = conn
|
||||
self.adapter = adapter
|
||||
self._local: dict[str, Any] = {}
|
||||
|
||||
def get(self, key: str, default: Any = None) -> Any:
|
||||
if self.conn is None:
|
||||
return self._local.get(key, default)
|
||||
row = self.conn.execute("SELECT value FROM state WHERE adapter = %s AND key = %s",
|
||||
(self.adapter, key)).fetchone()
|
||||
return row["value"] if row else default
|
||||
|
||||
def set(self, key: str, value: Any) -> None:
|
||||
if self.conn is None:
|
||||
self._local[key] = value
|
||||
return
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
self.conn.execute(
|
||||
"""INSERT INTO state (adapter, key, value) VALUES (%s, %s, %s)
|
||||
ON CONFLICT (adapter, key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()""",
|
||||
(self.adapter, key, Jsonb(value)),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunContext:
|
||||
adapter: str
|
||||
spec: SourceSpec
|
||||
state: StateStore
|
||||
stats: Counter = field(default_factory=Counter)
|
||||
since: datetime | None = None
|
||||
params: dict[str, Any] = field(default_factory=dict) # CLI/worker parameters, e.g. {"gp": "XXVIII"}
|
||||
mode: str = "fetch" # fetch | reparse | test
|
||||
conn: Connection | None = None # only for enrichers / the loader, never in parse()
|
||||
response: Response | None = None # set by the runner while parsing
|
||||
creates_persons: bool = False
|
||||
log: logging.Logger = field(default_factory=lambda: logging.getLogger("politrack"))
|
||||
|
||||
|
||||
class Adapter:
|
||||
"""Base class for source adapters. Subclass, set the class attributes, implement start()/parse()."""
|
||||
|
||||
spec: ClassVar[SourceSpec]
|
||||
name: ClassVar[str]
|
||||
description: ClassVar[str] = ""
|
||||
schedule: ClassVar[str | None] = None # cron, UTC; None = manual only
|
||||
queue: ClassVar[str] = "scrape" # scrape | bulk | social | enrich
|
||||
parser_version: ClassVar[int] = 1 # bump when parse() output changes -> `politrack reparse`
|
||||
creates_persons: ClassVar[bool] = False # may this adapter create new person entities?
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterable[Request]:
|
||||
raise NotImplementedError
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Record | Request]:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class Enricher:
|
||||
"""Derives records from data already in the database (linking, events, corroboration...)."""
|
||||
|
||||
spec: ClassVar[SourceSpec]
|
||||
name: ClassVar[str]
|
||||
description: ClassVar[str] = ""
|
||||
schedule: ClassVar[str | None] = None
|
||||
queue: ClassVar[str] = "enrich"
|
||||
creates_persons: ClassVar[bool] = False
|
||||
|
||||
def run(self, ctx: RunContext) -> Iterable[Record] | None:
|
||||
raise NotImplementedError
|
||||
Reference in new issue
Block a user