Politrack: track Austrian politicians from public sources

Initial import of Politrack, a self-hosted service that tracks Austrian
politicians from federal to municipal level - roles, votes, money flows,
statements, meetings, social media and legal proceedings - with every claim
linked to its sources.

- backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse ->
  load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive
  for reparsing, and a claim/evidence model with corroboration
- sources: Parliament, Wikidata, EU Parliament/Commission, Council of
  Ministers, Rechnungshof party reports and donations, procurement, RTR media
  transparency, political ad libraries, WKStA, news feeds, Wikipedia edits,
  Bluesky, Mastodon, public Telegram channels, YouTube and X (official API)
- web: Next.js frontend in German and English with typed API client
- mcp: MCP server for AI agents and RAG
- docs: architecture, data model, sources, legal notes, Unraid deployment and
  contributor guides; Docker Compose setup for Unraid

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
simonandClaude Opus 5.5 committed 2026-10-28 20:02:01 +01:00
commit 583ccba31c
187 files changed
+39890

No files matched your search

+11
View File
@@ -0,0 +1,11 @@
"""The adapter framework. Most adapter modules only need:
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Evidence, PersonRef, OrgRef, ...
"""
from politrack.core.adapter import Adapter, Enricher, Request, Response, RunContext
from politrack.core.registry import register, register_spec
from politrack.core.spec import SourceSpec
__all__ = ["Adapter", "Enricher", "Request", "Response", "RunContext", "SourceSpec", "register", "register_spec"]
+177
View File
@@ -0,0 +1,177 @@
"""Adapter / Enricher base classes and the request/response/context types they work with.
An Adapter turns one publisher's data into Records in two steps:
start(ctx) -> Request... entry points (lists, feeds, index pages)
parse(ctx, resp) -> Record | Request... PURE: no DB access, no network
The runner fetches requests (policy-checked, archived), calls parse() and hands records to the
Loader. See docs/contributing/adapters.md for a walkthrough.
"""
from __future__ import annotations
import hashlib
import json
import logging
from collections import Counter
from collections.abc import Iterable, Iterator
from dataclasses import dataclass, field
from datetime import datetime
from functools import cached_property
from typing import TYPE_CHECKING, Any, ClassVar
from bs4 import BeautifulSoup
from politrack.core.spec import SourceSpec
if TYPE_CHECKING:
from psycopg import Connection
from politrack.core.records import Record
SECRET_PARAMS = {"key", "api_key", "apikey", "token", "access_token"}
def redact_url(url: str) -> str:
"""Remove credentials from a URL before it is logged or stored."""
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
parts = urlsplit(url)
query = [(k, "<redacted>" if k.lower() in SECRET_PARAMS else v) for k, v in parse_qsl(parts.query, keep_blank_values=True)]
return urlunsplit(parts._replace(query=urlencode(query)))
@dataclass(frozen=True)
class Request:
url: str
kind: str = "default" # parse() dispatch key
method: str = "GET"
params: dict[str, Any] | None = None
json: Any = None
headers: dict[str, str] | None = None
meta: dict[str, Any] = field(default_factory=dict, compare=False, hash=False)
conditional: bool = True # send If-None-Match / If-Modified-Since
skip_unchanged: bool = True # body + parser_version unchanged: follow-ups only
@property
def fingerprint(self) -> str:
raw = json.dumps([self.method, self.url, self.params or {}, self.json], sort_keys=True, default=str)
return hashlib.sha256(raw.encode()).hexdigest()[:32]
def describe(self) -> dict[str, Any]:
"""What gets stored in the archive index (secrets removed)."""
params = {k: ("<redacted>" if k.lower() in SECRET_PARAMS else v) for k, v in (self.params or {}).items()}
return {"kind": self.kind, "params": params or None, "json": self.json, "meta": self.meta}
@dataclass
class Response:
request: Request
status: int
url: str
headers: dict[str, str]
body: bytes
fetched_at: datetime
sha256: str
fetch_id: int | None = None
tdm_reserved: bool = False
from_archive: bool = False
@cached_property
def text(self) -> str:
encoding = "utf-8"
content_type = self.headers.get("content-type", "")
if "charset=" in content_type:
encoding = content_type.split("charset=", 1)[1].split(";")[0].strip() or "utf-8"
try:
return self.body.decode(encoding)
except (LookupError, UnicodeDecodeError):
return self.body.decode("utf-8", errors="replace")
def json(self) -> Any:
return json.loads(self.text, strict=False)
@cached_property
def soup(self) -> BeautifulSoup:
return BeautifulSoup(self.text, "html.parser")
@property
def meta(self) -> dict[str, Any]:
return self.request.meta
class StateStore:
"""Cursors of incremental adapters (persisted in the `state` table)."""
def __init__(self, conn: Connection | None, adapter: str) -> None:
self.conn = conn
self.adapter = adapter
self._local: dict[str, Any] = {}
def get(self, key: str, default: Any = None) -> Any:
if self.conn is None:
return self._local.get(key, default)
row = self.conn.execute("SELECT value FROM state WHERE adapter = %s AND key = %s",
(self.adapter, key)).fetchone()
return row["value"] if row else default
def set(self, key: str, value: Any) -> None:
if self.conn is None:
self._local[key] = value
return
from psycopg.types.json import Jsonb
self.conn.execute(
"""INSERT INTO state (adapter, key, value) VALUES (%s, %s, %s)
ON CONFLICT (adapter, key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()""",
(self.adapter, key, Jsonb(value)),
)
@dataclass
class RunContext:
adapter: str
spec: SourceSpec
state: StateStore
stats: Counter = field(default_factory=Counter)
since: datetime | None = None
params: dict[str, Any] = field(default_factory=dict) # CLI/worker parameters, e.g. {"gp": "XXVIII"}
mode: str = "fetch" # fetch | reparse | test
conn: Connection | None = None # only for enrichers / the loader, never in parse()
response: Response | None = None # set by the runner while parsing
creates_persons: bool = False
log: logging.Logger = field(default_factory=lambda: logging.getLogger("politrack"))
class Adapter:
"""Base class for source adapters. Subclass, set the class attributes, implement start()/parse()."""
spec: ClassVar[SourceSpec]
name: ClassVar[str]
description: ClassVar[str] = ""
schedule: ClassVar[str | None] = None # cron, UTC; None = manual only
queue: ClassVar[str] = "scrape" # scrape | bulk | social | enrich
parser_version: ClassVar[int] = 1 # bump when parse() output changes -> `politrack reparse`
creates_persons: ClassVar[bool] = False # may this adapter create new person entities?
def start(self, ctx: RunContext) -> Iterable[Request]:
raise NotImplementedError
def parse(self, ctx: RunContext, response: Response) -> Iterator[Record | Request]:
raise NotImplementedError
class Enricher:
"""Derives records from data already in the database (linking, events, corroboration...)."""
spec: ClassVar[SourceSpec]
name: ClassVar[str]
description: ClassVar[str] = ""
schedule: ClassVar[str | None] = None
queue: ClassVar[str] = "enrich"
creates_persons: ClassVar[bool] = False
def run(self, ctx: RunContext) -> Iterable[Record] | None:
raise NotImplementedError
+94
View File
@@ -0,0 +1,94 @@
"""Raw archive: every fetch is logged in the `fetch_log` table; bodies are stored gzip-compressed and
content-addressed (sha256) below POLITRACK_RAW_DIR when the source allows it.
Used for: conditional requests (ETag/Last-Modified), change detection (skip re-parsing identical
bodies), `politrack reparse` after parser fixes, and as an audit trail.
"""
from __future__ import annotations
import gzip
import hashlib
import os
from pathlib import Path
from typing import Any
from psycopg import Connection
from psycopg.types.json import Jsonb
from politrack.config import get_settings
class Archive:
def __init__(self, raw_dir: Path | None = None) -> None:
self.raw_dir = Path(raw_dir or get_settings().raw_dir)
# -- blobs ----------------------------------------------------------------------------
def _path(self, sha256: str) -> Path:
return self.raw_dir / sha256[:2] / f"{sha256}.gz"
def write_blob(self, body: bytes) -> str:
sha = hashlib.sha256(body).hexdigest()
path = self._path(sha)
if not path.exists():
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(f".tmp{os.getpid()}")
with gzip.open(tmp, "wb") as fh:
fh.write(body)
tmp.replace(path)
return sha
def read_blob(self, sha256: str) -> bytes | None:
path = self._path(sha256)
if not path.exists():
return None
with gzip.open(path, "rb") as fh:
return fh.read()
# -- index ----------------------------------------------------------------------------
def last(self, conn: Connection, request_fp: str) -> dict[str, Any] | None:
return conn.execute(
"""SELECT id, sha256, etag, last_modified, stored, parsed_sha256, parser_version, status, url,
content_type, tdm_reserved, fetched_at
FROM fetch_log WHERE request_fp = %s AND status IS NOT NULL AND status < 400
ORDER BY fetched_at DESC LIMIT 1""",
(request_fp,),
).fetchone()
def log(self, conn: Connection, *, adapter: str, source_id: int | None, request_fp: str, url: str, method: str,
request: dict, status: int | None, body: bytes | None, headers: dict[str, str] | None,
keep: bool, tdm_reserved: bool, error: str | None = None) -> tuple[int, str | None]:
sha = hashlib.sha256(body).hexdigest() if body is not None else None
stored = False
if body is not None and keep and not tdm_reserved:
self.write_blob(body)
stored = True
headers = headers or {}
row = conn.execute(
"""INSERT INTO fetch_log (adapter, source_id, request_fp, url, method, request, status, sha256, size,
content_type, etag, last_modified, stored, tdm_reserved, error)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id""",
(adapter, source_id, request_fp, url, method, Jsonb(request), status, sha,
len(body) if body is not None else None, headers.get("content-type"), headers.get("etag"),
headers.get("last-modified"), stored, tdm_reserved, error),
).fetchone()
return row["id"], sha
def mark_parsed(self, conn: Connection, fetch_id: int, sha256: str, parser_version: int) -> None:
conn.execute("UPDATE fetch_log SET parsed_sha256 = %s, parser_version = %s WHERE id = %s",
(sha256, parser_version, fetch_id))
def mark_error(self, conn: Connection, fetch_id: int, error: str) -> None:
conn.execute("UPDATE fetch_log SET error = %s WHERE id = %s", (error[:2000], fetch_id))
def latest_stored(self, conn: Connection, adapter: str) -> list[dict[str, Any]]:
"""Latest stored fetch per request of an adapter (input for `reparse`)."""
return conn.execute(
"""SELECT DISTINCT ON (request_fp) id, request_fp, url, method, request, status, sha256, fetched_at,
content_type, tdm_reserved
FROM fetch_log WHERE adapter = %s AND stored AND status < 400
ORDER BY request_fp, fetched_at DESC""",
(adapter,),
).fetchall()
+158
View File
@@ -0,0 +1,158 @@
"""The Fetcher: the only component that talks to the network. It enforces the crawling policy:
* honest User-Agent with contact address,
* robots.txt (cached per origin); exemptions only for documented APIs listed in SourceSpec.robots_exempt,
* per-host throttling (max of SourceSpec.min_interval and robots.txt crawl-delay),
* conditional requests (ETag / Last-Modified from the raw archive),
* retries with exponential back-off for 429/5xx, honouring Retry-After,
* TDM reservations (W3C TDMRep: `tdm-reservation` header, /.well-known/tdmrep.json, /tdmrep.json),
* archiving of raw bodies (unless SourceSpec.keep_raw is False or TDM is reserved).
"""
from __future__ import annotations
import logging
import threading
import time
from datetime import datetime, timezone
from urllib.parse import urlsplit
import httpx
from protego import Protego
from psycopg import Connection
from politrack.config import get_settings
from politrack.core.adapter import Request, Response, redact_url
from politrack.core.archive import Archive
from politrack.core.spec import SourceSpec
log = logging.getLogger(__name__)
RETRY_STATUS = {429, 500, 502, 503, 504}
class Disallowed(Exception):
"""robots.txt forbids fetching this URL."""
class Fetcher:
def __init__(self, archive: Archive | None = None, client: httpx.Client | None = None) -> None:
settings = get_settings()
self.user_agent = settings.user_agent
self.default_interval = settings.min_request_interval
self.archive = archive or Archive()
self.client = client or httpx.Client(
headers={"User-Agent": self.user_agent, "Accept-Language": "de-AT,de;q=0.9,en;q=0.5"},
timeout=settings.http_timeout,
follow_redirects=True,
)
self._robots: dict[str, Protego | None] = {}
self._tdm: dict[str, bool] = {}
self._last: dict[str, float] = {}
self._lock = threading.Lock()
# -- policy -------------------------------------------------------------------------
@staticmethod
def origin(url: str) -> str:
parts = urlsplit(url)
return f"{parts.scheme}://{parts.netloc}"
def robots(self, url: str) -> Protego | None:
origin = self.origin(url)
if origin not in self._robots:
try:
resp = self.client.get(f"{origin}/robots.txt")
self._robots[origin] = Protego.parse(resp.text) if resp.status_code == 200 else None
except httpx.HTTPError:
self._robots[origin] = None
return self._robots[origin]
def allowed(self, url: str, spec: SourceSpec) -> bool:
if any(url.startswith(prefix) for prefix in spec.robots_exempt):
return True
rules = self.robots(url)
return rules is None or rules.can_fetch(url, self.user_agent)
def tdm_reserved(self, url: str) -> bool:
origin = self.origin(url)
if origin not in self._tdm:
reserved = False
for path in ("/.well-known/tdmrep.json", "/tdmrep.json"):
try:
resp = self.client.get(origin + path)
except httpx.HTTPError:
continue
if resp.status_code == 200:
try:
rules = resp.json()
except ValueError:
continue
reserved = any(str(r.get("tdm-reservation")) == "1" for r in rules if isinstance(r, dict))
break
self._tdm[origin] = reserved
return self._tdm[origin]
def _throttle(self, url: str, spec: SourceSpec) -> None:
rules = self._robots.get(self.origin(url))
delay = rules.crawl_delay(self.user_agent) if rules else None
interval = max(spec.min_interval or self.default_interval, float(delay or 0))
host = urlsplit(url).netloc
with self._lock:
wait = self._last.get(host, 0) + interval - time.monotonic()
if wait > 0:
time.sleep(wait)
self._last[host] = time.monotonic()
# -- fetching ---------------------------------------------------------------------
def fetch(self, conn: Connection, req: Request, *, adapter: str, spec: SourceSpec, source_id: int | None,
attempts: int = 5) -> Response:
if not self.allowed(req.url, spec):
raise Disallowed(req.url)
last = self.archive.last(conn, req.fingerprint) if req.conditional else None
headers = dict(req.headers or {})
if last and last["stored"]:
if last["etag"]:
headers["If-None-Match"] = last["etag"]
if last["last_modified"]:
headers["If-Modified-Since"] = last["last_modified"]
resp: httpx.Response | None = None
for attempt in range(attempts):
self._throttle(req.url, spec)
try:
resp = self.client.request(req.method, req.url, params=req.params, json=req.json, headers=headers)
except httpx.TransportError as exc:
if attempt == attempts - 1:
raise
log.warning("transport error %s (%s), retrying", req.url, exc)
time.sleep(min(5 * 2 ** attempt, 120))
continue
if resp.status_code in RETRY_STATUS and attempt < attempts - 1:
retry_after = resp.headers.get("Retry-After", "")
time.sleep(min(float(retry_after) if retry_after.isdigit() else 5 * 2 ** attempt, 300))
continue
break
assert resp is not None
tdm = resp.headers.get("tdm-reservation") == "1" or self.tdm_reserved(req.url)
now = datetime.now(timezone.utc)
if resp.status_code == 304 and last:
body = self.archive.read_blob(last["sha256"]) or b""
return Response(request=req, status=200, url=last["url"], headers={"content-type": last["content_type"] or ""},
body=body, fetched_at=now, sha256=last["sha256"], fetch_id=last["id"],
tdm_reserved=last["tdm_reserved"], from_archive=True)
lowered = {k.lower(): v for k, v in resp.headers.items()}
fetch_id, sha = self.archive.log(
conn, adapter=adapter, source_id=source_id, request_fp=req.fingerprint, url=redact_url(str(resp.url)),
method=req.method, request=req.describe(), status=resp.status_code, body=resp.content, headers=lowered,
keep=spec.keep_raw, tdm_reserved=tdm,
)
conn.commit()
resp.raise_for_status()
return Response(request=req, status=resp.status_code, url=redact_url(str(resp.url)), headers=lowered, body=resp.content,
fetched_at=now, sha256=sha or "", fetch_id=fetch_id, tdm_reserved=tdm)
def close(self) -> None:
self.client.close()
+448
View File
@@ -0,0 +1,448 @@
"""The Loader: the only code that writes canonical rows.
Invariants enforced here (not in adapters):
* no claim without evidence; claims whose person cannot be resolved are skipped (and counted),
* full text is kept only for sources with store_fulltext and responses without TDM reservation,
* claim identity is computed from *resolved* entity ids + type-specific fields, so the same fact
from different sources becomes one claim with several evidence rows ("corroborated"),
* claims marked `review=True` start as review_status='pending' (hidden until a human approves),
* persons are only created by adapters with creates_persons=True.
"""
from __future__ import annotations
from datetime import date, datetime
from decimal import Decimal, InvalidOperation
from typing import Any
from psycopg import Connection
from psycopg.types.json import Jsonb
from politrack.core.adapter import RunContext
from politrack.core.records import (
Claim,
Document,
Event,
Evidence,
Legal,
Money,
OrgRecord,
OrgRef,
Participation,
PersonRecord,
PersonRef,
Role,
SocialAccount,
Statement,
VoteEvent,
VotePosition,
)
from politrack.core.resolve import Resolver
from politrack.text import excerpt, fingerprint, make_slug, normalize, normalize_org_name, strip_titles
# Source policy is read from the DB once per process (synced from SourceSpecs).
_SOURCES: dict[str, dict[str, Any]] = {}
def source_row(conn: Connection, slug: str) -> dict[str, Any]:
if slug not in _SOURCES:
row = conn.execute("SELECT id, store_fulltext, kind FROM source WHERE slug = %s", (slug,)).fetchone()
if row is None:
raise KeyError(f"unknown source {slug!r}: run `politrack sources sync`")
_SOURCES[slug] = row
return _SOURCES[slug]
class Loader:
def __init__(self, conn: Connection, ctx: RunContext) -> None:
self.conn = conn
self.ctx = ctx
self.resolver = Resolver(conn)
# -- dispatch ---------------------------------------------------------------------
def load(self, record: Any) -> int | None:
if isinstance(record, Claim):
return self.claim(record)
if isinstance(record, Document):
return self.document(record.evidence)
if isinstance(record, PersonRecord):
return self.person(record)
if isinstance(record, OrgRecord):
return self.organization(record)
if isinstance(record, SocialAccount):
return self.social_account(record)
if isinstance(record, VoteEvent):
return self.vote_event(record)
if isinstance(record, Event):
return self.event(record)
raise TypeError(f"unknown record type {type(record).__name__}")
# -- documents --------------------------------------------------------------------
def document(self, ev: Evidence) -> int:
slug = ev.source or self.ctx.spec.slug
src = source_row(self.conn, slug)
response = self.ctx.response
tdm = bool(response and response.tdm_reserved)
full_text = ev.full_text if (src["store_fulltext"] and not tdm) else None
short = None if tdm else (excerpt(ev.excerpt, 400) if ev.excerpt else None)
# Link the document to the raw response only if it *is* that response (not e.g. a protocol it cites).
fetch_id = response.fetch_id if response and ev.url in (response.url, response.request.url) else None
row = self.conn.execute(
"""
INSERT INTO document (source_id, url, title, doc_type, published_at, excerpt, full_text, fetch_id, meta)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (url) DO UPDATE SET
title = COALESCE(EXCLUDED.title, document.title),
published_at = COALESCE(EXCLUDED.published_at, document.published_at),
excerpt = COALESCE(EXCLUDED.excerpt, document.excerpt),
full_text = COALESCE(EXCLUDED.full_text, document.full_text),
fetch_id = COALESCE(EXCLUDED.fetch_id, document.fetch_id),
meta = document.meta || EXCLUDED.meta,
retrieved_at = now()
RETURNING id
""",
(src["id"], ev.url, ev.title, ev.doc_type, ev.published_at, short, full_text, fetch_id,
Jsonb({**ev.meta, **({"tdm_reserved": True} if tdm else {})})),
).fetchone()
self.ctx.stats["documents"] += 1
return row["id"]
# -- entities ---------------------------------------------------------------------
def person(self, rec: PersonRecord) -> int | None:
person_id = self.resolver.person_by_ids(rec.ref.ids)
if person_id is None and rec.match_by_name:
person_id = self.resolver.person_by_name(rec.full_name)
if person_id is None:
if not self.ctx.creates_persons:
self.ctx.stats["persons_not_created"] += 1
return None
base = make_slug(rec.given_name or "", rec.family_name or strip_titles(rec.full_name)) or "person"
slug, n = base, 1
while self.conn.execute("SELECT 1 FROM person WHERE slug = %s", (slug,)).fetchone():
n += 1
slug = f"{base}-{n}"
person_id = self.conn.execute(
"""INSERT INTO person (slug, full_name, given_name, family_name, honorific_prefix, honorific_suffix,
birth_date, death_date, gender, photo_url)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id""",
(slug, rec.full_name, rec.given_name, rec.family_name, rec.honorific_prefix, rec.honorific_suffix,
rec.birth_date, rec.death_date, rec.gender, rec.photo_url),
).fetchone()["id"]
self.ctx.stats["persons_created"] += 1
else:
self.conn.execute(
"""UPDATE person SET full_name = %s,
given_name = COALESCE(%s, given_name), family_name = COALESCE(%s, family_name),
honorific_prefix = COALESCE(%s, honorific_prefix), honorific_suffix = COALESCE(%s, honorific_suffix),
birth_date = COALESCE(%s, birth_date), death_date = COALESCE(%s, death_date),
gender = COALESCE(%s, gender), photo_url = COALESCE(%s, photo_url), updated_at = now()
WHERE id = %s""",
(rec.full_name, rec.given_name, rec.family_name, rec.honorific_prefix, rec.honorific_suffix,
rec.birth_date, rec.death_date, rec.gender, rec.photo_url, person_id),
)
for scheme, value in rec.ref.ids.items():
if value:
self.conn.execute(
"""INSERT INTO identifier (entity_type, entity_id, scheme, value, source_slug)
VALUES ('person', %s, %s, %s, %s) ON CONFLICT DO NOTHING""",
(person_id, scheme, str(value), self.ctx.spec.slug),
)
names = {rec.full_name, strip_titles(rec.full_name), *rec.aliases}
if rec.given_name and rec.family_name:
names.add(f"{rec.given_name} {rec.family_name}")
for name in names:
if name:
self.conn.execute(
"""INSERT INTO alias (entity_type, entity_id, alias, alias_norm) VALUES ('person', %s, %s, %s)
ON CONFLICT DO NOTHING""",
(person_id, name, normalize(strip_titles(name))),
)
self.ctx.stats["persons"] += 1
return person_id
def organization(self, rec: OrgRecord) -> int | None:
org_id = self.resolver.org(rec.ref)
if org_id is None:
return None
if rec.color:
self.conn.execute("UPDATE organization SET color = COALESCE(color, %s) WHERE id = %s", (rec.color, org_id))
if rec.parent:
parent = self.resolver.org(rec.parent)
if parent and parent != org_id:
self.conn.execute("UPDATE organization SET parent_id = %s WHERE id = %s", (parent, org_id))
self.resolver.add_org_aliases(org_id, list(rec.aliases))
return org_id
def social_account(self, rec: SocialAccount) -> int | None:
person_id = self.resolver.person(rec.person)
org_id = self.resolver.org(rec.org) if rec.org else None # OrgRef.create decides
if person_id is None and org_id is None:
self.ctx.stats["social_unresolved"] += 1
return None
row = self.conn.execute(
"""INSERT INTO social_account (person_id, organization_id, platform, handle, url, verified, source_slug)
VALUES (%s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (platform, url) DO UPDATE SET
handle = COALESCE(EXCLUDED.handle, social_account.handle),
verified = social_account.verified OR EXCLUDED.verified,
person_id = COALESCE(social_account.person_id, EXCLUDED.person_id),
organization_id = COALESCE(social_account.organization_id, EXCLUDED.organization_id)
RETURNING id""",
(person_id, org_id, rec.platform.lower(), rec.handle, rec.url, rec.verified, self.ctx.spec.slug),
).fetchone()
self.ctx.stats["social_accounts"] += 1
return row["id"]
# -- standalone facets ------------------------------------------------------------
def vote_event(self, rec: VoteEvent) -> int:
chamber = self.resolver.org(rec.chamber, create=False) if rec.chamber else None
row = self.conn.execute(
"""INSERT INTO vote_event (event_key, chamber_id, item_ref, item_title, stage, voted_on, result, roll_call,
url, protocol_url, raw_text)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (event_key) DO UPDATE SET result = EXCLUDED.result, raw_text = EXCLUDED.raw_text,
roll_call = EXCLUDED.roll_call, item_title = EXCLUDED.item_title,
protocol_url = COALESCE(EXCLUDED.protocol_url, vote_event.protocol_url)
RETURNING id""",
(rec.key, chamber, rec.item_ref, rec.title, rec.stage, rec.voted_on, rec.result, rec.roll_call, rec.url,
rec.protocol_url, rec.raw_text),
).fetchone()
self.ctx.stats["vote_events"] += 1
return row["id"]
def event(self, rec: Event) -> int:
body = self.resolver.org(rec.body, create=False) if rec.body else None
source_id = source_row(self.conn, self.ctx.spec.slug)["id"]
row = self.conn.execute(
"""INSERT INTO event (event_key, kind, title, starts_at, ends_at, location_name, country, url, description,
organization_id, source_id)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (event_key) DO UPDATE SET title = EXCLUDED.title,
starts_at = COALESCE(EXCLUDED.starts_at, event.starts_at), ends_at = EXCLUDED.ends_at,
location_name = COALESCE(EXCLUDED.location_name, event.location_name),
description = COALESCE(EXCLUDED.description, event.description)
RETURNING id""",
(rec.key, rec.kind, rec.title, rec.starts_at, rec.ends_at, rec.location, rec.country, rec.url,
rec.description, body, source_id),
).fetchone()
self.ctx.stats["events"] += 1
return row["id"]
# -- claims -----------------------------------------------------------------------
def claim(self, c: Claim) -> int | None:
if not c.evidence:
raise ValueError("a claim needs at least one piece of evidence")
person_id = self.resolver.person(c.person)
if c.person is not None and person_id is None:
self.ctx.stats["unresolved_person"] += 1
self.ctx.log.debug("unresolved person %s for %s", c.person, c.summary_de[:80])
return None
org_id = self.resolver.org(c.org) if c.org else None
cp_person = self.resolver.person(c.counterpart_person)
cp_org = self.resolver.org(c.counterpart_org) if c.counterpart_org else None
if person_id is None and org_id is None and cp_org is None:
self.ctx.stats["claims_without_subject"] += 1
return None
fp = fingerprint(*self.identity(c, person_id, org_id, cp_org))
row = self.conn.execute(
"""
INSERT INTO claim (claim_type, fingerprint, person_id, organization_id, counterpart_person_id,
counterpart_org_id, occurred_at, valid_from, valid_to, summary_de, summary_en,
data, sensitive, review_status, created_by)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (fingerprint) DO UPDATE SET
summary_de = EXCLUDED.summary_de,
summary_en = COALESCE(EXCLUDED.summary_en, claim.summary_en),
valid_to = COALESCE(EXCLUDED.valid_to, claim.valid_to),
occurred_at = COALESCE(claim.occurred_at, EXCLUDED.occurred_at),
data = claim.data || EXCLUDED.data,
sensitive = claim.sensitive OR EXCLUDED.sensitive,
updated_at = now()
RETURNING id, (xmax = 0) AS inserted
""",
(c.claim_type, fp, person_id, org_id, cp_person, cp_org, c.occurred_at, c.valid_from, c.valid_to,
c.summary_de, c.summary_en, Jsonb(c.data), c.sensitive or c.claim_type == "legal_proceeding",
"pending" if c.review else "published", self.ctx.adapter),
).fetchone()
claim_id = row["id"]
self.ctx.stats["claims_new" if row["inserted"] else "claims_updated"] += 1
for ev in c.evidence:
doc_id = self.document(ev)
self.conn.execute(
"""INSERT INTO claim_evidence (claim_id, document_id, stance, quote, locator, extracted_by, confidence)
VALUES (%s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (claim_id, document_id, locator) DO UPDATE SET
quote = COALESCE(EXCLUDED.quote, claim_evidence.quote),
confidence = GREATEST(claim_evidence.confidence, EXCLUDED.confidence)""",
(claim_id, doc_id, ev.stance, ev.quote and excerpt(ev.quote, 600), ev.locator, self.ctx.adapter,
ev.confidence),
)
for topic in c.topics:
self.assign_topic(claim_id, topic)
if c.facet is not None:
self.facet(claim_id, c, person_id, org_id)
self.refresh_status(claim_id)
return claim_id
def identity(self, c: Claim, person_id: int | None, org_id: int | None, cp_org: int | None) -> tuple:
"""Semantic identity of a claim (see docs/architecture/01-design-from-scratch.md §3.3)."""
if c.key is not None:
return (c.claim_type, "key", *c.key)
f = c.facet
if isinstance(f, Role):
body = self.resolver.org(f.body) if f.body else cp_org
return ("role", person_id, body, normalize(f.role), f.start)
if isinstance(f, VotePosition):
return ("vote", f.event_key, person_id, org_id)
if isinstance(f, Money):
def side(ref: Any, name: str | None) -> Any:
if isinstance(ref, PersonRef):
return ("p", self.resolver.person(ref))
if isinstance(ref, OrgRef):
return ("o", self.resolver.org(ref))
return ("n", normalize_org_name(name or ""))
return ("money", f.kind, side(f.payer, f.payer_name), side(f.payee, f.payee_name), _amount(f.amount),
f.paid_on, f.period, f.reference)
if isinstance(f, Statement):
return ("statement", person_id, org_id, f.url)
if isinstance(f, Participation):
return ("participation", f.event_key, person_id, org_id)
if c.claim_type == "media_mention":
return ("mention", c.evidence[0].url, person_id, org_id)
raise ValueError(f"claim of type {c.claim_type} needs an explicit key")
def facet(self, claim_id: int, c: Claim, person_id: int | None, org_id: int | None) -> None:
f = c.facet
if isinstance(f, Role):
body = self.resolver.org(f.body) if f.body else None
party = self.resolver.org(f.party) if f.party else None
self.conn.execute(
"""INSERT INTO role (claim_id, person_id, organization_id, party_id, role, level, region, start_date, end_date)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (claim_id) DO UPDATE SET end_date = COALESCE(EXCLUDED.end_date, role.end_date),
party_id = COALESCE(EXCLUDED.party_id, role.party_id), region = COALESCE(EXCLUDED.region, role.region),
organization_id = COALESCE(EXCLUDED.organization_id, role.organization_id)""",
(claim_id, person_id, body, party, f.role, f.level, f.region, f.start, f.end),
)
elif isinstance(f, VotePosition):
event = self.conn.execute(
"""SELECT ve.id, ve.voted_on, o.slug AS chamber FROM vote_event ve
LEFT JOIN organization o ON o.id = ve.chamber_id WHERE ve.event_key = %s""",
(f.event_key,),
).fetchone()
if event is None:
raise ValueError(f"vote event {f.event_key} must be loaded before its positions")
party = self.resolver.org(f.party) if f.party else None
if person_id is not None and party is None and event["chamber"]:
# individual (roll-call) vote: remember the voter's faction on that day (dissent detection)
party = self.resolver.faction_of(person_id, event["chamber"], event["voted_on"])
self.conn.execute(
"""INSERT INTO vote_position (claim_id, vote_event_id, organization_id, person_id, party_id, position)
VALUES (%s, %s, %s, %s, %s, %s)
ON CONFLICT (claim_id) DO UPDATE SET position = EXCLUDED.position,
party_id = COALESCE(EXCLUDED.party_id, vote_position.party_id)""",
(claim_id, event["id"], org_id if person_id is None else None, person_id, party, f.position),
)
elif isinstance(f, Money):
payer_org = self.resolver.org(f.payer) if isinstance(f.payer, OrgRef) else None
payee_org = self.resolver.org(f.payee) if isinstance(f.payee, OrgRef) else None
payer_person = self.resolver.person(f.payer) if isinstance(f.payer, PersonRef) else None
payee_person = self.resolver.person(f.payee) if isinstance(f.payee, PersonRef) else None
self.conn.execute(
"""INSERT INTO money_flow (claim_id, kind, payer_name, payer_org_id, payer_person_id, payee_name,
payee_org_id, payee_person_id, amount, amount_category, currency, paid_on,
period, reference, category)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (claim_id) DO UPDATE SET amount = EXCLUDED.amount,
payer_org_id = COALESCE(EXCLUDED.payer_org_id, money_flow.payer_org_id),
payee_org_id = COALESCE(EXCLUDED.payee_org_id, money_flow.payee_org_id)""",
(claim_id, f.kind, f.payer_name, payer_org, payer_person, f.payee_name, payee_org, payee_person,
_amount(f.amount), f.amount_category, f.currency, f.paid_on, f.period, f.reference, f.category),
)
elif isinstance(f, Participation):
event = self.conn.execute("SELECT id FROM event WHERE event_key = %s", (f.event_key,)).fetchone()
if event is None:
raise ValueError(f"event {f.event_key} must be loaded before participations")
self.conn.execute(
"""INSERT INTO event_participant (claim_id, event_id, person_id, organization_id, participation)
VALUES (%s, %s, %s, %s, %s) ON CONFLICT (claim_id) DO NOTHING""",
(claim_id, event["id"], person_id, org_id, f.participation),
)
elif isinstance(f, Statement):
self.conn.execute(
"""INSERT INTO statement (claim_id, person_id, context, platform, medium, title, said_at, excerpt, url)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) ON CONFLICT (claim_id) DO UPDATE SET
excerpt = COALESCE(EXCLUDED.excerpt, statement.excerpt)""",
(claim_id, person_id, f.context, f.platform, f.medium, f.title, f.said_at,
f.excerpt and excerpt(f.excerpt, 600), f.url),
)
elif isinstance(f, Legal):
self.conn.execute(
"""INSERT INTO legal_proceeding (claim_id, person_id, organization_id, stage, offense, authority, case_ref,
stage_date)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (claim_id) DO UPDATE SET stage = EXCLUDED.stage,
offense = COALESCE(EXCLUDED.offense, legal_proceeding.offense),
authority = COALESCE(EXCLUDED.authority, legal_proceeding.authority),
case_ref = COALESCE(EXCLUDED.case_ref, legal_proceeding.case_ref)""",
(claim_id, person_id, org_id, f.stage, f.offense, f.authority, f.case_ref, f.stage_date),
)
# -- helpers ----------------------------------------------------------------------
def assign_topic(self, claim_id: int, topic_name: str, method: str | None = None, confidence: float = 1.0) -> None:
norm = normalize(topic_name)
row = self.conn.execute(
"SELECT id FROM topic WHERE lower(unaccent(name_de)) = %s OR slug = %s", (norm, topic_name)
).fetchone()
if row is None:
row = self.conn.execute(
"INSERT INTO topic (slug, name_de, name_en) VALUES (%s, %s, %s) ON CONFLICT (slug) DO UPDATE "
"SET name_de = topic.name_de RETURNING id",
(make_slug(topic_name), topic_name, topic_name),
).fetchone()
self.conn.execute(
"INSERT INTO topic_assignment (topic_id, claim_id, confidence, method) VALUES (%s, %s, %s, %s) "
"ON CONFLICT DO NOTHING",
(row["id"], claim_id, confidence, method or self.ctx.adapter),
)
def refresh_status(self, claim_id: int) -> None:
"""corroborated = backed by >= 2 different sources; disputed = any contradicting evidence."""
self.conn.execute(
"""
UPDATE claim SET status = CASE
WHEN status = 'retracted' THEN status
WHEN EXISTS (SELECT 1 FROM claim_evidence e WHERE e.claim_id = claim.id AND e.stance = 'contradicts')
THEN 'disputed'
WHEN (SELECT count(DISTINCT d.source_id) FROM claim_evidence e JOIN document d ON d.id = e.document_id
WHERE e.claim_id = claim.id AND e.stance IN ('supports', 'mentions')) >= 2 THEN 'corroborated'
ELSE 'reported' END
WHERE id = %s
""",
(claim_id,),
)
def _amount(value: Any) -> Decimal | None:
if value is None or value == "":
return None
try:
return Decimal(str(value)).quantize(Decimal("0.01"))
except InvalidOperation:
return None
def as_date(value: Any) -> date | None:
if isinstance(value, datetime):
return value.date()
if isinstance(value, date):
return value
return None
+265
View File
@@ -0,0 +1,265 @@
"""Typed records: the only way adapters and enrichers hand data to the Loader.
Records are plain dataclasses so parsers stay pure and testable (see tests/golden). They refer to
entities through PersonRef/OrgRef, which the Loader resolves (identifiers first, then hints, then
names). Data minimisation is enforced by schema: there is, for example, no address field anywhere,
so a parser cannot store a donor's street address even by accident.
"""
from __future__ import annotations
from dataclasses import dataclass, field, fields, is_dataclass
from datetime import date, datetime
from decimal import Decimal
from typing import Any, Literal, Union
# --------------------------------------------------------------------------------------------
# References to entities
# --------------------------------------------------------------------------------------------
@dataclass
class PersonRef:
"""How a record points at a person.
ids: strong identifiers, e.g. {"parlament": "2339"} or {"wikidata": "Q123"}
name: display/full name; used for alias matching if no id matches
hints: resolution strategies, e.g.
{"surname_in_body": "nationalrat", "on": date(...)} -> roll-call surname lists
{"office": "Bundesministerin für Justiz", "on": date(...)} -> office holder on a date
"""
ids: dict[str, str] = field(default_factory=dict)
name: str | None = None
hints: dict[str, Any] = field(default_factory=dict)
@dataclass
class OrgRef:
"""How a record points at an organization. `slug` addresses seeded orgs (e.g. "nationalrat")."""
ids: dict[str, str] = field(default_factory=dict)
name: str | None = None
short_name: str | None = None
kind: str = "other"
slug: str | None = None
create: bool = True
# --------------------------------------------------------------------------------------------
# Evidence & documents
# --------------------------------------------------------------------------------------------
DocType = Literal["article", "press_release", "api_record", "protocol", "social_post", "register_entry",
"web_page", "pdf", "dataset_row"]
@dataclass
class Evidence:
url: str
title: str | None = None
doc_type: str = "web_page"
published_at: datetime | date | str | None = None
excerpt: str | None = None
full_text: str | None = None
quote: str | None = None
locator: str = ""
stance: Literal["supports", "contradicts", "mentions"] = "supports"
confidence: float = 1.0
source: str | None = None # defaults to the adapter's source
meta: dict[str, Any] = field(default_factory=dict)
@dataclass
class Document:
"""A document without a claim yet (e.g. a headline); enrichers derive claims from it later."""
evidence: Evidence
# --------------------------------------------------------------------------------------------
# Entities
# --------------------------------------------------------------------------------------------
@dataclass
class PersonRecord:
ref: PersonRef
full_name: str
given_name: str | None = None
family_name: str | None = None
honorific_prefix: str | None = None
honorific_suffix: str | None = None
birth_date: date | None = None
death_date: date | None = None
gender: str | None = None
photo_url: str | None = None
aliases: tuple[str, ...] = ()
#: if no identifier matches, merge with an existing person of exactly this (unambiguous) name before
#: creating a new one - for authoritative sources that share no id with others (e.g. EP)
match_by_name: bool = False
@dataclass
class OrgRecord:
ref: OrgRef
aliases: tuple[str, ...] = ()
color: str | None = None
parent: OrgRef | None = None
@dataclass
class SocialAccount:
platform: str # bluesky | mastodon | telegram | youtube | twitter | facebook | ...
url: str
handle: str | None = None
person: PersonRef | None = None
org: OrgRef | None = None
verified: bool = False # True only for official sources (parliament profile, official website)
# --------------------------------------------------------------------------------------------
# Facets (structured payload of a claim)
# --------------------------------------------------------------------------------------------
@dataclass
class Role:
role: str # "mp" for chamber members, otherwise office title
level: Literal["eu", "federal", "state", "district", "municipal", "party", "other"]
body: OrgRef | None = None # chamber/government/party the role is in
party: OrgRef | None = None # party/faction of the holder in this role
region: str | None = None
start: date | None = None
end: date | None = None
@dataclass
class VoteEvent:
"""A vote (not a claim itself). Positions reference it by `key`."""
key: str
chamber: OrgRef | None
title: str
voted_on: date | None
result: Literal["accepted", "rejected", "unknown"] = "unknown"
stage: str | None = None
item_ref: str | None = None
roll_call: bool = False
url: str | None = None
protocol_url: str | None = None
raw_text: str | None = None
@dataclass
class VotePosition:
event_key: str
position: Literal["for", "against", "abstain", "absent"]
party: OrgRef | None = None # faction of an individual voter at that time
@dataclass
class Money:
kind: Literal["donation", "contract", "government_ad", "political_ad", "subsidy", "party_funding", "party_ad",
"loan", "side_income", "sponsorship", "other"]
amount: Decimal | float | None = None
currency: str = "EUR"
payer: OrgRef | PersonRef | None = None
payee: OrgRef | PersonRef | None = None
payer_name: str | None = None
payee_name: str | None = None
paid_on: date | None = None
period: str | None = None
reference: str | None = None # notice id, campaign, ...
category: str | None = None # CPV code, ad category, ...
amount_category: str | None = None # when only a bracket is published
@dataclass
class Event:
"""Something with time and place (not a claim). Participation claims reference it by `key`."""
key: str
kind: str
title: str
starts_at: datetime | date | str | None
ends_at: datetime | date | str | None = None
location: str | None = None
country: str | None = None
url: str | None = None
description: str | None = None
body: OrgRef | None = None
@dataclass
class Participation:
event_key: str
participation: str = "participant"
@dataclass
class Statement:
context: Literal["parliament_speech", "interview", "tv", "radio", "social_post", "press_release",
"event_speech", "op_ed", "testimony", "other"]
url: str
said_at: datetime | date | str | None = None
platform: str | None = None
medium: str | None = None
title: str | None = None
excerpt: str | None = None
@dataclass
class Legal:
stage: str
offense: str | None = None
authority: str | None = None
case_ref: str | None = None
stage_date: date | None = None
Facet = Union[Role, VotePosition, Money, Participation, Statement, Legal]
# --------------------------------------------------------------------------------------------
# Claims
# --------------------------------------------------------------------------------------------
@dataclass
class Claim:
claim_type: str
summary_de: str
evidence: list[Evidence]
person: PersonRef | None = None
org: OrgRef | None = None
counterpart_person: PersonRef | None = None
counterpart_org: OrgRef | None = None
occurred_at: datetime | date | str | None = None
valid_from: date | None = None
valid_to: date | None = None
summary_en: str | None = None
data: dict[str, Any] = field(default_factory=dict)
topics: tuple[str, ...] = ()
sensitive: bool = False
review: bool = False # True -> published only after human review
key: tuple | None = None # explicit identity (otherwise computed by the Loader)
facet: Facet | None = None
Record = Union[PersonRecord, OrgRecord, SocialAccount, VoteEvent, Event, Claim, Document]
def to_jsonable(obj: Any) -> Any:
"""Serialise records for golden tests and debugging (stable, sorted keys)."""
if is_dataclass(obj) and not isinstance(obj, type):
return {"__type__": type(obj).__name__, **{f.name: to_jsonable(getattr(obj, f.name)) for f in fields(obj)}}
if isinstance(obj, dict):
return {str(k): to_jsonable(v) for k, v in sorted(obj.items(), key=lambda kv: str(kv[0]))}
if isinstance(obj, (list, tuple)):
return [to_jsonable(v) for v in obj]
if isinstance(obj, (datetime, date)):
return obj.isoformat()
if isinstance(obj, Decimal):
return str(obj)
return obj
+95
View File
@@ -0,0 +1,95 @@
"""Registry of adapters, enrichers and source specs.
Built-in modules register themselves with @register when imported (see load_builtin()).
External packages can add adapters via the entry-point group `politrack.adapters`
(value: a module path whose import registers adapters), e.g. in their pyproject.toml:
[project.entry-points."politrack.adapters"]
my_sources = "my_package.sources"
"""
from __future__ import annotations
import importlib
import logging
import pkgutil
from importlib.metadata import entry_points
from politrack.core.adapter import Adapter, Enricher
from politrack.core.spec import DERIVED, SourceSpec
log = logging.getLogger(__name__)
ADAPTERS: dict[str, type[Adapter]] = {}
ENRICHERS: dict[str, type[Enricher]] = {}
SPECS: dict[str, SourceSpec] = {DERIVED.slug: DERIVED}
_loaded = False
def register(cls):
"""Class decorator for Adapter and Enricher subclasses."""
if issubclass(cls, Adapter):
target = ADAPTERS
elif issubclass(cls, Enricher):
target = ENRICHERS
else:
raise TypeError(f"{cls} is neither an Adapter nor an Enricher")
if cls.name in ADAPTERS or cls.name in ENRICHERS:
raise ValueError(f"duplicate adapter name {cls.name!r}")
target[cls.name] = cls
register_spec(cls.spec)
return cls
def register_spec(spec: SourceSpec) -> SourceSpec:
existing = SPECS.get(spec.slug)
if existing is not None and existing != spec:
raise ValueError(f"conflicting SourceSpec definitions for {spec.slug!r}")
SPECS[spec.slug] = spec
return spec
def load_builtin() -> None:
"""Import all built-in source and enricher modules (and plugins) exactly once."""
global _loaded
if _loaded:
return
_loaded = True
for package_name in ("politrack.sources", "politrack.enrich"):
package = importlib.import_module(package_name)
for module in pkgutil.walk_packages(package.__path__, prefix=f"{package_name}."):
importlib.import_module(module.name)
for ep in entry_points(group="politrack.adapters"):
try:
importlib.import_module(ep.value)
except Exception: # a broken plugin must not take the platform down
log.exception("failed to load adapter plugin %s", ep.value)
def get(name: str) -> type[Adapter] | type[Enricher]:
load_builtin()
if name in ADAPTERS:
return ADAPTERS[name]
if name in ENRICHERS:
return ENRICHERS[name]
raise KeyError(f"unknown adapter/enricher {name!r}; known: {', '.join(sorted(ADAPTERS) + sorted(ENRICHERS))}")
def sync_sources(conn) -> int:
"""Upsert all registered SourceSpecs into the `source` table."""
load_builtin()
for spec in SPECS.values():
conn.execute(
"""
INSERT INTO source (slug, name, kind, homepage, license, access_method, store_fulltext, keep_raw,
default_trust, tos_notes)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (slug) DO UPDATE SET name = EXCLUDED.name, kind = EXCLUDED.kind, homepage = EXCLUDED.homepage,
license = EXCLUDED.license, access_method = EXCLUDED.access_method,
store_fulltext = EXCLUDED.store_fulltext, keep_raw = EXCLUDED.keep_raw,
default_trust = EXCLUDED.default_trust, tos_notes = EXCLUDED.tos_notes, updated_at = now()
""",
(spec.slug, spec.name, spec.kind, spec.homepage, spec.license, spec.access, spec.store_fulltext,
spec.keep_raw, spec.trust, spec.tos_notes),
)
return len(SPECS)
+269
View File
@@ -0,0 +1,269 @@
"""Entity resolution: PersonRef/OrgRef -> database ids.
Order (first hit wins, ambiguity never guesses):
persons: identifiers -> hint strategies -> exact alias (titles stripped; prefer active)
organizations: identifiers -> slug -> exact alias -> normalised legal name -> create (if allowed)
Hint strategies for persons (documented in docs/data-model.md):
surname_in_body + on members of a chamber on a date with that surname (roll-call lists)
office + on holder of an office title on a date (Council of Ministers minutes)
"""
from __future__ import annotations
import re
from datetime import date
from typing import Any
from psycopg import Connection
from rapidfuzz import fuzz
from politrack.core.records import OrgRef, PersonRef
from politrack.text import make_slug, normalize, normalize_org_name, strip_titles
OFFICE_FILLER = re.compile(r"\b(der republik osterreich|der bundesrepublik osterreich|im bundeskanzleramt)\b|"
r"\(osterreich\)")
def office_key(title: str) -> str:
"""'Bundesministerin für Justiz' / 'Bundeskanzler der Republik Österreich' -> 'bundesminister fur justiz' /
'bundeskanzler' (gender-neutral, without filler, punctuation-free) for comparing office titles."""
text = normalize(title)
text = re.sub(r"\b(\w*(?:minister|kanzler|sekretar|prasident|rat))in\b", r"\1", text)
text = OFFICE_FILLER.sub(" ", text)
return re.sub(r"\s+", " ", re.sub(r"[^\w\s]", " ", text)).strip()
# Fallback for colloquial or English office labels (Wikidata: "Justizminister Österreichs", "Federal Minister for
# Arts, Culture, the Civil Service and Sport"): same kind of office + at least one common portfolio.
OFFICE_KINDS = [("vizekanzler", r"vizekanzler|vice.chancellor"), ("kanzler", r"kanzler|chancellor"),
("staatssekretar", r"staatssekretar|state secretary"), ("minister", r"minist")]
PORTFOLIOS = {
"justiz": r"justiz|justice", "finanzen": r"finanz|financ", "inneres": r"\binnere|\binnenmin|interior",
"aussen": r"au(ss|ß)en|auswartig|europaische und internationale|foreign|european and international",
"verteidigung": r"verteidigung|defen[cs]e", "soziales": r"sozial|\barbeit|labou?r|social",
"gesundheit": r"gesundheit|health", "bildung": r"bildung|education", "wissenschaft": r"wissenschaft|science",
"frauen": r"frauen|women", "landwirtschaft": r"landwirtschaft|forstwirtschaft|agricultur",
"umwelt": r"umwelt|klima|environment|climate", "kunst": r"kunst|kultur|\barts\b|culture|\bsport",
"mobilitat": r"mobilitat|verkehr|infrastruktur|mobility|transport|infrastructure",
"wirtschaft": r"\bwirtschaft|econom", "europa": r"\beuropa\b|integration|familie|famil",
"digitalisierung": r"digital", "energie": r"energie|energy", "tourismus": r"touris",
}
def office_kind(key: str) -> str | None:
return next((kind for kind, pattern in OFFICE_KINDS if re.search(pattern, key)), None)
def portfolios(key: str) -> set[str]:
return {name for name, pattern in PORTFOLIOS.items() if re.search(pattern, key)}
def lead_portfolio(key: str) -> str | None:
"""The portfolio named first ("Wirtschaft, Energie und Tourismus" -> wirtschaft): a role must cover it."""
found = [(m.start(), name) for name, pattern in PORTFOLIOS.items() if (m := re.search(pattern, key))]
return min(found)[1] if found else None
class Resolver:
def __init__(self, conn: Connection) -> None:
self.conn = conn
self._person_cache: dict[tuple, int | None] = {}
self._org_cache: dict[tuple, int | None] = {}
self._members_cache: dict[tuple, list[dict]] = {}
self._office_cache: dict[tuple, int | None] = {}
# -- persons ------------------------------------------------------------------------
def person_by_ids(self, ids: dict[str, str]) -> int | None:
for scheme, value in ids.items():
if value is None:
continue
if scheme == "politrack": # internal id, used by adapters that plan requests from the database
row = self.conn.execute("SELECT id FROM person WHERE id = %s", (int(value),)).fetchone()
if row:
return row["id"]
continue
row = self.conn.execute(
"SELECT entity_id FROM identifier WHERE entity_type = 'person' AND scheme = %s AND value = %s",
(scheme, str(value)),
).fetchone()
if row:
return row["entity_id"]
return None
def person(self, ref: PersonRef | None) -> int | None:
if ref is None:
return None
cache_key = (tuple(sorted(ref.ids.items())), ref.name, tuple(sorted((k, str(v)) for k, v in ref.hints.items())))
if cache_key in self._person_cache:
return self._person_cache[cache_key]
found = self.person_by_ids(ref.ids)
if found is None and ref.hints:
found = self._person_by_hints(ref)
if found is None and ref.name and not ref.hints.get("no_name_match"):
found = self.person_by_name(ref.name)
self._person_cache[cache_key] = found
return found
def person_by_name(self, name: str) -> int | None:
norm = normalize(strip_titles(name))
rows = self.conn.execute(
"""SELECT DISTINCT a.entity_id AS id, ps.status
FROM alias a LEFT JOIN person_status ps ON ps.person_id = a.entity_id
WHERE a.entity_type = 'person' AND a.alias_norm = %s""",
(norm,),
).fetchall()
if len(rows) == 1:
return rows[0]["id"]
active = [r for r in rows if r["status"] == "active"]
return active[0]["id"] if len(active) == 1 else None
def _person_by_hints(self, ref: PersonRef) -> int | None:
hints = ref.hints
on: date | None = hints.get("on")
if "surname_in_body" in hints and ref.name:
members = self.members(hints["surname_in_body"], on)
norm = normalize(strip_titles(ref.name))
exact = [m for m in members if m["family_name"] and normalize(m["family_name"]) == norm]
if len(exact) == 1:
return exact[0]["id"]
suffix = [m for m in members if normalize(strip_titles(m["full_name"])).endswith(norm)]
return suffix[0]["id"] if len(suffix) == 1 else None
if "office" in hints:
return self.office_holder(hints["office"], on)
return None
def office_holder(self, office: str, on: date | None) -> int | None:
"""Holder of a (single-holder) office on a date. Titles are compared whole, gender-neutral and without
filler ("Bundesministerin für Justiz" == "Bundesminister für Justiz"), so a generic "Bundesminister"
never matches a specific ministry. Several matches (open-ended roles from Wikidata) -> the most recent
appointment wins; no dated match -> None."""
key = (office, on)
if key not in self._office_cache:
rows = self.conn.execute(
"""SELECT DISTINCT r.person_id, r.role, r.start_date FROM role r JOIN claim c ON c.id = r.claim_id
WHERE c.merged_into IS NULL AND r.level IN ('federal', 'state', 'eu')
AND (%(on)s::date IS NULL OR ((r.start_date IS NULL OR r.start_date <= %(on)s)
AND (r.end_date IS NULL OR r.end_date >= %(on)s)))""",
{"on": on},
).fetchall()
target = office_key(office)
candidates = [(r["start_date"] or date.min, r["person_id"]) for r in rows
if fuzz.ratio(office_key(r["role"]), target) >= 92]
kind, lead = office_kind(target), lead_portfolio(target)
if not candidates and kind and (lead or kind in ("kanzler", "vizekanzler")):
# Fuzzy labels only for appointments since the current government took office (the latest
# chancellor appointment): stale open-ended roles of former ministers must not win.
formed = max((r["start_date"] for r in rows if r["start_date"]
and office_kind(office_key(r["role"])) == "kanzler"), default=date.min)
candidates = [(r["start_date"], r["person_id"]) for r in rows
if r["start_date"] and r["start_date"] >= formed
and office_kind(office_key(r["role"])) == kind
and (kind in ("kanzler", "vizekanzler") or lead in portfolios(office_key(r["role"])))]
persons = {p for _, p in candidates}
if len(persons) <= 1:
self._office_cache[key] = persons.pop() if persons else None
else:
latest = max(start for start, _ in candidates)
newest = {p for start, p in candidates if start == latest}
self._office_cache[key] = newest.pop() if len(newest) == 1 and latest != date.min else None
return self._office_cache[key]
def members(self, body_slug: str, on: date | None) -> list[dict[str, Any]]:
key = (body_slug, on)
if key not in self._members_cache:
self._members_cache[key] = self.conn.execute(
"""SELECT DISTINCT ON (p.id) p.id, p.full_name, p.family_name, r.party_id
FROM role r JOIN person p ON p.id = r.person_id JOIN organization o ON o.id = r.organization_id
JOIN claim c ON c.id = r.claim_id
WHERE o.slug = %(body)s AND r.role = 'mp' AND c.merged_into IS NULL
AND (%(on)s::date IS NULL OR (r.start_date <= %(on)s AND COALESCE(r.end_date, 'infinity') >= %(on)s))
ORDER BY p.id, r.start_date DESC""",
{"body": body_slug, "on": on},
).fetchall()
return self._members_cache[key]
def faction_of(self, person_id: int, body_slug: str, on: date | None) -> int | None:
return next((m["party_id"] for m in self.members(body_slug, on) if m["id"] == person_id), None)
# -- organizations ------------------------------------------------------------------
def org(self, ref: OrgRef | None, create: bool | None = None) -> int | None:
if ref is None:
return None
allow_create = ref.create if create is None else create
cache_key = (tuple(sorted(ref.ids.items())), ref.slug, ref.name, ref.short_name)
if cache_key in self._org_cache and (self._org_cache[cache_key] is not None or not allow_create):
return self._org_cache[cache_key]
found = self._org_lookup(ref)
if found is None and allow_create and (ref.name or ref.short_name):
found = self._org_create(ref)
elif found is not None:
self._add_org_ids(found, ref)
self._org_cache[cache_key] = found
return found
def _org_lookup(self, ref: OrgRef) -> int | None:
for scheme, value in ref.ids.items():
if scheme == "politrack_org": # internal id (enrichers)
row = self.conn.execute("SELECT id FROM organization WHERE id = %s", (int(value),)).fetchone()
if row:
return row["id"]
continue
row = self.conn.execute(
"SELECT entity_id FROM identifier WHERE entity_type = 'organization' AND scheme = %s AND value = %s",
(scheme, str(value)),
).fetchone()
if row:
return row["entity_id"]
if ref.slug:
row = self.conn.execute("SELECT id FROM organization WHERE slug = %s", (ref.slug,)).fetchone()
if row:
return row["id"]
for label in (ref.short_name, ref.name):
if not label:
continue
for norm in {normalize(label), normalize_org_name(label)} - {""}:
rows = self.conn.execute(
"SELECT DISTINCT entity_id FROM alias WHERE entity_type = 'organization' AND alias_norm = %s",
(norm,),
).fetchall()
if len(rows) == 1:
return rows[0]["entity_id"]
return None
def _org_create(self, ref: OrgRef) -> int:
base = ref.slug or make_slug(ref.short_name or ref.name or "org")[:60] or "org"
slug, n = base, 1
while self.conn.execute("SELECT 1 FROM organization WHERE slug = %s", (slug,)).fetchone():
n += 1
slug = f"{base}-{n}"
org_id = self.conn.execute(
"INSERT INTO organization (slug, name, short_name, kind) VALUES (%s, %s, %s, %s) RETURNING id",
(slug, ref.name or ref.short_name, ref.short_name, ref.kind if ref.kind else "other"),
).fetchone()["id"]
self.add_org_aliases(org_id, [ref.name, ref.short_name])
self._add_org_ids(org_id, ref)
return org_id
def add_org_aliases(self, org_id: int, names: list[str | None]) -> None:
for name in names:
if not name:
continue
for norm in {normalize(name), normalize_org_name(name)} - {""}:
self.conn.execute(
"INSERT INTO alias (entity_type, entity_id, alias, alias_norm) VALUES ('organization', %s, %s, %s) "
"ON CONFLICT DO NOTHING",
(org_id, name, norm),
)
def _add_org_ids(self, org_id: int, ref: OrgRef) -> None:
for scheme, value in ref.ids.items():
if value:
self.conn.execute(
"INSERT INTO identifier (entity_type, entity_id, scheme, value) VALUES ('organization', %s, %s, %s) "
"ON CONFLICT DO NOTHING",
(org_id, scheme, str(value)),
)
+232
View File
@@ -0,0 +1,232 @@
"""Runs adapters and enrichers.
fetch mode: start() -> fetch (policy, archive) -> parse() -> Loader, one transaction per response
reparse mode: replay the latest archived response of every request the adapter ever made
test mode: parse a given response without DB (used by golden tests)
"""
from __future__ import annotations
import logging
import traceback
from collections import deque
from collections.abc import Iterable
from datetime import datetime, timezone
from typing import Any
from psycopg import Connection
from psycopg.types.json import Jsonb
from politrack.core import registry
from politrack.core.adapter import Adapter, Enricher, Request, Response, RunContext, StateStore
from politrack.core.archive import Archive
from politrack.core.fetch import Disallowed, Fetcher
from politrack.core.loader import Loader, source_row
from politrack.db import connection
log = logging.getLogger(__name__)
ENRICH_COMMIT_EVERY = 200
def run(name: str, *, mode: str = "fetch", since: datetime | None = None, params: dict[str, Any] | None = None,
max_requests: int | None = None) -> dict:
cls = registry.get(name)
with connection() as conn:
run_id = conn.execute("INSERT INTO run (adapter, mode) VALUES (%s, %s) RETURNING id", (name, mode)).fetchone()["id"]
conn.commit()
ctx = RunContext(adapter=name, spec=cls.spec, state=StateStore(conn, name), since=since, params=params or {},
mode=mode, conn=conn, creates_persons=cls.creates_persons,
log=logging.getLogger(f"politrack.{name}"))
status, error = "ok", None
try:
if issubclass(cls, Enricher):
_run_enricher(cls(), ctx, conn)
elif mode == "reparse":
_reparse(cls(), ctx, conn)
else:
_fetch_run(cls(), ctx, conn, max_requests)
if ctx.stats.get("errors"):
status = "partial"
except Exception:
conn.rollback()
status, error = "error", traceback.format_exc()
ctx.log.exception("%s failed", name)
conn.execute("UPDATE run SET finished_at = now(), status = %s, stats = %s, error = %s WHERE id = %s",
(status, Jsonb(dict(ctx.stats)), error, run_id))
conn.commit()
ctx.log.info("%s %s: %s %s", name, mode, status, dict(ctx.stats))
if status == "error":
raise RuntimeError(f"{name} failed, see run {run_id}")
return {"status": status, **dict(ctx.stats)}
def _fetch_run(adapter: Adapter, ctx: RunContext, conn: Connection, max_requests: int | None) -> None:
fetcher = Fetcher()
loader = Loader(conn, ctx)
queue: deque[Request] = deque(adapter.start(ctx))
seen: set[str] = set()
try:
while queue:
req = queue.popleft()
if req.fingerprint in seen:
continue
seen.add(req.fingerprint)
if max_requests is not None and ctx.stats["requests"] >= max_requests:
ctx.stats["stopped_at_max_requests"] = 1
break
ctx.stats["requests"] += 1
# Multi-publisher adapters (news feeds, GDELT) name the publisher per request.
spec = registry.SPECS.get(req.meta.get("source") or adapter.spec.slug, adapter.spec)
try:
resp = fetcher.fetch(conn, req, adapter=adapter.name, spec=spec,
source_id=source_row(conn, spec.slug)["id"])
except Disallowed:
ctx.stats["disallowed_by_robots"] += 1
ctx.log.warning("robots.txt disallows %s", req.url)
continue
except Exception as exc:
conn.rollback()
ctx.stats["fetch_errors"] += 1
ctx.stats["errors"] += 1
ctx.log.warning("fetch failed %s: %s", req.url, exc)
continue
if req.skip_unchanged and _unchanged(conn, resp, adapter.parser_version):
# Same body, same parser: its records are already loaded. Still collect follow-up requests,
# so children an interrupted run never fetched are picked up (they have their own checks).
ctx.stats["unchanged"] += 1
queue.extend(_follow_ups(adapter, ctx, resp))
continue
queue.extend(_parse_and_load(adapter, ctx, conn, loader, resp))
finally:
fetcher.close()
def _unchanged(conn: Connection, resp: Response, parser_version: int) -> bool:
if resp.fetch_id is None:
return False
row = conn.execute(
"""SELECT 1 FROM fetch_log WHERE request_fp = %s AND parsed_sha256 = %s AND parser_version = %s LIMIT 1""",
(resp.request.fingerprint, resp.sha256, parser_version),
).fetchone()
return row is not None
def _follow_ups(adapter: Adapter, ctx: RunContext, resp: Response) -> list[Request]:
ctx.response = resp
try:
return [item for item in adapter.parse(ctx, resp) if isinstance(item, Request)]
except Exception:
ctx.stats["parse_errors"] += 1
ctx.log.exception("parse failed for unchanged %s", resp.url)
return []
finally:
ctx.response = None
def _parse_and_load(adapter: Adapter, ctx: RunContext, conn: Connection, loader: Loader,
resp: Response) -> list[Request]:
follow: list[Request] = []
ctx.response = resp
try:
for item in adapter.parse(ctx, resp):
if isinstance(item, Request):
follow.append(item)
else:
loader.load(item)
if resp.fetch_id is not None:
Archive().mark_parsed(conn, resp.fetch_id, resp.sha256, adapter.parser_version)
conn.commit()
except Exception as exc:
conn.rollback()
ctx.stats["errors"] += 1
ctx.stats["parse_errors"] += 1
ctx.log.exception("parse/load failed for %s", resp.url)
if resp.fetch_id is not None:
Archive().mark_error(conn, resp.fetch_id, f"{type(exc).__name__}: {exc}")
conn.commit()
loader.resolver = type(loader.resolver)(conn) # drop caches that may reference rolled-back rows
finally:
ctx.response = None
return follow
def _reparse(adapter: Adapter, ctx: RunContext, conn: Connection) -> None:
archive = Archive()
loader = Loader(conn, ctx)
started = conn.execute("SELECT now() AS t").fetchone()["t"]
rows = archive.latest_stored(conn, adapter.name)
ctx.log.info("reparsing %s archived responses", len(rows))
# Parse list/index responses before detail responses: records referencing others (vote positions ->
# vote events) are emitted by the parent's parse() in order, so the original order is preserved
# by sorting on fetch time.
for row in sorted(rows, key=lambda r: r["fetched_at"]):
body = archive.read_blob(row["sha256"])
if body is None:
ctx.stats["missing_blobs"] += 1
continue
req_info = row["request"] or {}
req = Request(url=row["url"], kind=req_info.get("kind", "default"), method=row["method"],
params=req_info.get("params"), json=req_info.get("json"), meta=req_info.get("meta") or {})
resp = Response(request=req, status=row["status"], url=row["url"],
headers={"content-type": row["content_type"] or ""}, body=body,
fetched_at=row["fetched_at"], sha256=row["sha256"], fetch_id=row["id"],
tdm_reserved=row["tdm_reserved"], from_archive=True)
ctx.stats["responses"] += 1
_parse_and_load(adapter, ctx, conn, loader, resp) # follow-ups are archived separately
if not ctx.stats.get("errors") and not ctx.stats.get("missing_blobs"):
_sweep_stale_claims(adapter, ctx, conn, started)
def _sweep_stale_claims(adapter: Adapter, ctx: RunContext, conn: Connection, started: datetime) -> None:
"""After a complete reparse, delete this adapter's claims that the current parser no longer produces
(e.g. a fixed parser emits a corrected fact with a new identity) - unless another source backs them.
Claims are reproducible from the archive, so nothing is lost; facets and evidence links cascade."""
deleted = conn.execute(
"""DELETE FROM claim c
WHERE c.created_by = %(adapter)s AND c.updated_at < %(started)s
AND NOT EXISTS (SELECT 1 FROM claim_evidence e JOIN document d ON d.id = e.document_id
JOIN source s ON s.id = d.source_id
WHERE e.claim_id = c.id AND s.slug <> %(source)s)""",
{"adapter": adapter.name, "started": started, "source": adapter.spec.slug},
).rowcount
conn.commit()
ctx.stats["stale_claims_deleted"] = deleted
def _run_enricher(enricher: Enricher, ctx: RunContext, conn: Connection) -> None:
loader = Loader(conn, ctx)
result = enricher.run(ctx)
pending = 0
for record in result or []:
try:
loader.load(record)
except Exception:
conn.rollback()
ctx.stats["errors"] += 1
ctx.log.exception("enricher %s failed to load a record", enricher.name)
loader = Loader(conn, ctx)
continue
pending += 1
if pending >= ENRICH_COMMIT_EVERY:
conn.commit()
pending = 0
conn.commit()
def parse_offline(adapter: Adapter, response: Response, params: dict[str, Any] | None = None) -> Iterable[Any]:
"""Run parse() without a database (golden tests, debugging)."""
ctx = RunContext(adapter=adapter.name, spec=adapter.spec, state=StateStore(None, adapter.name), mode="test",
params=params or {})
ctx.response = response
return list(adapter.parse(ctx, response))
def make_response(url: str, body: bytes, *, kind: str = "default", meta: dict | None = None,
content_type: str = "", method: str = "GET", json_body: Any = None,
params: dict | None = None) -> Response:
"""Build a Response for tests/fixtures."""
import hashlib
req = Request(url=url, kind=kind, meta=meta or {}, method=method, json=json_body, params=params)
return Response(request=req, status=200, url=url, headers={"content-type": content_type}, body=body,
fetched_at=datetime(2026, 1, 1, tzinfo=timezone.utc), sha256=hashlib.sha256(body).hexdigest())
+49
View File
@@ -0,0 +1,49 @@
"""SourceSpec: the single source of truth about a publisher we collect from.
Specs live next to the adapters that use them (backend/politrack/sources/*.py) and are synced
into the `source` table by `politrack migrate` / `politrack sources sync`.
"""
from dataclasses import dataclass
from typing import Literal
SourceKind = Literal["official", "ngo", "media", "social", "aggregator", "user", "derived"]
AccessMethod = Literal["api", "bulk", "rss", "scrape", "manual", "derived"]
@dataclass(frozen=True)
class SourceSpec:
slug: str
name: str
kind: SourceKind
homepage: str
license: str
access: AccessMethod
trust: int
#: may we keep body text (articles, posts, protocols)? False -> title, short excerpt and link only
store_fulltext: bool = True
#: may raw responses be archived? False for TDM-reserved publishers or restrictive terms
keep_raw: bool = True
#: minimum seconds between two requests to the same host (robots.txt crawl-delay can raise it)
min_interval: float = 2.0
#: URL prefixes of documented APIs whose operators allow programmatic use although robots.txt
#: lists them (justify in tos_notes). Everything else always obeys robots.txt.
robots_exempt: tuple[str, ...] = ()
tos_notes: str = ""
def __post_init__(self) -> None:
if not 0 <= self.trust <= 100:
raise ValueError(f"{self.slug}: trust must be 0-100")
# Sources used by the built-in enrichers for derived documents/claims.
DERIVED = SourceSpec(
slug="politrack",
name="Politrack (abgeleitet)",
kind="derived",
homepage="",
license="derived from cited sources",
access="derived",
trust=50,
tos_notes="Claims computed by Politrack from other sources; each cites its original evidence.",
)