Politrack: track Austrian politicians from public sources
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
583ccba31c
187 files changed
+39890
No files matched your search
@@ -0,0 +1,11 @@
|
||||
"""The adapter framework. Most adapter modules only need:
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Evidence, PersonRef, OrgRef, ...
|
||||
"""
|
||||
|
||||
from politrack.core.adapter import Adapter, Enricher, Request, Response, RunContext
|
||||
from politrack.core.registry import register, register_spec
|
||||
from politrack.core.spec import SourceSpec
|
||||
|
||||
__all__ = ["Adapter", "Enricher", "Request", "Response", "RunContext", "SourceSpec", "register", "register_spec"]
|
||||
@@ -0,0 +1,177 @@
|
||||
"""Adapter / Enricher base classes and the request/response/context types they work with.
|
||||
|
||||
An Adapter turns one publisher's data into Records in two steps:
|
||||
|
||||
start(ctx) -> Request... entry points (lists, feeds, index pages)
|
||||
parse(ctx, resp) -> Record | Request... PURE: no DB access, no network
|
||||
|
||||
The runner fetches requests (policy-checked, archived), calls parse() and hands records to the
|
||||
Loader. See docs/contributing/adapters.md for a walkthrough.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable, Iterator
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from functools import cached_property
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from politrack.core.spec import SourceSpec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from psycopg import Connection
|
||||
|
||||
from politrack.core.records import Record
|
||||
|
||||
|
||||
SECRET_PARAMS = {"key", "api_key", "apikey", "token", "access_token"}
|
||||
|
||||
|
||||
def redact_url(url: str) -> str:
|
||||
"""Remove credentials from a URL before it is logged or stored."""
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
|
||||
parts = urlsplit(url)
|
||||
query = [(k, "<redacted>" if k.lower() in SECRET_PARAMS else v) for k, v in parse_qsl(parts.query, keep_blank_values=True)]
|
||||
return urlunsplit(parts._replace(query=urlencode(query)))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Request:
|
||||
url: str
|
||||
kind: str = "default" # parse() dispatch key
|
||||
method: str = "GET"
|
||||
params: dict[str, Any] | None = None
|
||||
json: Any = None
|
||||
headers: dict[str, str] | None = None
|
||||
meta: dict[str, Any] = field(default_factory=dict, compare=False, hash=False)
|
||||
conditional: bool = True # send If-None-Match / If-Modified-Since
|
||||
skip_unchanged: bool = True # body + parser_version unchanged: follow-ups only
|
||||
|
||||
@property
|
||||
def fingerprint(self) -> str:
|
||||
raw = json.dumps([self.method, self.url, self.params or {}, self.json], sort_keys=True, default=str)
|
||||
return hashlib.sha256(raw.encode()).hexdigest()[:32]
|
||||
|
||||
def describe(self) -> dict[str, Any]:
|
||||
"""What gets stored in the archive index (secrets removed)."""
|
||||
params = {k: ("<redacted>" if k.lower() in SECRET_PARAMS else v) for k, v in (self.params or {}).items()}
|
||||
return {"kind": self.kind, "params": params or None, "json": self.json, "meta": self.meta}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Response:
|
||||
request: Request
|
||||
status: int
|
||||
url: str
|
||||
headers: dict[str, str]
|
||||
body: bytes
|
||||
fetched_at: datetime
|
||||
sha256: str
|
||||
fetch_id: int | None = None
|
||||
tdm_reserved: bool = False
|
||||
from_archive: bool = False
|
||||
|
||||
@cached_property
|
||||
def text(self) -> str:
|
||||
encoding = "utf-8"
|
||||
content_type = self.headers.get("content-type", "")
|
||||
if "charset=" in content_type:
|
||||
encoding = content_type.split("charset=", 1)[1].split(";")[0].strip() or "utf-8"
|
||||
try:
|
||||
return self.body.decode(encoding)
|
||||
except (LookupError, UnicodeDecodeError):
|
||||
return self.body.decode("utf-8", errors="replace")
|
||||
|
||||
def json(self) -> Any:
|
||||
return json.loads(self.text, strict=False)
|
||||
|
||||
@cached_property
|
||||
def soup(self) -> BeautifulSoup:
|
||||
return BeautifulSoup(self.text, "html.parser")
|
||||
|
||||
@property
|
||||
def meta(self) -> dict[str, Any]:
|
||||
return self.request.meta
|
||||
|
||||
|
||||
class StateStore:
|
||||
"""Cursors of incremental adapters (persisted in the `state` table)."""
|
||||
|
||||
def __init__(self, conn: Connection | None, adapter: str) -> None:
|
||||
self.conn = conn
|
||||
self.adapter = adapter
|
||||
self._local: dict[str, Any] = {}
|
||||
|
||||
def get(self, key: str, default: Any = None) -> Any:
|
||||
if self.conn is None:
|
||||
return self._local.get(key, default)
|
||||
row = self.conn.execute("SELECT value FROM state WHERE adapter = %s AND key = %s",
|
||||
(self.adapter, key)).fetchone()
|
||||
return row["value"] if row else default
|
||||
|
||||
def set(self, key: str, value: Any) -> None:
|
||||
if self.conn is None:
|
||||
self._local[key] = value
|
||||
return
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
self.conn.execute(
|
||||
"""INSERT INTO state (adapter, key, value) VALUES (%s, %s, %s)
|
||||
ON CONFLICT (adapter, key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()""",
|
||||
(self.adapter, key, Jsonb(value)),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunContext:
|
||||
adapter: str
|
||||
spec: SourceSpec
|
||||
state: StateStore
|
||||
stats: Counter = field(default_factory=Counter)
|
||||
since: datetime | None = None
|
||||
params: dict[str, Any] = field(default_factory=dict) # CLI/worker parameters, e.g. {"gp": "XXVIII"}
|
||||
mode: str = "fetch" # fetch | reparse | test
|
||||
conn: Connection | None = None # only for enrichers / the loader, never in parse()
|
||||
response: Response | None = None # set by the runner while parsing
|
||||
creates_persons: bool = False
|
||||
log: logging.Logger = field(default_factory=lambda: logging.getLogger("politrack"))
|
||||
|
||||
|
||||
class Adapter:
|
||||
"""Base class for source adapters. Subclass, set the class attributes, implement start()/parse()."""
|
||||
|
||||
spec: ClassVar[SourceSpec]
|
||||
name: ClassVar[str]
|
||||
description: ClassVar[str] = ""
|
||||
schedule: ClassVar[str | None] = None # cron, UTC; None = manual only
|
||||
queue: ClassVar[str] = "scrape" # scrape | bulk | social | enrich
|
||||
parser_version: ClassVar[int] = 1 # bump when parse() output changes -> `politrack reparse`
|
||||
creates_persons: ClassVar[bool] = False # may this adapter create new person entities?
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterable[Request]:
|
||||
raise NotImplementedError
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Record | Request]:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class Enricher:
|
||||
"""Derives records from data already in the database (linking, events, corroboration...)."""
|
||||
|
||||
spec: ClassVar[SourceSpec]
|
||||
name: ClassVar[str]
|
||||
description: ClassVar[str] = ""
|
||||
schedule: ClassVar[str | None] = None
|
||||
queue: ClassVar[str] = "enrich"
|
||||
creates_persons: ClassVar[bool] = False
|
||||
|
||||
def run(self, ctx: RunContext) -> Iterable[Record] | None:
|
||||
raise NotImplementedError
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Raw archive: every fetch is logged in the `fetch_log` table; bodies are stored gzip-compressed and
|
||||
content-addressed (sha256) below POLITRACK_RAW_DIR when the source allows it.
|
||||
|
||||
Used for: conditional requests (ETag/Last-Modified), change detection (skip re-parsing identical
|
||||
bodies), `politrack reparse` after parser fixes, and as an audit trail.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from psycopg import Connection
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from politrack.config import get_settings
|
||||
|
||||
|
||||
class Archive:
|
||||
def __init__(self, raw_dir: Path | None = None) -> None:
|
||||
self.raw_dir = Path(raw_dir or get_settings().raw_dir)
|
||||
|
||||
# -- blobs ----------------------------------------------------------------------------
|
||||
|
||||
def _path(self, sha256: str) -> Path:
|
||||
return self.raw_dir / sha256[:2] / f"{sha256}.gz"
|
||||
|
||||
def write_blob(self, body: bytes) -> str:
|
||||
sha = hashlib.sha256(body).hexdigest()
|
||||
path = self._path(sha)
|
||||
if not path.exists():
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(f".tmp{os.getpid()}")
|
||||
with gzip.open(tmp, "wb") as fh:
|
||||
fh.write(body)
|
||||
tmp.replace(path)
|
||||
return sha
|
||||
|
||||
def read_blob(self, sha256: str) -> bytes | None:
|
||||
path = self._path(sha256)
|
||||
if not path.exists():
|
||||
return None
|
||||
with gzip.open(path, "rb") as fh:
|
||||
return fh.read()
|
||||
|
||||
# -- index ----------------------------------------------------------------------------
|
||||
|
||||
def last(self, conn: Connection, request_fp: str) -> dict[str, Any] | None:
|
||||
return conn.execute(
|
||||
"""SELECT id, sha256, etag, last_modified, stored, parsed_sha256, parser_version, status, url,
|
||||
content_type, tdm_reserved, fetched_at
|
||||
FROM fetch_log WHERE request_fp = %s AND status IS NOT NULL AND status < 400
|
||||
ORDER BY fetched_at DESC LIMIT 1""",
|
||||
(request_fp,),
|
||||
).fetchone()
|
||||
|
||||
def log(self, conn: Connection, *, adapter: str, source_id: int | None, request_fp: str, url: str, method: str,
|
||||
request: dict, status: int | None, body: bytes | None, headers: dict[str, str] | None,
|
||||
keep: bool, tdm_reserved: bool, error: str | None = None) -> tuple[int, str | None]:
|
||||
sha = hashlib.sha256(body).hexdigest() if body is not None else None
|
||||
stored = False
|
||||
if body is not None and keep and not tdm_reserved:
|
||||
self.write_blob(body)
|
||||
stored = True
|
||||
headers = headers or {}
|
||||
row = conn.execute(
|
||||
"""INSERT INTO fetch_log (adapter, source_id, request_fp, url, method, request, status, sha256, size,
|
||||
content_type, etag, last_modified, stored, tdm_reserved, error)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id""",
|
||||
(adapter, source_id, request_fp, url, method, Jsonb(request), status, sha,
|
||||
len(body) if body is not None else None, headers.get("content-type"), headers.get("etag"),
|
||||
headers.get("last-modified"), stored, tdm_reserved, error),
|
||||
).fetchone()
|
||||
return row["id"], sha
|
||||
|
||||
def mark_parsed(self, conn: Connection, fetch_id: int, sha256: str, parser_version: int) -> None:
|
||||
conn.execute("UPDATE fetch_log SET parsed_sha256 = %s, parser_version = %s WHERE id = %s",
|
||||
(sha256, parser_version, fetch_id))
|
||||
|
||||
def mark_error(self, conn: Connection, fetch_id: int, error: str) -> None:
|
||||
conn.execute("UPDATE fetch_log SET error = %s WHERE id = %s", (error[:2000], fetch_id))
|
||||
|
||||
def latest_stored(self, conn: Connection, adapter: str) -> list[dict[str, Any]]:
|
||||
"""Latest stored fetch per request of an adapter (input for `reparse`)."""
|
||||
return conn.execute(
|
||||
"""SELECT DISTINCT ON (request_fp) id, request_fp, url, method, request, status, sha256, fetched_at,
|
||||
content_type, tdm_reserved
|
||||
FROM fetch_log WHERE adapter = %s AND stored AND status < 400
|
||||
ORDER BY request_fp, fetched_at DESC""",
|
||||
(adapter,),
|
||||
).fetchall()
|
||||
@@ -0,0 +1,158 @@
|
||||
"""The Fetcher: the only component that talks to the network. It enforces the crawling policy:
|
||||
|
||||
* honest User-Agent with contact address,
|
||||
* robots.txt (cached per origin); exemptions only for documented APIs listed in SourceSpec.robots_exempt,
|
||||
* per-host throttling (max of SourceSpec.min_interval and robots.txt crawl-delay),
|
||||
* conditional requests (ETag / Last-Modified from the raw archive),
|
||||
* retries with exponential back-off for 429/5xx, honouring Retry-After,
|
||||
* TDM reservations (W3C TDMRep: `tdm-reservation` header, /.well-known/tdmrep.json, /tdmrep.json),
|
||||
* archiving of raw bodies (unless SourceSpec.keep_raw is False or TDM is reserved).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import httpx
|
||||
from protego import Protego
|
||||
from psycopg import Connection
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core.adapter import Request, Response, redact_url
|
||||
from politrack.core.archive import Archive
|
||||
from politrack.core.spec import SourceSpec
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
RETRY_STATUS = {429, 500, 502, 503, 504}
|
||||
|
||||
|
||||
class Disallowed(Exception):
|
||||
"""robots.txt forbids fetching this URL."""
|
||||
|
||||
|
||||
class Fetcher:
|
||||
def __init__(self, archive: Archive | None = None, client: httpx.Client | None = None) -> None:
|
||||
settings = get_settings()
|
||||
self.user_agent = settings.user_agent
|
||||
self.default_interval = settings.min_request_interval
|
||||
self.archive = archive or Archive()
|
||||
self.client = client or httpx.Client(
|
||||
headers={"User-Agent": self.user_agent, "Accept-Language": "de-AT,de;q=0.9,en;q=0.5"},
|
||||
timeout=settings.http_timeout,
|
||||
follow_redirects=True,
|
||||
)
|
||||
self._robots: dict[str, Protego | None] = {}
|
||||
self._tdm: dict[str, bool] = {}
|
||||
self._last: dict[str, float] = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# -- policy -------------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def origin(url: str) -> str:
|
||||
parts = urlsplit(url)
|
||||
return f"{parts.scheme}://{parts.netloc}"
|
||||
|
||||
def robots(self, url: str) -> Protego | None:
|
||||
origin = self.origin(url)
|
||||
if origin not in self._robots:
|
||||
try:
|
||||
resp = self.client.get(f"{origin}/robots.txt")
|
||||
self._robots[origin] = Protego.parse(resp.text) if resp.status_code == 200 else None
|
||||
except httpx.HTTPError:
|
||||
self._robots[origin] = None
|
||||
return self._robots[origin]
|
||||
|
||||
def allowed(self, url: str, spec: SourceSpec) -> bool:
|
||||
if any(url.startswith(prefix) for prefix in spec.robots_exempt):
|
||||
return True
|
||||
rules = self.robots(url)
|
||||
return rules is None or rules.can_fetch(url, self.user_agent)
|
||||
|
||||
def tdm_reserved(self, url: str) -> bool:
|
||||
origin = self.origin(url)
|
||||
if origin not in self._tdm:
|
||||
reserved = False
|
||||
for path in ("/.well-known/tdmrep.json", "/tdmrep.json"):
|
||||
try:
|
||||
resp = self.client.get(origin + path)
|
||||
except httpx.HTTPError:
|
||||
continue
|
||||
if resp.status_code == 200:
|
||||
try:
|
||||
rules = resp.json()
|
||||
except ValueError:
|
||||
continue
|
||||
reserved = any(str(r.get("tdm-reservation")) == "1" for r in rules if isinstance(r, dict))
|
||||
break
|
||||
self._tdm[origin] = reserved
|
||||
return self._tdm[origin]
|
||||
|
||||
def _throttle(self, url: str, spec: SourceSpec) -> None:
|
||||
rules = self._robots.get(self.origin(url))
|
||||
delay = rules.crawl_delay(self.user_agent) if rules else None
|
||||
interval = max(spec.min_interval or self.default_interval, float(delay or 0))
|
||||
host = urlsplit(url).netloc
|
||||
with self._lock:
|
||||
wait = self._last.get(host, 0) + interval - time.monotonic()
|
||||
if wait > 0:
|
||||
time.sleep(wait)
|
||||
self._last[host] = time.monotonic()
|
||||
|
||||
# -- fetching ---------------------------------------------------------------------
|
||||
|
||||
def fetch(self, conn: Connection, req: Request, *, adapter: str, spec: SourceSpec, source_id: int | None,
|
||||
attempts: int = 5) -> Response:
|
||||
if not self.allowed(req.url, spec):
|
||||
raise Disallowed(req.url)
|
||||
last = self.archive.last(conn, req.fingerprint) if req.conditional else None
|
||||
headers = dict(req.headers or {})
|
||||
if last and last["stored"]:
|
||||
if last["etag"]:
|
||||
headers["If-None-Match"] = last["etag"]
|
||||
if last["last_modified"]:
|
||||
headers["If-Modified-Since"] = last["last_modified"]
|
||||
|
||||
resp: httpx.Response | None = None
|
||||
for attempt in range(attempts):
|
||||
self._throttle(req.url, spec)
|
||||
try:
|
||||
resp = self.client.request(req.method, req.url, params=req.params, json=req.json, headers=headers)
|
||||
except httpx.TransportError as exc:
|
||||
if attempt == attempts - 1:
|
||||
raise
|
||||
log.warning("transport error %s (%s), retrying", req.url, exc)
|
||||
time.sleep(min(5 * 2 ** attempt, 120))
|
||||
continue
|
||||
if resp.status_code in RETRY_STATUS and attempt < attempts - 1:
|
||||
retry_after = resp.headers.get("Retry-After", "")
|
||||
time.sleep(min(float(retry_after) if retry_after.isdigit() else 5 * 2 ** attempt, 300))
|
||||
continue
|
||||
break
|
||||
assert resp is not None
|
||||
tdm = resp.headers.get("tdm-reservation") == "1" or self.tdm_reserved(req.url)
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
if resp.status_code == 304 and last:
|
||||
body = self.archive.read_blob(last["sha256"]) or b""
|
||||
return Response(request=req, status=200, url=last["url"], headers={"content-type": last["content_type"] or ""},
|
||||
body=body, fetched_at=now, sha256=last["sha256"], fetch_id=last["id"],
|
||||
tdm_reserved=last["tdm_reserved"], from_archive=True)
|
||||
|
||||
lowered = {k.lower(): v for k, v in resp.headers.items()}
|
||||
fetch_id, sha = self.archive.log(
|
||||
conn, adapter=adapter, source_id=source_id, request_fp=req.fingerprint, url=redact_url(str(resp.url)),
|
||||
method=req.method, request=req.describe(), status=resp.status_code, body=resp.content, headers=lowered,
|
||||
keep=spec.keep_raw, tdm_reserved=tdm,
|
||||
)
|
||||
conn.commit()
|
||||
resp.raise_for_status()
|
||||
return Response(request=req, status=resp.status_code, url=redact_url(str(resp.url)), headers=lowered, body=resp.content,
|
||||
fetched_at=now, sha256=sha or "", fetch_id=fetch_id, tdm_reserved=tdm)
|
||||
|
||||
def close(self) -> None:
|
||||
self.client.close()
|
||||
@@ -0,0 +1,448 @@
|
||||
"""The Loader: the only code that writes canonical rows.
|
||||
|
||||
Invariants enforced here (not in adapters):
|
||||
* no claim without evidence; claims whose person cannot be resolved are skipped (and counted),
|
||||
* full text is kept only for sources with store_fulltext and responses without TDM reservation,
|
||||
* claim identity is computed from *resolved* entity ids + type-specific fields, so the same fact
|
||||
from different sources becomes one claim with several evidence rows ("corroborated"),
|
||||
* claims marked `review=True` start as review_status='pending' (hidden until a human approves),
|
||||
* persons are only created by adapters with creates_persons=True.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Any
|
||||
|
||||
from psycopg import Connection
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from politrack.core.adapter import RunContext
|
||||
from politrack.core.records import (
|
||||
Claim,
|
||||
Document,
|
||||
Event,
|
||||
Evidence,
|
||||
Legal,
|
||||
Money,
|
||||
OrgRecord,
|
||||
OrgRef,
|
||||
Participation,
|
||||
PersonRecord,
|
||||
PersonRef,
|
||||
Role,
|
||||
SocialAccount,
|
||||
Statement,
|
||||
VoteEvent,
|
||||
VotePosition,
|
||||
)
|
||||
from politrack.core.resolve import Resolver
|
||||
from politrack.text import excerpt, fingerprint, make_slug, normalize, normalize_org_name, strip_titles
|
||||
|
||||
# Source policy is read from the DB once per process (synced from SourceSpecs).
|
||||
_SOURCES: dict[str, dict[str, Any]] = {}
|
||||
|
||||
|
||||
def source_row(conn: Connection, slug: str) -> dict[str, Any]:
|
||||
if slug not in _SOURCES:
|
||||
row = conn.execute("SELECT id, store_fulltext, kind FROM source WHERE slug = %s", (slug,)).fetchone()
|
||||
if row is None:
|
||||
raise KeyError(f"unknown source {slug!r}: run `politrack sources sync`")
|
||||
_SOURCES[slug] = row
|
||||
return _SOURCES[slug]
|
||||
|
||||
|
||||
class Loader:
|
||||
def __init__(self, conn: Connection, ctx: RunContext) -> None:
|
||||
self.conn = conn
|
||||
self.ctx = ctx
|
||||
self.resolver = Resolver(conn)
|
||||
|
||||
# -- dispatch ---------------------------------------------------------------------
|
||||
|
||||
def load(self, record: Any) -> int | None:
|
||||
if isinstance(record, Claim):
|
||||
return self.claim(record)
|
||||
if isinstance(record, Document):
|
||||
return self.document(record.evidence)
|
||||
if isinstance(record, PersonRecord):
|
||||
return self.person(record)
|
||||
if isinstance(record, OrgRecord):
|
||||
return self.organization(record)
|
||||
if isinstance(record, SocialAccount):
|
||||
return self.social_account(record)
|
||||
if isinstance(record, VoteEvent):
|
||||
return self.vote_event(record)
|
||||
if isinstance(record, Event):
|
||||
return self.event(record)
|
||||
raise TypeError(f"unknown record type {type(record).__name__}")
|
||||
|
||||
# -- documents --------------------------------------------------------------------
|
||||
|
||||
def document(self, ev: Evidence) -> int:
|
||||
slug = ev.source or self.ctx.spec.slug
|
||||
src = source_row(self.conn, slug)
|
||||
response = self.ctx.response
|
||||
tdm = bool(response and response.tdm_reserved)
|
||||
full_text = ev.full_text if (src["store_fulltext"] and not tdm) else None
|
||||
short = None if tdm else (excerpt(ev.excerpt, 400) if ev.excerpt else None)
|
||||
# Link the document to the raw response only if it *is* that response (not e.g. a protocol it cites).
|
||||
fetch_id = response.fetch_id if response and ev.url in (response.url, response.request.url) else None
|
||||
row = self.conn.execute(
|
||||
"""
|
||||
INSERT INTO document (source_id, url, title, doc_type, published_at, excerpt, full_text, fetch_id, meta)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (url) DO UPDATE SET
|
||||
title = COALESCE(EXCLUDED.title, document.title),
|
||||
published_at = COALESCE(EXCLUDED.published_at, document.published_at),
|
||||
excerpt = COALESCE(EXCLUDED.excerpt, document.excerpt),
|
||||
full_text = COALESCE(EXCLUDED.full_text, document.full_text),
|
||||
fetch_id = COALESCE(EXCLUDED.fetch_id, document.fetch_id),
|
||||
meta = document.meta || EXCLUDED.meta,
|
||||
retrieved_at = now()
|
||||
RETURNING id
|
||||
""",
|
||||
(src["id"], ev.url, ev.title, ev.doc_type, ev.published_at, short, full_text, fetch_id,
|
||||
Jsonb({**ev.meta, **({"tdm_reserved": True} if tdm else {})})),
|
||||
).fetchone()
|
||||
self.ctx.stats["documents"] += 1
|
||||
return row["id"]
|
||||
|
||||
# -- entities ---------------------------------------------------------------------
|
||||
|
||||
def person(self, rec: PersonRecord) -> int | None:
|
||||
person_id = self.resolver.person_by_ids(rec.ref.ids)
|
||||
if person_id is None and rec.match_by_name:
|
||||
person_id = self.resolver.person_by_name(rec.full_name)
|
||||
if person_id is None:
|
||||
if not self.ctx.creates_persons:
|
||||
self.ctx.stats["persons_not_created"] += 1
|
||||
return None
|
||||
base = make_slug(rec.given_name or "", rec.family_name or strip_titles(rec.full_name)) or "person"
|
||||
slug, n = base, 1
|
||||
while self.conn.execute("SELECT 1 FROM person WHERE slug = %s", (slug,)).fetchone():
|
||||
n += 1
|
||||
slug = f"{base}-{n}"
|
||||
person_id = self.conn.execute(
|
||||
"""INSERT INTO person (slug, full_name, given_name, family_name, honorific_prefix, honorific_suffix,
|
||||
birth_date, death_date, gender, photo_url)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id""",
|
||||
(slug, rec.full_name, rec.given_name, rec.family_name, rec.honorific_prefix, rec.honorific_suffix,
|
||||
rec.birth_date, rec.death_date, rec.gender, rec.photo_url),
|
||||
).fetchone()["id"]
|
||||
self.ctx.stats["persons_created"] += 1
|
||||
else:
|
||||
self.conn.execute(
|
||||
"""UPDATE person SET full_name = %s,
|
||||
given_name = COALESCE(%s, given_name), family_name = COALESCE(%s, family_name),
|
||||
honorific_prefix = COALESCE(%s, honorific_prefix), honorific_suffix = COALESCE(%s, honorific_suffix),
|
||||
birth_date = COALESCE(%s, birth_date), death_date = COALESCE(%s, death_date),
|
||||
gender = COALESCE(%s, gender), photo_url = COALESCE(%s, photo_url), updated_at = now()
|
||||
WHERE id = %s""",
|
||||
(rec.full_name, rec.given_name, rec.family_name, rec.honorific_prefix, rec.honorific_suffix,
|
||||
rec.birth_date, rec.death_date, rec.gender, rec.photo_url, person_id),
|
||||
)
|
||||
for scheme, value in rec.ref.ids.items():
|
||||
if value:
|
||||
self.conn.execute(
|
||||
"""INSERT INTO identifier (entity_type, entity_id, scheme, value, source_slug)
|
||||
VALUES ('person', %s, %s, %s, %s) ON CONFLICT DO NOTHING""",
|
||||
(person_id, scheme, str(value), self.ctx.spec.slug),
|
||||
)
|
||||
names = {rec.full_name, strip_titles(rec.full_name), *rec.aliases}
|
||||
if rec.given_name and rec.family_name:
|
||||
names.add(f"{rec.given_name} {rec.family_name}")
|
||||
for name in names:
|
||||
if name:
|
||||
self.conn.execute(
|
||||
"""INSERT INTO alias (entity_type, entity_id, alias, alias_norm) VALUES ('person', %s, %s, %s)
|
||||
ON CONFLICT DO NOTHING""",
|
||||
(person_id, name, normalize(strip_titles(name))),
|
||||
)
|
||||
self.ctx.stats["persons"] += 1
|
||||
return person_id
|
||||
|
||||
def organization(self, rec: OrgRecord) -> int | None:
|
||||
org_id = self.resolver.org(rec.ref)
|
||||
if org_id is None:
|
||||
return None
|
||||
if rec.color:
|
||||
self.conn.execute("UPDATE organization SET color = COALESCE(color, %s) WHERE id = %s", (rec.color, org_id))
|
||||
if rec.parent:
|
||||
parent = self.resolver.org(rec.parent)
|
||||
if parent and parent != org_id:
|
||||
self.conn.execute("UPDATE organization SET parent_id = %s WHERE id = %s", (parent, org_id))
|
||||
self.resolver.add_org_aliases(org_id, list(rec.aliases))
|
||||
return org_id
|
||||
|
||||
def social_account(self, rec: SocialAccount) -> int | None:
|
||||
person_id = self.resolver.person(rec.person)
|
||||
org_id = self.resolver.org(rec.org) if rec.org else None # OrgRef.create decides
|
||||
if person_id is None and org_id is None:
|
||||
self.ctx.stats["social_unresolved"] += 1
|
||||
return None
|
||||
row = self.conn.execute(
|
||||
"""INSERT INTO social_account (person_id, organization_id, platform, handle, url, verified, source_slug)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (platform, url) DO UPDATE SET
|
||||
handle = COALESCE(EXCLUDED.handle, social_account.handle),
|
||||
verified = social_account.verified OR EXCLUDED.verified,
|
||||
person_id = COALESCE(social_account.person_id, EXCLUDED.person_id),
|
||||
organization_id = COALESCE(social_account.organization_id, EXCLUDED.organization_id)
|
||||
RETURNING id""",
|
||||
(person_id, org_id, rec.platform.lower(), rec.handle, rec.url, rec.verified, self.ctx.spec.slug),
|
||||
).fetchone()
|
||||
self.ctx.stats["social_accounts"] += 1
|
||||
return row["id"]
|
||||
|
||||
# -- standalone facets ------------------------------------------------------------
|
||||
|
||||
def vote_event(self, rec: VoteEvent) -> int:
|
||||
chamber = self.resolver.org(rec.chamber, create=False) if rec.chamber else None
|
||||
row = self.conn.execute(
|
||||
"""INSERT INTO vote_event (event_key, chamber_id, item_ref, item_title, stage, voted_on, result, roll_call,
|
||||
url, protocol_url, raw_text)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (event_key) DO UPDATE SET result = EXCLUDED.result, raw_text = EXCLUDED.raw_text,
|
||||
roll_call = EXCLUDED.roll_call, item_title = EXCLUDED.item_title,
|
||||
protocol_url = COALESCE(EXCLUDED.protocol_url, vote_event.protocol_url)
|
||||
RETURNING id""",
|
||||
(rec.key, chamber, rec.item_ref, rec.title, rec.stage, rec.voted_on, rec.result, rec.roll_call, rec.url,
|
||||
rec.protocol_url, rec.raw_text),
|
||||
).fetchone()
|
||||
self.ctx.stats["vote_events"] += 1
|
||||
return row["id"]
|
||||
|
||||
def event(self, rec: Event) -> int:
|
||||
body = self.resolver.org(rec.body, create=False) if rec.body else None
|
||||
source_id = source_row(self.conn, self.ctx.spec.slug)["id"]
|
||||
row = self.conn.execute(
|
||||
"""INSERT INTO event (event_key, kind, title, starts_at, ends_at, location_name, country, url, description,
|
||||
organization_id, source_id)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (event_key) DO UPDATE SET title = EXCLUDED.title,
|
||||
starts_at = COALESCE(EXCLUDED.starts_at, event.starts_at), ends_at = EXCLUDED.ends_at,
|
||||
location_name = COALESCE(EXCLUDED.location_name, event.location_name),
|
||||
description = COALESCE(EXCLUDED.description, event.description)
|
||||
RETURNING id""",
|
||||
(rec.key, rec.kind, rec.title, rec.starts_at, rec.ends_at, rec.location, rec.country, rec.url,
|
||||
rec.description, body, source_id),
|
||||
).fetchone()
|
||||
self.ctx.stats["events"] += 1
|
||||
return row["id"]
|
||||
|
||||
# -- claims -----------------------------------------------------------------------
|
||||
|
||||
def claim(self, c: Claim) -> int | None:
|
||||
if not c.evidence:
|
||||
raise ValueError("a claim needs at least one piece of evidence")
|
||||
person_id = self.resolver.person(c.person)
|
||||
if c.person is not None and person_id is None:
|
||||
self.ctx.stats["unresolved_person"] += 1
|
||||
self.ctx.log.debug("unresolved person %s for %s", c.person, c.summary_de[:80])
|
||||
return None
|
||||
org_id = self.resolver.org(c.org) if c.org else None
|
||||
cp_person = self.resolver.person(c.counterpart_person)
|
||||
cp_org = self.resolver.org(c.counterpart_org) if c.counterpart_org else None
|
||||
if person_id is None and org_id is None and cp_org is None:
|
||||
self.ctx.stats["claims_without_subject"] += 1
|
||||
return None
|
||||
|
||||
fp = fingerprint(*self.identity(c, person_id, org_id, cp_org))
|
||||
row = self.conn.execute(
|
||||
"""
|
||||
INSERT INTO claim (claim_type, fingerprint, person_id, organization_id, counterpart_person_id,
|
||||
counterpart_org_id, occurred_at, valid_from, valid_to, summary_de, summary_en,
|
||||
data, sensitive, review_status, created_by)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (fingerprint) DO UPDATE SET
|
||||
summary_de = EXCLUDED.summary_de,
|
||||
summary_en = COALESCE(EXCLUDED.summary_en, claim.summary_en),
|
||||
valid_to = COALESCE(EXCLUDED.valid_to, claim.valid_to),
|
||||
occurred_at = COALESCE(claim.occurred_at, EXCLUDED.occurred_at),
|
||||
data = claim.data || EXCLUDED.data,
|
||||
sensitive = claim.sensitive OR EXCLUDED.sensitive,
|
||||
updated_at = now()
|
||||
RETURNING id, (xmax = 0) AS inserted
|
||||
""",
|
||||
(c.claim_type, fp, person_id, org_id, cp_person, cp_org, c.occurred_at, c.valid_from, c.valid_to,
|
||||
c.summary_de, c.summary_en, Jsonb(c.data), c.sensitive or c.claim_type == "legal_proceeding",
|
||||
"pending" if c.review else "published", self.ctx.adapter),
|
||||
).fetchone()
|
||||
claim_id = row["id"]
|
||||
self.ctx.stats["claims_new" if row["inserted"] else "claims_updated"] += 1
|
||||
|
||||
for ev in c.evidence:
|
||||
doc_id = self.document(ev)
|
||||
self.conn.execute(
|
||||
"""INSERT INTO claim_evidence (claim_id, document_id, stance, quote, locator, extracted_by, confidence)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (claim_id, document_id, locator) DO UPDATE SET
|
||||
quote = COALESCE(EXCLUDED.quote, claim_evidence.quote),
|
||||
confidence = GREATEST(claim_evidence.confidence, EXCLUDED.confidence)""",
|
||||
(claim_id, doc_id, ev.stance, ev.quote and excerpt(ev.quote, 600), ev.locator, self.ctx.adapter,
|
||||
ev.confidence),
|
||||
)
|
||||
for topic in c.topics:
|
||||
self.assign_topic(claim_id, topic)
|
||||
if c.facet is not None:
|
||||
self.facet(claim_id, c, person_id, org_id)
|
||||
self.refresh_status(claim_id)
|
||||
return claim_id
|
||||
|
||||
def identity(self, c: Claim, person_id: int | None, org_id: int | None, cp_org: int | None) -> tuple:
|
||||
"""Semantic identity of a claim (see docs/architecture/01-design-from-scratch.md §3.3)."""
|
||||
if c.key is not None:
|
||||
return (c.claim_type, "key", *c.key)
|
||||
f = c.facet
|
||||
if isinstance(f, Role):
|
||||
body = self.resolver.org(f.body) if f.body else cp_org
|
||||
return ("role", person_id, body, normalize(f.role), f.start)
|
||||
if isinstance(f, VotePosition):
|
||||
return ("vote", f.event_key, person_id, org_id)
|
||||
if isinstance(f, Money):
|
||||
def side(ref: Any, name: str | None) -> Any:
|
||||
if isinstance(ref, PersonRef):
|
||||
return ("p", self.resolver.person(ref))
|
||||
if isinstance(ref, OrgRef):
|
||||
return ("o", self.resolver.org(ref))
|
||||
return ("n", normalize_org_name(name or ""))
|
||||
return ("money", f.kind, side(f.payer, f.payer_name), side(f.payee, f.payee_name), _amount(f.amount),
|
||||
f.paid_on, f.period, f.reference)
|
||||
if isinstance(f, Statement):
|
||||
return ("statement", person_id, org_id, f.url)
|
||||
if isinstance(f, Participation):
|
||||
return ("participation", f.event_key, person_id, org_id)
|
||||
if c.claim_type == "media_mention":
|
||||
return ("mention", c.evidence[0].url, person_id, org_id)
|
||||
raise ValueError(f"claim of type {c.claim_type} needs an explicit key")
|
||||
|
||||
def facet(self, claim_id: int, c: Claim, person_id: int | None, org_id: int | None) -> None:
|
||||
f = c.facet
|
||||
if isinstance(f, Role):
|
||||
body = self.resolver.org(f.body) if f.body else None
|
||||
party = self.resolver.org(f.party) if f.party else None
|
||||
self.conn.execute(
|
||||
"""INSERT INTO role (claim_id, person_id, organization_id, party_id, role, level, region, start_date, end_date)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (claim_id) DO UPDATE SET end_date = COALESCE(EXCLUDED.end_date, role.end_date),
|
||||
party_id = COALESCE(EXCLUDED.party_id, role.party_id), region = COALESCE(EXCLUDED.region, role.region),
|
||||
organization_id = COALESCE(EXCLUDED.organization_id, role.organization_id)""",
|
||||
(claim_id, person_id, body, party, f.role, f.level, f.region, f.start, f.end),
|
||||
)
|
||||
elif isinstance(f, VotePosition):
|
||||
event = self.conn.execute(
|
||||
"""SELECT ve.id, ve.voted_on, o.slug AS chamber FROM vote_event ve
|
||||
LEFT JOIN organization o ON o.id = ve.chamber_id WHERE ve.event_key = %s""",
|
||||
(f.event_key,),
|
||||
).fetchone()
|
||||
if event is None:
|
||||
raise ValueError(f"vote event {f.event_key} must be loaded before its positions")
|
||||
party = self.resolver.org(f.party) if f.party else None
|
||||
if person_id is not None and party is None and event["chamber"]:
|
||||
# individual (roll-call) vote: remember the voter's faction on that day (dissent detection)
|
||||
party = self.resolver.faction_of(person_id, event["chamber"], event["voted_on"])
|
||||
self.conn.execute(
|
||||
"""INSERT INTO vote_position (claim_id, vote_event_id, organization_id, person_id, party_id, position)
|
||||
VALUES (%s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (claim_id) DO UPDATE SET position = EXCLUDED.position,
|
||||
party_id = COALESCE(EXCLUDED.party_id, vote_position.party_id)""",
|
||||
(claim_id, event["id"], org_id if person_id is None else None, person_id, party, f.position),
|
||||
)
|
||||
elif isinstance(f, Money):
|
||||
payer_org = self.resolver.org(f.payer) if isinstance(f.payer, OrgRef) else None
|
||||
payee_org = self.resolver.org(f.payee) if isinstance(f.payee, OrgRef) else None
|
||||
payer_person = self.resolver.person(f.payer) if isinstance(f.payer, PersonRef) else None
|
||||
payee_person = self.resolver.person(f.payee) if isinstance(f.payee, PersonRef) else None
|
||||
self.conn.execute(
|
||||
"""INSERT INTO money_flow (claim_id, kind, payer_name, payer_org_id, payer_person_id, payee_name,
|
||||
payee_org_id, payee_person_id, amount, amount_category, currency, paid_on,
|
||||
period, reference, category)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (claim_id) DO UPDATE SET amount = EXCLUDED.amount,
|
||||
payer_org_id = COALESCE(EXCLUDED.payer_org_id, money_flow.payer_org_id),
|
||||
payee_org_id = COALESCE(EXCLUDED.payee_org_id, money_flow.payee_org_id)""",
|
||||
(claim_id, f.kind, f.payer_name, payer_org, payer_person, f.payee_name, payee_org, payee_person,
|
||||
_amount(f.amount), f.amount_category, f.currency, f.paid_on, f.period, f.reference, f.category),
|
||||
)
|
||||
elif isinstance(f, Participation):
|
||||
event = self.conn.execute("SELECT id FROM event WHERE event_key = %s", (f.event_key,)).fetchone()
|
||||
if event is None:
|
||||
raise ValueError(f"event {f.event_key} must be loaded before participations")
|
||||
self.conn.execute(
|
||||
"""INSERT INTO event_participant (claim_id, event_id, person_id, organization_id, participation)
|
||||
VALUES (%s, %s, %s, %s, %s) ON CONFLICT (claim_id) DO NOTHING""",
|
||||
(claim_id, event["id"], person_id, org_id, f.participation),
|
||||
)
|
||||
elif isinstance(f, Statement):
|
||||
self.conn.execute(
|
||||
"""INSERT INTO statement (claim_id, person_id, context, platform, medium, title, said_at, excerpt, url)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) ON CONFLICT (claim_id) DO UPDATE SET
|
||||
excerpt = COALESCE(EXCLUDED.excerpt, statement.excerpt)""",
|
||||
(claim_id, person_id, f.context, f.platform, f.medium, f.title, f.said_at,
|
||||
f.excerpt and excerpt(f.excerpt, 600), f.url),
|
||||
)
|
||||
elif isinstance(f, Legal):
|
||||
self.conn.execute(
|
||||
"""INSERT INTO legal_proceeding (claim_id, person_id, organization_id, stage, offense, authority, case_ref,
|
||||
stage_date)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (claim_id) DO UPDATE SET stage = EXCLUDED.stage,
|
||||
offense = COALESCE(EXCLUDED.offense, legal_proceeding.offense),
|
||||
authority = COALESCE(EXCLUDED.authority, legal_proceeding.authority),
|
||||
case_ref = COALESCE(EXCLUDED.case_ref, legal_proceeding.case_ref)""",
|
||||
(claim_id, person_id, org_id, f.stage, f.offense, f.authority, f.case_ref, f.stage_date),
|
||||
)
|
||||
|
||||
# -- helpers ----------------------------------------------------------------------
|
||||
|
||||
def assign_topic(self, claim_id: int, topic_name: str, method: str | None = None, confidence: float = 1.0) -> None:
|
||||
norm = normalize(topic_name)
|
||||
row = self.conn.execute(
|
||||
"SELECT id FROM topic WHERE lower(unaccent(name_de)) = %s OR slug = %s", (norm, topic_name)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
row = self.conn.execute(
|
||||
"INSERT INTO topic (slug, name_de, name_en) VALUES (%s, %s, %s) ON CONFLICT (slug) DO UPDATE "
|
||||
"SET name_de = topic.name_de RETURNING id",
|
||||
(make_slug(topic_name), topic_name, topic_name),
|
||||
).fetchone()
|
||||
self.conn.execute(
|
||||
"INSERT INTO topic_assignment (topic_id, claim_id, confidence, method) VALUES (%s, %s, %s, %s) "
|
||||
"ON CONFLICT DO NOTHING",
|
||||
(row["id"], claim_id, confidence, method or self.ctx.adapter),
|
||||
)
|
||||
|
||||
def refresh_status(self, claim_id: int) -> None:
|
||||
"""corroborated = backed by >= 2 different sources; disputed = any contradicting evidence."""
|
||||
self.conn.execute(
|
||||
"""
|
||||
UPDATE claim SET status = CASE
|
||||
WHEN status = 'retracted' THEN status
|
||||
WHEN EXISTS (SELECT 1 FROM claim_evidence e WHERE e.claim_id = claim.id AND e.stance = 'contradicts')
|
||||
THEN 'disputed'
|
||||
WHEN (SELECT count(DISTINCT d.source_id) FROM claim_evidence e JOIN document d ON d.id = e.document_id
|
||||
WHERE e.claim_id = claim.id AND e.stance IN ('supports', 'mentions')) >= 2 THEN 'corroborated'
|
||||
ELSE 'reported' END
|
||||
WHERE id = %s
|
||||
""",
|
||||
(claim_id,),
|
||||
)
|
||||
|
||||
|
||||
def _amount(value: Any) -> Decimal | None:
|
||||
if value is None or value == "":
|
||||
return None
|
||||
try:
|
||||
return Decimal(str(value)).quantize(Decimal("0.01"))
|
||||
except InvalidOperation:
|
||||
return None
|
||||
|
||||
|
||||
def as_date(value: Any) -> date | None:
|
||||
if isinstance(value, datetime):
|
||||
return value.date()
|
||||
if isinstance(value, date):
|
||||
return value
|
||||
return None
|
||||
@@ -0,0 +1,265 @@
|
||||
"""Typed records: the only way adapters and enrichers hand data to the Loader.
|
||||
|
||||
Records are plain dataclasses so parsers stay pure and testable (see tests/golden). They refer to
|
||||
entities through PersonRef/OrgRef, which the Loader resolves (identifiers first, then hints, then
|
||||
names). Data minimisation is enforced by schema: there is, for example, no address field anywhere,
|
||||
so a parser cannot store a donor's street address even by accident.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field, fields, is_dataclass
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any, Literal, Union
|
||||
|
||||
# --------------------------------------------------------------------------------------------
|
||||
# References to entities
|
||||
# --------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class PersonRef:
|
||||
"""How a record points at a person.
|
||||
|
||||
ids: strong identifiers, e.g. {"parlament": "2339"} or {"wikidata": "Q123"}
|
||||
name: display/full name; used for alias matching if no id matches
|
||||
hints: resolution strategies, e.g.
|
||||
{"surname_in_body": "nationalrat", "on": date(...)} -> roll-call surname lists
|
||||
{"office": "Bundesministerin für Justiz", "on": date(...)} -> office holder on a date
|
||||
"""
|
||||
|
||||
ids: dict[str, str] = field(default_factory=dict)
|
||||
name: str | None = None
|
||||
hints: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class OrgRef:
|
||||
"""How a record points at an organization. `slug` addresses seeded orgs (e.g. "nationalrat")."""
|
||||
|
||||
ids: dict[str, str] = field(default_factory=dict)
|
||||
name: str | None = None
|
||||
short_name: str | None = None
|
||||
kind: str = "other"
|
||||
slug: str | None = None
|
||||
create: bool = True
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------------
|
||||
# Evidence & documents
|
||||
# --------------------------------------------------------------------------------------------
|
||||
|
||||
DocType = Literal["article", "press_release", "api_record", "protocol", "social_post", "register_entry",
|
||||
"web_page", "pdf", "dataset_row"]
|
||||
|
||||
|
||||
@dataclass
|
||||
class Evidence:
|
||||
url: str
|
||||
title: str | None = None
|
||||
doc_type: str = "web_page"
|
||||
published_at: datetime | date | str | None = None
|
||||
excerpt: str | None = None
|
||||
full_text: str | None = None
|
||||
quote: str | None = None
|
||||
locator: str = ""
|
||||
stance: Literal["supports", "contradicts", "mentions"] = "supports"
|
||||
confidence: float = 1.0
|
||||
source: str | None = None # defaults to the adapter's source
|
||||
meta: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Document:
|
||||
"""A document without a claim yet (e.g. a headline); enrichers derive claims from it later."""
|
||||
|
||||
evidence: Evidence
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------------
|
||||
# Entities
|
||||
# --------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class PersonRecord:
|
||||
ref: PersonRef
|
||||
full_name: str
|
||||
given_name: str | None = None
|
||||
family_name: str | None = None
|
||||
honorific_prefix: str | None = None
|
||||
honorific_suffix: str | None = None
|
||||
birth_date: date | None = None
|
||||
death_date: date | None = None
|
||||
gender: str | None = None
|
||||
photo_url: str | None = None
|
||||
aliases: tuple[str, ...] = ()
|
||||
#: if no identifier matches, merge with an existing person of exactly this (unambiguous) name before
|
||||
#: creating a new one - for authoritative sources that share no id with others (e.g. EP)
|
||||
match_by_name: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class OrgRecord:
|
||||
ref: OrgRef
|
||||
aliases: tuple[str, ...] = ()
|
||||
color: str | None = None
|
||||
parent: OrgRef | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SocialAccount:
|
||||
platform: str # bluesky | mastodon | telegram | youtube | twitter | facebook | ...
|
||||
url: str
|
||||
handle: str | None = None
|
||||
person: PersonRef | None = None
|
||||
org: OrgRef | None = None
|
||||
verified: bool = False # True only for official sources (parliament profile, official website)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------------
|
||||
# Facets (structured payload of a claim)
|
||||
# --------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Role:
|
||||
role: str # "mp" for chamber members, otherwise office title
|
||||
level: Literal["eu", "federal", "state", "district", "municipal", "party", "other"]
|
||||
body: OrgRef | None = None # chamber/government/party the role is in
|
||||
party: OrgRef | None = None # party/faction of the holder in this role
|
||||
region: str | None = None
|
||||
start: date | None = None
|
||||
end: date | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class VoteEvent:
|
||||
"""A vote (not a claim itself). Positions reference it by `key`."""
|
||||
|
||||
key: str
|
||||
chamber: OrgRef | None
|
||||
title: str
|
||||
voted_on: date | None
|
||||
result: Literal["accepted", "rejected", "unknown"] = "unknown"
|
||||
stage: str | None = None
|
||||
item_ref: str | None = None
|
||||
roll_call: bool = False
|
||||
url: str | None = None
|
||||
protocol_url: str | None = None
|
||||
raw_text: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class VotePosition:
|
||||
event_key: str
|
||||
position: Literal["for", "against", "abstain", "absent"]
|
||||
party: OrgRef | None = None # faction of an individual voter at that time
|
||||
|
||||
|
||||
@dataclass
|
||||
class Money:
|
||||
kind: Literal["donation", "contract", "government_ad", "political_ad", "subsidy", "party_funding", "party_ad",
|
||||
"loan", "side_income", "sponsorship", "other"]
|
||||
amount: Decimal | float | None = None
|
||||
currency: str = "EUR"
|
||||
payer: OrgRef | PersonRef | None = None
|
||||
payee: OrgRef | PersonRef | None = None
|
||||
payer_name: str | None = None
|
||||
payee_name: str | None = None
|
||||
paid_on: date | None = None
|
||||
period: str | None = None
|
||||
reference: str | None = None # notice id, campaign, ...
|
||||
category: str | None = None # CPV code, ad category, ...
|
||||
amount_category: str | None = None # when only a bracket is published
|
||||
|
||||
|
||||
@dataclass
|
||||
class Event:
|
||||
"""Something with time and place (not a claim). Participation claims reference it by `key`."""
|
||||
|
||||
key: str
|
||||
kind: str
|
||||
title: str
|
||||
starts_at: datetime | date | str | None
|
||||
ends_at: datetime | date | str | None = None
|
||||
location: str | None = None
|
||||
country: str | None = None
|
||||
url: str | None = None
|
||||
description: str | None = None
|
||||
body: OrgRef | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Participation:
|
||||
event_key: str
|
||||
participation: str = "participant"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Statement:
|
||||
context: Literal["parliament_speech", "interview", "tv", "radio", "social_post", "press_release",
|
||||
"event_speech", "op_ed", "testimony", "other"]
|
||||
url: str
|
||||
said_at: datetime | date | str | None = None
|
||||
platform: str | None = None
|
||||
medium: str | None = None
|
||||
title: str | None = None
|
||||
excerpt: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Legal:
|
||||
stage: str
|
||||
offense: str | None = None
|
||||
authority: str | None = None
|
||||
case_ref: str | None = None
|
||||
stage_date: date | None = None
|
||||
|
||||
|
||||
Facet = Union[Role, VotePosition, Money, Participation, Statement, Legal]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------------
|
||||
# Claims
|
||||
# --------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Claim:
|
||||
claim_type: str
|
||||
summary_de: str
|
||||
evidence: list[Evidence]
|
||||
person: PersonRef | None = None
|
||||
org: OrgRef | None = None
|
||||
counterpart_person: PersonRef | None = None
|
||||
counterpart_org: OrgRef | None = None
|
||||
occurred_at: datetime | date | str | None = None
|
||||
valid_from: date | None = None
|
||||
valid_to: date | None = None
|
||||
summary_en: str | None = None
|
||||
data: dict[str, Any] = field(default_factory=dict)
|
||||
topics: tuple[str, ...] = ()
|
||||
sensitive: bool = False
|
||||
review: bool = False # True -> published only after human review
|
||||
key: tuple | None = None # explicit identity (otherwise computed by the Loader)
|
||||
facet: Facet | None = None
|
||||
|
||||
|
||||
Record = Union[PersonRecord, OrgRecord, SocialAccount, VoteEvent, Event, Claim, Document]
|
||||
|
||||
|
||||
def to_jsonable(obj: Any) -> Any:
|
||||
"""Serialise records for golden tests and debugging (stable, sorted keys)."""
|
||||
if is_dataclass(obj) and not isinstance(obj, type):
|
||||
return {"__type__": type(obj).__name__, **{f.name: to_jsonable(getattr(obj, f.name)) for f in fields(obj)}}
|
||||
if isinstance(obj, dict):
|
||||
return {str(k): to_jsonable(v) for k, v in sorted(obj.items(), key=lambda kv: str(kv[0]))}
|
||||
if isinstance(obj, (list, tuple)):
|
||||
return [to_jsonable(v) for v in obj]
|
||||
if isinstance(obj, (datetime, date)):
|
||||
return obj.isoformat()
|
||||
if isinstance(obj, Decimal):
|
||||
return str(obj)
|
||||
return obj
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Registry of adapters, enrichers and source specs.
|
||||
|
||||
Built-in modules register themselves with @register when imported (see load_builtin()).
|
||||
External packages can add adapters via the entry-point group `politrack.adapters`
|
||||
(value: a module path whose import registers adapters), e.g. in their pyproject.toml:
|
||||
|
||||
[project.entry-points."politrack.adapters"]
|
||||
my_sources = "my_package.sources"
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import logging
|
||||
import pkgutil
|
||||
from importlib.metadata import entry_points
|
||||
|
||||
from politrack.core.adapter import Adapter, Enricher
|
||||
from politrack.core.spec import DERIVED, SourceSpec
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
ADAPTERS: dict[str, type[Adapter]] = {}
|
||||
ENRICHERS: dict[str, type[Enricher]] = {}
|
||||
SPECS: dict[str, SourceSpec] = {DERIVED.slug: DERIVED}
|
||||
_loaded = False
|
||||
|
||||
|
||||
def register(cls):
|
||||
"""Class decorator for Adapter and Enricher subclasses."""
|
||||
if issubclass(cls, Adapter):
|
||||
target = ADAPTERS
|
||||
elif issubclass(cls, Enricher):
|
||||
target = ENRICHERS
|
||||
else:
|
||||
raise TypeError(f"{cls} is neither an Adapter nor an Enricher")
|
||||
if cls.name in ADAPTERS or cls.name in ENRICHERS:
|
||||
raise ValueError(f"duplicate adapter name {cls.name!r}")
|
||||
target[cls.name] = cls
|
||||
register_spec(cls.spec)
|
||||
return cls
|
||||
|
||||
|
||||
def register_spec(spec: SourceSpec) -> SourceSpec:
|
||||
existing = SPECS.get(spec.slug)
|
||||
if existing is not None and existing != spec:
|
||||
raise ValueError(f"conflicting SourceSpec definitions for {spec.slug!r}")
|
||||
SPECS[spec.slug] = spec
|
||||
return spec
|
||||
|
||||
|
||||
def load_builtin() -> None:
|
||||
"""Import all built-in source and enricher modules (and plugins) exactly once."""
|
||||
global _loaded
|
||||
if _loaded:
|
||||
return
|
||||
_loaded = True
|
||||
for package_name in ("politrack.sources", "politrack.enrich"):
|
||||
package = importlib.import_module(package_name)
|
||||
for module in pkgutil.walk_packages(package.__path__, prefix=f"{package_name}."):
|
||||
importlib.import_module(module.name)
|
||||
for ep in entry_points(group="politrack.adapters"):
|
||||
try:
|
||||
importlib.import_module(ep.value)
|
||||
except Exception: # a broken plugin must not take the platform down
|
||||
log.exception("failed to load adapter plugin %s", ep.value)
|
||||
|
||||
|
||||
def get(name: str) -> type[Adapter] | type[Enricher]:
|
||||
load_builtin()
|
||||
if name in ADAPTERS:
|
||||
return ADAPTERS[name]
|
||||
if name in ENRICHERS:
|
||||
return ENRICHERS[name]
|
||||
raise KeyError(f"unknown adapter/enricher {name!r}; known: {', '.join(sorted(ADAPTERS) + sorted(ENRICHERS))}")
|
||||
|
||||
|
||||
def sync_sources(conn) -> int:
|
||||
"""Upsert all registered SourceSpecs into the `source` table."""
|
||||
load_builtin()
|
||||
for spec in SPECS.values():
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO source (slug, name, kind, homepage, license, access_method, store_fulltext, keep_raw,
|
||||
default_trust, tos_notes)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (slug) DO UPDATE SET name = EXCLUDED.name, kind = EXCLUDED.kind, homepage = EXCLUDED.homepage,
|
||||
license = EXCLUDED.license, access_method = EXCLUDED.access_method,
|
||||
store_fulltext = EXCLUDED.store_fulltext, keep_raw = EXCLUDED.keep_raw,
|
||||
default_trust = EXCLUDED.default_trust, tos_notes = EXCLUDED.tos_notes, updated_at = now()
|
||||
""",
|
||||
(spec.slug, spec.name, spec.kind, spec.homepage, spec.license, spec.access, spec.store_fulltext,
|
||||
spec.keep_raw, spec.trust, spec.tos_notes),
|
||||
)
|
||||
return len(SPECS)
|
||||
@@ -0,0 +1,269 @@
|
||||
"""Entity resolution: PersonRef/OrgRef -> database ids.
|
||||
|
||||
Order (first hit wins, ambiguity never guesses):
|
||||
persons: identifiers -> hint strategies -> exact alias (titles stripped; prefer active)
|
||||
organizations: identifiers -> slug -> exact alias -> normalised legal name -> create (if allowed)
|
||||
|
||||
Hint strategies for persons (documented in docs/data-model.md):
|
||||
surname_in_body + on members of a chamber on a date with that surname (roll-call lists)
|
||||
office + on holder of an office title on a date (Council of Ministers minutes)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from psycopg import Connection
|
||||
from rapidfuzz import fuzz
|
||||
|
||||
from politrack.core.records import OrgRef, PersonRef
|
||||
from politrack.text import make_slug, normalize, normalize_org_name, strip_titles
|
||||
|
||||
|
||||
OFFICE_FILLER = re.compile(r"\b(der republik osterreich|der bundesrepublik osterreich|im bundeskanzleramt)\b|"
|
||||
r"\(osterreich\)")
|
||||
|
||||
|
||||
def office_key(title: str) -> str:
|
||||
"""'Bundesministerin für Justiz' / 'Bundeskanzler der Republik Österreich' -> 'bundesminister fur justiz' /
|
||||
'bundeskanzler' (gender-neutral, without filler, punctuation-free) for comparing office titles."""
|
||||
text = normalize(title)
|
||||
text = re.sub(r"\b(\w*(?:minister|kanzler|sekretar|prasident|rat))in\b", r"\1", text)
|
||||
text = OFFICE_FILLER.sub(" ", text)
|
||||
return re.sub(r"\s+", " ", re.sub(r"[^\w\s]", " ", text)).strip()
|
||||
|
||||
|
||||
# Fallback for colloquial or English office labels (Wikidata: "Justizminister Österreichs", "Federal Minister for
|
||||
# Arts, Culture, the Civil Service and Sport"): same kind of office + at least one common portfolio.
|
||||
OFFICE_KINDS = [("vizekanzler", r"vizekanzler|vice.chancellor"), ("kanzler", r"kanzler|chancellor"),
|
||||
("staatssekretar", r"staatssekretar|state secretary"), ("minister", r"minist")]
|
||||
PORTFOLIOS = {
|
||||
"justiz": r"justiz|justice", "finanzen": r"finanz|financ", "inneres": r"\binnere|\binnenmin|interior",
|
||||
"aussen": r"au(ss|ß)en|auswartig|europaische und internationale|foreign|european and international",
|
||||
"verteidigung": r"verteidigung|defen[cs]e", "soziales": r"sozial|\barbeit|labou?r|social",
|
||||
"gesundheit": r"gesundheit|health", "bildung": r"bildung|education", "wissenschaft": r"wissenschaft|science",
|
||||
"frauen": r"frauen|women", "landwirtschaft": r"landwirtschaft|forstwirtschaft|agricultur",
|
||||
"umwelt": r"umwelt|klima|environment|climate", "kunst": r"kunst|kultur|\barts\b|culture|\bsport",
|
||||
"mobilitat": r"mobilitat|verkehr|infrastruktur|mobility|transport|infrastructure",
|
||||
"wirtschaft": r"\bwirtschaft|econom", "europa": r"\beuropa\b|integration|familie|famil",
|
||||
"digitalisierung": r"digital", "energie": r"energie|energy", "tourismus": r"touris",
|
||||
}
|
||||
|
||||
|
||||
def office_kind(key: str) -> str | None:
|
||||
return next((kind for kind, pattern in OFFICE_KINDS if re.search(pattern, key)), None)
|
||||
|
||||
|
||||
def portfolios(key: str) -> set[str]:
|
||||
return {name for name, pattern in PORTFOLIOS.items() if re.search(pattern, key)}
|
||||
|
||||
|
||||
def lead_portfolio(key: str) -> str | None:
|
||||
"""The portfolio named first ("Wirtschaft, Energie und Tourismus" -> wirtschaft): a role must cover it."""
|
||||
found = [(m.start(), name) for name, pattern in PORTFOLIOS.items() if (m := re.search(pattern, key))]
|
||||
return min(found)[1] if found else None
|
||||
|
||||
|
||||
class Resolver:
|
||||
def __init__(self, conn: Connection) -> None:
|
||||
self.conn = conn
|
||||
self._person_cache: dict[tuple, int | None] = {}
|
||||
self._org_cache: dict[tuple, int | None] = {}
|
||||
self._members_cache: dict[tuple, list[dict]] = {}
|
||||
self._office_cache: dict[tuple, int | None] = {}
|
||||
|
||||
# -- persons ------------------------------------------------------------------------
|
||||
|
||||
def person_by_ids(self, ids: dict[str, str]) -> int | None:
|
||||
for scheme, value in ids.items():
|
||||
if value is None:
|
||||
continue
|
||||
if scheme == "politrack": # internal id, used by adapters that plan requests from the database
|
||||
row = self.conn.execute("SELECT id FROM person WHERE id = %s", (int(value),)).fetchone()
|
||||
if row:
|
||||
return row["id"]
|
||||
continue
|
||||
row = self.conn.execute(
|
||||
"SELECT entity_id FROM identifier WHERE entity_type = 'person' AND scheme = %s AND value = %s",
|
||||
(scheme, str(value)),
|
||||
).fetchone()
|
||||
if row:
|
||||
return row["entity_id"]
|
||||
return None
|
||||
|
||||
def person(self, ref: PersonRef | None) -> int | None:
|
||||
if ref is None:
|
||||
return None
|
||||
cache_key = (tuple(sorted(ref.ids.items())), ref.name, tuple(sorted((k, str(v)) for k, v in ref.hints.items())))
|
||||
if cache_key in self._person_cache:
|
||||
return self._person_cache[cache_key]
|
||||
found = self.person_by_ids(ref.ids)
|
||||
if found is None and ref.hints:
|
||||
found = self._person_by_hints(ref)
|
||||
if found is None and ref.name and not ref.hints.get("no_name_match"):
|
||||
found = self.person_by_name(ref.name)
|
||||
self._person_cache[cache_key] = found
|
||||
return found
|
||||
|
||||
def person_by_name(self, name: str) -> int | None:
|
||||
norm = normalize(strip_titles(name))
|
||||
rows = self.conn.execute(
|
||||
"""SELECT DISTINCT a.entity_id AS id, ps.status
|
||||
FROM alias a LEFT JOIN person_status ps ON ps.person_id = a.entity_id
|
||||
WHERE a.entity_type = 'person' AND a.alias_norm = %s""",
|
||||
(norm,),
|
||||
).fetchall()
|
||||
if len(rows) == 1:
|
||||
return rows[0]["id"]
|
||||
active = [r for r in rows if r["status"] == "active"]
|
||||
return active[0]["id"] if len(active) == 1 else None
|
||||
|
||||
def _person_by_hints(self, ref: PersonRef) -> int | None:
|
||||
hints = ref.hints
|
||||
on: date | None = hints.get("on")
|
||||
if "surname_in_body" in hints and ref.name:
|
||||
members = self.members(hints["surname_in_body"], on)
|
||||
norm = normalize(strip_titles(ref.name))
|
||||
exact = [m for m in members if m["family_name"] and normalize(m["family_name"]) == norm]
|
||||
if len(exact) == 1:
|
||||
return exact[0]["id"]
|
||||
suffix = [m for m in members if normalize(strip_titles(m["full_name"])).endswith(norm)]
|
||||
return suffix[0]["id"] if len(suffix) == 1 else None
|
||||
if "office" in hints:
|
||||
return self.office_holder(hints["office"], on)
|
||||
return None
|
||||
|
||||
def office_holder(self, office: str, on: date | None) -> int | None:
|
||||
"""Holder of a (single-holder) office on a date. Titles are compared whole, gender-neutral and without
|
||||
filler ("Bundesministerin für Justiz" == "Bundesminister für Justiz"), so a generic "Bundesminister"
|
||||
never matches a specific ministry. Several matches (open-ended roles from Wikidata) -> the most recent
|
||||
appointment wins; no dated match -> None."""
|
||||
key = (office, on)
|
||||
if key not in self._office_cache:
|
||||
rows = self.conn.execute(
|
||||
"""SELECT DISTINCT r.person_id, r.role, r.start_date FROM role r JOIN claim c ON c.id = r.claim_id
|
||||
WHERE c.merged_into IS NULL AND r.level IN ('federal', 'state', 'eu')
|
||||
AND (%(on)s::date IS NULL OR ((r.start_date IS NULL OR r.start_date <= %(on)s)
|
||||
AND (r.end_date IS NULL OR r.end_date >= %(on)s)))""",
|
||||
{"on": on},
|
||||
).fetchall()
|
||||
target = office_key(office)
|
||||
candidates = [(r["start_date"] or date.min, r["person_id"]) for r in rows
|
||||
if fuzz.ratio(office_key(r["role"]), target) >= 92]
|
||||
kind, lead = office_kind(target), lead_portfolio(target)
|
||||
if not candidates and kind and (lead or kind in ("kanzler", "vizekanzler")):
|
||||
# Fuzzy labels only for appointments since the current government took office (the latest
|
||||
# chancellor appointment): stale open-ended roles of former ministers must not win.
|
||||
formed = max((r["start_date"] for r in rows if r["start_date"]
|
||||
and office_kind(office_key(r["role"])) == "kanzler"), default=date.min)
|
||||
candidates = [(r["start_date"], r["person_id"]) for r in rows
|
||||
if r["start_date"] and r["start_date"] >= formed
|
||||
and office_kind(office_key(r["role"])) == kind
|
||||
and (kind in ("kanzler", "vizekanzler") or lead in portfolios(office_key(r["role"])))]
|
||||
persons = {p for _, p in candidates}
|
||||
if len(persons) <= 1:
|
||||
self._office_cache[key] = persons.pop() if persons else None
|
||||
else:
|
||||
latest = max(start for start, _ in candidates)
|
||||
newest = {p for start, p in candidates if start == latest}
|
||||
self._office_cache[key] = newest.pop() if len(newest) == 1 and latest != date.min else None
|
||||
return self._office_cache[key]
|
||||
|
||||
def members(self, body_slug: str, on: date | None) -> list[dict[str, Any]]:
|
||||
key = (body_slug, on)
|
||||
if key not in self._members_cache:
|
||||
self._members_cache[key] = self.conn.execute(
|
||||
"""SELECT DISTINCT ON (p.id) p.id, p.full_name, p.family_name, r.party_id
|
||||
FROM role r JOIN person p ON p.id = r.person_id JOIN organization o ON o.id = r.organization_id
|
||||
JOIN claim c ON c.id = r.claim_id
|
||||
WHERE o.slug = %(body)s AND r.role = 'mp' AND c.merged_into IS NULL
|
||||
AND (%(on)s::date IS NULL OR (r.start_date <= %(on)s AND COALESCE(r.end_date, 'infinity') >= %(on)s))
|
||||
ORDER BY p.id, r.start_date DESC""",
|
||||
{"body": body_slug, "on": on},
|
||||
).fetchall()
|
||||
return self._members_cache[key]
|
||||
|
||||
def faction_of(self, person_id: int, body_slug: str, on: date | None) -> int | None:
|
||||
return next((m["party_id"] for m in self.members(body_slug, on) if m["id"] == person_id), None)
|
||||
|
||||
# -- organizations ------------------------------------------------------------------
|
||||
|
||||
def org(self, ref: OrgRef | None, create: bool | None = None) -> int | None:
|
||||
if ref is None:
|
||||
return None
|
||||
allow_create = ref.create if create is None else create
|
||||
cache_key = (tuple(sorted(ref.ids.items())), ref.slug, ref.name, ref.short_name)
|
||||
if cache_key in self._org_cache and (self._org_cache[cache_key] is not None or not allow_create):
|
||||
return self._org_cache[cache_key]
|
||||
found = self._org_lookup(ref)
|
||||
if found is None and allow_create and (ref.name or ref.short_name):
|
||||
found = self._org_create(ref)
|
||||
elif found is not None:
|
||||
self._add_org_ids(found, ref)
|
||||
self._org_cache[cache_key] = found
|
||||
return found
|
||||
|
||||
def _org_lookup(self, ref: OrgRef) -> int | None:
|
||||
for scheme, value in ref.ids.items():
|
||||
if scheme == "politrack_org": # internal id (enrichers)
|
||||
row = self.conn.execute("SELECT id FROM organization WHERE id = %s", (int(value),)).fetchone()
|
||||
if row:
|
||||
return row["id"]
|
||||
continue
|
||||
row = self.conn.execute(
|
||||
"SELECT entity_id FROM identifier WHERE entity_type = 'organization' AND scheme = %s AND value = %s",
|
||||
(scheme, str(value)),
|
||||
).fetchone()
|
||||
if row:
|
||||
return row["entity_id"]
|
||||
if ref.slug:
|
||||
row = self.conn.execute("SELECT id FROM organization WHERE slug = %s", (ref.slug,)).fetchone()
|
||||
if row:
|
||||
return row["id"]
|
||||
for label in (ref.short_name, ref.name):
|
||||
if not label:
|
||||
continue
|
||||
for norm in {normalize(label), normalize_org_name(label)} - {""}:
|
||||
rows = self.conn.execute(
|
||||
"SELECT DISTINCT entity_id FROM alias WHERE entity_type = 'organization' AND alias_norm = %s",
|
||||
(norm,),
|
||||
).fetchall()
|
||||
if len(rows) == 1:
|
||||
return rows[0]["entity_id"]
|
||||
return None
|
||||
|
||||
def _org_create(self, ref: OrgRef) -> int:
|
||||
base = ref.slug or make_slug(ref.short_name or ref.name or "org")[:60] or "org"
|
||||
slug, n = base, 1
|
||||
while self.conn.execute("SELECT 1 FROM organization WHERE slug = %s", (slug,)).fetchone():
|
||||
n += 1
|
||||
slug = f"{base}-{n}"
|
||||
org_id = self.conn.execute(
|
||||
"INSERT INTO organization (slug, name, short_name, kind) VALUES (%s, %s, %s, %s) RETURNING id",
|
||||
(slug, ref.name or ref.short_name, ref.short_name, ref.kind if ref.kind else "other"),
|
||||
).fetchone()["id"]
|
||||
self.add_org_aliases(org_id, [ref.name, ref.short_name])
|
||||
self._add_org_ids(org_id, ref)
|
||||
return org_id
|
||||
|
||||
def add_org_aliases(self, org_id: int, names: list[str | None]) -> None:
|
||||
for name in names:
|
||||
if not name:
|
||||
continue
|
||||
for norm in {normalize(name), normalize_org_name(name)} - {""}:
|
||||
self.conn.execute(
|
||||
"INSERT INTO alias (entity_type, entity_id, alias, alias_norm) VALUES ('organization', %s, %s, %s) "
|
||||
"ON CONFLICT DO NOTHING",
|
||||
(org_id, name, norm),
|
||||
)
|
||||
|
||||
def _add_org_ids(self, org_id: int, ref: OrgRef) -> None:
|
||||
for scheme, value in ref.ids.items():
|
||||
if value:
|
||||
self.conn.execute(
|
||||
"INSERT INTO identifier (entity_type, entity_id, scheme, value) VALUES ('organization', %s, %s, %s) "
|
||||
"ON CONFLICT DO NOTHING",
|
||||
(org_id, scheme, str(value)),
|
||||
)
|
||||
@@ -0,0 +1,232 @@
|
||||
"""Runs adapters and enrichers.
|
||||
|
||||
fetch mode: start() -> fetch (policy, archive) -> parse() -> Loader, one transaction per response
|
||||
reparse mode: replay the latest archived response of every request the adapter ever made
|
||||
test mode: parse a given response without DB (used by golden tests)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import traceback
|
||||
from collections import deque
|
||||
from collections.abc import Iterable
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from psycopg import Connection
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from politrack.core import registry
|
||||
from politrack.core.adapter import Adapter, Enricher, Request, Response, RunContext, StateStore
|
||||
from politrack.core.archive import Archive
|
||||
from politrack.core.fetch import Disallowed, Fetcher
|
||||
from politrack.core.loader import Loader, source_row
|
||||
from politrack.db import connection
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
ENRICH_COMMIT_EVERY = 200
|
||||
|
||||
|
||||
def run(name: str, *, mode: str = "fetch", since: datetime | None = None, params: dict[str, Any] | None = None,
|
||||
max_requests: int | None = None) -> dict:
|
||||
cls = registry.get(name)
|
||||
with connection() as conn:
|
||||
run_id = conn.execute("INSERT INTO run (adapter, mode) VALUES (%s, %s) RETURNING id", (name, mode)).fetchone()["id"]
|
||||
conn.commit()
|
||||
ctx = RunContext(adapter=name, spec=cls.spec, state=StateStore(conn, name), since=since, params=params or {},
|
||||
mode=mode, conn=conn, creates_persons=cls.creates_persons,
|
||||
log=logging.getLogger(f"politrack.{name}"))
|
||||
status, error = "ok", None
|
||||
try:
|
||||
if issubclass(cls, Enricher):
|
||||
_run_enricher(cls(), ctx, conn)
|
||||
elif mode == "reparse":
|
||||
_reparse(cls(), ctx, conn)
|
||||
else:
|
||||
_fetch_run(cls(), ctx, conn, max_requests)
|
||||
if ctx.stats.get("errors"):
|
||||
status = "partial"
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
status, error = "error", traceback.format_exc()
|
||||
ctx.log.exception("%s failed", name)
|
||||
conn.execute("UPDATE run SET finished_at = now(), status = %s, stats = %s, error = %s WHERE id = %s",
|
||||
(status, Jsonb(dict(ctx.stats)), error, run_id))
|
||||
conn.commit()
|
||||
ctx.log.info("%s %s: %s %s", name, mode, status, dict(ctx.stats))
|
||||
if status == "error":
|
||||
raise RuntimeError(f"{name} failed, see run {run_id}")
|
||||
return {"status": status, **dict(ctx.stats)}
|
||||
|
||||
|
||||
def _fetch_run(adapter: Adapter, ctx: RunContext, conn: Connection, max_requests: int | None) -> None:
|
||||
fetcher = Fetcher()
|
||||
loader = Loader(conn, ctx)
|
||||
queue: deque[Request] = deque(adapter.start(ctx))
|
||||
seen: set[str] = set()
|
||||
try:
|
||||
while queue:
|
||||
req = queue.popleft()
|
||||
if req.fingerprint in seen:
|
||||
continue
|
||||
seen.add(req.fingerprint)
|
||||
if max_requests is not None and ctx.stats["requests"] >= max_requests:
|
||||
ctx.stats["stopped_at_max_requests"] = 1
|
||||
break
|
||||
ctx.stats["requests"] += 1
|
||||
# Multi-publisher adapters (news feeds, GDELT) name the publisher per request.
|
||||
spec = registry.SPECS.get(req.meta.get("source") or adapter.spec.slug, adapter.spec)
|
||||
try:
|
||||
resp = fetcher.fetch(conn, req, adapter=adapter.name, spec=spec,
|
||||
source_id=source_row(conn, spec.slug)["id"])
|
||||
except Disallowed:
|
||||
ctx.stats["disallowed_by_robots"] += 1
|
||||
ctx.log.warning("robots.txt disallows %s", req.url)
|
||||
continue
|
||||
except Exception as exc:
|
||||
conn.rollback()
|
||||
ctx.stats["fetch_errors"] += 1
|
||||
ctx.stats["errors"] += 1
|
||||
ctx.log.warning("fetch failed %s: %s", req.url, exc)
|
||||
continue
|
||||
if req.skip_unchanged and _unchanged(conn, resp, adapter.parser_version):
|
||||
# Same body, same parser: its records are already loaded. Still collect follow-up requests,
|
||||
# so children an interrupted run never fetched are picked up (they have their own checks).
|
||||
ctx.stats["unchanged"] += 1
|
||||
queue.extend(_follow_ups(adapter, ctx, resp))
|
||||
continue
|
||||
queue.extend(_parse_and_load(adapter, ctx, conn, loader, resp))
|
||||
finally:
|
||||
fetcher.close()
|
||||
|
||||
|
||||
def _unchanged(conn: Connection, resp: Response, parser_version: int) -> bool:
|
||||
if resp.fetch_id is None:
|
||||
return False
|
||||
row = conn.execute(
|
||||
"""SELECT 1 FROM fetch_log WHERE request_fp = %s AND parsed_sha256 = %s AND parser_version = %s LIMIT 1""",
|
||||
(resp.request.fingerprint, resp.sha256, parser_version),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
|
||||
def _follow_ups(adapter: Adapter, ctx: RunContext, resp: Response) -> list[Request]:
|
||||
ctx.response = resp
|
||||
try:
|
||||
return [item for item in adapter.parse(ctx, resp) if isinstance(item, Request)]
|
||||
except Exception:
|
||||
ctx.stats["parse_errors"] += 1
|
||||
ctx.log.exception("parse failed for unchanged %s", resp.url)
|
||||
return []
|
||||
finally:
|
||||
ctx.response = None
|
||||
|
||||
|
||||
def _parse_and_load(adapter: Adapter, ctx: RunContext, conn: Connection, loader: Loader,
|
||||
resp: Response) -> list[Request]:
|
||||
follow: list[Request] = []
|
||||
ctx.response = resp
|
||||
try:
|
||||
for item in adapter.parse(ctx, resp):
|
||||
if isinstance(item, Request):
|
||||
follow.append(item)
|
||||
else:
|
||||
loader.load(item)
|
||||
if resp.fetch_id is not None:
|
||||
Archive().mark_parsed(conn, resp.fetch_id, resp.sha256, adapter.parser_version)
|
||||
conn.commit()
|
||||
except Exception as exc:
|
||||
conn.rollback()
|
||||
ctx.stats["errors"] += 1
|
||||
ctx.stats["parse_errors"] += 1
|
||||
ctx.log.exception("parse/load failed for %s", resp.url)
|
||||
if resp.fetch_id is not None:
|
||||
Archive().mark_error(conn, resp.fetch_id, f"{type(exc).__name__}: {exc}")
|
||||
conn.commit()
|
||||
loader.resolver = type(loader.resolver)(conn) # drop caches that may reference rolled-back rows
|
||||
finally:
|
||||
ctx.response = None
|
||||
return follow
|
||||
|
||||
|
||||
def _reparse(adapter: Adapter, ctx: RunContext, conn: Connection) -> None:
|
||||
archive = Archive()
|
||||
loader = Loader(conn, ctx)
|
||||
started = conn.execute("SELECT now() AS t").fetchone()["t"]
|
||||
rows = archive.latest_stored(conn, adapter.name)
|
||||
ctx.log.info("reparsing %s archived responses", len(rows))
|
||||
# Parse list/index responses before detail responses: records referencing others (vote positions ->
|
||||
# vote events) are emitted by the parent's parse() in order, so the original order is preserved
|
||||
# by sorting on fetch time.
|
||||
for row in sorted(rows, key=lambda r: r["fetched_at"]):
|
||||
body = archive.read_blob(row["sha256"])
|
||||
if body is None:
|
||||
ctx.stats["missing_blobs"] += 1
|
||||
continue
|
||||
req_info = row["request"] or {}
|
||||
req = Request(url=row["url"], kind=req_info.get("kind", "default"), method=row["method"],
|
||||
params=req_info.get("params"), json=req_info.get("json"), meta=req_info.get("meta") or {})
|
||||
resp = Response(request=req, status=row["status"], url=row["url"],
|
||||
headers={"content-type": row["content_type"] or ""}, body=body,
|
||||
fetched_at=row["fetched_at"], sha256=row["sha256"], fetch_id=row["id"],
|
||||
tdm_reserved=row["tdm_reserved"], from_archive=True)
|
||||
ctx.stats["responses"] += 1
|
||||
_parse_and_load(adapter, ctx, conn, loader, resp) # follow-ups are archived separately
|
||||
if not ctx.stats.get("errors") and not ctx.stats.get("missing_blobs"):
|
||||
_sweep_stale_claims(adapter, ctx, conn, started)
|
||||
|
||||
|
||||
def _sweep_stale_claims(adapter: Adapter, ctx: RunContext, conn: Connection, started: datetime) -> None:
|
||||
"""After a complete reparse, delete this adapter's claims that the current parser no longer produces
|
||||
(e.g. a fixed parser emits a corrected fact with a new identity) - unless another source backs them.
|
||||
Claims are reproducible from the archive, so nothing is lost; facets and evidence links cascade."""
|
||||
deleted = conn.execute(
|
||||
"""DELETE FROM claim c
|
||||
WHERE c.created_by = %(adapter)s AND c.updated_at < %(started)s
|
||||
AND NOT EXISTS (SELECT 1 FROM claim_evidence e JOIN document d ON d.id = e.document_id
|
||||
JOIN source s ON s.id = d.source_id
|
||||
WHERE e.claim_id = c.id AND s.slug <> %(source)s)""",
|
||||
{"adapter": adapter.name, "started": started, "source": adapter.spec.slug},
|
||||
).rowcount
|
||||
conn.commit()
|
||||
ctx.stats["stale_claims_deleted"] = deleted
|
||||
|
||||
|
||||
def _run_enricher(enricher: Enricher, ctx: RunContext, conn: Connection) -> None:
|
||||
loader = Loader(conn, ctx)
|
||||
result = enricher.run(ctx)
|
||||
pending = 0
|
||||
for record in result or []:
|
||||
try:
|
||||
loader.load(record)
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
ctx.stats["errors"] += 1
|
||||
ctx.log.exception("enricher %s failed to load a record", enricher.name)
|
||||
loader = Loader(conn, ctx)
|
||||
continue
|
||||
pending += 1
|
||||
if pending >= ENRICH_COMMIT_EVERY:
|
||||
conn.commit()
|
||||
pending = 0
|
||||
conn.commit()
|
||||
|
||||
|
||||
def parse_offline(adapter: Adapter, response: Response, params: dict[str, Any] | None = None) -> Iterable[Any]:
|
||||
"""Run parse() without a database (golden tests, debugging)."""
|
||||
ctx = RunContext(adapter=adapter.name, spec=adapter.spec, state=StateStore(None, adapter.name), mode="test",
|
||||
params=params or {})
|
||||
ctx.response = response
|
||||
return list(adapter.parse(ctx, response))
|
||||
|
||||
|
||||
def make_response(url: str, body: bytes, *, kind: str = "default", meta: dict | None = None,
|
||||
content_type: str = "", method: str = "GET", json_body: Any = None,
|
||||
params: dict | None = None) -> Response:
|
||||
"""Build a Response for tests/fixtures."""
|
||||
import hashlib
|
||||
|
||||
req = Request(url=url, kind=kind, meta=meta or {}, method=method, json=json_body, params=params)
|
||||
return Response(request=req, status=200, url=url, headers={"content-type": content_type}, body=body,
|
||||
fetched_at=datetime(2026, 1, 1, tzinfo=timezone.utc), sha256=hashlib.sha256(body).hexdigest())
|
||||
@@ -0,0 +1,49 @@
|
||||
"""SourceSpec: the single source of truth about a publisher we collect from.
|
||||
|
||||
Specs live next to the adapters that use them (backend/politrack/sources/*.py) and are synced
|
||||
into the `source` table by `politrack migrate` / `politrack sources sync`.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
|
||||
SourceKind = Literal["official", "ngo", "media", "social", "aggregator", "user", "derived"]
|
||||
AccessMethod = Literal["api", "bulk", "rss", "scrape", "manual", "derived"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceSpec:
|
||||
slug: str
|
||||
name: str
|
||||
kind: SourceKind
|
||||
homepage: str
|
||||
license: str
|
||||
access: AccessMethod
|
||||
trust: int
|
||||
#: may we keep body text (articles, posts, protocols)? False -> title, short excerpt and link only
|
||||
store_fulltext: bool = True
|
||||
#: may raw responses be archived? False for TDM-reserved publishers or restrictive terms
|
||||
keep_raw: bool = True
|
||||
#: minimum seconds between two requests to the same host (robots.txt crawl-delay can raise it)
|
||||
min_interval: float = 2.0
|
||||
#: URL prefixes of documented APIs whose operators allow programmatic use although robots.txt
|
||||
#: lists them (justify in tos_notes). Everything else always obeys robots.txt.
|
||||
robots_exempt: tuple[str, ...] = ()
|
||||
tos_notes: str = ""
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not 0 <= self.trust <= 100:
|
||||
raise ValueError(f"{self.slug}: trust must be 0-100")
|
||||
|
||||
|
||||
# Sources used by the built-in enrichers for derived documents/claims.
|
||||
DERIVED = SourceSpec(
|
||||
slug="politrack",
|
||||
name="Politrack (abgeleitet)",
|
||||
kind="derived",
|
||||
homepage="",
|
||||
license="derived from cited sources",
|
||||
access="derived",
|
||||
trust=50,
|
||||
tos_notes="Claims computed by Politrack from other sources; each cites its original evidence.",
|
||||
)
|
||||
Reference in new issue
Block a user