simonandClaude Opus 5.5
583ccba31c
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
233 lines
10 KiB
Python
233 lines
10 KiB
Python
"""Runs adapters and enrichers.
|
|
|
|
fetch mode: start() -> fetch (policy, archive) -> parse() -> Loader, one transaction per response
|
|
reparse mode: replay the latest archived response of every request the adapter ever made
|
|
test mode: parse a given response without DB (used by golden tests)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import traceback
|
|
from collections import deque
|
|
from collections.abc import Iterable
|
|
from datetime import datetime, timezone
|
|
from typing import Any
|
|
|
|
from psycopg import Connection
|
|
from psycopg.types.json import Jsonb
|
|
|
|
from politrack.core import registry
|
|
from politrack.core.adapter import Adapter, Enricher, Request, Response, RunContext, StateStore
|
|
from politrack.core.archive import Archive
|
|
from politrack.core.fetch import Disallowed, Fetcher
|
|
from politrack.core.loader import Loader, source_row
|
|
from politrack.db import connection
|
|
|
|
log = logging.getLogger(__name__)
|
|
ENRICH_COMMIT_EVERY = 200
|
|
|
|
|
|
def run(name: str, *, mode: str = "fetch", since: datetime | None = None, params: dict[str, Any] | None = None,
|
|
max_requests: int | None = None) -> dict:
|
|
cls = registry.get(name)
|
|
with connection() as conn:
|
|
run_id = conn.execute("INSERT INTO run (adapter, mode) VALUES (%s, %s) RETURNING id", (name, mode)).fetchone()["id"]
|
|
conn.commit()
|
|
ctx = RunContext(adapter=name, spec=cls.spec, state=StateStore(conn, name), since=since, params=params or {},
|
|
mode=mode, conn=conn, creates_persons=cls.creates_persons,
|
|
log=logging.getLogger(f"politrack.{name}"))
|
|
status, error = "ok", None
|
|
try:
|
|
if issubclass(cls, Enricher):
|
|
_run_enricher(cls(), ctx, conn)
|
|
elif mode == "reparse":
|
|
_reparse(cls(), ctx, conn)
|
|
else:
|
|
_fetch_run(cls(), ctx, conn, max_requests)
|
|
if ctx.stats.get("errors"):
|
|
status = "partial"
|
|
except Exception:
|
|
conn.rollback()
|
|
status, error = "error", traceback.format_exc()
|
|
ctx.log.exception("%s failed", name)
|
|
conn.execute("UPDATE run SET finished_at = now(), status = %s, stats = %s, error = %s WHERE id = %s",
|
|
(status, Jsonb(dict(ctx.stats)), error, run_id))
|
|
conn.commit()
|
|
ctx.log.info("%s %s: %s %s", name, mode, status, dict(ctx.stats))
|
|
if status == "error":
|
|
raise RuntimeError(f"{name} failed, see run {run_id}")
|
|
return {"status": status, **dict(ctx.stats)}
|
|
|
|
|
|
def _fetch_run(adapter: Adapter, ctx: RunContext, conn: Connection, max_requests: int | None) -> None:
|
|
fetcher = Fetcher()
|
|
loader = Loader(conn, ctx)
|
|
queue: deque[Request] = deque(adapter.start(ctx))
|
|
seen: set[str] = set()
|
|
try:
|
|
while queue:
|
|
req = queue.popleft()
|
|
if req.fingerprint in seen:
|
|
continue
|
|
seen.add(req.fingerprint)
|
|
if max_requests is not None and ctx.stats["requests"] >= max_requests:
|
|
ctx.stats["stopped_at_max_requests"] = 1
|
|
break
|
|
ctx.stats["requests"] += 1
|
|
# Multi-publisher adapters (news feeds, GDELT) name the publisher per request.
|
|
spec = registry.SPECS.get(req.meta.get("source") or adapter.spec.slug, adapter.spec)
|
|
try:
|
|
resp = fetcher.fetch(conn, req, adapter=adapter.name, spec=spec,
|
|
source_id=source_row(conn, spec.slug)["id"])
|
|
except Disallowed:
|
|
ctx.stats["disallowed_by_robots"] += 1
|
|
ctx.log.warning("robots.txt disallows %s", req.url)
|
|
continue
|
|
except Exception as exc:
|
|
conn.rollback()
|
|
ctx.stats["fetch_errors"] += 1
|
|
ctx.stats["errors"] += 1
|
|
ctx.log.warning("fetch failed %s: %s", req.url, exc)
|
|
continue
|
|
if req.skip_unchanged and _unchanged(conn, resp, adapter.parser_version):
|
|
# Same body, same parser: its records are already loaded. Still collect follow-up requests,
|
|
# so children an interrupted run never fetched are picked up (they have their own checks).
|
|
ctx.stats["unchanged"] += 1
|
|
queue.extend(_follow_ups(adapter, ctx, resp))
|
|
continue
|
|
queue.extend(_parse_and_load(adapter, ctx, conn, loader, resp))
|
|
finally:
|
|
fetcher.close()
|
|
|
|
|
|
def _unchanged(conn: Connection, resp: Response, parser_version: int) -> bool:
|
|
if resp.fetch_id is None:
|
|
return False
|
|
row = conn.execute(
|
|
"""SELECT 1 FROM fetch_log WHERE request_fp = %s AND parsed_sha256 = %s AND parser_version = %s LIMIT 1""",
|
|
(resp.request.fingerprint, resp.sha256, parser_version),
|
|
).fetchone()
|
|
return row is not None
|
|
|
|
|
|
def _follow_ups(adapter: Adapter, ctx: RunContext, resp: Response) -> list[Request]:
|
|
ctx.response = resp
|
|
try:
|
|
return [item for item in adapter.parse(ctx, resp) if isinstance(item, Request)]
|
|
except Exception:
|
|
ctx.stats["parse_errors"] += 1
|
|
ctx.log.exception("parse failed for unchanged %s", resp.url)
|
|
return []
|
|
finally:
|
|
ctx.response = None
|
|
|
|
|
|
def _parse_and_load(adapter: Adapter, ctx: RunContext, conn: Connection, loader: Loader,
|
|
resp: Response) -> list[Request]:
|
|
follow: list[Request] = []
|
|
ctx.response = resp
|
|
try:
|
|
for item in adapter.parse(ctx, resp):
|
|
if isinstance(item, Request):
|
|
follow.append(item)
|
|
else:
|
|
loader.load(item)
|
|
if resp.fetch_id is not None:
|
|
Archive().mark_parsed(conn, resp.fetch_id, resp.sha256, adapter.parser_version)
|
|
conn.commit()
|
|
except Exception as exc:
|
|
conn.rollback()
|
|
ctx.stats["errors"] += 1
|
|
ctx.stats["parse_errors"] += 1
|
|
ctx.log.exception("parse/load failed for %s", resp.url)
|
|
if resp.fetch_id is not None:
|
|
Archive().mark_error(conn, resp.fetch_id, f"{type(exc).__name__}: {exc}")
|
|
conn.commit()
|
|
loader.resolver = type(loader.resolver)(conn) # drop caches that may reference rolled-back rows
|
|
finally:
|
|
ctx.response = None
|
|
return follow
|
|
|
|
|
|
def _reparse(adapter: Adapter, ctx: RunContext, conn: Connection) -> None:
|
|
archive = Archive()
|
|
loader = Loader(conn, ctx)
|
|
started = conn.execute("SELECT now() AS t").fetchone()["t"]
|
|
rows = archive.latest_stored(conn, adapter.name)
|
|
ctx.log.info("reparsing %s archived responses", len(rows))
|
|
# Parse list/index responses before detail responses: records referencing others (vote positions ->
|
|
# vote events) are emitted by the parent's parse() in order, so the original order is preserved
|
|
# by sorting on fetch time.
|
|
for row in sorted(rows, key=lambda r: r["fetched_at"]):
|
|
body = archive.read_blob(row["sha256"])
|
|
if body is None:
|
|
ctx.stats["missing_blobs"] += 1
|
|
continue
|
|
req_info = row["request"] or {}
|
|
req = Request(url=row["url"], kind=req_info.get("kind", "default"), method=row["method"],
|
|
params=req_info.get("params"), json=req_info.get("json"), meta=req_info.get("meta") or {})
|
|
resp = Response(request=req, status=row["status"], url=row["url"],
|
|
headers={"content-type": row["content_type"] or ""}, body=body,
|
|
fetched_at=row["fetched_at"], sha256=row["sha256"], fetch_id=row["id"],
|
|
tdm_reserved=row["tdm_reserved"], from_archive=True)
|
|
ctx.stats["responses"] += 1
|
|
_parse_and_load(adapter, ctx, conn, loader, resp) # follow-ups are archived separately
|
|
if not ctx.stats.get("errors") and not ctx.stats.get("missing_blobs"):
|
|
_sweep_stale_claims(adapter, ctx, conn, started)
|
|
|
|
|
|
def _sweep_stale_claims(adapter: Adapter, ctx: RunContext, conn: Connection, started: datetime) -> None:
|
|
"""After a complete reparse, delete this adapter's claims that the current parser no longer produces
|
|
(e.g. a fixed parser emits a corrected fact with a new identity) - unless another source backs them.
|
|
Claims are reproducible from the archive, so nothing is lost; facets and evidence links cascade."""
|
|
deleted = conn.execute(
|
|
"""DELETE FROM claim c
|
|
WHERE c.created_by = %(adapter)s AND c.updated_at < %(started)s
|
|
AND NOT EXISTS (SELECT 1 FROM claim_evidence e JOIN document d ON d.id = e.document_id
|
|
JOIN source s ON s.id = d.source_id
|
|
WHERE e.claim_id = c.id AND s.slug <> %(source)s)""",
|
|
{"adapter": adapter.name, "started": started, "source": adapter.spec.slug},
|
|
).rowcount
|
|
conn.commit()
|
|
ctx.stats["stale_claims_deleted"] = deleted
|
|
|
|
|
|
def _run_enricher(enricher: Enricher, ctx: RunContext, conn: Connection) -> None:
|
|
loader = Loader(conn, ctx)
|
|
result = enricher.run(ctx)
|
|
pending = 0
|
|
for record in result or []:
|
|
try:
|
|
loader.load(record)
|
|
except Exception:
|
|
conn.rollback()
|
|
ctx.stats["errors"] += 1
|
|
ctx.log.exception("enricher %s failed to load a record", enricher.name)
|
|
loader = Loader(conn, ctx)
|
|
continue
|
|
pending += 1
|
|
if pending >= ENRICH_COMMIT_EVERY:
|
|
conn.commit()
|
|
pending = 0
|
|
conn.commit()
|
|
|
|
|
|
def parse_offline(adapter: Adapter, response: Response, params: dict[str, Any] | None = None) -> Iterable[Any]:
|
|
"""Run parse() without a database (golden tests, debugging)."""
|
|
ctx = RunContext(adapter=adapter.name, spec=adapter.spec, state=StateStore(None, adapter.name), mode="test",
|
|
params=params or {})
|
|
ctx.response = response
|
|
return list(adapter.parse(ctx, response))
|
|
|
|
|
|
def make_response(url: str, body: bytes, *, kind: str = "default", meta: dict | None = None,
|
|
content_type: str = "", method: str = "GET", json_body: Any = None,
|
|
params: dict | None = None) -> Response:
|
|
"""Build a Response for tests/fixtures."""
|
|
import hashlib
|
|
|
|
req = Request(url=url, kind=kind, meta=meta or {}, method=method, json=json_body, params=params)
|
|
return Response(request=req, status=200, url=url, headers={"content-type": content_type}, body=body,
|
|
fetched_at=datetime(2026, 1, 1, tzinfo=timezone.utc), sha256=hashlib.sha256(body).hexdigest())
|