Politrack: track Austrian politicians from public sources
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
583ccba31c
187 files changed
+39890
No files matched your search
@@ -0,0 +1,94 @@
|
||||
"""Raw archive: every fetch is logged in the `fetch_log` table; bodies are stored gzip-compressed and
|
||||
content-addressed (sha256) below POLITRACK_RAW_DIR when the source allows it.
|
||||
|
||||
Used for: conditional requests (ETag/Last-Modified), change detection (skip re-parsing identical
|
||||
bodies), `politrack reparse` after parser fixes, and as an audit trail.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from psycopg import Connection
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from politrack.config import get_settings
|
||||
|
||||
|
||||
class Archive:
|
||||
def __init__(self, raw_dir: Path | None = None) -> None:
|
||||
self.raw_dir = Path(raw_dir or get_settings().raw_dir)
|
||||
|
||||
# -- blobs ----------------------------------------------------------------------------
|
||||
|
||||
def _path(self, sha256: str) -> Path:
|
||||
return self.raw_dir / sha256[:2] / f"{sha256}.gz"
|
||||
|
||||
def write_blob(self, body: bytes) -> str:
|
||||
sha = hashlib.sha256(body).hexdigest()
|
||||
path = self._path(sha)
|
||||
if not path.exists():
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(f".tmp{os.getpid()}")
|
||||
with gzip.open(tmp, "wb") as fh:
|
||||
fh.write(body)
|
||||
tmp.replace(path)
|
||||
return sha
|
||||
|
||||
def read_blob(self, sha256: str) -> bytes | None:
|
||||
path = self._path(sha256)
|
||||
if not path.exists():
|
||||
return None
|
||||
with gzip.open(path, "rb") as fh:
|
||||
return fh.read()
|
||||
|
||||
# -- index ----------------------------------------------------------------------------
|
||||
|
||||
def last(self, conn: Connection, request_fp: str) -> dict[str, Any] | None:
|
||||
return conn.execute(
|
||||
"""SELECT id, sha256, etag, last_modified, stored, parsed_sha256, parser_version, status, url,
|
||||
content_type, tdm_reserved, fetched_at
|
||||
FROM fetch_log WHERE request_fp = %s AND status IS NOT NULL AND status < 400
|
||||
ORDER BY fetched_at DESC LIMIT 1""",
|
||||
(request_fp,),
|
||||
).fetchone()
|
||||
|
||||
def log(self, conn: Connection, *, adapter: str, source_id: int | None, request_fp: str, url: str, method: str,
|
||||
request: dict, status: int | None, body: bytes | None, headers: dict[str, str] | None,
|
||||
keep: bool, tdm_reserved: bool, error: str | None = None) -> tuple[int, str | None]:
|
||||
sha = hashlib.sha256(body).hexdigest() if body is not None else None
|
||||
stored = False
|
||||
if body is not None and keep and not tdm_reserved:
|
||||
self.write_blob(body)
|
||||
stored = True
|
||||
headers = headers or {}
|
||||
row = conn.execute(
|
||||
"""INSERT INTO fetch_log (adapter, source_id, request_fp, url, method, request, status, sha256, size,
|
||||
content_type, etag, last_modified, stored, tdm_reserved, error)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id""",
|
||||
(adapter, source_id, request_fp, url, method, Jsonb(request), status, sha,
|
||||
len(body) if body is not None else None, headers.get("content-type"), headers.get("etag"),
|
||||
headers.get("last-modified"), stored, tdm_reserved, error),
|
||||
).fetchone()
|
||||
return row["id"], sha
|
||||
|
||||
def mark_parsed(self, conn: Connection, fetch_id: int, sha256: str, parser_version: int) -> None:
|
||||
conn.execute("UPDATE fetch_log SET parsed_sha256 = %s, parser_version = %s WHERE id = %s",
|
||||
(sha256, parser_version, fetch_id))
|
||||
|
||||
def mark_error(self, conn: Connection, fetch_id: int, error: str) -> None:
|
||||
conn.execute("UPDATE fetch_log SET error = %s WHERE id = %s", (error[:2000], fetch_id))
|
||||
|
||||
def latest_stored(self, conn: Connection, adapter: str) -> list[dict[str, Any]]:
|
||||
"""Latest stored fetch per request of an adapter (input for `reparse`)."""
|
||||
return conn.execute(
|
||||
"""SELECT DISTINCT ON (request_fp) id, request_fp, url, method, request, status, sha256, fetched_at,
|
||||
content_type, tdm_reserved
|
||||
FROM fetch_log WHERE adapter = %s AND stored AND status < 400
|
||||
ORDER BY request_fp, fetched_at DESC""",
|
||||
(adapter,),
|
||||
).fetchall()
|
||||
Reference in new issue
Block a user