Simon DiesenreiterandClaude Opus 5.5
9da5a1dc48
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
95 lines
4.2 KiB
Python
95 lines
4.2 KiB
Python
"""Raw archive: every fetch is logged in the `fetch_log` table; bodies are stored gzip-compressed and
|
|
content-addressed (sha256) below POLITRACK_RAW_DIR when the source allows it.
|
|
|
|
Used for: conditional requests (ETag/Last-Modified), change detection (skip re-parsing identical
|
|
bodies), `politrack reparse` after parser fixes, and as an audit trail.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import gzip
|
|
import hashlib
|
|
import os
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from psycopg import Connection
|
|
from psycopg.types.json import Jsonb
|
|
|
|
from politrack.config import get_settings
|
|
|
|
|
|
class Archive:
|
|
def __init__(self, raw_dir: Path | None = None) -> None:
|
|
self.raw_dir = Path(raw_dir or get_settings().raw_dir)
|
|
|
|
# -- blobs ----------------------------------------------------------------------------
|
|
|
|
def _path(self, sha256: str) -> Path:
|
|
return self.raw_dir / sha256[:2] / f"{sha256}.gz"
|
|
|
|
def write_blob(self, body: bytes) -> str:
|
|
sha = hashlib.sha256(body).hexdigest()
|
|
path = self._path(sha)
|
|
if not path.exists():
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp = path.with_suffix(f".tmp{os.getpid()}")
|
|
with gzip.open(tmp, "wb") as fh:
|
|
fh.write(body)
|
|
tmp.replace(path)
|
|
return sha
|
|
|
|
def read_blob(self, sha256: str) -> bytes | None:
|
|
path = self._path(sha256)
|
|
if not path.exists():
|
|
return None
|
|
with gzip.open(path, "rb") as fh:
|
|
return fh.read()
|
|
|
|
# -- index ----------------------------------------------------------------------------
|
|
|
|
def last(self, conn: Connection, request_fp: str) -> dict[str, Any] | None:
|
|
return conn.execute(
|
|
"""SELECT id, sha256, etag, last_modified, stored, parsed_sha256, parser_version, status, url,
|
|
content_type, tdm_reserved, fetched_at
|
|
FROM fetch_log WHERE request_fp = %s AND status IS NOT NULL AND status < 400
|
|
ORDER BY fetched_at DESC LIMIT 1""",
|
|
(request_fp,),
|
|
).fetchone()
|
|
|
|
def log(self, conn: Connection, *, adapter: str, source_id: int | None, request_fp: str, url: str, method: str,
|
|
request: dict, status: int | None, body: bytes | None, headers: dict[str, str] | None,
|
|
keep: bool, tdm_reserved: bool, error: str | None = None) -> tuple[int, str | None]:
|
|
sha = hashlib.sha256(body).hexdigest() if body is not None else None
|
|
stored = False
|
|
if body is not None and keep and not tdm_reserved:
|
|
self.write_blob(body)
|
|
stored = True
|
|
headers = headers or {}
|
|
row = conn.execute(
|
|
"""INSERT INTO fetch_log (adapter, source_id, request_fp, url, method, request, status, sha256, size,
|
|
content_type, etag, last_modified, stored, tdm_reserved, error)
|
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id""",
|
|
(adapter, source_id, request_fp, url, method, Jsonb(request), status, sha,
|
|
len(body) if body is not None else None, headers.get("content-type"), headers.get("etag"),
|
|
headers.get("last-modified"), stored, tdm_reserved, error),
|
|
).fetchone()
|
|
return row["id"], sha
|
|
|
|
def mark_parsed(self, conn: Connection, fetch_id: int, sha256: str, parser_version: int) -> None:
|
|
conn.execute("UPDATE fetch_log SET parsed_sha256 = %s, parser_version = %s WHERE id = %s",
|
|
(sha256, parser_version, fetch_id))
|
|
|
|
def mark_error(self, conn: Connection, fetch_id: int, error: str) -> None:
|
|
conn.execute("UPDATE fetch_log SET error = %s WHERE id = %s", (error[:2000], fetch_id))
|
|
|
|
def latest_stored(self, conn: Connection, adapter: str) -> list[dict[str, Any]]:
|
|
"""Latest stored fetch per request of an adapter (input for `reparse`)."""
|
|
return conn.execute(
|
|
"""SELECT DISTINCT ON (request_fp) id, request_fp, url, method, request, status, sha256, fetched_at,
|
|
content_type, tdm_reserved
|
|
FROM fetch_log WHERE adapter = %s AND stored AND status < 400
|
|
ORDER BY request_fp, fetched_at DESC""",
|
|
(adapter,),
|
|
).fetchall()
|