"""Raw archive: every fetch is logged in the `fetch_log` table; bodies are stored gzip-compressed and content-addressed (sha256) below POLITRACK_RAW_DIR when the source allows it. Used for: conditional requests (ETag/Last-Modified), change detection (skip re-parsing identical bodies), `politrack reparse` after parser fixes, and as an audit trail. """ from __future__ import annotations import gzip import hashlib import os from pathlib import Path from typing import Any from psycopg import Connection from psycopg.types.json import Jsonb from politrack.config import get_settings class Archive: def __init__(self, raw_dir: Path | None = None) -> None: self.raw_dir = Path(raw_dir or get_settings().raw_dir) # -- blobs ---------------------------------------------------------------------------- def _path(self, sha256: str) -> Path: return self.raw_dir / sha256[:2] / f"{sha256}.gz" def write_blob(self, body: bytes) -> str: sha = hashlib.sha256(body).hexdigest() path = self._path(sha) if not path.exists(): path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_suffix(f".tmp{os.getpid()}") with gzip.open(tmp, "wb") as fh: fh.write(body) tmp.replace(path) return sha def read_blob(self, sha256: str) -> bytes | None: path = self._path(sha256) if not path.exists(): return None with gzip.open(path, "rb") as fh: return fh.read() # -- index ---------------------------------------------------------------------------- def last(self, conn: Connection, request_fp: str) -> dict[str, Any] | None: return conn.execute( """SELECT id, sha256, etag, last_modified, stored, parsed_sha256, parser_version, status, url, content_type, tdm_reserved, fetched_at FROM fetch_log WHERE request_fp = %s AND status IS NOT NULL AND status < 400 ORDER BY fetched_at DESC LIMIT 1""", (request_fp,), ).fetchone() def log(self, conn: Connection, *, adapter: str, source_id: int | None, request_fp: str, url: str, method: str, request: dict, status: int | None, body: bytes | None, headers: dict[str, str] | None, keep: bool, tdm_reserved: bool, error: str | None = None) -> tuple[int, str | None]: sha = hashlib.sha256(body).hexdigest() if body is not None else None stored = False if body is not None and keep and not tdm_reserved: self.write_blob(body) stored = True headers = headers or {} row = conn.execute( """INSERT INTO fetch_log (adapter, source_id, request_fp, url, method, request, status, sha256, size, content_type, etag, last_modified, stored, tdm_reserved, error) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id""", (adapter, source_id, request_fp, url, method, Jsonb(request), status, sha, len(body) if body is not None else None, headers.get("content-type"), headers.get("etag"), headers.get("last-modified"), stored, tdm_reserved, error), ).fetchone() return row["id"], sha def mark_parsed(self, conn: Connection, fetch_id: int, sha256: str, parser_version: int) -> None: conn.execute("UPDATE fetch_log SET parsed_sha256 = %s, parser_version = %s WHERE id = %s", (sha256, parser_version, fetch_id)) def mark_error(self, conn: Connection, fetch_id: int, error: str) -> None: conn.execute("UPDATE fetch_log SET error = %s WHERE id = %s", (error[:2000], fetch_id)) def latest_stored(self, conn: Connection, adapter: str) -> list[dict[str, Any]]: """Latest stored fetch per request of an adapter (input for `reparse`).""" return conn.execute( """SELECT DISTINCT ON (request_fp) id, request_fp, url, method, request, status, sha256, fetched_at, content_type, tdm_reserved FROM fetch_log WHERE adapter = %s AND stored AND status < 400 ORDER BY request_fp, fetched_at DESC""", (adapter,), ).fetchall()