Backups, freshness monitoring, persistent logs, procurement cleanup

- backup service: nightly pg_dump with retention, recorded as a run
- GET /api/v1/status/sources: 503 when a scheduled source or the backup
  missed its last two slots (for Uptime Kuma)
- worker logs to ${DATA_DIR}/logs, so they survive redeploys
- procurement: reject implausible award dates (1900, contract end dates),
  skip placeholder contractors, credit framework agreements to the BBG

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
simonandClaude Opus 5.5 committed 2026-11-13 00:46:18 +01:00
1 parent 79639e5319
commit 405c21a8c5
12 files changed
+251 -18

No files matched your search

+18
View File
@@ -12,6 +12,7 @@ from typing import Annotated
from fastapi import Depends, FastAPI, Header, HTTPException, Query
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
from psycopg import Connection
from politrack import queries
@@ -66,6 +67,23 @@ def healthz(conn: DB) -> dict:
return {"ok": True}
BACKUP_SCHEDULE = "30 2 * * *" # the `backup` service in docker-compose.yml logs each dump as a run
@app.get("/api/v1/status/sources", include_in_schema=False)
def source_status(conn: DB) -> JSONResponse:
"""For monitoring (e.g. Uptime Kuma): HTTP 503 as soon as a scheduled source or the nightly backup missed its
last two slots. Lists every task with its last successful run."""
from politrack.core import registry
registry.load_builtin()
schedules = {name: cls.schedule for name, cls in {**registry.ADAPTERS, **registry.ENRICHERS}.items() if cls.schedule}
schedules["backup"] = BACKUP_SCHEDULE
rows = queries.source_freshness(conn, schedules)
stale = [r["adapter"] for r in rows if r["stale"]]
return JSONResponse({"ok": not stale, "stale": stale, "sources": rows}, status_code=503 if stale else 200)
# -- reference data -----------------------------------------------------------------------
@app.get("/api/v1/stats", response_model=s.Stats)
+10 -2
View File
@@ -1,6 +1,14 @@
"""Read queries shared by the REST API and the MCP server (one module per domain)."""
from politrack.queries.admin import list_sources, list_topics, pending_claims, recent_runs, set_review, stats
from politrack.queries.admin import (
list_sources,
list_topics,
pending_claims,
recent_runs,
set_review,
source_freshness,
stats,
)
from politrack.queries.claims import calendar, list_claims, timeline_histogram
from politrack.queries.common import SourceFilter
from politrack.queries.money import list_money, money_summary, social_feed, wiki_edits
@@ -13,5 +21,5 @@ __all__ = [
"SourceFilter", "calendar", "former_overview", "get_organization", "get_party", "get_person", "list_claims",
"list_money", "list_organizations", "list_parties", "list_persons", "list_sources", "list_topics", "list_votes",
"money_summary", "pending_claims", "recent_runs", "person_network", "person_votes", "search", "set_review", "social_feed",
"stats", "timeline_histogram", "vote_detail", "wiki_edits",
"source_freshness", "stats", "timeline_histogram", "vote_detail", "wiki_edits",
]
+23
View File
@@ -2,6 +2,8 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from psycopg import Connection
@@ -61,6 +63,27 @@ def stats(conn: Connection) -> dict:
).fetchone()
def source_freshness(conn: Connection, schedules: dict[str, str], now: datetime | None = None) -> list[dict]:
"""Last successful run per scheduled task. Stale = its two latest scheduled slots passed without a
successful run (plus an hour's grace for long runs), so one slow or failed run alone never alerts."""
from croniter import croniter
now = now or datetime.now(timezone.utc)
last_ok = {r["adapter"]: r["at"] for r in conn.execute(
"SELECT adapter, max(COALESCE(finished_at, started_at)) AS at FROM run "
"WHERE status IN ('ok', 'partial') GROUP BY adapter")}
rows = []
for name, schedule in sorted(schedules.items()):
it = croniter(schedule, now)
due = it.get_prev(datetime)
before = it.get_prev(datetime)
deadline = before - timedelta(hours=1)
last = last_ok.get(name)
rows.append({"adapter": name, "schedule": schedule, "last_success": last.isoformat() if last else None,
"stale": last is None or last < deadline, "latest_slot": due.isoformat()})
return rows
def recent_runs(conn: Connection, limit: int = 50) -> list[dict]:
return conn.execute(
"SELECT adapter, mode, status, started_at, finished_at, stats, left(error, 2000) AS error FROM run "
+36 -5
View File
@@ -14,6 +14,7 @@ fetch items whose lastmod changed. Runs on the `bulk` queue.
from __future__ import annotations
import re
import xml.etree.ElementTree as ET
from collections.abc import Iterator
from datetime import date
@@ -139,8 +140,28 @@ def parse_award_notice(xml: bytes) -> dict[str, Any] | None:
}
PLACEHOLDER_NAME = re.compile(r"^[\W_]*$|^(?:n/?a|k\.?\s?a\.?|unbekannt|keine angabe|tbd|xxx+)$", re.IGNORECASE)
FRAMEWORK_BUYER = re.compile(r"^Auftraggeber (?:sind|ist)\b", re.IGNORECASE)
def plausible_date(value: str | None, latest: date) -> date | None:
"""An award date, or None if it can't be one: spreadsheet placeholders (1899-12-30, 1900-01-01) and dates after
the notice was fetched (some publishers put the contract's end date in the conclusion field)."""
d = as_date(value)
return d if d and date(1995, 1, 1) <= d <= latest else None
def buyer_name(name: str) -> tuple[str, str | None]:
"""Framework agreements name every eligible buyer in one sentence ("Auftraggeber sind die Republik Österreich
(Bund), die Bundesbeschaffung GmbH sowie alle weiteren ..."). They are concluded by the federal procurement
agency, so it is the buyer; the sentence is kept as the original wording."""
if FRAMEWORK_BUYER.match(name) and "Bundesbeschaffung" in name:
return "Bundesbeschaffung GmbH", name
return name, None
def org_ref(org: dict[str, Any], kind: str) -> OrgRef | None:
if not org.get("name"):
if not org.get("name") or PLACEHOLDER_NAME.match(org["name"].strip()):
return None
company_id = (org.get("company_id") or "").replace(" ", "")
ids = {}
@@ -167,7 +188,7 @@ class PublicContracts(Adapter):
description = "Contract awards of Austrian public bodies (eForms and BVergG core data via data.gv.at)"
schedule = "0 2 * * *"
queue = "bulk"
parser_version = 2
parser_version = 3 # 3: implausible award dates, placeholder contractors, framework-agreement buyers
def start(self, ctx: RunContext) -> Iterator[Request]:
for query in CATALOG_QUERIES:
@@ -230,22 +251,32 @@ class PublicContracts(Adapter):
if notice is None:
ctx.stats["not_an_award"] += 1
return
buyer = next((org_ref(b, "public_body") for b in notice["buyers"] if b.get("name")), None)
named = next((b for b in notice["buyers"] if b.get("name")), None)
buyer_text = None
if named is not None:
name, buyer_text = buyer_name(named["name"])
named = {**named, "name": name}
buyer = org_ref(named, "public_body") if named else None
if buyer is None:
return
latest = response.fetched_at.date()
for contract in notice["contracts"]:
for winner in contract["winners"]:
supplier = org_ref(winner, "company")
if supplier is None:
ctx.stats["contractor_unnamed"] += 1
continue
amount = contract["amount"] or (notice["total"] if len(notice["contracts"]) == 1 else None)
awarded = as_date(contract["awarded"]) or as_date(notice["issued"])
awarded = plausible_date(contract["awarded"], latest) or plausible_date(notice["issued"], latest)
if contract["awarded"] and awarded is None:
ctx.stats["implausible_award_date"] += 1
title = contract["title"] or notice["title"] or "Auftrag"
yield Claim(
claim_type="money_flow", org=buyer, counterpart_org=supplier, occurred_at=awarded,
summary_de=f"Auftrag „{title}“ an {supplier.name}" + (f" – {format_eur_de(amount)} €" if amount else ""),
summary_en=f"Contract \"{title}\" awarded to {supplier.name}" + (f" – EUR {amount}" if amount else ""),
data={"notice_id": notice["notice_id"], "cpv": notice["cpv"], "publisher": response.meta.get("publisher")},
data={"notice_id": notice["notice_id"], "cpv": notice["cpv"], "publisher": response.meta.get("publisher"),
**({"buyer_text": buyer_text} if buyer_text else {})},
evidence=[Evidence(url=response.request.url, doc_type="register_entry", title=f"Vergabe: {title}",
published_at=awarded, quote=f"{buyer.name} → {supplier.name}: {title}")],
facet=Money(kind="contract", amount=amount, payer=buyer, payer_name=buyer.name, payee=supplier,
+25 -1
View File
@@ -19,7 +19,11 @@ from __future__ import annotations
import asyncio
import logging
import os
import socket
from datetime import datetime, timedelta, timezone
from logging.handlers import RotatingFileHandler
from pathlib import Path
from croniter import croniter
from procrastinate import App, PsycopgConnector
@@ -31,9 +35,29 @@ from politrack.core import registry
from politrack.core.runner import run
from politrack.db import connection
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
LOG_FORMAT = "%(asctime)s %(levelname)s %(name)s: %(message)s"
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
log = logging.getLogger("politrack.watchdog")
def _log_to_file() -> None:
"""Container logs vanish when a container is recreated (every deploy); keep a rotating copy in
$POLITRACK_LOG_DIR/<hostname>.log so failures can still be traced afterwards."""
log_dir = os.environ.get("POLITRACK_LOG_DIR")
if not log_dir:
return
try:
handler = RotatingFileHandler(Path(log_dir) / f"{socket.gethostname()}.log", maxBytes=20_000_000,
backupCount=5, encoding="utf-8")
except OSError as exc: # e.g. the host folder belongs to root: logs stay on stdout only
log.warning("cannot write logs to %s: %s", log_dir, exc)
return
handler.setFormatter(logging.Formatter(LOG_FORMAT))
logging.getLogger().addHandler(handler)
_log_to_file()
app = App(connector=PsycopgConnector(conninfo=get_settings().database_url))
TASKS: dict = {}
SCHEDULES: dict[str, str] = {}