Backups, freshness monitoring, persistent logs, procurement cleanup
- backup service: nightly pg_dump with retention, recorded as a run
- GET /api/v1/status/sources: 503 when a scheduled source or the backup
missed its last two slots (for Uptime Kuma)
- worker logs to ${DATA_DIR}/logs, so they survive redeploys
- procurement: reject implausible award dates (1900, contract end dates),
skip placeholder contractors, credit framework agreements to the BBG
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
79639e5319
commit
405c21a8c5
12 files changed
+251
-18
No files matched your search
@@ -12,6 +12,7 @@ from typing import Annotated
|
||||
|
||||
from fastapi import Depends, FastAPI, Header, HTTPException, Query
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
from psycopg import Connection
|
||||
|
||||
from politrack import queries
|
||||
@@ -66,6 +67,23 @@ def healthz(conn: DB) -> dict:
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
BACKUP_SCHEDULE = "30 2 * * *" # the `backup` service in docker-compose.yml logs each dump as a run
|
||||
|
||||
|
||||
@app.get("/api/v1/status/sources", include_in_schema=False)
|
||||
def source_status(conn: DB) -> JSONResponse:
|
||||
"""For monitoring (e.g. Uptime Kuma): HTTP 503 as soon as a scheduled source or the nightly backup missed its
|
||||
last two slots. Lists every task with its last successful run."""
|
||||
from politrack.core import registry
|
||||
|
||||
registry.load_builtin()
|
||||
schedules = {name: cls.schedule for name, cls in {**registry.ADAPTERS, **registry.ENRICHERS}.items() if cls.schedule}
|
||||
schedules["backup"] = BACKUP_SCHEDULE
|
||||
rows = queries.source_freshness(conn, schedules)
|
||||
stale = [r["adapter"] for r in rows if r["stale"]]
|
||||
return JSONResponse({"ok": not stale, "stale": stale, "sources": rows}, status_code=503 if stale else 200)
|
||||
|
||||
|
||||
# -- reference data -----------------------------------------------------------------------
|
||||
|
||||
@app.get("/api/v1/stats", response_model=s.Stats)
|
||||
|
||||
@@ -1,6 +1,14 @@
|
||||
"""Read queries shared by the REST API and the MCP server (one module per domain)."""
|
||||
|
||||
from politrack.queries.admin import list_sources, list_topics, pending_claims, recent_runs, set_review, stats
|
||||
from politrack.queries.admin import (
|
||||
list_sources,
|
||||
list_topics,
|
||||
pending_claims,
|
||||
recent_runs,
|
||||
set_review,
|
||||
source_freshness,
|
||||
stats,
|
||||
)
|
||||
from politrack.queries.claims import calendar, list_claims, timeline_histogram
|
||||
from politrack.queries.common import SourceFilter
|
||||
from politrack.queries.money import list_money, money_summary, social_feed, wiki_edits
|
||||
@@ -13,5 +21,5 @@ __all__ = [
|
||||
"SourceFilter", "calendar", "former_overview", "get_organization", "get_party", "get_person", "list_claims",
|
||||
"list_money", "list_organizations", "list_parties", "list_persons", "list_sources", "list_topics", "list_votes",
|
||||
"money_summary", "pending_claims", "recent_runs", "person_network", "person_votes", "search", "set_review", "social_feed",
|
||||
"stats", "timeline_histogram", "vote_detail", "wiki_edits",
|
||||
"source_freshness", "stats", "timeline_histogram", "vote_detail", "wiki_edits",
|
||||
]
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from psycopg import Connection
|
||||
|
||||
|
||||
@@ -61,6 +63,27 @@ def stats(conn: Connection) -> dict:
|
||||
).fetchone()
|
||||
|
||||
|
||||
def source_freshness(conn: Connection, schedules: dict[str, str], now: datetime | None = None) -> list[dict]:
|
||||
"""Last successful run per scheduled task. Stale = its two latest scheduled slots passed without a
|
||||
successful run (plus an hour's grace for long runs), so one slow or failed run alone never alerts."""
|
||||
from croniter import croniter
|
||||
|
||||
now = now or datetime.now(timezone.utc)
|
||||
last_ok = {r["adapter"]: r["at"] for r in conn.execute(
|
||||
"SELECT adapter, max(COALESCE(finished_at, started_at)) AS at FROM run "
|
||||
"WHERE status IN ('ok', 'partial') GROUP BY adapter")}
|
||||
rows = []
|
||||
for name, schedule in sorted(schedules.items()):
|
||||
it = croniter(schedule, now)
|
||||
due = it.get_prev(datetime)
|
||||
before = it.get_prev(datetime)
|
||||
deadline = before - timedelta(hours=1)
|
||||
last = last_ok.get(name)
|
||||
rows.append({"adapter": name, "schedule": schedule, "last_success": last.isoformat() if last else None,
|
||||
"stale": last is None or last < deadline, "latest_slot": due.isoformat()})
|
||||
return rows
|
||||
|
||||
|
||||
def recent_runs(conn: Connection, limit: int = 50) -> list[dict]:
|
||||
return conn.execute(
|
||||
"SELECT adapter, mode, status, started_at, finished_at, stats, left(error, 2000) AS error FROM run "
|
||||
|
||||
@@ -14,6 +14,7 @@ fetch items whose lastmod changed. Runs on the `bulk` queue.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
@@ -139,8 +140,28 @@ def parse_award_notice(xml: bytes) -> dict[str, Any] | None:
|
||||
}
|
||||
|
||||
|
||||
PLACEHOLDER_NAME = re.compile(r"^[\W_]*$|^(?:n/?a|k\.?\s?a\.?|unbekannt|keine angabe|tbd|xxx+)$", re.IGNORECASE)
|
||||
FRAMEWORK_BUYER = re.compile(r"^Auftraggeber (?:sind|ist)\b", re.IGNORECASE)
|
||||
|
||||
|
||||
def plausible_date(value: str | None, latest: date) -> date | None:
|
||||
"""An award date, or None if it can't be one: spreadsheet placeholders (1899-12-30, 1900-01-01) and dates after
|
||||
the notice was fetched (some publishers put the contract's end date in the conclusion field)."""
|
||||
d = as_date(value)
|
||||
return d if d and date(1995, 1, 1) <= d <= latest else None
|
||||
|
||||
|
||||
def buyer_name(name: str) -> tuple[str, str | None]:
|
||||
"""Framework agreements name every eligible buyer in one sentence ("Auftraggeber sind die Republik Österreich
|
||||
(Bund), die Bundesbeschaffung GmbH sowie alle weiteren ..."). They are concluded by the federal procurement
|
||||
agency, so it is the buyer; the sentence is kept as the original wording."""
|
||||
if FRAMEWORK_BUYER.match(name) and "Bundesbeschaffung" in name:
|
||||
return "Bundesbeschaffung GmbH", name
|
||||
return name, None
|
||||
|
||||
|
||||
def org_ref(org: dict[str, Any], kind: str) -> OrgRef | None:
|
||||
if not org.get("name"):
|
||||
if not org.get("name") or PLACEHOLDER_NAME.match(org["name"].strip()):
|
||||
return None
|
||||
company_id = (org.get("company_id") or "").replace(" ", "")
|
||||
ids = {}
|
||||
@@ -167,7 +188,7 @@ class PublicContracts(Adapter):
|
||||
description = "Contract awards of Austrian public bodies (eForms and BVergG core data via data.gv.at)"
|
||||
schedule = "0 2 * * *"
|
||||
queue = "bulk"
|
||||
parser_version = 2
|
||||
parser_version = 3 # 3: implausible award dates, placeholder contractors, framework-agreement buyers
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
for query in CATALOG_QUERIES:
|
||||
@@ -230,22 +251,32 @@ class PublicContracts(Adapter):
|
||||
if notice is None:
|
||||
ctx.stats["not_an_award"] += 1
|
||||
return
|
||||
buyer = next((org_ref(b, "public_body") for b in notice["buyers"] if b.get("name")), None)
|
||||
named = next((b for b in notice["buyers"] if b.get("name")), None)
|
||||
buyer_text = None
|
||||
if named is not None:
|
||||
name, buyer_text = buyer_name(named["name"])
|
||||
named = {**named, "name": name}
|
||||
buyer = org_ref(named, "public_body") if named else None
|
||||
if buyer is None:
|
||||
return
|
||||
latest = response.fetched_at.date()
|
||||
for contract in notice["contracts"]:
|
||||
for winner in contract["winners"]:
|
||||
supplier = org_ref(winner, "company")
|
||||
if supplier is None:
|
||||
ctx.stats["contractor_unnamed"] += 1
|
||||
continue
|
||||
amount = contract["amount"] or (notice["total"] if len(notice["contracts"]) == 1 else None)
|
||||
awarded = as_date(contract["awarded"]) or as_date(notice["issued"])
|
||||
awarded = plausible_date(contract["awarded"], latest) or plausible_date(notice["issued"], latest)
|
||||
if contract["awarded"] and awarded is None:
|
||||
ctx.stats["implausible_award_date"] += 1
|
||||
title = contract["title"] or notice["title"] or "Auftrag"
|
||||
yield Claim(
|
||||
claim_type="money_flow", org=buyer, counterpart_org=supplier, occurred_at=awarded,
|
||||
summary_de=f"Auftrag „{title}“ an {supplier.name}" + (f" – {format_eur_de(amount)} €" if amount else ""),
|
||||
summary_en=f"Contract \"{title}\" awarded to {supplier.name}" + (f" – EUR {amount}" if amount else ""),
|
||||
data={"notice_id": notice["notice_id"], "cpv": notice["cpv"], "publisher": response.meta.get("publisher")},
|
||||
data={"notice_id": notice["notice_id"], "cpv": notice["cpv"], "publisher": response.meta.get("publisher"),
|
||||
**({"buyer_text": buyer_text} if buyer_text else {})},
|
||||
evidence=[Evidence(url=response.request.url, doc_type="register_entry", title=f"Vergabe: {title}",
|
||||
published_at=awarded, quote=f"{buyer.name} → {supplier.name}: {title}")],
|
||||
facet=Money(kind="contract", amount=amount, payer=buyer, payer_name=buyer.name, payee=supplier,
|
||||
|
||||
@@ -19,7 +19,11 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from logging.handlers import RotatingFileHandler
|
||||
from pathlib import Path
|
||||
|
||||
from croniter import croniter
|
||||
from procrastinate import App, PsycopgConnector
|
||||
@@ -31,9 +35,29 @@ from politrack.core import registry
|
||||
from politrack.core.runner import run
|
||||
from politrack.db import connection
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||||
LOG_FORMAT = "%(asctime)s %(levelname)s %(name)s: %(message)s"
|
||||
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
|
||||
log = logging.getLogger("politrack.watchdog")
|
||||
|
||||
|
||||
def _log_to_file() -> None:
|
||||
"""Container logs vanish when a container is recreated (every deploy); keep a rotating copy in
|
||||
$POLITRACK_LOG_DIR/<hostname>.log so failures can still be traced afterwards."""
|
||||
log_dir = os.environ.get("POLITRACK_LOG_DIR")
|
||||
if not log_dir:
|
||||
return
|
||||
try:
|
||||
handler = RotatingFileHandler(Path(log_dir) / f"{socket.gethostname()}.log", maxBytes=20_000_000,
|
||||
backupCount=5, encoding="utf-8")
|
||||
except OSError as exc: # e.g. the host folder belongs to root: logs stay on stdout only
|
||||
log.warning("cannot write logs to %s: %s", log_dir, exc)
|
||||
return
|
||||
handler.setFormatter(logging.Formatter(LOG_FORMAT))
|
||||
logging.getLogger().addHandler(handler)
|
||||
|
||||
|
||||
_log_to_file()
|
||||
|
||||
app = App(connector=PsycopgConnector(conninfo=get_settings().database_url))
|
||||
TASKS: dict = {}
|
||||
SCHEDULES: dict[str, str] = {}
|
||||
|
||||
Reference in new issue
Block a user