Backups, freshness monitoring, persistent logs, procurement cleanup

- backup service: nightly pg_dump with retention, recorded as a run
- GET /api/v1/status/sources: 503 when a scheduled source or the backup
  missed its last two slots (for Uptime Kuma)
- worker logs to ${DATA_DIR}/logs, so they survive redeploys
- procurement: reject implausible award dates (1900, contract end dates),
  skip placeholder contractors, credit framework agreements to the BBG

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
simonandClaude Opus 5.5 committed 2026-11-13 00:46:18 +01:00
1 parent 79639e5319
commit 405c21a8c5
12 files changed
+251 -18

No files matched your search

+36 -5
View File
@@ -14,6 +14,7 @@ fetch items whose lastmod changed. Runs on the `bulk` queue.
from __future__ import annotations
import re
import xml.etree.ElementTree as ET
from collections.abc import Iterator
from datetime import date
@@ -139,8 +140,28 @@ def parse_award_notice(xml: bytes) -> dict[str, Any] | None:
}
PLACEHOLDER_NAME = re.compile(r"^[\W_]*$|^(?:n/?a|k\.?\s?a\.?|unbekannt|keine angabe|tbd|xxx+)$", re.IGNORECASE)
FRAMEWORK_BUYER = re.compile(r"^Auftraggeber (?:sind|ist)\b", re.IGNORECASE)
def plausible_date(value: str | None, latest: date) -> date | None:
"""An award date, or None if it can't be one: spreadsheet placeholders (1899-12-30, 1900-01-01) and dates after
the notice was fetched (some publishers put the contract's end date in the conclusion field)."""
d = as_date(value)
return d if d and date(1995, 1, 1) <= d <= latest else None
def buyer_name(name: str) -> tuple[str, str | None]:
"""Framework agreements name every eligible buyer in one sentence ("Auftraggeber sind die Republik Österreich
(Bund), die Bundesbeschaffung GmbH sowie alle weiteren ..."). They are concluded by the federal procurement
agency, so it is the buyer; the sentence is kept as the original wording."""
if FRAMEWORK_BUYER.match(name) and "Bundesbeschaffung" in name:
return "Bundesbeschaffung GmbH", name
return name, None
def org_ref(org: dict[str, Any], kind: str) -> OrgRef | None:
if not org.get("name"):
if not org.get("name") or PLACEHOLDER_NAME.match(org["name"].strip()):
return None
company_id = (org.get("company_id") or "").replace(" ", "")
ids = {}
@@ -167,7 +188,7 @@ class PublicContracts(Adapter):
description = "Contract awards of Austrian public bodies (eForms and BVergG core data via data.gv.at)"
schedule = "0 2 * * *"
queue = "bulk"
parser_version = 2
parser_version = 3 # 3: implausible award dates, placeholder contractors, framework-agreement buyers
def start(self, ctx: RunContext) -> Iterator[Request]:
for query in CATALOG_QUERIES:
@@ -230,22 +251,32 @@ class PublicContracts(Adapter):
if notice is None:
ctx.stats["not_an_award"] += 1
return
buyer = next((org_ref(b, "public_body") for b in notice["buyers"] if b.get("name")), None)
named = next((b for b in notice["buyers"] if b.get("name")), None)
buyer_text = None
if named is not None:
name, buyer_text = buyer_name(named["name"])
named = {**named, "name": name}
buyer = org_ref(named, "public_body") if named else None
if buyer is None:
return
latest = response.fetched_at.date()
for contract in notice["contracts"]:
for winner in contract["winners"]:
supplier = org_ref(winner, "company")
if supplier is None:
ctx.stats["contractor_unnamed"] += 1
continue
amount = contract["amount"] or (notice["total"] if len(notice["contracts"]) == 1 else None)
awarded = as_date(contract["awarded"]) or as_date(notice["issued"])
awarded = plausible_date(contract["awarded"], latest) or plausible_date(notice["issued"], latest)
if contract["awarded"] and awarded is None:
ctx.stats["implausible_award_date"] += 1
title = contract["title"] or notice["title"] or "Auftrag"
yield Claim(
claim_type="money_flow", org=buyer, counterpart_org=supplier, occurred_at=awarded,
summary_de=f"Auftrag „{title}“ an {supplier.name}" + (f" – {format_eur_de(amount)} €" if amount else ""),
summary_en=f"Contract \"{title}\" awarded to {supplier.name}" + (f" – EUR {amount}" if amount else ""),
data={"notice_id": notice["notice_id"], "cpv": notice["cpv"], "publisher": response.meta.get("publisher")},
data={"notice_id": notice["notice_id"], "cpv": notice["cpv"], "publisher": response.meta.get("publisher"),
**({"buyer_text": buyer_text} if buyer_text else {})},
evidence=[Evidence(url=response.request.url, doc_type="register_entry", title=f"Vergabe: {title}",
published_at=awarded, quote=f"{buyer.name} → {supplier.name}: {title}")],
facet=Money(kind="contract", amount=amount, payer=buyer, payer_name=buyer.name, payee=supplier,