Backups, freshness monitoring, persistent logs, procurement cleanup
- backup service: nightly pg_dump with retention, recorded as a run
- GET /api/v1/status/sources: 503 when a scheduled source or the backup
missed its last two slots (for Uptime Kuma)
- worker logs to ${DATA_DIR}/logs, so they survive redeploys
- procurement: reject implausible award dates (1900, contract end dates),
skip placeholder contractors, credit framework agreements to the BBG
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
79639e5319
commit
405c21a8c5
12 files changed
+251
-18
No files matched your search
@@ -14,6 +14,7 @@ fetch items whose lastmod changed. Runs on the `bulk` queue.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
@@ -139,8 +140,28 @@ def parse_award_notice(xml: bytes) -> dict[str, Any] | None:
|
||||
}
|
||||
|
||||
|
||||
PLACEHOLDER_NAME = re.compile(r"^[\W_]*$|^(?:n/?a|k\.?\s?a\.?|unbekannt|keine angabe|tbd|xxx+)$", re.IGNORECASE)
|
||||
FRAMEWORK_BUYER = re.compile(r"^Auftraggeber (?:sind|ist)\b", re.IGNORECASE)
|
||||
|
||||
|
||||
def plausible_date(value: str | None, latest: date) -> date | None:
|
||||
"""An award date, or None if it can't be one: spreadsheet placeholders (1899-12-30, 1900-01-01) and dates after
|
||||
the notice was fetched (some publishers put the contract's end date in the conclusion field)."""
|
||||
d = as_date(value)
|
||||
return d if d and date(1995, 1, 1) <= d <= latest else None
|
||||
|
||||
|
||||
def buyer_name(name: str) -> tuple[str, str | None]:
|
||||
"""Framework agreements name every eligible buyer in one sentence ("Auftraggeber sind die Republik Österreich
|
||||
(Bund), die Bundesbeschaffung GmbH sowie alle weiteren ..."). They are concluded by the federal procurement
|
||||
agency, so it is the buyer; the sentence is kept as the original wording."""
|
||||
if FRAMEWORK_BUYER.match(name) and "Bundesbeschaffung" in name:
|
||||
return "Bundesbeschaffung GmbH", name
|
||||
return name, None
|
||||
|
||||
|
||||
def org_ref(org: dict[str, Any], kind: str) -> OrgRef | None:
|
||||
if not org.get("name"):
|
||||
if not org.get("name") or PLACEHOLDER_NAME.match(org["name"].strip()):
|
||||
return None
|
||||
company_id = (org.get("company_id") or "").replace(" ", "")
|
||||
ids = {}
|
||||
@@ -167,7 +188,7 @@ class PublicContracts(Adapter):
|
||||
description = "Contract awards of Austrian public bodies (eForms and BVergG core data via data.gv.at)"
|
||||
schedule = "0 2 * * *"
|
||||
queue = "bulk"
|
||||
parser_version = 2
|
||||
parser_version = 3 # 3: implausible award dates, placeholder contractors, framework-agreement buyers
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
for query in CATALOG_QUERIES:
|
||||
@@ -230,22 +251,32 @@ class PublicContracts(Adapter):
|
||||
if notice is None:
|
||||
ctx.stats["not_an_award"] += 1
|
||||
return
|
||||
buyer = next((org_ref(b, "public_body") for b in notice["buyers"] if b.get("name")), None)
|
||||
named = next((b for b in notice["buyers"] if b.get("name")), None)
|
||||
buyer_text = None
|
||||
if named is not None:
|
||||
name, buyer_text = buyer_name(named["name"])
|
||||
named = {**named, "name": name}
|
||||
buyer = org_ref(named, "public_body") if named else None
|
||||
if buyer is None:
|
||||
return
|
||||
latest = response.fetched_at.date()
|
||||
for contract in notice["contracts"]:
|
||||
for winner in contract["winners"]:
|
||||
supplier = org_ref(winner, "company")
|
||||
if supplier is None:
|
||||
ctx.stats["contractor_unnamed"] += 1
|
||||
continue
|
||||
amount = contract["amount"] or (notice["total"] if len(notice["contracts"]) == 1 else None)
|
||||
awarded = as_date(contract["awarded"]) or as_date(notice["issued"])
|
||||
awarded = plausible_date(contract["awarded"], latest) or plausible_date(notice["issued"], latest)
|
||||
if contract["awarded"] and awarded is None:
|
||||
ctx.stats["implausible_award_date"] += 1
|
||||
title = contract["title"] or notice["title"] or "Auftrag"
|
||||
yield Claim(
|
||||
claim_type="money_flow", org=buyer, counterpart_org=supplier, occurred_at=awarded,
|
||||
summary_de=f"Auftrag „{title}“ an {supplier.name}" + (f" – {format_eur_de(amount)} €" if amount else ""),
|
||||
summary_en=f"Contract \"{title}\" awarded to {supplier.name}" + (f" – EUR {amount}" if amount else ""),
|
||||
data={"notice_id": notice["notice_id"], "cpv": notice["cpv"], "publisher": response.meta.get("publisher")},
|
||||
data={"notice_id": notice["notice_id"], "cpv": notice["cpv"], "publisher": response.meta.get("publisher"),
|
||||
**({"buyer_text": buyer_text} if buyer_text else {})},
|
||||
evidence=[Evidence(url=response.request.url, doc_type="register_entry", title=f"Vergabe: {title}",
|
||||
published_at=awarded, quote=f"{buyer.name} → {supplier.name}: {title}")],
|
||||
facet=Money(kind="contract", amount=amount, payer=buyer, payer_name=buyer.name, payee=supplier,
|
||||
|
||||
Reference in new issue
Block a user