Politrack: track Austrian politicians from public sources
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
583ccba31c
187 files changed
+39890
No files matched your search
@@ -0,0 +1,254 @@
|
||||
"""Public procurement: contract award notices published as open "core data" (Kerndaten, §62/64 BVergG 2018).
|
||||
|
||||
Every contracting authority (or the platform it uses) publishes a "Kerndatenquelle" (KDQ): an XML index
|
||||
<kdq><item id lastmod><url>…</url></item>…</kdq> listed as a dataset on data.gv.at. Each item is one
|
||||
notice; since 2023/24 notices use the EU eForms UBL schema. We keep ContractAwardNotice documents
|
||||
(who bought what from whom, for how much) and skip calls for tenders.
|
||||
|
||||
Correlations this enables: suppliers that are also party donors, sponsors or side-job employers of
|
||||
politicians; contracts by ministries led by a given minister (organisation pages).
|
||||
|
||||
Volume: tens of thousands of notices; first run backfills from POLITRACK_PROCUREMENT_SINCE, later runs only
|
||||
fetch items whose lastmod changed. Runs on the `bulk` queue.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Evidence, Money, OrgRef
|
||||
from politrack.text import firmenbuch_number, format_eur_de
|
||||
|
||||
HUB = "https://www.data.gv.at/api/hub/search/search"
|
||||
SPEC = SourceSpec(
|
||||
slug="vergabe", name="Öffentliche Aufträge (Kerndaten, data.gv.at)", kind="official",
|
||||
homepage="https://www.data.gv.at", license="Open data (free licence required by §64 BVergG 2018)", access="bulk",
|
||||
trust=90, min_interval=1.0,
|
||||
tos_notes="Contracting authorities must publish award core data as open, machine-readable data under a free "
|
||||
"licence (BVergG 2018). We read each publisher's Kerndatenquelle (KDQ) XML and its eForms notices.",
|
||||
)
|
||||
NS = {
|
||||
"cbc": "urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2",
|
||||
"cac": "urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2",
|
||||
"efac": "http://data.europa.eu/p27/eforms-ubl-extension-aggregate-components/1",
|
||||
"efbc": "http://data.europa.eu/p27/eforms-ubl-extension-basic-components/1",
|
||||
"efext": "http://data.europa.eu/p27/eforms-ubl-extensions/1",
|
||||
"ext": "urn:oasis:names:specification:ubl:schema:xsd:CommonExtensionComponents-2",
|
||||
}
|
||||
CATALOG_QUERIES = ("Kerndaten", "Kerndatenquelle", "KDQ BVergG")
|
||||
|
||||
|
||||
def local(tag: str) -> str:
|
||||
return tag.rsplit("}", 1)[-1]
|
||||
|
||||
|
||||
def text(node: ET.Element | None, path: str) -> str | None:
|
||||
if node is None:
|
||||
return None
|
||||
found = node.find(path, NS)
|
||||
return found.text.strip() if found is not None and found.text else None
|
||||
|
||||
|
||||
def parse_kerndaten(root: ET.Element) -> dict[str, Any] | None:
|
||||
"""Austrian core-data format (BVergG 2018, schema brz.gv.at/eproc/kd): KD_8_2_* = awarded contracts.
|
||||
|
||||
Still used by many publishers (e-shops, below-threshold awards). Contractor NATIONALID is usually the
|
||||
company register number ("183286p"), occasionally the VAT id.
|
||||
"""
|
||||
if not local(root.tag).startswith("KD_8_2"):
|
||||
return None
|
||||
|
||||
def t(node: ET.Element | None, path: str) -> str | None:
|
||||
found = node.find(path) if node is not None else None
|
||||
return found.text.strip() if found is not None and found.text else None
|
||||
|
||||
buyer = root.find("{*}CONTRACTING_BODY/{*}ADDRESS_CONTRACTING_BODY")
|
||||
title = " ".join(p.text.strip() for p in root.iterfind("{*}OBJECT_CONTRACT/{*}TITLE/{*}P") if p.text)
|
||||
cpv = root.find("{*}OBJECT_CONTRACT/{*}CPV_MAIN/{*}CPV_CODE")
|
||||
contracts = []
|
||||
for awarded in root.iterfind(".//{*}AWARD_CONTRACT/{*}AWARDED_CONTRACT"):
|
||||
contracts.append({
|
||||
"id": None,
|
||||
"title": None,
|
||||
"awarded": t(awarded, "{*}DATE_CONCLUSION_CONTRACT"),
|
||||
"amount": t(awarded, "{*}VAL_TOTAL"),
|
||||
"winners": [{"name": t(c, "{*}ADDRESS_CONTRACTOR/{*}OFFICIALNAME"),
|
||||
"company_id": t(c, "{*}ADDRESS_CONTRACTOR/{*}NATIONALID")}
|
||||
for c in awarded.iterfind("{*}CONTRACTOR")],
|
||||
})
|
||||
return {
|
||||
"notice_id": t(root, "{*}EXTENSION[@KEY='Referenz']") or t(root, "{*}OBJECT_CONTRACT/{*}REFERENCE_NUMBER"),
|
||||
"issued": None,
|
||||
"title": title or None,
|
||||
"cpv": cpv.get("CODE") if cpv is not None else None,
|
||||
"total": None,
|
||||
"buyers": [{"name": t(buyer, "{*}OFFICIALNAME"), "company_id": None}] if buyer is not None else [],
|
||||
"contracts": contracts,
|
||||
}
|
||||
|
||||
|
||||
def parse_award_notice(xml: bytes) -> dict[str, Any] | None:
|
||||
"""Extract buyer, winners, amounts, title, CPV and dates from an award notice: eForms
|
||||
ContractAwardNotice, or the Austrian core-data format (parse_kerndaten)."""
|
||||
root = ET.fromstring(xml)
|
||||
if local(root.tag).startswith("KD_"):
|
||||
return parse_kerndaten(root)
|
||||
if local(root.tag) != "ContractAwardNotice":
|
||||
return None
|
||||
orgs: dict[str, dict[str, Any]] = {}
|
||||
for org in root.iterfind(".//efac:Organizations/efac:Organization/efac:Company", NS):
|
||||
org_id = text(org, "cac:PartyIdentification/cbc:ID")
|
||||
if org_id:
|
||||
orgs[org_id] = {"name": text(org, "cac:PartyName/cbc:Name"),
|
||||
"company_id": text(org, "cac:PartyLegalEntity/cbc:CompanyID"),
|
||||
"city": text(org, "cac:PostalAddress/cbc:CityName")}
|
||||
buyer_ids = [e.text for e in root.iterfind("cac:ContractingParty/cac:Party/cac:PartyIdentification/cbc:ID", NS)
|
||||
if e.text]
|
||||
result = root.find(".//efac:NoticeResult", NS)
|
||||
tender_amounts = {text(t, "cbc:ID"): text(t, "cac:LegalMonetaryTotal/cbc:PayableAmount")
|
||||
for t in (result.iterfind("efac:LotTender", NS) if result is not None else [])}
|
||||
tendering_parties = {text(tp, "cbc:ID"): [e.text for e in tp.iterfind("efac:Tenderer/cbc:ID", NS) if e.text]
|
||||
for tp in (result.iterfind("efac:TenderingParty", NS) if result is not None else [])}
|
||||
lot_tender_party = {text(t, "cbc:ID"): text(t, "efac:TenderingParty/cbc:ID")
|
||||
for t in (result.iterfind("efac:LotTender", NS) if result is not None else [])}
|
||||
contracts = []
|
||||
for contract in (result.iterfind("efac:SettledContract", NS) if result is not None else []):
|
||||
tender_id = text(contract, "efac:LotTender/cbc:ID")
|
||||
winners = tendering_parties.get(lot_tender_party.get(tender_id), [])
|
||||
contracts.append({
|
||||
"id": text(contract, "cbc:ID"),
|
||||
"title": text(contract, "cbc:Title"),
|
||||
"awarded": text(contract, "cbc:AwardDate") or text(contract, "cbc:IssueDate"),
|
||||
"amount": tender_amounts.get(tender_id),
|
||||
"winners": [orgs.get(w, {"name": None}) | {"ref": w} for w in winners],
|
||||
})
|
||||
project_title = text(root, "cac:ProcurementProject/cbc:Name")
|
||||
return {
|
||||
"notice_id": text(root, "cbc:ID"),
|
||||
"issued": text(root, "cbc:IssueDate"),
|
||||
"title": project_title,
|
||||
"cpv": text(root, "cac:ProcurementProject/cac:MainCommodityClassification/cbc:ItemClassificationCode"),
|
||||
"total": text(root, ".//efac:NoticeResult/cbc:TotalAmount"),
|
||||
"buyers": [orgs.get(b, {"name": None}) | {"ref": b} for b in buyer_ids],
|
||||
"contracts": contracts,
|
||||
}
|
||||
|
||||
|
||||
def org_ref(org: dict[str, Any], kind: str) -> OrgRef | None:
|
||||
if not org.get("name"):
|
||||
return None
|
||||
company_id = (org.get("company_id") or "").replace(" ", "")
|
||||
ids = {}
|
||||
if company_id.upper().startswith("ATU"):
|
||||
ids["uid"] = company_id.upper()
|
||||
elif fn := firmenbuch_number(company_id):
|
||||
ids["firmenbuch"] = fn
|
||||
return OrgRef(name=org["name"], kind=kind, ids=ids)
|
||||
|
||||
|
||||
def as_date(value: str | None) -> date | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return date.fromisoformat(value[:10])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
@register
|
||||
class PublicContracts(Adapter):
|
||||
spec = SPEC
|
||||
name = "vergabe.awards"
|
||||
description = "Contract awards of Austrian public bodies (eForms and BVergG core data via data.gv.at)"
|
||||
schedule = "0 2 * * *"
|
||||
queue = "bulk"
|
||||
parser_version = 2
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
for query in CATALOG_QUERIES:
|
||||
yield Request(url=HUB, params={"q": query, "limit": 100, "page": 0}, kind="catalog", conditional=False)
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
kind = response.request.kind
|
||||
if kind == "catalog":
|
||||
result = response.json().get("result") or {}
|
||||
for dataset in result.get("results") or []:
|
||||
for dist in dataset.get("distributions") or []:
|
||||
url = (dist.get("access_url") or dist.get("download_url") or [None])
|
||||
url = url[0] if isinstance(url, list) else url
|
||||
if url and url.startswith("http") and "data.gv.at/katalog" not in url:
|
||||
if ctx.params.get("only_url") and ctx.params["only_url"] not in url:
|
||||
continue # targeted runs: -p only_url=bbg.gv.at
|
||||
title = (dataset.get("title") or {}).get("de") or ""
|
||||
yield Request(url=url, kind="kdq", meta={"publisher": title.split("von", 1)[-1].strip(" '")})
|
||||
if result.get("count", 0) > (response.request.params["page"] + 1) * 100:
|
||||
yield Request(url=HUB, params={**response.request.params, "page": response.request.params["page"] + 1},
|
||||
kind="catalog", conditional=False)
|
||||
elif kind == "kdq":
|
||||
yield from self.parse_kdq(ctx, response)
|
||||
elif kind == "notice":
|
||||
yield from self.parse_notice(ctx, response)
|
||||
|
||||
def parse_kdq(self, ctx: RunContext, response: Response) -> Iterator[Request]:
|
||||
try:
|
||||
root = ET.fromstring(response.body)
|
||||
except ET.ParseError:
|
||||
ctx.stats["kdq_not_xml"] += 1
|
||||
return
|
||||
state_key = f"kdq:{response.request.url}"
|
||||
since = ctx.params.get("since") or ctx.state.get(state_key) or get_settings().procurement_since
|
||||
limit = int(ctx.params.get("max_notices_per_source", 3000))
|
||||
items = [(item.get("lastmod") or "", item.findtext("{*}url") or item.findtext("url"))
|
||||
for item in root.iter() if local(item.tag) == "item"]
|
||||
# Oldest first so an interrupted backfill resumes where it stopped (cursor = newest lastmod handled).
|
||||
items = sorted(i for i in items if i[1] and i[0] > since)[:limit]
|
||||
for lastmod, url in items:
|
||||
# lastmod in meta (not in the URL) + conditional requests: unchanged notices cost a 304 at most.
|
||||
# The cursor advances only when a notice has been parsed (see advance_cursor), so an interrupted
|
||||
# run resumes with the first notice it did not get to.
|
||||
yield Request(url=url.strip(), kind="notice", meta={"publisher": response.meta.get("publisher"),
|
||||
"lastmod": lastmod, "state_key": state_key})
|
||||
|
||||
@staticmethod
|
||||
def advance_cursor(ctx: RunContext, response: Response) -> None:
|
||||
key, lastmod = response.meta.get("state_key"), response.meta.get("lastmod")
|
||||
if key and lastmod and lastmod > (ctx.state.get(key) or ""):
|
||||
ctx.state.set(key, lastmod)
|
||||
|
||||
def parse_notice(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||||
self.advance_cursor(ctx, response) # notices arrive oldest-first per publisher
|
||||
try:
|
||||
notice = parse_award_notice(response.body)
|
||||
except ET.ParseError:
|
||||
ctx.stats["notice_not_xml"] += 1
|
||||
return
|
||||
if notice is None:
|
||||
ctx.stats["not_an_award"] += 1
|
||||
return
|
||||
buyer = next((org_ref(b, "public_body") for b in notice["buyers"] if b.get("name")), None)
|
||||
if buyer is None:
|
||||
return
|
||||
for contract in notice["contracts"]:
|
||||
for winner in contract["winners"]:
|
||||
supplier = org_ref(winner, "company")
|
||||
if supplier is None:
|
||||
continue
|
||||
amount = contract["amount"] or (notice["total"] if len(notice["contracts"]) == 1 else None)
|
||||
awarded = as_date(contract["awarded"]) or as_date(notice["issued"])
|
||||
title = contract["title"] or notice["title"] or "Auftrag"
|
||||
yield Claim(
|
||||
claim_type="money_flow", org=buyer, counterpart_org=supplier, occurred_at=awarded,
|
||||
summary_de=f"Auftrag „{title}“ an {supplier.name}" + (f" – {format_eur_de(amount)} €" if amount else ""),
|
||||
summary_en=f"Contract \"{title}\" awarded to {supplier.name}" + (f" – EUR {amount}" if amount else ""),
|
||||
data={"notice_id": notice["notice_id"], "cpv": notice["cpv"], "publisher": response.meta.get("publisher")},
|
||||
evidence=[Evidence(url=response.request.url, doc_type="register_entry", title=f"Vergabe: {title}",
|
||||
published_at=awarded, quote=f"{buyer.name} → {supplier.name}: {title}")],
|
||||
facet=Money(kind="contract", amount=amount, payer=buyer, payer_name=buyer.name, payee=supplier,
|
||||
payee_name=supplier.name, paid_on=awarded,
|
||||
reference=f"{notice['notice_id']}/{contract['id']}", category=notice["cpv"]),
|
||||
)
|
||||
Reference in new issue
Block a user