Politrack: track Austrian politicians from public sources

Initial import of Politrack, a self-hosted service that tracks Austrian
politicians from federal to municipal level - roles, votes, money flows,
statements, meetings, social media and legal proceedings - with every claim
linked to its sources.

- backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse ->
  load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive
  for reparsing, and a claim/evidence model with corroboration
- sources: Parliament, Wikidata, EU Parliament/Commission, Council of
  Ministers, Rechnungshof party reports and donations, procurement, RTR media
  transparency, political ad libraries, WKStA, news feeds, Wikipedia edits,
  Bluesky, Mastodon, public Telegram channels, YouTube and X (official API)
- web: Next.js frontend in German and English with typed API client
- mcp: MCP server for AI agents and RAG
- docs: architecture, data model, sources, legal notes, Unraid deployment and
  contributor guides; Docker Compose setup for Unraid

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
simonandClaude Opus 5.5 committed 2026-10-28 20:02:01 +01:00
commit 583ccba31c
187 files changed
+39890

No files matched your search

+254
View File
@@ -0,0 +1,254 @@
"""Public procurement: contract award notices published as open "core data" (Kerndaten, §62/64 BVergG 2018).
Every contracting authority (or the platform it uses) publishes a "Kerndatenquelle" (KDQ): an XML index
<kdq><item id lastmod><url>…</url></item>…</kdq> listed as a dataset on data.gv.at. Each item is one
notice; since 2023/24 notices use the EU eForms UBL schema. We keep ContractAwardNotice documents
(who bought what from whom, for how much) and skip calls for tenders.
Correlations this enables: suppliers that are also party donors, sponsors or side-job employers of
politicians; contracts by ministries led by a given minister (organisation pages).
Volume: tens of thousands of notices; first run backfills from POLITRACK_PROCUREMENT_SINCE, later runs only
fetch items whose lastmod changed. Runs on the `bulk` queue.
"""
from __future__ import annotations
import xml.etree.ElementTree as ET
from collections.abc import Iterator
from datetime import date
from typing import Any
from politrack.config import get_settings
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Evidence, Money, OrgRef
from politrack.text import firmenbuch_number, format_eur_de
HUB = "https://www.data.gv.at/api/hub/search/search"
SPEC = SourceSpec(
slug="vergabe", name="Öffentliche Aufträge (Kerndaten, data.gv.at)", kind="official",
homepage="https://www.data.gv.at", license="Open data (free licence required by §64 BVergG 2018)", access="bulk",
trust=90, min_interval=1.0,
tos_notes="Contracting authorities must publish award core data as open, machine-readable data under a free "
"licence (BVergG 2018). We read each publisher's Kerndatenquelle (KDQ) XML and its eForms notices.",
)
NS = {
"cbc": "urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2",
"cac": "urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2",
"efac": "http://data.europa.eu/p27/eforms-ubl-extension-aggregate-components/1",
"efbc": "http://data.europa.eu/p27/eforms-ubl-extension-basic-components/1",
"efext": "http://data.europa.eu/p27/eforms-ubl-extensions/1",
"ext": "urn:oasis:names:specification:ubl:schema:xsd:CommonExtensionComponents-2",
}
CATALOG_QUERIES = ("Kerndaten", "Kerndatenquelle", "KDQ BVergG")
def local(tag: str) -> str:
return tag.rsplit("}", 1)[-1]
def text(node: ET.Element | None, path: str) -> str | None:
if node is None:
return None
found = node.find(path, NS)
return found.text.strip() if found is not None and found.text else None
def parse_kerndaten(root: ET.Element) -> dict[str, Any] | None:
"""Austrian core-data format (BVergG 2018, schema brz.gv.at/eproc/kd): KD_8_2_* = awarded contracts.
Still used by many publishers (e-shops, below-threshold awards). Contractor NATIONALID is usually the
company register number ("183286p"), occasionally the VAT id.
"""
if not local(root.tag).startswith("KD_8_2"):
return None
def t(node: ET.Element | None, path: str) -> str | None:
found = node.find(path) if node is not None else None
return found.text.strip() if found is not None and found.text else None
buyer = root.find("{*}CONTRACTING_BODY/{*}ADDRESS_CONTRACTING_BODY")
title = " ".join(p.text.strip() for p in root.iterfind("{*}OBJECT_CONTRACT/{*}TITLE/{*}P") if p.text)
cpv = root.find("{*}OBJECT_CONTRACT/{*}CPV_MAIN/{*}CPV_CODE")
contracts = []
for awarded in root.iterfind(".//{*}AWARD_CONTRACT/{*}AWARDED_CONTRACT"):
contracts.append({
"id": None,
"title": None,
"awarded": t(awarded, "{*}DATE_CONCLUSION_CONTRACT"),
"amount": t(awarded, "{*}VAL_TOTAL"),
"winners": [{"name": t(c, "{*}ADDRESS_CONTRACTOR/{*}OFFICIALNAME"),
"company_id": t(c, "{*}ADDRESS_CONTRACTOR/{*}NATIONALID")}
for c in awarded.iterfind("{*}CONTRACTOR")],
})
return {
"notice_id": t(root, "{*}EXTENSION[@KEY='Referenz']") or t(root, "{*}OBJECT_CONTRACT/{*}REFERENCE_NUMBER"),
"issued": None,
"title": title or None,
"cpv": cpv.get("CODE") if cpv is not None else None,
"total": None,
"buyers": [{"name": t(buyer, "{*}OFFICIALNAME"), "company_id": None}] if buyer is not None else [],
"contracts": contracts,
}
def parse_award_notice(xml: bytes) -> dict[str, Any] | None:
"""Extract buyer, winners, amounts, title, CPV and dates from an award notice: eForms
ContractAwardNotice, or the Austrian core-data format (parse_kerndaten)."""
root = ET.fromstring(xml)
if local(root.tag).startswith("KD_"):
return parse_kerndaten(root)
if local(root.tag) != "ContractAwardNotice":
return None
orgs: dict[str, dict[str, Any]] = {}
for org in root.iterfind(".//efac:Organizations/efac:Organization/efac:Company", NS):
org_id = text(org, "cac:PartyIdentification/cbc:ID")
if org_id:
orgs[org_id] = {"name": text(org, "cac:PartyName/cbc:Name"),
"company_id": text(org, "cac:PartyLegalEntity/cbc:CompanyID"),
"city": text(org, "cac:PostalAddress/cbc:CityName")}
buyer_ids = [e.text for e in root.iterfind("cac:ContractingParty/cac:Party/cac:PartyIdentification/cbc:ID", NS)
if e.text]
result = root.find(".//efac:NoticeResult", NS)
tender_amounts = {text(t, "cbc:ID"): text(t, "cac:LegalMonetaryTotal/cbc:PayableAmount")
for t in (result.iterfind("efac:LotTender", NS) if result is not None else [])}
tendering_parties = {text(tp, "cbc:ID"): [e.text for e in tp.iterfind("efac:Tenderer/cbc:ID", NS) if e.text]
for tp in (result.iterfind("efac:TenderingParty", NS) if result is not None else [])}
lot_tender_party = {text(t, "cbc:ID"): text(t, "efac:TenderingParty/cbc:ID")
for t in (result.iterfind("efac:LotTender", NS) if result is not None else [])}
contracts = []
for contract in (result.iterfind("efac:SettledContract", NS) if result is not None else []):
tender_id = text(contract, "efac:LotTender/cbc:ID")
winners = tendering_parties.get(lot_tender_party.get(tender_id), [])
contracts.append({
"id": text(contract, "cbc:ID"),
"title": text(contract, "cbc:Title"),
"awarded": text(contract, "cbc:AwardDate") or text(contract, "cbc:IssueDate"),
"amount": tender_amounts.get(tender_id),
"winners": [orgs.get(w, {"name": None}) | {"ref": w} for w in winners],
})
project_title = text(root, "cac:ProcurementProject/cbc:Name")
return {
"notice_id": text(root, "cbc:ID"),
"issued": text(root, "cbc:IssueDate"),
"title": project_title,
"cpv": text(root, "cac:ProcurementProject/cac:MainCommodityClassification/cbc:ItemClassificationCode"),
"total": text(root, ".//efac:NoticeResult/cbc:TotalAmount"),
"buyers": [orgs.get(b, {"name": None}) | {"ref": b} for b in buyer_ids],
"contracts": contracts,
}
def org_ref(org: dict[str, Any], kind: str) -> OrgRef | None:
if not org.get("name"):
return None
company_id = (org.get("company_id") or "").replace(" ", "")
ids = {}
if company_id.upper().startswith("ATU"):
ids["uid"] = company_id.upper()
elif fn := firmenbuch_number(company_id):
ids["firmenbuch"] = fn
return OrgRef(name=org["name"], kind=kind, ids=ids)
def as_date(value: str | None) -> date | None:
if not value:
return None
try:
return date.fromisoformat(value[:10])
except ValueError:
return None
@register
class PublicContracts(Adapter):
spec = SPEC
name = "vergabe.awards"
description = "Contract awards of Austrian public bodies (eForms and BVergG core data via data.gv.at)"
schedule = "0 2 * * *"
queue = "bulk"
parser_version = 2
def start(self, ctx: RunContext) -> Iterator[Request]:
for query in CATALOG_QUERIES:
yield Request(url=HUB, params={"q": query, "limit": 100, "page": 0}, kind="catalog", conditional=False)
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
kind = response.request.kind
if kind == "catalog":
result = response.json().get("result") or {}
for dataset in result.get("results") or []:
for dist in dataset.get("distributions") or []:
url = (dist.get("access_url") or dist.get("download_url") or [None])
url = url[0] if isinstance(url, list) else url
if url and url.startswith("http") and "data.gv.at/katalog" not in url:
if ctx.params.get("only_url") and ctx.params["only_url"] not in url:
continue # targeted runs: -p only_url=bbg.gv.at
title = (dataset.get("title") or {}).get("de") or ""
yield Request(url=url, kind="kdq", meta={"publisher": title.split("von", 1)[-1].strip(" '")})
if result.get("count", 0) > (response.request.params["page"] + 1) * 100:
yield Request(url=HUB, params={**response.request.params, "page": response.request.params["page"] + 1},
kind="catalog", conditional=False)
elif kind == "kdq":
yield from self.parse_kdq(ctx, response)
elif kind == "notice":
yield from self.parse_notice(ctx, response)
def parse_kdq(self, ctx: RunContext, response: Response) -> Iterator[Request]:
try:
root = ET.fromstring(response.body)
except ET.ParseError:
ctx.stats["kdq_not_xml"] += 1
return
state_key = f"kdq:{response.request.url}"
since = ctx.params.get("since") or ctx.state.get(state_key) or get_settings().procurement_since
limit = int(ctx.params.get("max_notices_per_source", 3000))
items = [(item.get("lastmod") or "", item.findtext("{*}url") or item.findtext("url"))
for item in root.iter() if local(item.tag) == "item"]
# Oldest first so an interrupted backfill resumes where it stopped (cursor = newest lastmod handled).
items = sorted(i for i in items if i[1] and i[0] > since)[:limit]
for lastmod, url in items:
# lastmod in meta (not in the URL) + conditional requests: unchanged notices cost a 304 at most.
# The cursor advances only when a notice has been parsed (see advance_cursor), so an interrupted
# run resumes with the first notice it did not get to.
yield Request(url=url.strip(), kind="notice", meta={"publisher": response.meta.get("publisher"),
"lastmod": lastmod, "state_key": state_key})
@staticmethod
def advance_cursor(ctx: RunContext, response: Response) -> None:
key, lastmod = response.meta.get("state_key"), response.meta.get("lastmod")
if key and lastmod and lastmod > (ctx.state.get(key) or ""):
ctx.state.set(key, lastmod)
def parse_notice(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
self.advance_cursor(ctx, response) # notices arrive oldest-first per publisher
try:
notice = parse_award_notice(response.body)
except ET.ParseError:
ctx.stats["notice_not_xml"] += 1
return
if notice is None:
ctx.stats["not_an_award"] += 1
return
buyer = next((org_ref(b, "public_body") for b in notice["buyers"] if b.get("name")), None)
if buyer is None:
return
for contract in notice["contracts"]:
for winner in contract["winners"]:
supplier = org_ref(winner, "company")
if supplier is None:
continue
amount = contract["amount"] or (notice["total"] if len(notice["contracts"]) == 1 else None)
awarded = as_date(contract["awarded"]) or as_date(notice["issued"])
title = contract["title"] or notice["title"] or "Auftrag"
yield Claim(
claim_type="money_flow", org=buyer, counterpart_org=supplier, occurred_at=awarded,
summary_de=f"Auftrag „{title}“ an {supplier.name}" + (f" – {format_eur_de(amount)} €" if amount else ""),
summary_en=f"Contract \"{title}\" awarded to {supplier.name}" + (f" – EUR {amount}" if amount else ""),
data={"notice_id": notice["notice_id"], "cpv": notice["cpv"], "publisher": response.meta.get("publisher")},
evidence=[Evidence(url=response.request.url, doc_type="register_entry", title=f"Vergabe: {title}",
published_at=awarded, quote=f"{buyer.name} → {supplier.name}: {title}")],
facet=Money(kind="contract", amount=amount, payer=buyer, payer_name=buyer.name, payee=supplier,
payee_name=supplier.name, paid_on=awarded,
reference=f"{notice['notice_id']}/{contract['id']}", category=notice["cpv"]),
)