Simon DiesenreiterandClaude Opus 5.5
9da5a1dc48
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
552 lines
28 KiB
Python
552 lines
28 KiB
Python
"""Rechnungshof (Court of Audit): published party donations and parties' annual financial reports.
|
||
|
||
Official works (§7 UrhG); no robots restrictions.
|
||
|
||
Donations: HTML pages + one CSV per year/election. The CSVs come in UTF-8, Windows-1252 and Mac Roman,
|
||
with or without title rows, and one older file without header (date; amount; donor; address; party).
|
||
|
||
Annual reports (Rechenschaftsberichte, 2023+ as XLSX): income statement per organisation level
|
||
(public funding "Fördermittel", membership fees, donations, sponsoring, ad income...) and named lists:
|
||
donations > EUR 500, sponsors, advertisers in party media, party-owned companies, loans, affiliated
|
||
organisations.
|
||
|
||
Data minimisation: donor addresses and postcodes are never mapped; private donors are stored as names
|
||
only (no person entity is created); company donors become organisation entities (contract/donor correlation).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import csv
|
||
import io
|
||
import re
|
||
from collections.abc import Iterator
|
||
from datetime import date, datetime
|
||
from decimal import Decimal, InvalidOperation
|
||
from typing import Any
|
||
from urllib.parse import urljoin
|
||
|
||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||
from politrack.core.records import Claim, Document, Evidence, Money, OrgRef
|
||
from politrack.text import firmenbuch_number, format_eur_de, normalize, normalize_org_name
|
||
|
||
BASE = "https://www.rechnungshof.gv.at"
|
||
SPEC = SourceSpec(
|
||
slug="rechnungshof",
|
||
name="Rechnungshof Österreich",
|
||
kind="official",
|
||
homepage=BASE,
|
||
license="Amtliches Werk (§7 UrhG)",
|
||
access="scrape",
|
||
trust=95,
|
||
min_interval=2.0,
|
||
tos_notes="Party donation lists and audited annual party reports. Official works; no robots restrictions.",
|
||
)
|
||
|
||
DONATION_INDEX = [
|
||
"/rh/home/was-wir-tun/was-wir-tun_5/Kontrolle_der_Parteien.html",
|
||
"/rh/home/was-wir-tun/was-wir-tun_5/was-wir-tun_5/Parteispenden/Parteispenden_2024.html",
|
||
]
|
||
REPORTS_INDEX = "/rh/home/was-wir-tun/was-wir-tun_5/was-wir-tun_6/Kontrolle_der_Parteien.html"
|
||
SECTION_PREFIX = "/rh/home/was-wir-tun/"
|
||
DONATION_LINK = re.compile(r"spende", re.IGNORECASE)
|
||
PARTY_PREFIXES = {"ovp": "oevp", "osterreichische volkspartei": "oevp", "die neue volkspartei": "oevp",
|
||
"spo": "spoe", "sozialdemokratische": "spoe", "fpo": "fpoe", "freiheitliche": "fpoe",
|
||
"die freiheitliche": "fpoe", "neos": "neos", "die grunen": "gruene", "grune": "gruene",
|
||
"kpo": "kpoe", "kommunistische": "kpoe", "bier": "bier", "die bierpartei": "bier", "mfg": "mfg"}
|
||
COMPANY = re.compile(r"\b(GmbH|AG|KG|OG|e\.U\.|GesmbH|Ges\.m\.b\.H|Gesellschaft|Verein|Stiftung|Bank|Holding|"
|
||
r"Genossenschaft|Verband|Kammer|Sparkasse|Versicherung|Privatstiftung)\b", re.IGNORECASE)
|
||
|
||
|
||
# -- shared helpers -----------------------------------------------------------------------
|
||
|
||
def decode(raw: bytes) -> str:
|
||
"""UTF-8, Windows-1252 or Mac Roman: pick the decoding that yields real umlauts."""
|
||
try:
|
||
return raw.decode("utf-8-sig")
|
||
except UnicodeDecodeError:
|
||
pass
|
||
candidates = [raw.decode(enc, errors="replace") for enc in ("cp1252", "mac_roman")]
|
||
return max(candidates, key=lambda t: sum(t.count(c) for c in "äöüÄÖÜß"))
|
||
|
||
|
||
def parse_amount(value: Any) -> Decimal | None:
|
||
if isinstance(value, (int, float, Decimal)):
|
||
return Decimal(str(value)).quantize(Decimal("0.01"))
|
||
value = (value or "").strip().replace(".", "").replace(",", ".").replace("€", "").strip()
|
||
try:
|
||
return Decimal(value)
|
||
except InvalidOperation:
|
||
return None
|
||
|
||
|
||
def parse_de_date(value: str) -> date | None:
|
||
for fmt in ("%d.%m.%Y", "%d.%m.%y"):
|
||
try:
|
||
return datetime.strptime((value or "").strip(), fmt).date()
|
||
except ValueError:
|
||
continue
|
||
return None
|
||
|
||
|
||
def format_de(amount: Decimal) -> str:
|
||
return format_eur_de(amount)
|
||
|
||
|
||
POSTAL_ADDRESS = re.compile(r"^\d{4}\s")
|
||
|
||
|
||
def clean_name(value: str) -> str:
|
||
"""Guard against malformed cells ('"2522 Ort Straße 4;Partei"') - an address never ends up in a name."""
|
||
value = (value or "").split(";")[-1].strip()
|
||
return "" if POSTAL_ADDRESS.match(value) else value
|
||
|
||
|
||
def party_ref(label: str) -> OrgRef | None:
|
||
if not label:
|
||
return None
|
||
norm = normalize(label)
|
||
for prefix, slug in PARTY_PREFIXES.items():
|
||
if norm.startswith(prefix):
|
||
return OrgRef(slug=slug)
|
||
return OrgRef(name=label.strip(), kind="party")
|
||
|
||
|
||
def donor_ref(name: str) -> OrgRef | None:
|
||
"""Organisations become entities (for correlations); private persons stay a plain name."""
|
||
return OrgRef(name=name, kind="company") if COMPANY.search(name) else None
|
||
|
||
|
||
def column(header: list[str], *needles: str) -> int | None:
|
||
for i, h in enumerate(header):
|
||
h = normalize(h)
|
||
if any(n in h for n in needles):
|
||
return i
|
||
return None
|
||
|
||
|
||
DATE_CELL = re.compile(r"^\d{1,2}\.\d{1,2}\.\d{2,4}$")
|
||
AMOUNT_CELL = re.compile(r"^[\d.]+(,\d{1,2})?$")
|
||
|
||
|
||
def find_columns(rows: list[list[str]]) -> tuple[int, dict[str, int | None]]:
|
||
"""Locate the header row (older files have a title row above it) and map the columns we use.
|
||
Address/postcode columns are never mapped."""
|
||
for i, row in enumerate(rows[:5]):
|
||
norm = [normalize(c) for c in row]
|
||
if any("betrag" in c for c in norm) and any("name" in c or "spender" in c for c in norm):
|
||
return i + 1, {
|
||
"party": column(row, "partei"),
|
||
"date": column(row, "eingang") if column(row, "eingang") is not None else column(row, "datum"),
|
||
"donor": column(row, "name", "spender"),
|
||
"amount": column(row, "betrag"),
|
||
"currency": column(row, "wahrung", "waehrung"),
|
||
"recipient": column(row, "empfang", "empfaeng"),
|
||
"note": column(row, "anmerk"),
|
||
}
|
||
first = next((r for r in rows if any(c.strip() for c in r)), [])
|
||
date_i = next((i for i, c in enumerate(first) if DATE_CELL.match(c.strip())), None)
|
||
amount_i = next((i for i, c in enumerate(first) if AMOUNT_CELL.match(c.strip()) and i != date_i), None)
|
||
if date_i is None or amount_i is None:
|
||
return 0, {}
|
||
return 0, {"party": len(first) - 1, "date": date_i, "donor": amount_i + 1, "amount": amount_i,
|
||
"currency": None, "recipient": len(first) - 1, "note": None}
|
||
|
||
|
||
def read_rows(text: str) -> tuple[dict[str, int | None], list[list[str]]]:
|
||
"""Parse the semicolon CSV; repairs rows broken by stray line breaks."""
|
||
reader = list(csv.reader(io.StringIO(text), delimiter=";"))
|
||
if not reader:
|
||
return {}, []
|
||
start, idx = find_columns(reader)
|
||
if not idx:
|
||
return {}, []
|
||
width = max(i for i in idx.values() if i is not None) + 1
|
||
rows, pending = [], []
|
||
for row in reader[start:]:
|
||
if pending:
|
||
pending[-1] += " " + (row[0] if row else "")
|
||
pending.extend(row[1:])
|
||
else:
|
||
pending = list(row)
|
||
if len(pending) >= width - 1:
|
||
rows.append(pending)
|
||
pending = []
|
||
return idx, [r for r in rows if any(c.strip() for c in r)]
|
||
|
||
|
||
# -- donations ----------------------------------------------------------------------------
|
||
|
||
@register
|
||
class RechnungshofDonations(Adapter):
|
||
spec = SPEC
|
||
name = "rechnungshof.donations"
|
||
description = "Published party donations (CSV per year / election)"
|
||
schedule = "30 5 * * *"
|
||
|
||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||
for path in DONATION_INDEX:
|
||
yield Request(url=urljoin(BASE, path), kind="page", meta={"depth": 0})
|
||
|
||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||
if response.request.kind == "page":
|
||
depth = response.meta.get("depth", 0)
|
||
for a in response.soup.find_all("a", href=True):
|
||
href = urljoin(response.url, a["href"])
|
||
path = href.removeprefix(BASE)
|
||
if href.lower().endswith(".csv") and DONATION_LINK.search(href):
|
||
yield Request(url=href, kind="csv", meta={"page": response.url})
|
||
elif (depth < 2 and path.startswith(SECTION_PREFIX) and path.endswith(".html")
|
||
and DONATION_LINK.search(path)):
|
||
yield Request(url=href, kind="page", meta={"depth": depth + 1})
|
||
return
|
||
yield from self.parse_csv(ctx, response)
|
||
|
||
def parse_csv(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||
idx, rows = read_rows(decode(response.body))
|
||
if idx.get("donor") is None or idx.get("amount") is None:
|
||
ctx.stats["unparsed_files"] += 1
|
||
return
|
||
|
||
def cell(row: list[str], key: str) -> str:
|
||
i = idx[key]
|
||
value = row[i].strip() if i is not None and i < len(row) else ""
|
||
return clean_name(value) if key in ("party", "donor", "recipient") else value
|
||
|
||
for n, row in enumerate(rows):
|
||
party_label, donor, recipient = cell(row, "party"), cell(row, "donor"), cell(row, "recipient")
|
||
amount, received = parse_amount(cell(row, "amount")), parse_de_date(cell(row, "date"))
|
||
if not donor or amount is None:
|
||
continue
|
||
currency = cell(row, "currency") or "EUR"
|
||
currency = "EUR" if currency.lower().startswith("eur") else currency
|
||
party = party_ref(party_label) or party_ref(recipient)
|
||
line = " | ".join(c for c in (cell(row, "date"), party_label, donor, cell(row, "amount"), currency,
|
||
recipient) if c)
|
||
yield Claim(
|
||
claim_type="donation", org=party, occurred_at=received,
|
||
summary_de=f"{recipient or party_label} erhielt {format_de(amount)} {currency} von {donor}",
|
||
summary_en=f"{recipient or party_label} received {currency} {amount:,.2f} from {donor}",
|
||
data={"party_label": party_label, "recipient": recipient, "note": cell(row, "note")},
|
||
evidence=[
|
||
Evidence(url=response.meta["page"], doc_type="register_entry",
|
||
title="Rechnungshof: veröffentlichte Parteispenden", quote=line, locator=f"row:{line}"),
|
||
Evidence(url=response.request.url, doc_type="register_entry",
|
||
title="Rechnungshof: Parteispenden (CSV)", locator=f"row:{n + 1}"),
|
||
],
|
||
topics=("Transparenz und Korruption",),
|
||
facet=Money(kind="donation", amount=amount, currency=currency, payer=donor_ref(donor), payer_name=donor,
|
||
payee=party, payee_name=recipient or party_label, paid_on=received),
|
||
)
|
||
|
||
|
||
# -- annual party reports -------------------------------------------------------------------
|
||
|
||
# Lines of the income statement (§ 5 Abs. 4/5 PartG), matched on the normalised label. Order matters:
|
||
# the first matching pattern wins ("Inserate und Werbeeinschaltungen" is an expense, "Inserate" income).
|
||
FINANCE_LINES = [
|
||
(re.compile(r"^(gesamt)?summe (der )?ertrage"), "total_income"),
|
||
(re.compile(r"^(gesamt)?summe (der )?aufwendungen"), "total_expenses"),
|
||
(re.compile(r"^fordermittel"), "public_funding"),
|
||
(re.compile(r"^mitgliedsbeitrage"), "membership_fees"),
|
||
(re.compile(r"^beitrage der .*angehorenden mandatare"), "mandate_holder_levies"),
|
||
(re.compile(r"^geldspenden"), "donations"),
|
||
(re.compile(r"^sponsoring"), "sponsoring"),
|
||
(re.compile(r"^inserate und werbeeinschaltungen"), "ad_spending"),
|
||
(re.compile(r"^inserate\b"), "ad_income"),
|
||
(re.compile(r"^ertrage aus nahestehenden organisationen"), "from_affiliates"),
|
||
(re.compile(r"^personalaufwand"), "staff_costs"),
|
||
(re.compile(r"^au(ss|ß)enwerbung"), "outdoor_advertising"),
|
||
(re.compile(r"^direktwerbung"), "direct_marketing"),
|
||
]
|
||
|
||
|
||
def finance_lines(rows: list[list[Any]], year: str) -> dict[str, Decimal]:
|
||
"""Income/expense lines of one sheet, summed over all blocks in it.
|
||
|
||
Some parties stack one block per organisation in a sheet (e.g. every state below the other); each block
|
||
has its own header row with the years. The value is taken from the reporting-year column of the latest
|
||
header (columns are "2024, 2023" in some files and "2023, 2024" in others); without a header, the first
|
||
number after the label.
|
||
"""
|
||
total: dict[str, Decimal] = {}
|
||
block: dict[str, Decimal] = {}
|
||
year_col: int | None = None
|
||
for row in rows:
|
||
years = [i for i, c in enumerate(row) if str(c).strip() in (year, f"{year}.0")]
|
||
if years and sum(1 for c in row if re.fullmatch(r"(19|20)\d\d(\.0)?", str(c).strip() if c is not None else "")) >= 1:
|
||
year_col = years[0]
|
||
for i, cell in enumerate(row):
|
||
if not isinstance(cell, str) or len(cell.strip()) < 4:
|
||
continue
|
||
label = re.sub(r"\s+", " ", normalize(cell))
|
||
key = next((k for pattern, k in FINANCE_LINES if pattern.search(label)), None)
|
||
if key:
|
||
value = number(row[year_col]) if year_col is not None and year_col > i and year_col < len(row) else None
|
||
if value is None:
|
||
value = next((number(c) for c in row[i + 1:] if number(c) is not None), None)
|
||
if value is not None:
|
||
if key in block: # the same line again: a new block (next organisation) starts
|
||
for k, v in block.items():
|
||
total[k] = total.get(k, Decimal(0)) + v
|
||
block = {}
|
||
block[key] = value
|
||
break # only the first text cell of a row is its label
|
||
for k, v in block.items():
|
||
total[k] = total.get(k, Decimal(0)) + v
|
||
return total
|
||
|
||
|
||
def sum_levels(levels: dict[str, dict[str, Decimal]]) -> tuple[dict[str, Decimal], dict[str, dict[str, Decimal]]]:
|
||
"""Sum all organisation levels (federal, states, capitals, districts ...) -> (totals, levels used).
|
||
|
||
Overview sheets that add up other levels are left out: titles listing several levels ("BO&LOs&LH-St.&NTG")
|
||
and sheets whose public funding equals the sum of all the others. Totals (income, expenses) are only
|
||
reported when every level states them - a partial sum would be misleading.
|
||
"""
|
||
levels = {t: v for t, v in levels.items() if "&" not in t}
|
||
funding = {title: v.get("public_funding", Decimal(0)) for title, v in levels.items()}
|
||
for title, value in funding.items():
|
||
others = sum(v for t, v in funding.items() if t != title)
|
||
if value > 0 and len(funding) > 2 and abs(value - others) <= Decimal(1):
|
||
levels = {t: v for t, v in levels.items() if t != title}
|
||
break
|
||
totals: dict[str, Decimal] = {}
|
||
for values in levels.values():
|
||
for key, value in values.items():
|
||
totals[key] = totals.get(key, Decimal(0)) + value
|
||
for key in ("total_income", "total_expenses"):
|
||
if key in totals and not all(key in v for v in levels.values()):
|
||
del totals[key]
|
||
return totals, levels
|
||
|
||
|
||
AFFILIATE_SKIP = re.compile(
|
||
r"^(anlage|liste|ertrag|gem\.|gemass|\(?§|name$|zvr|reg\.? ?nr|organisationseinheit|nahestehende|"
|
||
r"bundesorganisation|landesorganisation|zweigvereine:?$|[a-z]\)|[a-z]\. anlage|keine|leermeldung|rechtsform|zweck)")
|
||
ZVR = re.compile(r"\(?ZVR(?:-Zahl)?:?\s*(\d{6,10})\)?", re.IGNORECASE)
|
||
|
||
|
||
def affiliate_from_row(row: list[Any]) -> tuple[str, str | None] | None:
|
||
"""Name (and association-register number) of an affiliated organisation from one row of the list."""
|
||
cells = [str(c).strip() for c in row if c is not None and str(c).strip()]
|
||
number_cell = next((c for c in cells if re.fullmatch(r"\d{6,10}", c)), None)
|
||
for cell in cells:
|
||
if re.fullmatch(r"[\d\s./-]+", cell) or cell.lower() in ("partei", "verein", "gmbh", "stiftung"):
|
||
continue
|
||
if AFFILIATE_SKIP.search(normalize(cell)) or len(cell) <= 3:
|
||
return None
|
||
zvr = ZVR.search(cell)
|
||
name = ZVR.sub("", cell)
|
||
name = re.sub(r'"?,?\s*Kurzbezeichnung:?\s*"?([^"]+)"?\s*$', r" (\1)", name).strip(' "„“,;-')
|
||
return (name, (zvr.group(1) if zvr else number_cell)) if len(name) > 3 else None
|
||
return None
|
||
|
||
|
||
def transliterate(text: str) -> str:
|
||
"""'Kaernten Koefer Gruenen' -> 'Kärnten Köfer Grünen' (file names; not after vowels: 'Neue', 'Steuer')."""
|
||
for plain, umlaut in (("ae", "ä"), ("oe", "ö"), ("ue", "ü")):
|
||
text = re.sub(rf"(?<![aeiouqAEIOUQ]){plain}", umlaut, text)
|
||
text = re.sub(rf"(?<![aeiouqAEIOUQ]){plain.capitalize()}", umlaut.upper(), text)
|
||
return text
|
||
|
||
|
||
def report_party_name(sheets: dict[str, list[list[Any]]], url: str) -> str:
|
||
"""Cover sheet, else the 'Territoriale Gliederung ... : <party>' line, else the file name."""
|
||
cover = [str(c).strip(" -–") for row in sheets.get("Deckblatt", []) for c in row if c and str(c).strip(" -–")]
|
||
names = [c for c in cover if len(c) > 2 and not re.search(
|
||
r"(?i)rechenschaftsbericht|bericht|kalenderjahr|jahr\b|partg|gemäß|name der|politische partei$|^\d", c)]
|
||
if names:
|
||
return names[0]
|
||
for rows in sheets.values():
|
||
for row in rows:
|
||
for cell in row:
|
||
if isinstance(cell, str) and (m := re.search(r"Territoriale Gliederung[^:]*:\s*(.+)$", cell.strip())):
|
||
return m.group(1).strip()
|
||
stem = url.rsplit("/", 1)[-1].rsplit(".", 1)[0]
|
||
stem = re.sub(r"(?i)^Rechenschaftsbericht[_-]?\d{4}[_-]?|[_-](gepr(ue|ü)ft|Anlagen|Verm(oe|ö)gensaufstellung)$", "", stem)
|
||
return transliterate(re.sub(r"[_-]+", " ", stem)).strip()
|
||
|
||
|
||
def sheets_titled(sheets: dict[str, list[list[Any]]], *keywords: str) -> list[list[Any]]:
|
||
"""All rows of every sheet whose (normalised) title contains one of the keywords."""
|
||
return [row for title, rows in sheets.items() if any(k in normalize(title) for k in keywords) for row in rows]
|
||
|
||
|
||
def number(value: Any) -> Decimal | None:
|
||
if value is None or isinstance(value, bool):
|
||
return None
|
||
if isinstance(value, (int, float, Decimal)):
|
||
return Decimal(str(value)).quantize(Decimal("0.01"))
|
||
text = str(value).strip()
|
||
if re.fullmatch(r"-?\d+\.\d{1,2}", text): # '301561.19' - plain decimal point (some exports)
|
||
return Decimal(text).quantize(Decimal("0.01"))
|
||
return parse_amount(text) if re.fullmatch(r"[\d.,\s€-]+", text or "x") else None
|
||
|
||
|
||
def sheet_lists(rows: list[list[Any]], header_needles: tuple[str, ...]) -> Iterator[dict[str, Any]]:
|
||
"""Yield dict rows of every table in a sheet whose header contains all needles (case-insensitive)."""
|
||
header: list[str] | None = None
|
||
for row in rows:
|
||
cells = ["" if c is None else str(c).strip() for c in row]
|
||
lowered = [normalize(c) for c in cells]
|
||
if all(any(n in c for c in lowered) for n in header_needles):
|
||
header = lowered
|
||
continue
|
||
if header is None or not any(cells):
|
||
continue
|
||
if any("leermeldung" in c or "keine" in c for c in lowered[:2]):
|
||
header = None
|
||
continue
|
||
record = {header[i]: row[i] for i in range(min(len(header), len(row))) if header[i]}
|
||
amount = next((number(row[i]) for i in range(len(row) - 1, -1, -1) if number(row[i]) is not None), None)
|
||
if amount is not None and any(isinstance(v, str) and v.strip() for v in record.values()):
|
||
record["__amount"] = amount
|
||
yield record
|
||
|
||
|
||
def pick(record: dict[str, Any], *needles: str) -> str:
|
||
for key, value in record.items():
|
||
if any(n in key for n in needles) and isinstance(value, str) and value.strip():
|
||
return value.strip()
|
||
return ""
|
||
|
||
|
||
@register
|
||
class RechnungshofPartyReports(Adapter):
|
||
spec = SPEC
|
||
name = "rechnungshof.party_reports"
|
||
description = "Parties' annual financial reports: funding, income, sponsors, advertisers, companies, loans"
|
||
schedule = "0 6 * * 1"
|
||
parser_version = 3
|
||
|
||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||
yield Request(url=f"{BASE}{REPORTS_INDEX}", kind="index")
|
||
|
||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||
if response.request.kind == "index":
|
||
for a in response.soup.find_all("a", href=True):
|
||
href = urljoin(response.url, a["href"])
|
||
if re.search(r"Rechenschaftsbericht[_-]?\d{4}.*\.xlsx$", href, re.I):
|
||
yield Request(url=href, kind="xlsx")
|
||
elif re.search(r"(Ergebnis[_-]der[_-]Pruefung|Rechenschaftsbericht).*\.pdf$", href, re.I):
|
||
yield Document(Evidence(url=href, doc_type="pdf", title=a.get_text(" ", strip=True) or href.rsplit("/", 1)[-1],
|
||
meta={"kind": "party_report_pdf"}))
|
||
return
|
||
yield from self.parse_report(ctx, response)
|
||
|
||
def parse_report(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||
import openpyxl
|
||
|
||
wb = openpyxl.load_workbook(io.BytesIO(response.body), data_only=True, read_only=True)
|
||
sheets = {ws.title.strip(): [list(r) for r in ws.iter_rows(values_only=True)] for ws in wb.worksheets}
|
||
url = response.request.url
|
||
year_match = re.search(r"Rechenschaftsbericht[_-]?(\d{4})", url) or re.search(r"(\d{4})", url)
|
||
year = year_match.group(1) if year_match else "?"
|
||
audited = bool(re.search(r"gepr(ue|ü)ft", url, re.IGNORECASE))
|
||
party_name = report_party_name(sheets, url)
|
||
party = party_ref(party_name)
|
||
version_de = "vom Rechnungshof geprüft" if audited else "eingereicht"
|
||
evidence = [Evidence(url=url, doc_type="register_entry",
|
||
title=f"Rechenschaftsbericht {year} ({version_de}): {party_name}")]
|
||
|
||
# Income statement per organisation level (federal, states, capitals, districts...), then summed.
|
||
levels = {title: lines for title, rows in sheets.items()
|
||
if (lines := finance_lines(rows, year)) and ("public_funding" in lines or "total_income" in lines)}
|
||
totals, levels = sum_levels(levels)
|
||
if totals:
|
||
income = totals.get("total_income")
|
||
yield Claim(
|
||
claim_type="party_finance", org=party,
|
||
key=("rh-report", normalize_org_name(party_name), year, "audited" if audited else "submitted"),
|
||
valid_from=date(int(year), 1, 1) if year.isdigit() else None,
|
||
valid_to=date(int(year), 12, 31) if year.isdigit() else None,
|
||
summary_de=f"Rechenschaftsbericht {year} ({version_de}): "
|
||
+ (f"Erträge {format_de(income)} €, " if income is not None else "")
|
||
+ f"davon Fördermittel {format_de(totals.get('public_funding', Decimal(0)))} €"
|
||
+ " (Summe aller ausgewiesenen Gliederungen)",
|
||
summary_en=f"Annual report {year} ({'audited' if audited else 'submitted'}): "
|
||
+ (f"income EUR {income:,.2f}, " if income is not None else "")
|
||
+ f"public funding EUR {totals.get('public_funding', 0):,.2f}",
|
||
data={"year": year, "audited": audited, **{k: str(v) for k, v in totals.items()},
|
||
"levels": {t: {k: str(v) for k, v in lines.items()} for t, lines in levels.items()}},
|
||
evidence=evidence, topics=("Transparenz und Korruption",),
|
||
)
|
||
if totals.get("public_funding"):
|
||
yield Claim(
|
||
claim_type="money_flow", org=party, valid_from=date(int(year), 1, 1) if year.isdigit() else None,
|
||
summary_de=f"{party_name} erhielt {year} öffentliche Fördermittel von {format_de(totals['public_funding'])} €",
|
||
summary_en=f"{party_name} received EUR {totals['public_funding']:,.2f} in public funding in {year}",
|
||
evidence=evidence, topics=("Transparenz und Korruption",),
|
||
facet=Money(kind="party_funding", amount=totals["public_funding"], payee=party,
|
||
payee_name=party_name, payer_name="Öffentliche Parteienförderung", period=year),
|
||
)
|
||
else:
|
||
ctx.stats["reports_without_income_statement"] += 1
|
||
|
||
def money_claims(rows: list[list[Any]], needles: tuple[str, ...], kind: str, who_keys: tuple[str, ...],
|
||
label_de: str) -> Iterator[Claim]:
|
||
for rec in sheet_lists(rows, needles):
|
||
who = clean_name(pick(rec, *who_keys))
|
||
if not who:
|
||
continue
|
||
recipient = pick(rec, "empfanger", "empfager")
|
||
medium = pick(rec, "medium")
|
||
amount = rec["__amount"]
|
||
yield Claim(
|
||
claim_type="donation" if kind == "donation" else "money_flow", org=party,
|
||
valid_from=date(int(year), 1, 1) if year.isdigit() else None,
|
||
summary_de=f"{label_de}: {who} → {recipient or party_name}"
|
||
+ (f" ({medium})" if medium else "") + f", {format_de(amount)} € ({year})",
|
||
data={"year": year, "recipient": recipient, "medium": medium},
|
||
evidence=[Evidence(url=url, doc_type="register_entry", title=evidence[0].title,
|
||
quote=f"{who} | {recipient} | {medium} | {amount}", locator=f"{kind}:{who}:{amount}")],
|
||
topics=("Transparenz und Korruption",),
|
||
facet=Money(kind=kind, amount=amount, payer=donor_ref(who), payer_name=who, payee=party,
|
||
payee_name=recipient or party_name, period=year, category=medium or None),
|
||
)
|
||
|
||
# List sheets are named differently by every party ("Spenden", "Spendenliste (§ 6 PartG)", "Spenden,
|
||
# Sponsoring, Inserate" ...): select by title keyword, find the tables by their header.
|
||
yield from money_claims(sheets_titled(sheets, "spende"), ("empfanger", "name", "plz"), "donation", ("name",),
|
||
"Spende laut Rechenschaftsbericht")
|
||
yield from money_claims(sheets_titled(sheets, "sponsor"), ("sponsor",), "sponsorship", ("sponsor",), "Sponsoring")
|
||
yield from money_claims(sheets_titled(sheets, "inserat"), ("inserent",), "party_ad", ("inserent",),
|
||
"Inserat in Parteimedium")
|
||
for rec in sheet_lists(sheets_titled(sheets, "kredit", "darlehen", "verbindlichkeit"), ("kreditgeber",)):
|
||
lender = clean_name(pick(rec, "kreditgeber"))
|
||
if lender:
|
||
yield Claim(
|
||
claim_type="money_flow", org=party,
|
||
summary_de=f"Kredit/Darlehen von {lender} an {party_name}: {format_de(rec['__amount'])} € ({year})",
|
||
evidence=evidence, facet=Money(kind="loan", amount=rec["__amount"], payer=OrgRef(name=lender, kind="company"),
|
||
payer_name=lender, payee=party, payee_name=party_name, period=year),
|
||
)
|
||
for row in sheets_titled(sheets, "beteiligung"):
|
||
cells = [str(c).strip() for c in row if c is not None and str(c).strip()]
|
||
fn = next((f for f in (firmenbuch_number(c) for c in cells[1:3]) if f), None)
|
||
if fn:
|
||
company = re.sub(r"\s*\d\)$", "", cells[0]) # footnote markers: "Werbeagentur 1)"
|
||
yield Claim(
|
||
claim_type="ownership", org=party,
|
||
counterpart_org=OrgRef(name=company, kind="company", ids={"firmenbuch": fn}),
|
||
key=("rh-ownership", normalize_org_name(party_name), year, fn),
|
||
summary_de=f"{party_name} hält eine Beteiligung an {company} (FN {fn}) – laut Rechenschaftsbericht {year}",
|
||
evidence=evidence, topics=("Transparenz und Korruption",),
|
||
)
|
||
for title, rows in sheets.items():
|
||
# list of affiliated organisations (not the sheet of income received from them)
|
||
if "nahestehend" not in normalize(title) or "ertr" in normalize(title):
|
||
continue
|
||
for row in rows:
|
||
affiliate = affiliate_from_row(row)
|
||
if affiliate:
|
||
name, zvr = affiliate
|
||
yield Claim(
|
||
claim_type="affiliation", org=party,
|
||
counterpart_org=OrgRef(name=name, kind="ngo", ids={"zvr": zvr} if zvr else {}),
|
||
key=("rh-affiliate", normalize_org_name(party_name), year, normalize_org_name(name)),
|
||
summary_de=f"Nahestehende Organisation von {party_name}: {name} ({year})",
|
||
evidence=evidence,
|
||
)
|
||
ctx.stats["reports"] += 1
|