- europarl.meps also loads the Austrian MEPs of the 2019-2024 term; roles end with the last membership, so former MEPs don't show as current - bmi.elections: National Council 2017 and EU 2019 preference votes - eu.commission_meetings: the 2014-2019 Commission - social.bluesky / social.mastodon: -p backfill_pages=N pages older posts Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
236 lines
13 KiB
Python
236 lines
13 KiB
Python
"""Official election results from the Federal Ministry of the Interior (BMI, Bundeswahlbehörde).
|
||
|
||
Per election: the final result (XLSX, one row per region with votes per party) and the preference votes of the
|
||
candidates (XLSX, per list). Only the federal and state totals of the result are kept; municipal rows stay in the
|
||
raw archive. Candidates become claims only if they resolve to a known politician: by exact name plus birth year
|
||
where the file has one, otherwise by exact, unambiguous name. Unknown candidates are counted, never stored, and
|
||
their occupation is never kept.
|
||
|
||
Not covered yet: the 2019 state and regional lists (a print layout with side-by-side columns), elections before
|
||
2019, Landtag and municipal elections (published by the states).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import io
|
||
import re
|
||
from collections.abc import Iterator
|
||
from dataclasses import dataclass
|
||
from datetime import date
|
||
from typing import Any
|
||
|
||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||
from politrack.core.records import Claim, Evidence, OrgRef, PersonRef
|
||
from politrack.text import strip_titles
|
||
|
||
BMI = "https://www.bmi.gv.at/412"
|
||
SPEC = SourceSpec(
|
||
slug="bmi-wahlen", name="Bundesministerium für Inneres – Wahlergebnisse", kind="official",
|
||
homepage=f"{BMI}/start.aspx", license="Amtliches Werk (§7 UrhG); Ergebnisse auch als CC BY 4.0 auf data.gv.at",
|
||
access="bulk", trust=95, min_interval=3.0,
|
||
tos_notes="Final results and preference votes as published by the Bundeswahlbehörde (XLSX). robots.txt sets "
|
||
"no rules. Only candidates who resolve to a known politician are stored; occupations are never kept.",
|
||
)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Election:
|
||
key: str
|
||
name_de: str
|
||
name_en: str
|
||
held_on: date
|
||
body: str # slug of the elected body
|
||
path: str # folder on bmi.gv.at
|
||
result: str | None # None: only in a format we can't read (EU 2019: legacy .xls)
|
||
preferences: tuple[tuple[str, str], ...] # (layout, file): layouts "bund", "land", "eu"
|
||
|
||
|
||
ELECTIONS = (
|
||
Election("nrw2024", "Nationalratswahl 2024", "2024 National Council election", date(2024, 9, 29), "nationalrat",
|
||
"nationalratswahlen/nationalratswahl_2024/files", "endgueltiges_ergebnis_beschluss_bundeswahlbehoerde_16102024.xlsx",
|
||
(("bund", "nrw2024_aufstellung_der_vorzugsstimmen_der_bundesbewerber_16102024.xlsx"),
|
||
("land", "nrw2024_aufstellung_der_vorzugsstimmen_der_landes_undregionalbewerber_16102024.xlsx"))),
|
||
Election("nrw2019", "Nationalratswahl 2019", "2019 National Council election", date(2019, 9, 29), "nationalrat",
|
||
"nationalratswahlen/nationalratswahl_2019/files", "endgultiges_gesamtergebnis_nrw19_16102019.xlsx",
|
||
(("bund", "nrw19_vorzugsstimmen_bundesweit_16102019.xlsx"),)),
|
||
Election("euw2024", "Europawahl 2024", "2024 European Parliament election", date(2024, 6, 9),
|
||
"europaeisches-parlament", "europawahlen/europawahl_2024/files", "endgueltiges_ergebnis_europawahl_26062024.xlsx",
|
||
(("eu", "gesamtaufstellung_erzielte_vorzugsstimmen_eu_wahl_2024_parteien.xlsx"),)),
|
||
Election("nrw2017", "Nationalratswahl 2017", "2017 National Council election", date(2017, 10, 15), "nationalrat",
|
||
"nationalratswahlen/nationalratswahl_2017/files", "nrw17_endgueltiges_gesamtergebnis.xlsx",
|
||
(("bund", "nrw17_vorzugsstimmen_bundesweit.xlsx"),)),
|
||
Election("euw2019", "Europawahl 2019", "2019 European Parliament election", date(2019, 5, 26),
|
||
"europaeisches-parlament", "europawahlen/europawahl_2019/files/endgueltiges_ergebnis", None,
|
||
(("eu", "euw_2019_vorzugsstimmen_gesamthomepage.xlsx"),)),
|
||
)
|
||
BY_KEY = {e.key: e for e in ELECTIONS}
|
||
|
||
REGION_CODE = re.compile(r"^G\d0000$") # G00000 Österreich, G10000 Burgenland ... G90000 Wien
|
||
PARTY_HEAD = re.compile(r"\(([^()]+)\)\s*$")
|
||
LIST_HEAD = re.compile(r"^(Landes|Regional)parteilisten im (?:Landes|Regional)wahlkreis\s+(.+?)\s*$")
|
||
|
||
|
||
def de_int(n: int) -> str:
|
||
return f"{n:,}".replace(",", ".")
|
||
|
||
|
||
def de_pct(p: float) -> str:
|
||
return f"{p:.2f}".replace(".", ",")
|
||
|
||
|
||
def sheets(body: bytes) -> list[tuple[str, list[tuple]]]:
|
||
import openpyxl
|
||
|
||
wb = openpyxl.load_workbook(io.BytesIO(body), read_only=True, data_only=True)
|
||
return [(ws.title, list(ws.iter_rows(values_only=True))) for ws in wb.worksheets]
|
||
|
||
|
||
def label(*cells: Any) -> str:
|
||
return " ".join(str(c) for c in cells if c not in (None, "")).replace("\n", "").strip().lower()
|
||
|
||
|
||
def parse_results(rows: list[tuple]) -> list[dict[str, Any]]:
|
||
"""Votes per party for Austria and the nine states. The header has party names in the first row, either
|
||
with the totals in the same row (2024) or split over two rows ("Stimmen" / "gültige", 2019 and EU 2024)."""
|
||
head, sub = rows[0], rows[1] if rows[1] and rows[1][0] is None else ()
|
||
labels = [label(head[i], sub[i] if i < len(sub) else None) for i in range(len(head))]
|
||
valid_col = next(i for i, text in enumerate(labels) if text.endswith("gültige") and "ungültige" not in text)
|
||
parties = [(i, str(name).strip()) for i, name in enumerate(head)
|
||
if i > valid_col and isinstance(name, str) and name.strip() not in ("", "%")]
|
||
out = []
|
||
for row in rows:
|
||
code = row[0]
|
||
if not (isinstance(code, str) and REGION_CODE.match(code)):
|
||
continue
|
||
valid = int(row[valid_col] or 0)
|
||
for col, party in parties:
|
||
votes = int(row[col] or 0)
|
||
if not votes:
|
||
continue
|
||
# the share as published (next column), so the figures match the official result exactly
|
||
published = row[col + 1] if col + 1 < len(row) and head[col + 1] == "%" else None
|
||
percent = (round(float(published), 2) if isinstance(published, (int, float))
|
||
else round(votes * 100 / valid, 2) if valid else None)
|
||
out.append({"code": code, "region": str(row[1]).strip(), "electorate": int(row[2] or 0),
|
||
"valid": valid, "party": party, "votes": votes, "percent": percent})
|
||
return out
|
||
|
||
|
||
def split_name(text: str) -> tuple[str, str]:
|
||
"""'Brunner, Dr. Magnus' -> ('Brunner', 'Dr. Magnus'); 'Babler, MSc, Andreas' -> ('Babler', 'Andreas')."""
|
||
family, _, rest = str(text).partition(",")
|
||
parts = [p.strip() for p in rest.split(",")]
|
||
return family.strip(), next((p for p in parts if strip_titles(p)), "")
|
||
|
||
|
||
def parse_preferences(layout: str, workbook: list[tuple[str, list[tuple]]]) -> list[dict[str, Any]]:
|
||
"""Candidates with their preference votes. Returns rank, name, birth year (if published), party, list."""
|
||
out: list[dict[str, Any]] = []
|
||
for sheet, rows in workbook:
|
||
party, party_name, list_label = None, None, "Bundesliste"
|
||
total_col = None
|
||
if layout == "eu":
|
||
party, party_name, list_label = sheet.strip(), sheet.strip(), "Bundesliste"
|
||
for row in rows:
|
||
cells = [c for c in row if c is not None and c != ""]
|
||
if not cells:
|
||
continue
|
||
first = row[0]
|
||
if layout == "eu" and any(isinstance(c, str) and re.match(r"Bewerber|Kandidat", c) for c in row):
|
||
# header: "LP | Bewerber(in) | BGLD ... | Gesamt" (2024) or "_ | Kandidat(in) | ... | Gesamt" (2019).
|
||
# 2024 repeats a re-sorted copy of the table further right: its header must not win.
|
||
if total_col is None:
|
||
total_col = next((i for i, c in enumerate(row) if isinstance(c, str) and c.strip() == "Gesamt"),
|
||
None)
|
||
continue
|
||
if isinstance(first, str) and len(cells) == 1:
|
||
if (m := LIST_HEAD.match(first)):
|
||
region = re.sub(r"^\w+\s*-\s*", "", m.group(2)) # "1A - Burgenland Nord" -> "Burgenland Nord"
|
||
list_label = f"{m.group(1)}parteiliste {region}"
|
||
elif (m := PARTY_HEAD.search(first)) and layout != "eu":
|
||
party, party_name, list_label = m.group(1).strip(), first.strip(), "Bundesliste"
|
||
continue
|
||
rank = str(first).rstrip(".") if first is not None else ""
|
||
if not rank.isdigit() or party is None:
|
||
continue
|
||
if layout == "land":
|
||
family, given, year, votes = row[1], row[2], row[4], row[6]
|
||
elif layout == "eu":
|
||
family, given = split_name(row[1])
|
||
year, votes = None, row[total_col] if total_col is not None else None
|
||
else:
|
||
family, given = split_name(row[1])
|
||
year, votes = row[2], row[4]
|
||
if not family or not isinstance(votes, (int, float)):
|
||
continue
|
||
out.append({"rank": int(rank), "name": f"{given} {family}".strip(), "birth_year": int(year)
|
||
if isinstance(year, (int, float)) else None, "votes": int(votes), "party": party,
|
||
"party_name": party_name, "list": list_label})
|
||
return out
|
||
|
||
|
||
@register
|
||
class ElectionResults(Adapter):
|
||
spec = SPEC
|
||
name = "bmi.elections"
|
||
description = "Final results of federal and EU elections and candidates' preference votes (BMI)"
|
||
schedule = "0 6 1 * *" # the files are final; monthly re-checks cost a conditional request each
|
||
queue = "bulk"
|
||
|
||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||
wanted = set(ctx.params.get("elections") or BY_KEY)
|
||
for election in ELECTIONS:
|
||
if election.key not in wanted:
|
||
continue
|
||
base = f"{BMI}/{election.path}"
|
||
if election.result:
|
||
yield Request(url=f"{base}/{election.result}", kind="result", meta={"election": election.key})
|
||
for layout, file in election.preferences:
|
||
yield Request(url=f"{base}/{file}", kind="preferences", meta={"election": election.key, "layout": layout})
|
||
|
||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||
election = BY_KEY[response.meta["election"]]
|
||
workbook = sheets(response.body)
|
||
if response.request.kind == "result":
|
||
yield from self.result_claims(ctx, election, response.request.url, workbook[0][1])
|
||
else:
|
||
yield from self.preference_claims(ctx, election, response.request.url,
|
||
parse_preferences(response.meta["layout"], workbook))
|
||
|
||
def result_claims(self, ctx: RunContext, election: Election, url: str, rows: list[tuple]) -> Iterator[Claim]:
|
||
for r in parse_results(rows):
|
||
pct = f"{de_pct(r['percent'])} %" if r["percent"] is not None else "?"
|
||
yield Claim(
|
||
claim_type="election_result", org=OrgRef(short_name=r["party"], kind="party", create=False),
|
||
counterpart_org=OrgRef(slug=election.body), occurred_at=election.held_on,
|
||
key=("election", election.key, r["code"], r["party"]),
|
||
summary_de=f"{r['party']}: {pct} ({de_int(r['votes'])} Stimmen) bei der {election.name_de} – {r['region']}",
|
||
summary_en=f"{r['party']}: {pct.replace(',', '.')} ({r['votes']:,} votes) in the {election.name_en} "
|
||
f"– {r['region']}",
|
||
data={"election": election.key, "party": r["party"], "region": r["region"], "region_code": r["code"],
|
||
"votes": r["votes"], "percent": r["percent"], "valid_votes": r["valid"],
|
||
"electorate": r["electorate"]},
|
||
evidence=[Evidence(url=url, doc_type="dataset_row", title=f"BMI: {election.name_de}, endgültiges Ergebnis",
|
||
published_at=election.held_on, locator=f"{r['code']}:{r['party']}",
|
||
quote=f"{r['region']} | {r['party']} | {r['votes']} | {r['valid']} gültige")],
|
||
)
|
||
|
||
def preference_claims(self, ctx: RunContext, election: Election, url: str,
|
||
candidates: list[dict[str, Any]]) -> Iterator[Claim]:
|
||
for c in candidates:
|
||
hints = {"birth_year": c["birth_year"], "no_name_match": True} if c["birth_year"] else {}
|
||
yield Claim(
|
||
claim_type="election_result", person=PersonRef(name=c["name"], hints=hints),
|
||
counterpart_org=OrgRef(short_name=c["party"], kind="party", create=False), occurred_at=election.held_on,
|
||
key=("preference-votes", election.key, c["list"], c["party"], c["rank"], c["name"]),
|
||
summary_de=f"{de_int(c['votes'])} Vorzugsstimmen bei der {election.name_de} "
|
||
f"({c['list']} {c['party']}, Platz {c['rank']})",
|
||
summary_en=f"{c['votes']:,} preference votes in the {election.name_en} "
|
||
f"({c['party']}, {c['list']}, position {c['rank']})",
|
||
data={"election": election.key, "preference_votes": c["votes"], "rank": c["rank"], "list": c["list"],
|
||
"party": c["party"]},
|
||
evidence=[Evidence(url=url, doc_type="dataset_row", title=f"BMI: {election.name_de}, Vorzugsstimmen",
|
||
published_at=election.held_on, locator=f"{c['list']}:{c['party']}:{c['rank']}",
|
||
quote=f"{c['list']} {c['party']}, Platz {c['rank']}: {c['votes']} Vorzugsstimmen")],
|
||
)
|