Files
simonandClaude Opus 5.5 79639e5319 Backfill: former MEPs, older elections, EC 2014-2019, social history paging
- europarl.meps also loads the Austrian MEPs of the 2019-2024 term; roles end
  with the last membership, so former MEPs don't show as current
- bmi.elections: National Council 2017 and EU 2019 preference votes
- eu.commission_meetings: the 2014-2019 Commission
- social.bluesky / social.mastodon: -p backfill_pages=N pages older posts

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 00:34:24 +01:00

236 lines
13 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Official election results from the Federal Ministry of the Interior (BMI, Bundeswahlbehörde).
Per election: the final result (XLSX, one row per region with votes per party) and the preference votes of the
candidates (XLSX, per list). Only the federal and state totals of the result are kept; municipal rows stay in the
raw archive. Candidates become claims only if they resolve to a known politician: by exact name plus birth year
where the file has one, otherwise by exact, unambiguous name. Unknown candidates are counted, never stored, and
their occupation is never kept.
Not covered yet: the 2019 state and regional lists (a print layout with side-by-side columns), elections before
2019, Landtag and municipal elections (published by the states).
"""
from __future__ import annotations
import io
import re
from collections.abc import Iterator
from dataclasses import dataclass
from datetime import date
from typing import Any
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Evidence, OrgRef, PersonRef
from politrack.text import strip_titles
BMI = "https://www.bmi.gv.at/412"
SPEC = SourceSpec(
slug="bmi-wahlen", name="Bundesministerium für Inneres – Wahlergebnisse", kind="official",
homepage=f"{BMI}/start.aspx", license="Amtliches Werk (§7 UrhG); Ergebnisse auch als CC BY 4.0 auf data.gv.at",
access="bulk", trust=95, min_interval=3.0,
tos_notes="Final results and preference votes as published by the Bundeswahlbehörde (XLSX). robots.txt sets "
"no rules. Only candidates who resolve to a known politician are stored; occupations are never kept.",
)
@dataclass(frozen=True)
class Election:
key: str
name_de: str
name_en: str
held_on: date
body: str # slug of the elected body
path: str # folder on bmi.gv.at
result: str | None # None: only in a format we can't read (EU 2019: legacy .xls)
preferences: tuple[tuple[str, str], ...] # (layout, file): layouts "bund", "land", "eu"
ELECTIONS = (
Election("nrw2024", "Nationalratswahl 2024", "2024 National Council election", date(2024, 9, 29), "nationalrat",
"nationalratswahlen/nationalratswahl_2024/files", "endgueltiges_ergebnis_beschluss_bundeswahlbehoerde_16102024.xlsx",
(("bund", "nrw2024_aufstellung_der_vorzugsstimmen_der_bundesbewerber_16102024.xlsx"),
("land", "nrw2024_aufstellung_der_vorzugsstimmen_der_landes_undregionalbewerber_16102024.xlsx"))),
Election("nrw2019", "Nationalratswahl 2019", "2019 National Council election", date(2019, 9, 29), "nationalrat",
"nationalratswahlen/nationalratswahl_2019/files", "endgultiges_gesamtergebnis_nrw19_16102019.xlsx",
(("bund", "nrw19_vorzugsstimmen_bundesweit_16102019.xlsx"),)),
Election("euw2024", "Europawahl 2024", "2024 European Parliament election", date(2024, 6, 9),
"europaeisches-parlament", "europawahlen/europawahl_2024/files", "endgueltiges_ergebnis_europawahl_26062024.xlsx",
(("eu", "gesamtaufstellung_erzielte_vorzugsstimmen_eu_wahl_2024_parteien.xlsx"),)),
Election("nrw2017", "Nationalratswahl 2017", "2017 National Council election", date(2017, 10, 15), "nationalrat",
"nationalratswahlen/nationalratswahl_2017/files", "nrw17_endgueltiges_gesamtergebnis.xlsx",
(("bund", "nrw17_vorzugsstimmen_bundesweit.xlsx"),)),
Election("euw2019", "Europawahl 2019", "2019 European Parliament election", date(2019, 5, 26),
"europaeisches-parlament", "europawahlen/europawahl_2019/files/endgueltiges_ergebnis", None,
(("eu", "euw_2019_vorzugsstimmen_gesamthomepage.xlsx"),)),
)
BY_KEY = {e.key: e for e in ELECTIONS}
REGION_CODE = re.compile(r"^G\d0000$") # G00000 Österreich, G10000 Burgenland ... G90000 Wien
PARTY_HEAD = re.compile(r"\(([^()]+)\)\s*$")
LIST_HEAD = re.compile(r"^(Landes|Regional)parteilisten im (?:Landes|Regional)wahlkreis\s+(.+?)\s*$")
def de_int(n: int) -> str:
return f"{n:,}".replace(",", ".")
def de_pct(p: float) -> str:
return f"{p:.2f}".replace(".", ",")
def sheets(body: bytes) -> list[tuple[str, list[tuple]]]:
import openpyxl
wb = openpyxl.load_workbook(io.BytesIO(body), read_only=True, data_only=True)
return [(ws.title, list(ws.iter_rows(values_only=True))) for ws in wb.worksheets]
def label(*cells: Any) -> str:
return " ".join(str(c) for c in cells if c not in (None, "")).replace("\n", "").strip().lower()
def parse_results(rows: list[tuple]) -> list[dict[str, Any]]:
"""Votes per party for Austria and the nine states. The header has party names in the first row, either
with the totals in the same row (2024) or split over two rows ("Stimmen" / "gültige", 2019 and EU 2024)."""
head, sub = rows[0], rows[1] if rows[1] and rows[1][0] is None else ()
labels = [label(head[i], sub[i] if i < len(sub) else None) for i in range(len(head))]
valid_col = next(i for i, text in enumerate(labels) if text.endswith("gültige") and "ungültige" not in text)
parties = [(i, str(name).strip()) for i, name in enumerate(head)
if i > valid_col and isinstance(name, str) and name.strip() not in ("", "%")]
out = []
for row in rows:
code = row[0]
if not (isinstance(code, str) and REGION_CODE.match(code)):
continue
valid = int(row[valid_col] or 0)
for col, party in parties:
votes = int(row[col] or 0)
if not votes:
continue
# the share as published (next column), so the figures match the official result exactly
published = row[col + 1] if col + 1 < len(row) and head[col + 1] == "%" else None
percent = (round(float(published), 2) if isinstance(published, (int, float))
else round(votes * 100 / valid, 2) if valid else None)
out.append({"code": code, "region": str(row[1]).strip(), "electorate": int(row[2] or 0),
"valid": valid, "party": party, "votes": votes, "percent": percent})
return out
def split_name(text: str) -> tuple[str, str]:
"""'Brunner, Dr. Magnus' -> ('Brunner', 'Dr. Magnus'); 'Babler, MSc, Andreas' -> ('Babler', 'Andreas')."""
family, _, rest = str(text).partition(",")
parts = [p.strip() for p in rest.split(",")]
return family.strip(), next((p for p in parts if strip_titles(p)), "")
def parse_preferences(layout: str, workbook: list[tuple[str, list[tuple]]]) -> list[dict[str, Any]]:
"""Candidates with their preference votes. Returns rank, name, birth year (if published), party, list."""
out: list[dict[str, Any]] = []
for sheet, rows in workbook:
party, party_name, list_label = None, None, "Bundesliste"
total_col = None
if layout == "eu":
party, party_name, list_label = sheet.strip(), sheet.strip(), "Bundesliste"
for row in rows:
cells = [c for c in row if c is not None and c != ""]
if not cells:
continue
first = row[0]
if layout == "eu" and any(isinstance(c, str) and re.match(r"Bewerber|Kandidat", c) for c in row):
# header: "LP | Bewerber(in) | BGLD ... | Gesamt" (2024) or "_ | Kandidat(in) | ... | Gesamt" (2019).
# 2024 repeats a re-sorted copy of the table further right: its header must not win.
if total_col is None:
total_col = next((i for i, c in enumerate(row) if isinstance(c, str) and c.strip() == "Gesamt"),
None)
continue
if isinstance(first, str) and len(cells) == 1:
if (m := LIST_HEAD.match(first)):
region = re.sub(r"^\w+\s*-\s*", "", m.group(2)) # "1A - Burgenland Nord" -> "Burgenland Nord"
list_label = f"{m.group(1)}parteiliste {region}"
elif (m := PARTY_HEAD.search(first)) and layout != "eu":
party, party_name, list_label = m.group(1).strip(), first.strip(), "Bundesliste"
continue
rank = str(first).rstrip(".") if first is not None else ""
if not rank.isdigit() or party is None:
continue
if layout == "land":
family, given, year, votes = row[1], row[2], row[4], row[6]
elif layout == "eu":
family, given = split_name(row[1])
year, votes = None, row[total_col] if total_col is not None else None
else:
family, given = split_name(row[1])
year, votes = row[2], row[4]
if not family or not isinstance(votes, (int, float)):
continue
out.append({"rank": int(rank), "name": f"{given} {family}".strip(), "birth_year": int(year)
if isinstance(year, (int, float)) else None, "votes": int(votes), "party": party,
"party_name": party_name, "list": list_label})
return out
@register
class ElectionResults(Adapter):
spec = SPEC
name = "bmi.elections"
description = "Final results of federal and EU elections and candidates' preference votes (BMI)"
schedule = "0 6 1 * *" # the files are final; monthly re-checks cost a conditional request each
queue = "bulk"
def start(self, ctx: RunContext) -> Iterator[Request]:
wanted = set(ctx.params.get("elections") or BY_KEY)
for election in ELECTIONS:
if election.key not in wanted:
continue
base = f"{BMI}/{election.path}"
if election.result:
yield Request(url=f"{base}/{election.result}", kind="result", meta={"election": election.key})
for layout, file in election.preferences:
yield Request(url=f"{base}/{file}", kind="preferences", meta={"election": election.key, "layout": layout})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
election = BY_KEY[response.meta["election"]]
workbook = sheets(response.body)
if response.request.kind == "result":
yield from self.result_claims(ctx, election, response.request.url, workbook[0][1])
else:
yield from self.preference_claims(ctx, election, response.request.url,
parse_preferences(response.meta["layout"], workbook))
def result_claims(self, ctx: RunContext, election: Election, url: str, rows: list[tuple]) -> Iterator[Claim]:
for r in parse_results(rows):
pct = f"{de_pct(r['percent'])} %" if r["percent"] is not None else "?"
yield Claim(
claim_type="election_result", org=OrgRef(short_name=r["party"], kind="party", create=False),
counterpart_org=OrgRef(slug=election.body), occurred_at=election.held_on,
key=("election", election.key, r["code"], r["party"]),
summary_de=f"{r['party']}: {pct} ({de_int(r['votes'])} Stimmen) bei der {election.name_de} – {r['region']}",
summary_en=f"{r['party']}: {pct.replace(',', '.')} ({r['votes']:,} votes) in the {election.name_en} "
f"– {r['region']}",
data={"election": election.key, "party": r["party"], "region": r["region"], "region_code": r["code"],
"votes": r["votes"], "percent": r["percent"], "valid_votes": r["valid"],
"electorate": r["electorate"]},
evidence=[Evidence(url=url, doc_type="dataset_row", title=f"BMI: {election.name_de}, endgültiges Ergebnis",
published_at=election.held_on, locator=f"{r['code']}:{r['party']}",
quote=f"{r['region']} | {r['party']} | {r['votes']} | {r['valid']} gültige")],
)
def preference_claims(self, ctx: RunContext, election: Election, url: str,
candidates: list[dict[str, Any]]) -> Iterator[Claim]:
for c in candidates:
hints = {"birth_year": c["birth_year"], "no_name_match": True} if c["birth_year"] else {}
yield Claim(
claim_type="election_result", person=PersonRef(name=c["name"], hints=hints),
counterpart_org=OrgRef(short_name=c["party"], kind="party", create=False), occurred_at=election.held_on,
key=("preference-votes", election.key, c["list"], c["party"], c["rank"], c["name"]),
summary_de=f"{de_int(c['votes'])} Vorzugsstimmen bei der {election.name_de} "
f"({c['list']} {c['party']}, Platz {c['rank']})",
summary_en=f"{c['votes']:,} preference votes in the {election.name_en} "
f"({c['party']}, {c['list']}, position {c['rank']})",
data={"election": election.key, "preference_votes": c["votes"], "rank": c["rank"], "list": c["list"],
"party": c["party"]},
evidence=[Evidence(url=url, doc_type="dataset_row", title=f"BMI: {election.name_de}, Vorzugsstimmen",
published_at=election.held_on, locator=f"{c['list']}:{c['party']}:{c['rank']}",
quote=f"{c['list']} {c['party']}, Platz {c['rank']}: {c['votes']} Vorzugsstimmen")],
)