Add official election results and candidates' preference votes (BMI)
National Council 2024 and 2019 and EU 2024: party results for Austria and the states, and preference votes of candidates who resolve to a known politician (exact name plus birth year where published). New claim type election_result, new resolver hint birth_year. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b06f17e587
commit
a59d475e6b
29 files changed
+1696
-9
No files matched your search
@@ -11,7 +11,7 @@ from pydantic import BaseModel, ConfigDict
|
||||
ClaimType = Literal["role", "vote", "donation", "money_flow", "statement", "appearance", "meeting", "travel",
|
||||
"side_job", "legal_proceeding", "lobbying", "media_mention", "career", "parliamentary_question",
|
||||
"government_decision", "wiki_edit", "declaration", "ownership", "affiliation", "party_finance",
|
||||
"other"]
|
||||
"election_result", "other"]
|
||||
PersonStatus = Literal["active", "former", "party_member", "deceased"]
|
||||
|
||||
|
||||
|
||||
@@ -7,6 +7,8 @@ Order (first hit wins, ambiguity never guesses):
|
||||
Hint strategies for persons (documented in docs/data-model.md):
|
||||
surname_in_body + on members of a chamber on a date with that surname (roll-call lists)
|
||||
office + on holder of an office title on a date (Council of Ministers minutes)
|
||||
birth_year exact name and year of birth (candidate lists); add no_name_match so that a name
|
||||
alone never matches when the year doesn't
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -133,6 +135,14 @@ class Resolver:
|
||||
return suffix[0]["id"] if len(suffix) == 1 else None
|
||||
if "office" in hints:
|
||||
return self.office_holder(hints["office"], on)
|
||||
if "birth_year" in hints and ref.name:
|
||||
rows = self.conn.execute(
|
||||
"""SELECT DISTINCT a.entity_id AS id FROM alias a JOIN person p ON p.id = a.entity_id
|
||||
WHERE a.entity_type = 'person' AND a.alias_norm = %s
|
||||
AND EXTRACT(YEAR FROM p.birth_date) = %s""",
|
||||
(normalize(strip_titles(ref.name)), int(hints["birth_year"])),
|
||||
).fetchall()
|
||||
return rows[0]["id"] if len(rows) == 1 else None
|
||||
return None
|
||||
|
||||
def office_holder(self, office: str, on: date | None) -> int | None:
|
||||
|
||||
@@ -115,7 +115,8 @@ def list_claims(
|
||||
) -> str:
|
||||
"""Claims about a person/party/organisation, newest first. claim_types: role, vote, donation, money_flow,
|
||||
statement, appearance, meeting, travel, side_job, legal_proceeding, lobbying, media_mention, career,
|
||||
parliamentary_question, government_decision, wiki_edit, declaration, ownership, affiliation, party_finance.
|
||||
parliamentary_question, government_decision, wiki_edit, declaration, ownership, affiliation, party_finance,
|
||||
election_result.
|
||||
Dates as YYYY-MM-DD."""
|
||||
with connection() as conn:
|
||||
return _dump(queries.list_claims(
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
"""Official election results from the Federal Ministry of the Interior (BMI, Bundeswahlbehörde).
|
||||
|
||||
Per election: the final result (XLSX, one row per region with votes per party) and the preference votes of the
|
||||
candidates (XLSX, per list). Only the federal and state totals of the result are kept; municipal rows stay in the
|
||||
raw archive. Candidates become claims only if they resolve to a known politician: by exact name plus birth year
|
||||
where the file has one, otherwise by exact, unambiguous name. Unknown candidates are counted, never stored, and
|
||||
their occupation is never kept.
|
||||
|
||||
Not covered yet: the 2019 state and regional lists (a print layout with side-by-side columns), elections before
|
||||
2019, Landtag and municipal elections (published by the states).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Evidence, OrgRef, PersonRef
|
||||
from politrack.text import strip_titles
|
||||
|
||||
BMI = "https://www.bmi.gv.at/412"
|
||||
SPEC = SourceSpec(
|
||||
slug="bmi-wahlen", name="Bundesministerium für Inneres – Wahlergebnisse", kind="official",
|
||||
homepage=f"{BMI}/start.aspx", license="Amtliches Werk (§7 UrhG); Ergebnisse auch als CC BY 4.0 auf data.gv.at",
|
||||
access="bulk", trust=95, min_interval=3.0,
|
||||
tos_notes="Final results and preference votes as published by the Bundeswahlbehörde (XLSX). robots.txt sets "
|
||||
"no rules. Only candidates who resolve to a known politician are stored; occupations are never kept.",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Election:
|
||||
key: str
|
||||
name_de: str
|
||||
name_en: str
|
||||
held_on: date
|
||||
body: str # slug of the elected body
|
||||
path: str # folder on bmi.gv.at
|
||||
result: str
|
||||
preferences: tuple[tuple[str, str], ...] # (layout, file): layouts "bund", "land", "eu"
|
||||
|
||||
|
||||
ELECTIONS = (
|
||||
Election("nrw2024", "Nationalratswahl 2024", "2024 National Council election", date(2024, 9, 29), "nationalrat",
|
||||
"nationalratswahlen/nationalratswahl_2024/files", "endgueltiges_ergebnis_beschluss_bundeswahlbehoerde_16102024.xlsx",
|
||||
(("bund", "nrw2024_aufstellung_der_vorzugsstimmen_der_bundesbewerber_16102024.xlsx"),
|
||||
("land", "nrw2024_aufstellung_der_vorzugsstimmen_der_landes_undregionalbewerber_16102024.xlsx"))),
|
||||
Election("nrw2019", "Nationalratswahl 2019", "2019 National Council election", date(2019, 9, 29), "nationalrat",
|
||||
"nationalratswahlen/nationalratswahl_2019/files", "endgultiges_gesamtergebnis_nrw19_16102019.xlsx",
|
||||
(("bund", "nrw19_vorzugsstimmen_bundesweit_16102019.xlsx"),)),
|
||||
Election("euw2024", "Europawahl 2024", "2024 European Parliament election", date(2024, 6, 9),
|
||||
"europaeisches-parlament", "europawahlen/europawahl_2024/files", "endgueltiges_ergebnis_europawahl_26062024.xlsx",
|
||||
(("eu", "gesamtaufstellung_erzielte_vorzugsstimmen_eu_wahl_2024_parteien.xlsx"),)),
|
||||
)
|
||||
BY_KEY = {e.key: e for e in ELECTIONS}
|
||||
|
||||
REGION_CODE = re.compile(r"^G\d0000$") # G00000 Österreich, G10000 Burgenland ... G90000 Wien
|
||||
PARTY_HEAD = re.compile(r"\(([^()]+)\)\s*$")
|
||||
LIST_HEAD = re.compile(r"^(Landes|Regional)parteilisten im (?:Landes|Regional)wahlkreis\s+(.+?)\s*$")
|
||||
|
||||
|
||||
def de_int(n: int) -> str:
|
||||
return f"{n:,}".replace(",", ".")
|
||||
|
||||
|
||||
def de_pct(p: float) -> str:
|
||||
return f"{p:.2f}".replace(".", ",")
|
||||
|
||||
|
||||
def sheets(body: bytes) -> list[tuple[str, list[tuple]]]:
|
||||
import openpyxl
|
||||
|
||||
wb = openpyxl.load_workbook(io.BytesIO(body), read_only=True, data_only=True)
|
||||
return [(ws.title, list(ws.iter_rows(values_only=True))) for ws in wb.worksheets]
|
||||
|
||||
|
||||
def label(*cells: Any) -> str:
|
||||
return " ".join(str(c) for c in cells if c not in (None, "")).replace("\n", "").strip().lower()
|
||||
|
||||
|
||||
def parse_results(rows: list[tuple]) -> list[dict[str, Any]]:
|
||||
"""Votes per party for Austria and the nine states. The header has party names in the first row, either
|
||||
with the totals in the same row (2024) or split over two rows ("Stimmen" / "gültige", 2019 and EU 2024)."""
|
||||
head, sub = rows[0], rows[1] if rows[1] and rows[1][0] is None else ()
|
||||
labels = [label(head[i], sub[i] if i < len(sub) else None) for i in range(len(head))]
|
||||
valid_col = next(i for i, text in enumerate(labels) if text.endswith("gültige") and "ungültige" not in text)
|
||||
parties = [(i, str(name).strip()) for i, name in enumerate(head)
|
||||
if i > valid_col and isinstance(name, str) and name.strip() not in ("", "%")]
|
||||
out = []
|
||||
for row in rows:
|
||||
code = row[0]
|
||||
if not (isinstance(code, str) and REGION_CODE.match(code)):
|
||||
continue
|
||||
valid = int(row[valid_col] or 0)
|
||||
for col, party in parties:
|
||||
votes = int(row[col] or 0)
|
||||
if not votes:
|
||||
continue
|
||||
# the share as published (next column), so the figures match the official result exactly
|
||||
published = row[col + 1] if col + 1 < len(row) and head[col + 1] == "%" else None
|
||||
percent = (round(float(published), 2) if isinstance(published, (int, float))
|
||||
else round(votes * 100 / valid, 2) if valid else None)
|
||||
out.append({"code": code, "region": str(row[1]).strip(), "electorate": int(row[2] or 0),
|
||||
"valid": valid, "party": party, "votes": votes, "percent": percent})
|
||||
return out
|
||||
|
||||
|
||||
def split_name(text: str) -> tuple[str, str]:
|
||||
"""'Brunner, Dr. Magnus' -> ('Brunner', 'Dr. Magnus'); 'Babler, MSc, Andreas' -> ('Babler', 'Andreas')."""
|
||||
family, _, rest = str(text).partition(",")
|
||||
parts = [p.strip() for p in rest.split(",")]
|
||||
return family.strip(), next((p for p in parts if strip_titles(p)), "")
|
||||
|
||||
|
||||
def parse_preferences(layout: str, workbook: list[tuple[str, list[tuple]]]) -> list[dict[str, Any]]:
|
||||
"""Candidates with their preference votes. Returns rank, name, birth year (if published), party, list."""
|
||||
out: list[dict[str, Any]] = []
|
||||
for sheet, rows in workbook:
|
||||
party, party_name, list_label = None, None, "Bundesliste"
|
||||
total_col = None
|
||||
if layout == "eu":
|
||||
party, party_name, list_label = sheet.strip(), sheet.strip(), "Bundesliste"
|
||||
for row in rows:
|
||||
cells = [c for c in row if c is not None and c != ""]
|
||||
if not cells:
|
||||
continue
|
||||
first = row[0]
|
||||
if layout == "eu" and first == "LP":
|
||||
total_col = next((i for i, c in enumerate(row) if c == "Gesamt"), None)
|
||||
continue
|
||||
if isinstance(first, str) and len(cells) == 1:
|
||||
if (m := LIST_HEAD.match(first)):
|
||||
region = re.sub(r"^\w+\s*-\s*", "", m.group(2)) # "1A - Burgenland Nord" -> "Burgenland Nord"
|
||||
list_label = f"{m.group(1)}parteiliste {region}"
|
||||
elif (m := PARTY_HEAD.search(first)) and layout != "eu":
|
||||
party, party_name, list_label = m.group(1).strip(), first.strip(), "Bundesliste"
|
||||
continue
|
||||
rank = str(first).rstrip(".") if first is not None else ""
|
||||
if not rank.isdigit() or party is None:
|
||||
continue
|
||||
if layout == "land":
|
||||
family, given, year, votes = row[1], row[2], row[4], row[6]
|
||||
elif layout == "eu":
|
||||
family, given = split_name(row[1])
|
||||
year, votes = None, row[total_col] if total_col is not None else None
|
||||
else:
|
||||
family, given = split_name(row[1])
|
||||
year, votes = row[2], row[4]
|
||||
if not family or not isinstance(votes, (int, float)):
|
||||
continue
|
||||
out.append({"rank": int(rank), "name": f"{given} {family}".strip(), "birth_year": int(year)
|
||||
if isinstance(year, (int, float)) else None, "votes": int(votes), "party": party,
|
||||
"party_name": party_name, "list": list_label})
|
||||
return out
|
||||
|
||||
|
||||
@register
|
||||
class ElectionResults(Adapter):
|
||||
spec = SPEC
|
||||
name = "bmi.elections"
|
||||
description = "Final results of federal and EU elections and candidates' preference votes (BMI)"
|
||||
schedule = "0 6 1 * *" # the files are final; monthly re-checks cost a conditional request each
|
||||
queue = "bulk"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
wanted = set(ctx.params.get("elections") or BY_KEY)
|
||||
for election in ELECTIONS:
|
||||
if election.key not in wanted:
|
||||
continue
|
||||
base = f"{BMI}/{election.path}"
|
||||
yield Request(url=f"{base}/{election.result}", kind="result", meta={"election": election.key})
|
||||
for layout, file in election.preferences:
|
||||
yield Request(url=f"{base}/{file}", kind="preferences", meta={"election": election.key, "layout": layout})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||||
election = BY_KEY[response.meta["election"]]
|
||||
workbook = sheets(response.body)
|
||||
if response.request.kind == "result":
|
||||
yield from self.result_claims(ctx, election, response.request.url, workbook[0][1])
|
||||
else:
|
||||
yield from self.preference_claims(ctx, election, response.request.url,
|
||||
parse_preferences(response.meta["layout"], workbook))
|
||||
|
||||
def result_claims(self, ctx: RunContext, election: Election, url: str, rows: list[tuple]) -> Iterator[Claim]:
|
||||
for r in parse_results(rows):
|
||||
pct = f"{de_pct(r['percent'])} %" if r["percent"] is not None else "?"
|
||||
yield Claim(
|
||||
claim_type="election_result", org=OrgRef(short_name=r["party"], kind="party", create=False),
|
||||
counterpart_org=OrgRef(slug=election.body), occurred_at=election.held_on,
|
||||
key=("election", election.key, r["code"], r["party"]),
|
||||
summary_de=f"{r['party']}: {pct} ({de_int(r['votes'])} Stimmen) bei der {election.name_de} – {r['region']}",
|
||||
summary_en=f"{r['party']}: {pct.replace(',', '.')} ({r['votes']:,} votes) in the {election.name_en} "
|
||||
f"– {r['region']}",
|
||||
data={"election": election.key, "party": r["party"], "region": r["region"], "region_code": r["code"],
|
||||
"votes": r["votes"], "percent": r["percent"], "valid_votes": r["valid"],
|
||||
"electorate": r["electorate"]},
|
||||
evidence=[Evidence(url=url, doc_type="dataset_row", title=f"BMI: {election.name_de}, endgültiges Ergebnis",
|
||||
published_at=election.held_on, locator=f"{r['code']}:{r['party']}",
|
||||
quote=f"{r['region']} | {r['party']} | {r['votes']} | {r['valid']} gültige")],
|
||||
)
|
||||
|
||||
def preference_claims(self, ctx: RunContext, election: Election, url: str,
|
||||
candidates: list[dict[str, Any]]) -> Iterator[Claim]:
|
||||
for c in candidates:
|
||||
hints = {"birth_year": c["birth_year"], "no_name_match": True} if c["birth_year"] else {}
|
||||
yield Claim(
|
||||
claim_type="election_result", person=PersonRef(name=c["name"], hints=hints),
|
||||
counterpart_org=OrgRef(short_name=c["party"], kind="party", create=False), occurred_at=election.held_on,
|
||||
key=("preference-votes", election.key, c["list"], c["party"], c["rank"], c["name"]),
|
||||
summary_de=f"{de_int(c['votes'])} Vorzugsstimmen bei der {election.name_de} "
|
||||
f"({c['list']} {c['party']}, Platz {c['rank']})",
|
||||
summary_en=f"{c['votes']:,} preference votes in the {election.name_en} "
|
||||
f"({c['party']}, {c['list']}, position {c['rank']})",
|
||||
data={"election": election.key, "preference_votes": c["votes"], "rank": c["rank"], "list": c["list"],
|
||||
"party": c["party"]},
|
||||
evidence=[Evidence(url=url, doc_type="dataset_row", title=f"BMI: {election.name_de}, Vorzugsstimmen",
|
||||
published_at=election.held_on, locator=f"{c['list']}:{c['party']}:{c['rank']}",
|
||||
quote=f"{c['list']} {c['party']}, Platz {c['rank']}: {c['votes']} Vorzugsstimmen")],
|
||||
)
|
||||
Reference in new issue
Block a user