Add EP final roll-call votes of Austrian MEPs

Uses the European Parliament's open data (CC BY 4.0) rather than
HowTheyVote, whose dump is ODbL share-alike and behind a robots.txt block.
Also documents the raw archive ownership step for Unraid.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
simonandClaude Opus 5.5 committed 2026-11-09 01:18:58 +01:00
1 parent 693c32a5e9
commit cd44d8fbba
14 files changed
+727 -2

No files matched your search

+122
View File
@@ -25,6 +25,8 @@ from politrack.core.records import (
PersonRecord,
PersonRef,
Role,
VoteEvent,
VotePosition,
)
from politrack.text import excerpt, fingerprint, normalize
@@ -191,3 +193,123 @@ class EuropeanParliament(Adapter):
evidence=[Evidence(url=url, doc_type="pdf", title=title, published_at=doc.get("document_date"))],
topics=("Transparenz und Korruption",),
)
LD_JSON = "application/ld+json"
EP_DECISIONS_PAGE = 500
EP_OUTCOME = {"ADOPTED": ("accepted", "angenommen", "adopted"), "REJECTED": ("rejected", "abgelehnt", "rejected")}
EP_VOTERS = (("had_voter_favor", "for", "Stimmte für", "Voted for"),
("had_voter_against", "against", "Stimmte gegen", "Voted against"),
("had_voter_abstention", "abstain", "Enthielt sich bei", "Abstained on"))
EP_INTENDED = (("had_voter_intended_favor", "for"), ("had_voter_intended_against", "against"),
("had_voter_intended_abstention", "abstain"))
def ep_label(value: Any) -> str:
"""German text of a multilingual EP field, falling back to English."""
text = (value.get("de") or value.get("en") or "") if isinstance(value, dict) else (value or "")
return re.sub(r"\s+", " ", str(text)).strip()
def ep_person_ids(values: list[str] | None) -> set[str]:
return {v.rsplit("/", 1)[-1] for v in values or []}
@register
class ParliamentVotes(Adapter):
"""Final roll-call votes (whole texts, no amendments) of the Austrian MEPs known to the database.
Amendment votes are left out on purpose: a sitting has over a hundred of them, which would bury
the final votes on a member's page. The official result counts; corrections MEPs file afterwards are
kept in `data.intended`, because they don't change the result.
"""
spec = EP_SPEC
name = "europarl.votes"
description = "Final roll-call votes of Austrian MEPs in the European Parliament plenary"
schedule = "45 5 * * *"
queue = "scrape"
def start(self, ctx: RunContext) -> Iterator[Request]:
ep_ids = []
if ctx.conn is not None:
ep_ids = sorted(row["value"] for row in ctx.conn.execute(
"SELECT value FROM identifier WHERE entity_type = 'person' AND scheme = 'ep'"))
if not ep_ids:
ctx.stats["no_meps_loaded"] = 1
return
for year in range(int(ctx.params.get("since_year", 2024)), date.today().year + 1):
yield Request(url=f"{EP_API}/meetings", kind="meetings", params={"year": year, "format": LD_JSON, "limit": 200},
meta={"ep_ids": ep_ids})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
kind = response.request.kind
data = response.json().get("data") or []
meta = response.meta
if kind == "meetings":
today = response.fetched_at.date().isoformat()
for meeting in data: # the list includes future sittings, which have no results yet
if str(meeting.get("activity_date", "9999"))[:10] <= today:
yield Request(url=f"{EP_API}/meetings/{meeting['activity_id']}/vote-results", kind="results",
params={"format": LD_JSON, "limit": 500},
meta={"ep_ids": meta["ep_ids"], "meeting": meeting["activity_id"]})
elif kind == "results":
titles = {item["id"]: ep_label(item.get("activity_label")) for item in data if item.get("activity_date")}
if titles:
yield self.decisions_request(meta["meeting"], 0, meta["ep_ids"], titles)
else:
yield from self.parse_decisions(ctx, response, data)
@staticmethod
def decisions_request(meeting: str, offset: int, ep_ids: list[str], titles: dict[str, str]) -> Request:
return Request(url=f"{EP_API}/meetings/{meeting}/decisions", kind="decisions",
params={"format": LD_JSON, "limit": EP_DECISIONS_PAGE, "offset": offset},
meta={"ep_ids": ep_ids, "meeting": meeting, "offset": offset, "titles": titles})
def parse_decisions(self, ctx: RunContext, response: Response, data: list[dict]) -> Iterator[Any]:
meta = response.meta
austrian = set(meta["ep_ids"])
chamber = OrgRef(slug="europaeisches-parlament")
for decision in data:
if "ROLLCALL" not in str(decision.get("decision_method")) or "decisionAboutId" in decision:
ctx.stats["not_final_rollcall"] += 1
continue
voters = {position: ep_person_ids(decision.get(field)) & austrian for field, position, _, _ in EP_VOTERS}
if not any(voters.values()):
ctx.stats["no_austrian_voters"] += 1
continue
voted_on = date.fromisoformat(decision["activity_date"][:10])
label = ep_label(decision.get("activity_label"))
items = decision.get("inverse_consists_of") or [None]
title = meta["titles"].get(items[0]) or label
stage = ep_label(decision.get("referenceText"))
status = str(decision.get("decision_outcome", "")).rsplit("/", 1)[-1]
result, outcome_de, outcome_en = EP_OUTCOME.get(status, ("unknown", "Ergebnis unbekannt", "outcome unknown"))
counts = {"yes": decision.get("number_of_votes_favor"), "no": decision.get("number_of_votes_against"),
"abstained": decision.get("number_of_votes_abstention")}
tally = f"{counts['yes']} Ja, {counts['no']} Nein, {counts['abstained']} Enthaltungen"
url = f"https://data.europarl.europa.eu/{decision['id']}"
record = next(iter(decision.get("recorded_in_a_realization_of") or []), "")
pv = re.search(r"PV-(\d+)-(\d{4}-\d{2}-\d{2})-RCV", record)
protocol = (f"https://www.europarl.europa.eu/doceo/document/PV-{pv.group(1)}-{pv.group(2)}-RCV_DE.html"
if pv else None)
event_key = fingerprint("ep-vote", decision["id"])
yield VoteEvent(key=event_key, chamber=chamber, title=title, voted_on=voted_on, result=result, stage=stage,
item_ref=label.split(" – ", 1)[0], roll_call=True, url=url, protocol_url=protocol,
raw_text=f"{stage}: {tally}")
intended = {mep: position for field, position in EP_INTENDED for mep in ep_person_ids(decision.get(field))}
for _, position, verb_de, verb_en in EP_VOTERS:
for mep in sorted(voters[position]):
yield Claim(
claim_type="vote", person=PersonRef(ids={"ep": mep}), counterpart_org=chamber, occurred_at=voted_on,
summary_de=f"{verb_de}: {title} (Europäisches Parlament) – {outcome_de}",
summary_en=f"{verb_en}: {title} (European Parliament) – {outcome_en}",
data={"position": position, "result": result, "roll_call": True, "chamber": "EP", **counts,
**({"intended": intended[mep]} if mep in intended else {})},
evidence=[Evidence(url=url, doc_type="api_record", title=title, published_at=voted_on,
quote=f"{stage}: {tally}", locator=f"vote:{decision['id']}")],
topics=("Europäische Union",),
facet=VotePosition(event_key=event_key, position=position),
)
if len(data) == EP_DECISIONS_PAGE:
yield self.decisions_request(meta["meeting"], meta["offset"] + EP_DECISIONS_PAGE, meta["ep_ids"], meta["titles"])