Files
simonandClaude Opus 5.5 79639e5319 Backfill: former MEPs, older elections, EC 2014-2019, social history paging
- europarl.meps also loads the Austrian MEPs of the 2019-2024 term; roles end
  with the last membership, so former MEPs don't show as current
- bmi.elections: National Council 2017 and EU 2019 preference votes
- eu.commission_meetings: the 2014-2019 Commission
- social.bluesky / social.mastodon: -p backfill_pages=N pages older posts

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 00:34:24 +01:00

340 lines
20 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""EU institutions: Commission lobby meetings and the European Parliament (Austrian MEPs, declarations).
* European Commission - meetings with interest representatives (open data XLSX per Commission term).
Every row names the cabinet ("Cabinet of Commissioner Magnus Brunner"). Only meetings of Austrian
Commissioners' cabinets are kept (names passed from start(): Commission roles in the DB + a fallback list).
* European Parliament Open Data API v2 (CC BY 4.0): current Austrian MEPs (authoritative mandate data),
and their declarations (financial interests, conflicts of interest, gifts, ...).
"""
from __future__ import annotations
import io
import re
from collections.abc import Iterator
from datetime import date
from typing import Any
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import (
Claim,
Event,
Evidence,
OrgRef,
Participation,
PersonRecord,
PersonRef,
Role,
VoteEvent,
VotePosition,
)
from politrack.text import excerpt, fingerprint, normalize
EC_SPEC = SourceSpec(
slug="ec-meetings", name="Europäische Kommission – Treffen mit Interessenvertretern", kind="official",
homepage="https://commission.europa.eu/about/service-standards-and-principles/transparency_en",
license="EU open data (reuse permitted, attribution)", access="bulk", trust=90, min_interval=2.0,
tos_notes="Open data export of meetings of Commissioners and their cabinets with interest representatives.",
)
EP_SPEC = SourceSpec(
slug="europarl", name="Europäisches Parlament Open Data", kind="official",
homepage="https://data.europarl.europa.eu", license="CC BY 4.0", access="api", trust=95, min_interval=1.0,
tos_notes="EP Open Data API v2 (limit 500 requests / 5 min / endpoint).",
)
EC_EXPORT = "https://ec.europa.eu/transparency-initiative/meetings/data/meetings/dataxlsx"
EC_TERMS = ("meetingscommissionrepresentatives2429", "meetingscommissionrepresentatives1924",
"meetingscommissionrepresentatives1419")
EP_PAST_TERMS = (9,) # Austrian MEPs of earlier terms: their votes and declarations, roles with end dates
EP_API = "https://data.europarl.europa.eu/api/v2"
# Austrian Commissioners in the terms covered by EC_TERMS (fallback if the DB has no Commission roles yet).
AUSTRIAN_COMMISSIONERS = ("Johannes Hahn", "Magnus Brunner")
CABINET = re.compile(r"Cabinet of (?:Executive Vice-President|Vice-President|Commissioner|President)\s+(?P<name>.+)$")
@register
class CommissionMeetings(Adapter):
spec = EC_SPEC
name = "eu.commission_meetings"
description = "Meetings of (Austrian) EU Commissioners and their cabinets with interest representatives"
schedule = "0 5 * * 1"
queue = "bulk"
def start(self, ctx: RunContext) -> Iterator[Request]:
names = set(AUSTRIAN_COMMISSIONERS)
if ctx.conn is not None:
names |= {row["full_name"] for row in ctx.conn.execute(
"""SELECT DISTINCT p.full_name FROM role r JOIN person p ON p.id = r.person_id
JOIN organization o ON o.id = r.organization_id WHERE o.slug = 'europaeische-kommission'""")}
for term in EC_TERMS:
yield Request(url=EC_EXPORT, params={"name": term}, kind="xlsx", meta={"commissioners": sorted(names)})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
import openpyxl
wb = openpyxl.load_workbook(io.BytesIO(response.body), read_only=True, data_only=True)
rows = list(wb.worksheets[0].iter_rows(values_only=True))
header_index = next((i for i, r in enumerate(rows) if r and r[0] == "Name of cabinet"), None)
if header_index is None:
ctx.stats["unexpected_layout"] += 1
return
header = [str(h) for h in rows[header_index]]
url = f"{EC_EXPORT}?name={response.request.params['name']}"
wanted = {normalize(n) for n in response.meta.get("commissioners") or AUSTRIAN_COMMISSIONERS}
for row in rows[header_index + 1:]:
rec = dict(zip(header, row))
cabinet = rec.get("Name of cabinet") or ""
m = CABINET.search(cabinet.strip())
if not m or not rec.get("Date of meeting"):
continue
commissioner = m.group("name").strip()
if normalize(commissioner) not in wanted:
ctx.stats["other_commissioners"] += 1
continue
org_name = (rec.get("Name of interest representative") or "").strip()
held = str(rec["Date of meeting"])[:10]
subject = (rec.get("Subject of the meeting") or "").strip()
reps = rec.get("Name of EC representative") or ""
titles = rec.get("Title of EC representative") or ""
register_id = (rec.get("Transparency register ID") or "").strip()
event_key = fingerprint("ec-meeting", cabinet, held, org_name, subject)
yield Event(key=event_key, kind="meeting", title=f"{cabinet}: {org_name}", starts_at=held,
location=rec.get("Location"), url=url, description=subject,
body=OrgRef(slug="europaeische-kommission"))
yield Claim(
claim_type="meeting", person=PersonRef(name=commissioner), occurred_at=held,
counterpart_org=OrgRef(name=org_name, kind="lobby",
ids={"eu_transparency": register_id} if register_id else {}) if org_name else None,
summary_de=f"Treffen ({'Kommissar persönlich' if 'Commissioner' in titles else 'Kabinett'}) mit "
f"{org_name}: {excerpt(subject, 160)}",
summary_en=f"Meeting ({'Commissioner' if 'Commissioner' in titles else 'cabinet'}) with {org_name}: "
f"{excerpt(subject, 160)}",
data={"cabinet": cabinet, "representatives": reps, "titles": titles, "location": rec.get("Location"),
"register_id": register_id, "date_precision": "day"},
evidence=[Evidence(url=url, doc_type="dataset_row", title=f"EU-Kommission: Treffen {cabinet}",
published_at=held, quote=f"{held} | {org_name} | {subject}",
locator=f"{held}:{org_name}:{subject[:60]}")],
topics=("Europäische Union",),
facet=Participation(event_key=event_key, participation="host"),
)
@register
class EuropeanParliament(Adapter):
spec = EP_SPEC
name = "europarl.meps"
description = "Current Austrian MEPs (mandates) and their declarations"
schedule = "30 4 * * 2"
creates_persons = True
parser_version = 2 # roles of former MEPs end with their last membership
def start(self, ctx: RunContext) -> Iterator[Request]:
yield Request(url=f"{EP_API}/meps/show-current", kind="meps",
params={"country-of-representation": "AT", "format": "application/ld+json", "limit": 100})
for term in EP_PAST_TERMS:
yield Request(url=f"{EP_API}/meps", kind="meps",
params={"parliamentary-term": term, "country-of-representation": "AT",
"format": "application/ld+json", "limit": 100})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if not response.body.strip(): # the API answers some MEPs' declaration requests with an empty body
ctx.stats["empty_responses"] += 1
return
data = response.json().get("data") or []
if response.request.kind == "meps":
for mep in data:
mep_id = str(mep.get("identifier"))
given, family = mep.get("givenName"), mep.get("familyName")
name = f"{given} {family}" if given and family else mep.get("label")
ref = PersonRef(ids={"ep": mep_id}, name=name)
yield PersonRecord(ref=ref, full_name=name, given_name=given, family_name=family,
aliases=(mep.get("label") or "",), match_by_name=True)
yield Request(url=f"{EP_API}/meps/{mep_id}", params={"format": "application/ld+json"}, kind="mep",
meta={"mep_id": mep_id, "name": name, "group": mep.get("api:political-group")})
yield Request(url=f"{EP_API}/meps-declarations", kind="declarations",
params={"person-id": mep_id, "format": "application/ld+json", "limit": 100},
meta={"mep_id": mep_id, "name": name})
elif response.request.kind == "mep":
yield from self.parse_mep(response, data[0] if data else {})
else:
yield from self.parse_declarations(response, data)
def parse_mep(self, response: Response, mep: dict) -> Iterator[Any]:
meta = response.meta
ref = PersonRef(ids={"ep": meta["mep_id"]})
if mep.get("img"):
yield PersonRecord(ref=ref, full_name=meta["name"], photo_url=mep["img"],
birth_date=date.fromisoformat(mep["bday"][:10]) if mep.get("bday") else None)
periods = sorted(((m.get("memberDuring") or {}).get("startDate"), (m.get("memberDuring") or {}).get("endDate"))
for m in mep.get("hasMembership") or []
if "EU_PARLIAMENT" in str(m.get("membershipClassification", ""))
and (m.get("memberDuring") or {}).get("startDate"))
body = OrgRef(slug="europaeisches-parlament")
start = date.fromisoformat(periods[-1][0][:10]) if periods else None
# former MEPs: the latest membership has ended; current ones have an open end
end = date.fromisoformat(periods[-1][1][:10]) if periods and periods[-1][1] else None
url = f"https://www.europarl.europa.eu/meps/de/{meta['mep_id']}"
yield Claim(
claim_type="role", person=ref, org=body, valid_from=start, valid_to=end,
summary_de=f"Mitglied des Europäischen Parlaments (Fraktion {meta.get('group') or '?'})",
summary_en=f"Member of the European Parliament ({meta.get('group') or '?'})",
data={"ep_group": meta.get("group")},
evidence=[Evidence(url=url, doc_type="api_record", title=f"Europäisches Parlament: {meta['name']}")],
facet=Role(role="mp", level="eu", body=body, start=start, end=end),
)
def parse_declarations(self, response: Response, docs: list[dict]) -> Iterator[Claim]:
meta = response.meta
for doc in docs:
titles = []
pdf = None
for expression in doc.get("is_realized_by") or []:
title = (expression.get("title") or {})
titles.extend(v for v in title.values() if v)
for manifestation in expression.get("is_embodied_by") or []:
if "pdf" in str(manifestation.get("media_type", "")).lower() and not pdf:
pdf = manifestation.get("is_exemplified_by")
title = next((t for t in titles if t), doc.get("id"))
url = f"https://www.europarl.europa.eu/{pdf}" if pdf and not pdf.startswith("http") else (
pdf or f"https://data.europarl.europa.eu/{doc.get('id')}")
kind = str(doc.get("work_type", "")).rsplit("/", 1)[-1]
yield Claim(
claim_type="declaration", person=PersonRef(ids={"ep": meta["mep_id"]}),
key=("ep-declaration", doc.get("id")), occurred_at=doc.get("document_date"),
summary_de=f"Erklärung im Europäischen Parlament: {excerpt(title, 200)}",
summary_en=f"Declaration filed with the European Parliament: {excerpt(title, 200)}",
data={"work_type": kind},
evidence=[Evidence(url=url, doc_type="pdf", title=title, published_at=doc.get("document_date"))],
topics=("Transparenz und Korruption",),
)
LD_JSON = "application/ld+json"
EP_DECISIONS_PAGE = 500
EP_REFERENCE = re.compile(r"^(?:RC-)?[A-C]\d+-\d+/\d{4}$") # A10-0272/2026, RC-B9-0253/2024, C9-0010/2024
EP_OUTCOME = {"ADOPTED": ("accepted", "angenommen", "adopted"), "REJECTED": ("rejected", "abgelehnt", "rejected")}
EP_VOTERS = (("had_voter_favor", "for", "Stimmte für", "Voted for"),
("had_voter_against", "against", "Stimmte gegen", "Voted against"),
("had_voter_abstention", "abstain", "Enthielt sich bei", "Abstained on"))
EP_INTENDED = (("had_voter_intended_favor", "for"), ("had_voter_intended_against", "against"),
("had_voter_intended_abstention", "abstain"))
def ep_label(value: Any) -> str:
"""German text of a multilingual EP field, falling back to English."""
text = (value.get("de") or value.get("en") or "") if isinstance(value, dict) else (value or "")
return re.sub(r"\s+", " ", str(text)).strip()
def ep_person_ids(values: list[str] | None) -> set[str]:
return {v.rsplit("/", 1)[-1] for v in values or []}
@register
class ParliamentVotes(Adapter):
"""Roll-call votes of the Austrian MEPs known to the database: on whole texts and on amendments or parts.
Part votes are flagged (`VoteEvent.part_vote`, `data.part_vote`), because a sitting has over a hundred of
them and they would otherwise bury the final votes. The official result counts; corrections MEPs file
afterwards are kept in `data.intended`, because they don't change the result.
"""
spec = EP_SPEC
name = "europarl.votes"
description = "Roll-call votes of Austrian MEPs in the European Parliament plenary, including amendments"
schedule = "45 5 * * *"
queue = "scrape"
parser_version = 3 # 2: amendment and part votes; 3: references of joint motions and Council documents
def start(self, ctx: RunContext) -> Iterator[Request]:
ep_ids = []
if ctx.conn is not None:
ep_ids = sorted(row["value"] for row in ctx.conn.execute(
"SELECT value FROM identifier WHERE entity_type = 'person' AND scheme = 'ep'"))
if not ep_ids:
ctx.stats["no_meps_loaded"] = 1
return
for year in range(int(ctx.params.get("since_year", 2024)), date.today().year + 1):
yield Request(url=f"{EP_API}/meetings", kind="meetings", params={"year": year, "format": LD_JSON, "limit": 200},
meta={"ep_ids": ep_ids})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if not response.body.strip(): # sittings without recorded votes come back with an empty body
ctx.stats["empty_responses"] += 1
return
kind = response.request.kind
data = response.json().get("data") or []
meta = response.meta
if kind == "meetings":
today = response.fetched_at.date().isoformat()
for meeting in data: # the list includes future sittings, which have no results yet
if str(meeting.get("activity_date", "9999"))[:10] <= today:
yield Request(url=f"{EP_API}/meetings/{meeting['activity_id']}/vote-results", kind="results",
params={"format": LD_JSON, "limit": 500},
meta={"ep_ids": meta["ep_ids"], "meeting": meeting["activity_id"]})
elif kind == "results":
titles = {item["id"]: ep_label(item.get("activity_label")) for item in data if item.get("activity_date")}
if titles:
yield self.decisions_request(meta["meeting"], 0, meta["ep_ids"], titles)
else:
yield from self.parse_decisions(ctx, response, data)
@staticmethod
def decisions_request(meeting: str, offset: int, ep_ids: list[str], titles: dict[str, str]) -> Request:
return Request(url=f"{EP_API}/meetings/{meeting}/decisions", kind="decisions",
params={"format": LD_JSON, "limit": EP_DECISIONS_PAGE, "offset": offset},
meta={"ep_ids": ep_ids, "meeting": meeting, "offset": offset, "titles": titles})
def parse_decisions(self, ctx: RunContext, response: Response, data: list[dict]) -> Iterator[Any]:
meta = response.meta
austrian = set(meta["ep_ids"])
chamber = OrgRef(slug="europaeisches-parlament")
for decision in data:
if "ROLLCALL" not in str(decision.get("decision_method")):
ctx.stats["not_rollcall"] += 1
continue
part_vote = "decisionAboutId" in decision
voters = {position: ep_person_ids(decision.get(field)) & austrian for field, position, _, _ in EP_VOTERS}
if not any(voters.values()):
ctx.stats["no_austrian_voters"] += 1
continue
voted_on = date.fromisoformat(decision["activity_date"][:10])
label = ep_label(decision.get("activity_label"))
items = decision.get("inverse_consists_of") or [None]
title = meta["titles"].get(items[0]) or label
stage = ep_label(decision.get("referenceText"))
label_parts = label.split(" – ", 2) # "A10-0272/2026 – rapporteur – § 13, nach Buchstabe a – Änd. 9"
if part_vote and len(label_parts) == 3:
stage = label_parts[2]
status = str(decision.get("decision_outcome", "")).rsplit("/", 1)[-1]
result, outcome_de, outcome_en = EP_OUTCOME.get(status, ("unknown", "Ergebnis unbekannt", "outcome unknown"))
counts = {"yes": decision.get("number_of_votes_favor"), "no": decision.get("number_of_votes_against"),
"abstained": decision.get("number_of_votes_abstention")}
tally = f"{counts['yes']} Ja, {counts['no']} Nein, {counts['abstained']} Enthaltungen"
url = f"https://data.europarl.europa.eu/{decision['id']}"
record = next(iter(decision.get("recorded_in_a_realization_of") or []), "")
pv = re.search(r"PV-(\d+)-(\d{4}-\d{2}-\d{2})-RCV", record)
protocol = (f"https://www.europarl.europa.eu/doceo/document/PV-{pv.group(1)}-{pv.group(2)}-RCV_DE.html"
if pv else None)
event_key = fingerprint("ep-vote", decision["id"])
yield VoteEvent(key=event_key, chamber=chamber, title=title, voted_on=voted_on, result=result, stage=stage,
item_ref=label_parts[0] if EP_REFERENCE.match(label_parts[0]) else None, roll_call=True,
url=url, protocol_url=protocol, raw_text=f"{stage}: {tally}", part_vote=part_vote)
subject = f"{title}, {stage}" if part_vote and title != label else title # a label already names the part
intended = {mep: position for field, position in EP_INTENDED for mep in ep_person_ids(decision.get(field))}
for _, position, verb_de, verb_en in EP_VOTERS:
for mep in sorted(voters[position]):
yield Claim(
claim_type="vote", person=PersonRef(ids={"ep": mep}), counterpart_org=chamber, occurred_at=voted_on,
summary_de=f"{verb_de}: {subject} (Europäisches Parlament) – {outcome_de}",
summary_en=f"{verb_en}: {subject} (European Parliament) – {outcome_en}",
data={"position": position, "result": result, "roll_call": True, "chamber": "EP", **counts,
**({"intended": intended[mep]} if mep in intended else {}),
**({"part_vote": True} if part_vote else {})},
evidence=[Evidence(url=url, doc_type="api_record", title=title, published_at=voted_on,
quote=f"{stage}: {tally}", locator=f"vote:{decision['id']}")],
topics=("Europäische Union",),
facet=VotePosition(event_key=event_key, position=position),
)
if len(data) == EP_DECISIONS_PAGE:
yield self.decisions_request(meta["meeting"], meta["offset"] + EP_DECISIONS_PAGE, meta["ep_ids"], meta["titles"])