- europarl.meps also loads the Austrian MEPs of the 2019-2024 term; roles end with the last membership, so former MEPs don't show as current - bmi.elections: National Council 2017 and EU 2019 preference votes - eu.commission_meetings: the 2014-2019 Commission - social.bluesky / social.mastodon: -p backfill_pages=N pages older posts Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
340 lines
20 KiB
Python
340 lines
20 KiB
Python
"""EU institutions: Commission lobby meetings and the European Parliament (Austrian MEPs, declarations).
|
||
|
||
* European Commission - meetings with interest representatives (open data XLSX per Commission term).
|
||
Every row names the cabinet ("Cabinet of Commissioner Magnus Brunner"). Only meetings of Austrian
|
||
Commissioners' cabinets are kept (names passed from start(): Commission roles in the DB + a fallback list).
|
||
* European Parliament Open Data API v2 (CC BY 4.0): current Austrian MEPs (authoritative mandate data),
|
||
and their declarations (financial interests, conflicts of interest, gifts, ...).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import io
|
||
import re
|
||
from collections.abc import Iterator
|
||
from datetime import date
|
||
from typing import Any
|
||
|
||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||
from politrack.core.records import (
|
||
Claim,
|
||
Event,
|
||
Evidence,
|
||
OrgRef,
|
||
Participation,
|
||
PersonRecord,
|
||
PersonRef,
|
||
Role,
|
||
VoteEvent,
|
||
VotePosition,
|
||
)
|
||
from politrack.text import excerpt, fingerprint, normalize
|
||
|
||
EC_SPEC = SourceSpec(
|
||
slug="ec-meetings", name="Europäische Kommission – Treffen mit Interessenvertretern", kind="official",
|
||
homepage="https://commission.europa.eu/about/service-standards-and-principles/transparency_en",
|
||
license="EU open data (reuse permitted, attribution)", access="bulk", trust=90, min_interval=2.0,
|
||
tos_notes="Open data export of meetings of Commissioners and their cabinets with interest representatives.",
|
||
)
|
||
EP_SPEC = SourceSpec(
|
||
slug="europarl", name="Europäisches Parlament Open Data", kind="official",
|
||
homepage="https://data.europarl.europa.eu", license="CC BY 4.0", access="api", trust=95, min_interval=1.0,
|
||
tos_notes="EP Open Data API v2 (limit 500 requests / 5 min / endpoint).",
|
||
)
|
||
EC_EXPORT = "https://ec.europa.eu/transparency-initiative/meetings/data/meetings/dataxlsx"
|
||
EC_TERMS = ("meetingscommissionrepresentatives2429", "meetingscommissionrepresentatives1924",
|
||
"meetingscommissionrepresentatives1419")
|
||
EP_PAST_TERMS = (9,) # Austrian MEPs of earlier terms: their votes and declarations, roles with end dates
|
||
EP_API = "https://data.europarl.europa.eu/api/v2"
|
||
# Austrian Commissioners in the terms covered by EC_TERMS (fallback if the DB has no Commission roles yet).
|
||
AUSTRIAN_COMMISSIONERS = ("Johannes Hahn", "Magnus Brunner")
|
||
CABINET = re.compile(r"Cabinet of (?:Executive Vice-President|Vice-President|Commissioner|President)\s+(?P<name>.+)$")
|
||
|
||
|
||
@register
|
||
class CommissionMeetings(Adapter):
|
||
spec = EC_SPEC
|
||
name = "eu.commission_meetings"
|
||
description = "Meetings of (Austrian) EU Commissioners and their cabinets with interest representatives"
|
||
schedule = "0 5 * * 1"
|
||
queue = "bulk"
|
||
|
||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||
names = set(AUSTRIAN_COMMISSIONERS)
|
||
if ctx.conn is not None:
|
||
names |= {row["full_name"] for row in ctx.conn.execute(
|
||
"""SELECT DISTINCT p.full_name FROM role r JOIN person p ON p.id = r.person_id
|
||
JOIN organization o ON o.id = r.organization_id WHERE o.slug = 'europaeische-kommission'""")}
|
||
for term in EC_TERMS:
|
||
yield Request(url=EC_EXPORT, params={"name": term}, kind="xlsx", meta={"commissioners": sorted(names)})
|
||
|
||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||
import openpyxl
|
||
|
||
wb = openpyxl.load_workbook(io.BytesIO(response.body), read_only=True, data_only=True)
|
||
rows = list(wb.worksheets[0].iter_rows(values_only=True))
|
||
header_index = next((i for i, r in enumerate(rows) if r and r[0] == "Name of cabinet"), None)
|
||
if header_index is None:
|
||
ctx.stats["unexpected_layout"] += 1
|
||
return
|
||
header = [str(h) for h in rows[header_index]]
|
||
url = f"{EC_EXPORT}?name={response.request.params['name']}"
|
||
wanted = {normalize(n) for n in response.meta.get("commissioners") or AUSTRIAN_COMMISSIONERS}
|
||
for row in rows[header_index + 1:]:
|
||
rec = dict(zip(header, row))
|
||
cabinet = rec.get("Name of cabinet") or ""
|
||
m = CABINET.search(cabinet.strip())
|
||
if not m or not rec.get("Date of meeting"):
|
||
continue
|
||
commissioner = m.group("name").strip()
|
||
if normalize(commissioner) not in wanted:
|
||
ctx.stats["other_commissioners"] += 1
|
||
continue
|
||
org_name = (rec.get("Name of interest representative") or "").strip()
|
||
held = str(rec["Date of meeting"])[:10]
|
||
subject = (rec.get("Subject of the meeting") or "").strip()
|
||
reps = rec.get("Name of EC representative") or ""
|
||
titles = rec.get("Title of EC representative") or ""
|
||
register_id = (rec.get("Transparency register ID") or "").strip()
|
||
event_key = fingerprint("ec-meeting", cabinet, held, org_name, subject)
|
||
yield Event(key=event_key, kind="meeting", title=f"{cabinet}: {org_name}", starts_at=held,
|
||
location=rec.get("Location"), url=url, description=subject,
|
||
body=OrgRef(slug="europaeische-kommission"))
|
||
yield Claim(
|
||
claim_type="meeting", person=PersonRef(name=commissioner), occurred_at=held,
|
||
counterpart_org=OrgRef(name=org_name, kind="lobby",
|
||
ids={"eu_transparency": register_id} if register_id else {}) if org_name else None,
|
||
summary_de=f"Treffen ({'Kommissar persönlich' if 'Commissioner' in titles else 'Kabinett'}) mit "
|
||
f"{org_name}: {excerpt(subject, 160)}",
|
||
summary_en=f"Meeting ({'Commissioner' if 'Commissioner' in titles else 'cabinet'}) with {org_name}: "
|
||
f"{excerpt(subject, 160)}",
|
||
data={"cabinet": cabinet, "representatives": reps, "titles": titles, "location": rec.get("Location"),
|
||
"register_id": register_id, "date_precision": "day"},
|
||
evidence=[Evidence(url=url, doc_type="dataset_row", title=f"EU-Kommission: Treffen {cabinet}",
|
||
published_at=held, quote=f"{held} | {org_name} | {subject}",
|
||
locator=f"{held}:{org_name}:{subject[:60]}")],
|
||
topics=("Europäische Union",),
|
||
facet=Participation(event_key=event_key, participation="host"),
|
||
)
|
||
|
||
|
||
@register
|
||
class EuropeanParliament(Adapter):
|
||
spec = EP_SPEC
|
||
name = "europarl.meps"
|
||
description = "Current Austrian MEPs (mandates) and their declarations"
|
||
schedule = "30 4 * * 2"
|
||
creates_persons = True
|
||
parser_version = 2 # roles of former MEPs end with their last membership
|
||
|
||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||
yield Request(url=f"{EP_API}/meps/show-current", kind="meps",
|
||
params={"country-of-representation": "AT", "format": "application/ld+json", "limit": 100})
|
||
for term in EP_PAST_TERMS:
|
||
yield Request(url=f"{EP_API}/meps", kind="meps",
|
||
params={"parliamentary-term": term, "country-of-representation": "AT",
|
||
"format": "application/ld+json", "limit": 100})
|
||
|
||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||
if not response.body.strip(): # the API answers some MEPs' declaration requests with an empty body
|
||
ctx.stats["empty_responses"] += 1
|
||
return
|
||
data = response.json().get("data") or []
|
||
if response.request.kind == "meps":
|
||
for mep in data:
|
||
mep_id = str(mep.get("identifier"))
|
||
given, family = mep.get("givenName"), mep.get("familyName")
|
||
name = f"{given} {family}" if given and family else mep.get("label")
|
||
ref = PersonRef(ids={"ep": mep_id}, name=name)
|
||
yield PersonRecord(ref=ref, full_name=name, given_name=given, family_name=family,
|
||
aliases=(mep.get("label") or "",), match_by_name=True)
|
||
yield Request(url=f"{EP_API}/meps/{mep_id}", params={"format": "application/ld+json"}, kind="mep",
|
||
meta={"mep_id": mep_id, "name": name, "group": mep.get("api:political-group")})
|
||
yield Request(url=f"{EP_API}/meps-declarations", kind="declarations",
|
||
params={"person-id": mep_id, "format": "application/ld+json", "limit": 100},
|
||
meta={"mep_id": mep_id, "name": name})
|
||
elif response.request.kind == "mep":
|
||
yield from self.parse_mep(response, data[0] if data else {})
|
||
else:
|
||
yield from self.parse_declarations(response, data)
|
||
|
||
def parse_mep(self, response: Response, mep: dict) -> Iterator[Any]:
|
||
meta = response.meta
|
||
ref = PersonRef(ids={"ep": meta["mep_id"]})
|
||
if mep.get("img"):
|
||
yield PersonRecord(ref=ref, full_name=meta["name"], photo_url=mep["img"],
|
||
birth_date=date.fromisoformat(mep["bday"][:10]) if mep.get("bday") else None)
|
||
periods = sorted(((m.get("memberDuring") or {}).get("startDate"), (m.get("memberDuring") or {}).get("endDate"))
|
||
for m in mep.get("hasMembership") or []
|
||
if "EU_PARLIAMENT" in str(m.get("membershipClassification", ""))
|
||
and (m.get("memberDuring") or {}).get("startDate"))
|
||
body = OrgRef(slug="europaeisches-parlament")
|
||
start = date.fromisoformat(periods[-1][0][:10]) if periods else None
|
||
# former MEPs: the latest membership has ended; current ones have an open end
|
||
end = date.fromisoformat(periods[-1][1][:10]) if periods and periods[-1][1] else None
|
||
url = f"https://www.europarl.europa.eu/meps/de/{meta['mep_id']}"
|
||
yield Claim(
|
||
claim_type="role", person=ref, org=body, valid_from=start, valid_to=end,
|
||
summary_de=f"Mitglied des Europäischen Parlaments (Fraktion {meta.get('group') or '?'})",
|
||
summary_en=f"Member of the European Parliament ({meta.get('group') or '?'})",
|
||
data={"ep_group": meta.get("group")},
|
||
evidence=[Evidence(url=url, doc_type="api_record", title=f"Europäisches Parlament: {meta['name']}")],
|
||
facet=Role(role="mp", level="eu", body=body, start=start, end=end),
|
||
)
|
||
|
||
def parse_declarations(self, response: Response, docs: list[dict]) -> Iterator[Claim]:
|
||
meta = response.meta
|
||
for doc in docs:
|
||
titles = []
|
||
pdf = None
|
||
for expression in doc.get("is_realized_by") or []:
|
||
title = (expression.get("title") or {})
|
||
titles.extend(v for v in title.values() if v)
|
||
for manifestation in expression.get("is_embodied_by") or []:
|
||
if "pdf" in str(manifestation.get("media_type", "")).lower() and not pdf:
|
||
pdf = manifestation.get("is_exemplified_by")
|
||
title = next((t for t in titles if t), doc.get("id"))
|
||
url = f"https://www.europarl.europa.eu/{pdf}" if pdf and not pdf.startswith("http") else (
|
||
pdf or f"https://data.europarl.europa.eu/{doc.get('id')}")
|
||
kind = str(doc.get("work_type", "")).rsplit("/", 1)[-1]
|
||
yield Claim(
|
||
claim_type="declaration", person=PersonRef(ids={"ep": meta["mep_id"]}),
|
||
key=("ep-declaration", doc.get("id")), occurred_at=doc.get("document_date"),
|
||
summary_de=f"Erklärung im Europäischen Parlament: {excerpt(title, 200)}",
|
||
summary_en=f"Declaration filed with the European Parliament: {excerpt(title, 200)}",
|
||
data={"work_type": kind},
|
||
evidence=[Evidence(url=url, doc_type="pdf", title=title, published_at=doc.get("document_date"))],
|
||
topics=("Transparenz und Korruption",),
|
||
)
|
||
|
||
|
||
LD_JSON = "application/ld+json"
|
||
EP_DECISIONS_PAGE = 500
|
||
EP_REFERENCE = re.compile(r"^(?:RC-)?[A-C]\d+-\d+/\d{4}$") # A10-0272/2026, RC-B9-0253/2024, C9-0010/2024
|
||
EP_OUTCOME = {"ADOPTED": ("accepted", "angenommen", "adopted"), "REJECTED": ("rejected", "abgelehnt", "rejected")}
|
||
EP_VOTERS = (("had_voter_favor", "for", "Stimmte für", "Voted for"),
|
||
("had_voter_against", "against", "Stimmte gegen", "Voted against"),
|
||
("had_voter_abstention", "abstain", "Enthielt sich bei", "Abstained on"))
|
||
EP_INTENDED = (("had_voter_intended_favor", "for"), ("had_voter_intended_against", "against"),
|
||
("had_voter_intended_abstention", "abstain"))
|
||
|
||
|
||
def ep_label(value: Any) -> str:
|
||
"""German text of a multilingual EP field, falling back to English."""
|
||
text = (value.get("de") or value.get("en") or "") if isinstance(value, dict) else (value or "")
|
||
return re.sub(r"\s+", " ", str(text)).strip()
|
||
|
||
|
||
def ep_person_ids(values: list[str] | None) -> set[str]:
|
||
return {v.rsplit("/", 1)[-1] for v in values or []}
|
||
|
||
|
||
@register
|
||
class ParliamentVotes(Adapter):
|
||
"""Roll-call votes of the Austrian MEPs known to the database: on whole texts and on amendments or parts.
|
||
|
||
Part votes are flagged (`VoteEvent.part_vote`, `data.part_vote`), because a sitting has over a hundred of
|
||
them and they would otherwise bury the final votes. The official result counts; corrections MEPs file
|
||
afterwards are kept in `data.intended`, because they don't change the result.
|
||
"""
|
||
|
||
spec = EP_SPEC
|
||
name = "europarl.votes"
|
||
description = "Roll-call votes of Austrian MEPs in the European Parliament plenary, including amendments"
|
||
schedule = "45 5 * * *"
|
||
queue = "scrape"
|
||
parser_version = 3 # 2: amendment and part votes; 3: references of joint motions and Council documents
|
||
|
||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||
ep_ids = []
|
||
if ctx.conn is not None:
|
||
ep_ids = sorted(row["value"] for row in ctx.conn.execute(
|
||
"SELECT value FROM identifier WHERE entity_type = 'person' AND scheme = 'ep'"))
|
||
if not ep_ids:
|
||
ctx.stats["no_meps_loaded"] = 1
|
||
return
|
||
for year in range(int(ctx.params.get("since_year", 2024)), date.today().year + 1):
|
||
yield Request(url=f"{EP_API}/meetings", kind="meetings", params={"year": year, "format": LD_JSON, "limit": 200},
|
||
meta={"ep_ids": ep_ids})
|
||
|
||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||
if not response.body.strip(): # sittings without recorded votes come back with an empty body
|
||
ctx.stats["empty_responses"] += 1
|
||
return
|
||
kind = response.request.kind
|
||
data = response.json().get("data") or []
|
||
meta = response.meta
|
||
if kind == "meetings":
|
||
today = response.fetched_at.date().isoformat()
|
||
for meeting in data: # the list includes future sittings, which have no results yet
|
||
if str(meeting.get("activity_date", "9999"))[:10] <= today:
|
||
yield Request(url=f"{EP_API}/meetings/{meeting['activity_id']}/vote-results", kind="results",
|
||
params={"format": LD_JSON, "limit": 500},
|
||
meta={"ep_ids": meta["ep_ids"], "meeting": meeting["activity_id"]})
|
||
elif kind == "results":
|
||
titles = {item["id"]: ep_label(item.get("activity_label")) for item in data if item.get("activity_date")}
|
||
if titles:
|
||
yield self.decisions_request(meta["meeting"], 0, meta["ep_ids"], titles)
|
||
else:
|
||
yield from self.parse_decisions(ctx, response, data)
|
||
|
||
@staticmethod
|
||
def decisions_request(meeting: str, offset: int, ep_ids: list[str], titles: dict[str, str]) -> Request:
|
||
return Request(url=f"{EP_API}/meetings/{meeting}/decisions", kind="decisions",
|
||
params={"format": LD_JSON, "limit": EP_DECISIONS_PAGE, "offset": offset},
|
||
meta={"ep_ids": ep_ids, "meeting": meeting, "offset": offset, "titles": titles})
|
||
|
||
def parse_decisions(self, ctx: RunContext, response: Response, data: list[dict]) -> Iterator[Any]:
|
||
meta = response.meta
|
||
austrian = set(meta["ep_ids"])
|
||
chamber = OrgRef(slug="europaeisches-parlament")
|
||
for decision in data:
|
||
if "ROLLCALL" not in str(decision.get("decision_method")):
|
||
ctx.stats["not_rollcall"] += 1
|
||
continue
|
||
part_vote = "decisionAboutId" in decision
|
||
voters = {position: ep_person_ids(decision.get(field)) & austrian for field, position, _, _ in EP_VOTERS}
|
||
if not any(voters.values()):
|
||
ctx.stats["no_austrian_voters"] += 1
|
||
continue
|
||
voted_on = date.fromisoformat(decision["activity_date"][:10])
|
||
label = ep_label(decision.get("activity_label"))
|
||
items = decision.get("inverse_consists_of") or [None]
|
||
title = meta["titles"].get(items[0]) or label
|
||
stage = ep_label(decision.get("referenceText"))
|
||
label_parts = label.split(" – ", 2) # "A10-0272/2026 – rapporteur – § 13, nach Buchstabe a – Änd. 9"
|
||
if part_vote and len(label_parts) == 3:
|
||
stage = label_parts[2]
|
||
status = str(decision.get("decision_outcome", "")).rsplit("/", 1)[-1]
|
||
result, outcome_de, outcome_en = EP_OUTCOME.get(status, ("unknown", "Ergebnis unbekannt", "outcome unknown"))
|
||
counts = {"yes": decision.get("number_of_votes_favor"), "no": decision.get("number_of_votes_against"),
|
||
"abstained": decision.get("number_of_votes_abstention")}
|
||
tally = f"{counts['yes']} Ja, {counts['no']} Nein, {counts['abstained']} Enthaltungen"
|
||
url = f"https://data.europarl.europa.eu/{decision['id']}"
|
||
record = next(iter(decision.get("recorded_in_a_realization_of") or []), "")
|
||
pv = re.search(r"PV-(\d+)-(\d{4}-\d{2}-\d{2})-RCV", record)
|
||
protocol = (f"https://www.europarl.europa.eu/doceo/document/PV-{pv.group(1)}-{pv.group(2)}-RCV_DE.html"
|
||
if pv else None)
|
||
event_key = fingerprint("ep-vote", decision["id"])
|
||
yield VoteEvent(key=event_key, chamber=chamber, title=title, voted_on=voted_on, result=result, stage=stage,
|
||
item_ref=label_parts[0] if EP_REFERENCE.match(label_parts[0]) else None, roll_call=True,
|
||
url=url, protocol_url=protocol, raw_text=f"{stage}: {tally}", part_vote=part_vote)
|
||
subject = f"{title}, {stage}" if part_vote and title != label else title # a label already names the part
|
||
intended = {mep: position for field, position in EP_INTENDED for mep in ep_person_ids(decision.get(field))}
|
||
for _, position, verb_de, verb_en in EP_VOTERS:
|
||
for mep in sorted(voters[position]):
|
||
yield Claim(
|
||
claim_type="vote", person=PersonRef(ids={"ep": mep}), counterpart_org=chamber, occurred_at=voted_on,
|
||
summary_de=f"{verb_de}: {subject} (Europäisches Parlament) – {outcome_de}",
|
||
summary_en=f"{verb_en}: {subject} (European Parliament) – {outcome_en}",
|
||
data={"position": position, "result": result, "roll_call": True, "chamber": "EP", **counts,
|
||
**({"intended": intended[mep]} if mep in intended else {}),
|
||
**({"part_vote": True} if part_vote else {})},
|
||
evidence=[Evidence(url=url, doc_type="api_record", title=title, published_at=voted_on,
|
||
quote=f"{stage}: {tally}", locator=f"vote:{decision['id']}")],
|
||
topics=("Europäische Union",),
|
||
facet=VotePosition(event_key=event_key, position=position),
|
||
)
|
||
if len(data) == EP_DECISIONS_PAGE:
|
||
yield self.decisions_request(meta["meeting"], meta["offset"] + EP_DECISIONS_PAGE, meta["ep_ids"], meta["titles"])
|