"""EU institutions: Commission lobby meetings and the European Parliament (Austrian MEPs, declarations). * European Commission - meetings with interest representatives (open data XLSX per Commission term). Every row names the cabinet ("Cabinet of Commissioner Magnus Brunner"). Only meetings of Austrian Commissioners' cabinets are kept (names passed from start(): Commission roles in the DB + a fallback list). * European Parliament Open Data API v2 (CC BY 4.0): current Austrian MEPs (authoritative mandate data), and their declarations (financial interests, conflicts of interest, gifts, ...). """ from __future__ import annotations import io import re from collections.abc import Iterator from datetime import date from typing import Any from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register from politrack.core.records import ( Claim, Event, Evidence, OrgRef, Participation, PersonRecord, PersonRef, Role, VoteEvent, VotePosition, ) from politrack.text import excerpt, fingerprint, normalize EC_SPEC = SourceSpec( slug="ec-meetings", name="Europäische Kommission – Treffen mit Interessenvertretern", kind="official", homepage="https://commission.europa.eu/about/service-standards-and-principles/transparency_en", license="EU open data (reuse permitted, attribution)", access="bulk", trust=90, min_interval=2.0, tos_notes="Open data export of meetings of Commissioners and their cabinets with interest representatives.", ) EP_SPEC = SourceSpec( slug="europarl", name="Europäisches Parlament Open Data", kind="official", homepage="https://data.europarl.europa.eu", license="CC BY 4.0", access="api", trust=95, min_interval=1.0, tos_notes="EP Open Data API v2 (limit 500 requests / 5 min / endpoint).", ) EC_EXPORT = "https://ec.europa.eu/transparency-initiative/meetings/data/meetings/dataxlsx" EC_TERMS = ("meetingscommissionrepresentatives2429", "meetingscommissionrepresentatives1924", "meetingscommissionrepresentatives1419") EP_PAST_TERMS = (9,) # Austrian MEPs of earlier terms: their votes and declarations, roles with end dates EP_API = "https://data.europarl.europa.eu/api/v2" # Austrian Commissioners in the terms covered by EC_TERMS (fallback if the DB has no Commission roles yet). AUSTRIAN_COMMISSIONERS = ("Johannes Hahn", "Magnus Brunner") CABINET = re.compile(r"Cabinet of (?:Executive Vice-President|Vice-President|Commissioner|President)\s+(?P.+)$") @register class CommissionMeetings(Adapter): spec = EC_SPEC name = "eu.commission_meetings" description = "Meetings of (Austrian) EU Commissioners and their cabinets with interest representatives" schedule = "0 5 * * 1" queue = "bulk" def start(self, ctx: RunContext) -> Iterator[Request]: names = set(AUSTRIAN_COMMISSIONERS) if ctx.conn is not None: names |= {row["full_name"] for row in ctx.conn.execute( """SELECT DISTINCT p.full_name FROM role r JOIN person p ON p.id = r.person_id JOIN organization o ON o.id = r.organization_id WHERE o.slug = 'europaeische-kommission'""")} for term in EC_TERMS: yield Request(url=EC_EXPORT, params={"name": term}, kind="xlsx", meta={"commissioners": sorted(names)}) def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: import openpyxl wb = openpyxl.load_workbook(io.BytesIO(response.body), read_only=True, data_only=True) rows = list(wb.worksheets[0].iter_rows(values_only=True)) header_index = next((i for i, r in enumerate(rows) if r and r[0] == "Name of cabinet"), None) if header_index is None: ctx.stats["unexpected_layout"] += 1 return header = [str(h) for h in rows[header_index]] url = f"{EC_EXPORT}?name={response.request.params['name']}" wanted = {normalize(n) for n in response.meta.get("commissioners") or AUSTRIAN_COMMISSIONERS} for row in rows[header_index + 1:]: rec = dict(zip(header, row)) cabinet = rec.get("Name of cabinet") or "" m = CABINET.search(cabinet.strip()) if not m or not rec.get("Date of meeting"): continue commissioner = m.group("name").strip() if normalize(commissioner) not in wanted: ctx.stats["other_commissioners"] += 1 continue org_name = (rec.get("Name of interest representative") or "").strip() held = str(rec["Date of meeting"])[:10] subject = (rec.get("Subject of the meeting") or "").strip() reps = rec.get("Name of EC representative") or "" titles = rec.get("Title of EC representative") or "" register_id = (rec.get("Transparency register ID") or "").strip() event_key = fingerprint("ec-meeting", cabinet, held, org_name, subject) yield Event(key=event_key, kind="meeting", title=f"{cabinet}: {org_name}", starts_at=held, location=rec.get("Location"), url=url, description=subject, body=OrgRef(slug="europaeische-kommission")) yield Claim( claim_type="meeting", person=PersonRef(name=commissioner), occurred_at=held, counterpart_org=OrgRef(name=org_name, kind="lobby", ids={"eu_transparency": register_id} if register_id else {}) if org_name else None, summary_de=f"Treffen ({'Kommissar persönlich' if 'Commissioner' in titles else 'Kabinett'}) mit " f"{org_name}: {excerpt(subject, 160)}", summary_en=f"Meeting ({'Commissioner' if 'Commissioner' in titles else 'cabinet'}) with {org_name}: " f"{excerpt(subject, 160)}", data={"cabinet": cabinet, "representatives": reps, "titles": titles, "location": rec.get("Location"), "register_id": register_id, "date_precision": "day"}, evidence=[Evidence(url=url, doc_type="dataset_row", title=f"EU-Kommission: Treffen {cabinet}", published_at=held, quote=f"{held} | {org_name} | {subject}", locator=f"{held}:{org_name}:{subject[:60]}")], topics=("Europäische Union",), facet=Participation(event_key=event_key, participation="host"), ) @register class EuropeanParliament(Adapter): spec = EP_SPEC name = "europarl.meps" description = "Current Austrian MEPs (mandates) and their declarations" schedule = "30 4 * * 2" creates_persons = True parser_version = 2 # roles of former MEPs end with their last membership def start(self, ctx: RunContext) -> Iterator[Request]: yield Request(url=f"{EP_API}/meps/show-current", kind="meps", params={"country-of-representation": "AT", "format": "application/ld+json", "limit": 100}) for term in EP_PAST_TERMS: yield Request(url=f"{EP_API}/meps", kind="meps", params={"parliamentary-term": term, "country-of-representation": "AT", "format": "application/ld+json", "limit": 100}) def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: if not response.body.strip(): # the API answers some MEPs' declaration requests with an empty body ctx.stats["empty_responses"] += 1 return data = response.json().get("data") or [] if response.request.kind == "meps": for mep in data: mep_id = str(mep.get("identifier")) given, family = mep.get("givenName"), mep.get("familyName") name = f"{given} {family}" if given and family else mep.get("label") ref = PersonRef(ids={"ep": mep_id}, name=name) yield PersonRecord(ref=ref, full_name=name, given_name=given, family_name=family, aliases=(mep.get("label") or "",), match_by_name=True) yield Request(url=f"{EP_API}/meps/{mep_id}", params={"format": "application/ld+json"}, kind="mep", meta={"mep_id": mep_id, "name": name, "group": mep.get("api:political-group")}) yield Request(url=f"{EP_API}/meps-declarations", kind="declarations", params={"person-id": mep_id, "format": "application/ld+json", "limit": 100}, meta={"mep_id": mep_id, "name": name}) elif response.request.kind == "mep": yield from self.parse_mep(response, data[0] if data else {}) else: yield from self.parse_declarations(response, data) def parse_mep(self, response: Response, mep: dict) -> Iterator[Any]: meta = response.meta ref = PersonRef(ids={"ep": meta["mep_id"]}) if mep.get("img"): yield PersonRecord(ref=ref, full_name=meta["name"], photo_url=mep["img"], birth_date=date.fromisoformat(mep["bday"][:10]) if mep.get("bday") else None) periods = sorted(((m.get("memberDuring") or {}).get("startDate"), (m.get("memberDuring") or {}).get("endDate")) for m in mep.get("hasMembership") or [] if "EU_PARLIAMENT" in str(m.get("membershipClassification", "")) and (m.get("memberDuring") or {}).get("startDate")) body = OrgRef(slug="europaeisches-parlament") start = date.fromisoformat(periods[-1][0][:10]) if periods else None # former MEPs: the latest membership has ended; current ones have an open end end = date.fromisoformat(periods[-1][1][:10]) if periods and periods[-1][1] else None url = f"https://www.europarl.europa.eu/meps/de/{meta['mep_id']}" yield Claim( claim_type="role", person=ref, org=body, valid_from=start, valid_to=end, summary_de=f"Mitglied des Europäischen Parlaments (Fraktion {meta.get('group') or '?'})", summary_en=f"Member of the European Parliament ({meta.get('group') or '?'})", data={"ep_group": meta.get("group")}, evidence=[Evidence(url=url, doc_type="api_record", title=f"Europäisches Parlament: {meta['name']}")], facet=Role(role="mp", level="eu", body=body, start=start, end=end), ) def parse_declarations(self, response: Response, docs: list[dict]) -> Iterator[Claim]: meta = response.meta for doc in docs: titles = [] pdf = None for expression in doc.get("is_realized_by") or []: title = (expression.get("title") or {}) titles.extend(v for v in title.values() if v) for manifestation in expression.get("is_embodied_by") or []: if "pdf" in str(manifestation.get("media_type", "")).lower() and not pdf: pdf = manifestation.get("is_exemplified_by") title = next((t for t in titles if t), doc.get("id")) url = f"https://www.europarl.europa.eu/{pdf}" if pdf and not pdf.startswith("http") else ( pdf or f"https://data.europarl.europa.eu/{doc.get('id')}") kind = str(doc.get("work_type", "")).rsplit("/", 1)[-1] yield Claim( claim_type="declaration", person=PersonRef(ids={"ep": meta["mep_id"]}), key=("ep-declaration", doc.get("id")), occurred_at=doc.get("document_date"), summary_de=f"Erklärung im Europäischen Parlament: {excerpt(title, 200)}", summary_en=f"Declaration filed with the European Parliament: {excerpt(title, 200)}", data={"work_type": kind}, evidence=[Evidence(url=url, doc_type="pdf", title=title, published_at=doc.get("document_date"))], topics=("Transparenz und Korruption",), ) LD_JSON = "application/ld+json" EP_DECISIONS_PAGE = 500 EP_REFERENCE = re.compile(r"^(?:RC-)?[A-C]\d+-\d+/\d{4}$") # A10-0272/2026, RC-B9-0253/2024, C9-0010/2024 EP_OUTCOME = {"ADOPTED": ("accepted", "angenommen", "adopted"), "REJECTED": ("rejected", "abgelehnt", "rejected")} EP_VOTERS = (("had_voter_favor", "for", "Stimmte für", "Voted for"), ("had_voter_against", "against", "Stimmte gegen", "Voted against"), ("had_voter_abstention", "abstain", "Enthielt sich bei", "Abstained on")) EP_INTENDED = (("had_voter_intended_favor", "for"), ("had_voter_intended_against", "against"), ("had_voter_intended_abstention", "abstain")) def ep_label(value: Any) -> str: """German text of a multilingual EP field, falling back to English.""" text = (value.get("de") or value.get("en") or "") if isinstance(value, dict) else (value or "") return re.sub(r"\s+", " ", str(text)).strip() def ep_person_ids(values: list[str] | None) -> set[str]: return {v.rsplit("/", 1)[-1] for v in values or []} @register class ParliamentVotes(Adapter): """Roll-call votes of the Austrian MEPs known to the database: on whole texts and on amendments or parts. Part votes are flagged (`VoteEvent.part_vote`, `data.part_vote`), because a sitting has over a hundred of them and they would otherwise bury the final votes. The official result counts; corrections MEPs file afterwards are kept in `data.intended`, because they don't change the result. """ spec = EP_SPEC name = "europarl.votes" description = "Roll-call votes of Austrian MEPs in the European Parliament plenary, including amendments" schedule = "45 5 * * *" queue = "scrape" parser_version = 3 # 2: amendment and part votes; 3: references of joint motions and Council documents def start(self, ctx: RunContext) -> Iterator[Request]: ep_ids = [] if ctx.conn is not None: ep_ids = sorted(row["value"] for row in ctx.conn.execute( "SELECT value FROM identifier WHERE entity_type = 'person' AND scheme = 'ep'")) if not ep_ids: ctx.stats["no_meps_loaded"] = 1 return for year in range(int(ctx.params.get("since_year", 2024)), date.today().year + 1): yield Request(url=f"{EP_API}/meetings", kind="meetings", params={"year": year, "format": LD_JSON, "limit": 200}, meta={"ep_ids": ep_ids}) def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: if not response.body.strip(): # sittings without recorded votes come back with an empty body ctx.stats["empty_responses"] += 1 return kind = response.request.kind data = response.json().get("data") or [] meta = response.meta if kind == "meetings": today = response.fetched_at.date().isoformat() for meeting in data: # the list includes future sittings, which have no results yet if str(meeting.get("activity_date", "9999"))[:10] <= today: yield Request(url=f"{EP_API}/meetings/{meeting['activity_id']}/vote-results", kind="results", params={"format": LD_JSON, "limit": 500}, meta={"ep_ids": meta["ep_ids"], "meeting": meeting["activity_id"]}) elif kind == "results": titles = {item["id"]: ep_label(item.get("activity_label")) for item in data if item.get("activity_date")} if titles: yield self.decisions_request(meta["meeting"], 0, meta["ep_ids"], titles) else: yield from self.parse_decisions(ctx, response, data) @staticmethod def decisions_request(meeting: str, offset: int, ep_ids: list[str], titles: dict[str, str]) -> Request: return Request(url=f"{EP_API}/meetings/{meeting}/decisions", kind="decisions", params={"format": LD_JSON, "limit": EP_DECISIONS_PAGE, "offset": offset}, meta={"ep_ids": ep_ids, "meeting": meeting, "offset": offset, "titles": titles}) def parse_decisions(self, ctx: RunContext, response: Response, data: list[dict]) -> Iterator[Any]: meta = response.meta austrian = set(meta["ep_ids"]) chamber = OrgRef(slug="europaeisches-parlament") for decision in data: if "ROLLCALL" not in str(decision.get("decision_method")): ctx.stats["not_rollcall"] += 1 continue part_vote = "decisionAboutId" in decision voters = {position: ep_person_ids(decision.get(field)) & austrian for field, position, _, _ in EP_VOTERS} if not any(voters.values()): ctx.stats["no_austrian_voters"] += 1 continue voted_on = date.fromisoformat(decision["activity_date"][:10]) label = ep_label(decision.get("activity_label")) items = decision.get("inverse_consists_of") or [None] title = meta["titles"].get(items[0]) or label stage = ep_label(decision.get("referenceText")) label_parts = label.split(" – ", 2) # "A10-0272/2026 – rapporteur – § 13, nach Buchstabe a – Änd. 9" if part_vote and len(label_parts) == 3: stage = label_parts[2] status = str(decision.get("decision_outcome", "")).rsplit("/", 1)[-1] result, outcome_de, outcome_en = EP_OUTCOME.get(status, ("unknown", "Ergebnis unbekannt", "outcome unknown")) counts = {"yes": decision.get("number_of_votes_favor"), "no": decision.get("number_of_votes_against"), "abstained": decision.get("number_of_votes_abstention")} tally = f"{counts['yes']} Ja, {counts['no']} Nein, {counts['abstained']} Enthaltungen" url = f"https://data.europarl.europa.eu/{decision['id']}" record = next(iter(decision.get("recorded_in_a_realization_of") or []), "") pv = re.search(r"PV-(\d+)-(\d{4}-\d{2}-\d{2})-RCV", record) protocol = (f"https://www.europarl.europa.eu/doceo/document/PV-{pv.group(1)}-{pv.group(2)}-RCV_DE.html" if pv else None) event_key = fingerprint("ep-vote", decision["id"]) yield VoteEvent(key=event_key, chamber=chamber, title=title, voted_on=voted_on, result=result, stage=stage, item_ref=label_parts[0] if EP_REFERENCE.match(label_parts[0]) else None, roll_call=True, url=url, protocol_url=protocol, raw_text=f"{stage}: {tally}", part_vote=part_vote) subject = f"{title}, {stage}" if part_vote and title != label else title # a label already names the part intended = {mep: position for field, position in EP_INTENDED for mep in ep_person_ids(decision.get(field))} for _, position, verb_de, verb_en in EP_VOTERS: for mep in sorted(voters[position]): yield Claim( claim_type="vote", person=PersonRef(ids={"ep": mep}), counterpart_org=chamber, occurred_at=voted_on, summary_de=f"{verb_de}: {subject} (Europäisches Parlament) – {outcome_de}", summary_en=f"{verb_en}: {subject} (European Parliament) – {outcome_en}", data={"position": position, "result": result, "roll_call": True, "chamber": "EP", **counts, **({"intended": intended[mep]} if mep in intended else {}), **({"part_vote": True} if part_vote else {})}, evidence=[Evidence(url=url, doc_type="api_record", title=title, published_at=voted_on, quote=f"{stage}: {tally}", locator=f"vote:{decision['id']}")], topics=("Europäische Union",), facet=VotePosition(event_key=event_key, position=position), ) if len(data) == EP_DECISIONS_PAGE: yield self.decisions_request(meta["meeting"], meta["offset"] + EP_DECISIONS_PAGE, meta["ep_ids"], meta["titles"])