"""EU institutions: Commission lobby meetings and the European Parliament (Austrian MEPs, declarations). * European Commission - meetings with interest representatives (open data XLSX per Commission term). Every row names the cabinet ("Cabinet of Commissioner Magnus Brunner"). Only meetings of Austrian Commissioners' cabinets are kept (names passed from start(): Commission roles in the DB + a fallback list). * European Parliament Open Data API v2 (CC BY 4.0): current Austrian MEPs (authoritative mandate data), and their declarations (financial interests, conflicts of interest, gifts, ...). """ from __future__ import annotations import io import re from collections.abc import Iterator from datetime import date from typing import Any from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register from politrack.core.records import ( Claim, Event, Evidence, OrgRef, Participation, PersonRecord, PersonRef, Role, ) from politrack.text import excerpt, fingerprint, normalize EC_SPEC = SourceSpec( slug="ec-meetings", name="Europäische Kommission – Treffen mit Interessenvertretern", kind="official", homepage="https://commission.europa.eu/about/service-standards-and-principles/transparency_en", license="EU open data (reuse permitted, attribution)", access="bulk", trust=90, min_interval=2.0, tos_notes="Open data export of meetings of Commissioners and their cabinets with interest representatives.", ) EP_SPEC = SourceSpec( slug="europarl", name="Europäisches Parlament Open Data", kind="official", homepage="https://data.europarl.europa.eu", license="CC BY 4.0", access="api", trust=95, min_interval=1.0, tos_notes="EP Open Data API v2 (limit 500 requests / 5 min / endpoint).", ) EC_EXPORT = "https://ec.europa.eu/transparency-initiative/meetings/data/meetings/dataxlsx" EC_TERMS = ("meetingscommissionrepresentatives2429", "meetingscommissionrepresentatives1924") EP_API = "https://data.europarl.europa.eu/api/v2" # Austrian Commissioners in the terms covered by EC_TERMS (fallback if the DB has no Commission roles yet). AUSTRIAN_COMMISSIONERS = ("Johannes Hahn", "Magnus Brunner") CABINET = re.compile(r"Cabinet of (?:Executive Vice-President|Vice-President|Commissioner|President)\s+(?P.+)$") @register class CommissionMeetings(Adapter): spec = EC_SPEC name = "eu.commission_meetings" description = "Meetings of (Austrian) EU Commissioners and their cabinets with interest representatives" schedule = "0 5 * * 1" queue = "bulk" def start(self, ctx: RunContext) -> Iterator[Request]: names = set(AUSTRIAN_COMMISSIONERS) if ctx.conn is not None: names |= {row["full_name"] for row in ctx.conn.execute( """SELECT DISTINCT p.full_name FROM role r JOIN person p ON p.id = r.person_id JOIN organization o ON o.id = r.organization_id WHERE o.slug = 'europaeische-kommission'""")} for term in EC_TERMS: yield Request(url=EC_EXPORT, params={"name": term}, kind="xlsx", meta={"commissioners": sorted(names)}) def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: import openpyxl wb = openpyxl.load_workbook(io.BytesIO(response.body), read_only=True, data_only=True) rows = list(wb.worksheets[0].iter_rows(values_only=True)) header_index = next((i for i, r in enumerate(rows) if r and r[0] == "Name of cabinet"), None) if header_index is None: ctx.stats["unexpected_layout"] += 1 return header = [str(h) for h in rows[header_index]] url = f"{EC_EXPORT}?name={response.request.params['name']}" wanted = {normalize(n) for n in response.meta.get("commissioners") or AUSTRIAN_COMMISSIONERS} for row in rows[header_index + 1:]: rec = dict(zip(header, row)) cabinet = rec.get("Name of cabinet") or "" m = CABINET.search(cabinet.strip()) if not m or not rec.get("Date of meeting"): continue commissioner = m.group("name").strip() if normalize(commissioner) not in wanted: ctx.stats["other_commissioners"] += 1 continue org_name = (rec.get("Name of interest representative") or "").strip() held = str(rec["Date of meeting"])[:10] subject = (rec.get("Subject of the meeting") or "").strip() reps = rec.get("Name of EC representative") or "" titles = rec.get("Title of EC representative") or "" register_id = (rec.get("Transparency register ID") or "").strip() event_key = fingerprint("ec-meeting", cabinet, held, org_name, subject) yield Event(key=event_key, kind="meeting", title=f"{cabinet}: {org_name}", starts_at=held, location=rec.get("Location"), url=url, description=subject, body=OrgRef(slug="europaeische-kommission")) yield Claim( claim_type="meeting", person=PersonRef(name=commissioner), occurred_at=held, counterpart_org=OrgRef(name=org_name, kind="lobby", ids={"eu_transparency": register_id} if register_id else {}) if org_name else None, summary_de=f"Treffen ({'Kommissar persönlich' if 'Commissioner' in titles else 'Kabinett'}) mit " f"{org_name}: {excerpt(subject, 160)}", summary_en=f"Meeting ({'Commissioner' if 'Commissioner' in titles else 'cabinet'}) with {org_name}: " f"{excerpt(subject, 160)}", data={"cabinet": cabinet, "representatives": reps, "titles": titles, "location": rec.get("Location"), "register_id": register_id, "date_precision": "day"}, evidence=[Evidence(url=url, doc_type="dataset_row", title=f"EU-Kommission: Treffen {cabinet}", published_at=held, quote=f"{held} | {org_name} | {subject}", locator=f"{held}:{org_name}:{subject[:60]}")], topics=("Europäische Union",), facet=Participation(event_key=event_key, participation="host"), ) @register class EuropeanParliament(Adapter): spec = EP_SPEC name = "europarl.meps" description = "Current Austrian MEPs (mandates) and their declarations" schedule = "30 4 * * 2" creates_persons = True def start(self, ctx: RunContext) -> Iterator[Request]: yield Request(url=f"{EP_API}/meps/show-current", kind="meps", params={"country-of-representation": "AT", "format": "application/ld+json", "limit": 100}) def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: data = response.json().get("data") or [] if response.request.kind == "meps": for mep in data: mep_id = str(mep.get("identifier")) given, family = mep.get("givenName"), mep.get("familyName") name = f"{given} {family}" if given and family else mep.get("label") ref = PersonRef(ids={"ep": mep_id}, name=name) yield PersonRecord(ref=ref, full_name=name, given_name=given, family_name=family, aliases=(mep.get("label") or "",), match_by_name=True) yield Request(url=f"{EP_API}/meps/{mep_id}", params={"format": "application/ld+json"}, kind="mep", meta={"mep_id": mep_id, "name": name, "group": mep.get("api:political-group")}) yield Request(url=f"{EP_API}/meps-declarations", kind="declarations", params={"person-id": mep_id, "format": "application/ld+json", "limit": 100}, meta={"mep_id": mep_id, "name": name}) elif response.request.kind == "mep": yield from self.parse_mep(response, data[0] if data else {}) else: yield from self.parse_declarations(response, data) def parse_mep(self, response: Response, mep: dict) -> Iterator[Any]: meta = response.meta ref = PersonRef(ids={"ep": meta["mep_id"]}) if mep.get("img"): yield PersonRecord(ref=ref, full_name=meta["name"], photo_url=mep["img"], birth_date=date.fromisoformat(mep["bday"][:10]) if mep.get("bday") else None) starts = [m.get("memberDuring", {}).get("startDate") for m in mep.get("hasMembership") or [] if "EU_PARLIAMENT" in str(m.get("membershipClassification", ""))] starts = sorted(s for s in starts if s) body = OrgRef(slug="europaeisches-parlament") start = date.fromisoformat(starts[-1]) if starts else None url = f"https://www.europarl.europa.eu/meps/de/{meta['mep_id']}" yield Claim( claim_type="role", person=ref, org=body, valid_from=start, summary_de=f"Mitglied des Europäischen Parlaments (Fraktion {meta.get('group') or '?'})", summary_en=f"Member of the European Parliament ({meta.get('group') or '?'})", data={"ep_group": meta.get("group")}, evidence=[Evidence(url=url, doc_type="api_record", title=f"Europäisches Parlament: {meta['name']}")], facet=Role(role="mp", level="eu", body=body, start=start), ) def parse_declarations(self, response: Response, docs: list[dict]) -> Iterator[Claim]: meta = response.meta for doc in docs: titles = [] pdf = None for expression in doc.get("is_realized_by") or []: title = (expression.get("title") or {}) titles.extend(v for v in title.values() if v) for manifestation in expression.get("is_embodied_by") or []: if "pdf" in str(manifestation.get("media_type", "")).lower() and not pdf: pdf = manifestation.get("is_exemplified_by") title = next((t for t in titles if t), doc.get("id")) url = f"https://www.europarl.europa.eu/{pdf}" if pdf and not pdf.startswith("http") else ( pdf or f"https://data.europarl.europa.eu/{doc.get('id')}") kind = str(doc.get("work_type", "")).rsplit("/", 1)[-1] yield Claim( claim_type="declaration", person=PersonRef(ids={"ep": meta["mep_id"]}), key=("ep-declaration", doc.get("id")), occurred_at=doc.get("document_date"), summary_de=f"Erklärung im Europäischen Parlament: {excerpt(title, 200)}", summary_en=f"Declaration filed with the European Parliament: {excerpt(title, 200)}", data={"work_type": kind}, evidence=[Evidence(url=url, doc_type="pdf", title=title, published_at=doc.get("document_date"))], topics=("Transparenz und Korruption",), )