Simon DiesenreiterandClaude Opus 5.5
9da5a1dc48
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
194 lines
11 KiB
Python
194 lines
11 KiB
Python
"""EU institutions: Commission lobby meetings and the European Parliament (Austrian MEPs, declarations).
|
||
|
||
* European Commission - meetings with interest representatives (open data XLSX per Commission term).
|
||
Every row names the cabinet ("Cabinet of Commissioner Magnus Brunner"). Only meetings of Austrian
|
||
Commissioners' cabinets are kept (names passed from start(): Commission roles in the DB + a fallback list).
|
||
* European Parliament Open Data API v2 (CC BY 4.0): current Austrian MEPs (authoritative mandate data),
|
||
and their declarations (financial interests, conflicts of interest, gifts, ...).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import io
|
||
import re
|
||
from collections.abc import Iterator
|
||
from datetime import date
|
||
from typing import Any
|
||
|
||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||
from politrack.core.records import (
|
||
Claim,
|
||
Event,
|
||
Evidence,
|
||
OrgRef,
|
||
Participation,
|
||
PersonRecord,
|
||
PersonRef,
|
||
Role,
|
||
)
|
||
from politrack.text import excerpt, fingerprint, normalize
|
||
|
||
EC_SPEC = SourceSpec(
|
||
slug="ec-meetings", name="Europäische Kommission – Treffen mit Interessenvertretern", kind="official",
|
||
homepage="https://commission.europa.eu/about/service-standards-and-principles/transparency_en",
|
||
license="EU open data (reuse permitted, attribution)", access="bulk", trust=90, min_interval=2.0,
|
||
tos_notes="Open data export of meetings of Commissioners and their cabinets with interest representatives.",
|
||
)
|
||
EP_SPEC = SourceSpec(
|
||
slug="europarl", name="Europäisches Parlament Open Data", kind="official",
|
||
homepage="https://data.europarl.europa.eu", license="CC BY 4.0", access="api", trust=95, min_interval=1.0,
|
||
tos_notes="EP Open Data API v2 (limit 500 requests / 5 min / endpoint).",
|
||
)
|
||
EC_EXPORT = "https://ec.europa.eu/transparency-initiative/meetings/data/meetings/dataxlsx"
|
||
EC_TERMS = ("meetingscommissionrepresentatives2429", "meetingscommissionrepresentatives1924")
|
||
EP_API = "https://data.europarl.europa.eu/api/v2"
|
||
# Austrian Commissioners in the terms covered by EC_TERMS (fallback if the DB has no Commission roles yet).
|
||
AUSTRIAN_COMMISSIONERS = ("Johannes Hahn", "Magnus Brunner")
|
||
CABINET = re.compile(r"Cabinet of (?:Executive Vice-President|Vice-President|Commissioner|President)\s+(?P<name>.+)$")
|
||
|
||
|
||
@register
|
||
class CommissionMeetings(Adapter):
|
||
spec = EC_SPEC
|
||
name = "eu.commission_meetings"
|
||
description = "Meetings of (Austrian) EU Commissioners and their cabinets with interest representatives"
|
||
schedule = "0 5 * * 1"
|
||
queue = "bulk"
|
||
|
||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||
names = set(AUSTRIAN_COMMISSIONERS)
|
||
if ctx.conn is not None:
|
||
names |= {row["full_name"] for row in ctx.conn.execute(
|
||
"""SELECT DISTINCT p.full_name FROM role r JOIN person p ON p.id = r.person_id
|
||
JOIN organization o ON o.id = r.organization_id WHERE o.slug = 'europaeische-kommission'""")}
|
||
for term in EC_TERMS:
|
||
yield Request(url=EC_EXPORT, params={"name": term}, kind="xlsx", meta={"commissioners": sorted(names)})
|
||
|
||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||
import openpyxl
|
||
|
||
wb = openpyxl.load_workbook(io.BytesIO(response.body), read_only=True, data_only=True)
|
||
rows = list(wb.worksheets[0].iter_rows(values_only=True))
|
||
header_index = next((i for i, r in enumerate(rows) if r and r[0] == "Name of cabinet"), None)
|
||
if header_index is None:
|
||
ctx.stats["unexpected_layout"] += 1
|
||
return
|
||
header = [str(h) for h in rows[header_index]]
|
||
url = f"{EC_EXPORT}?name={response.request.params['name']}"
|
||
wanted = {normalize(n) for n in response.meta.get("commissioners") or AUSTRIAN_COMMISSIONERS}
|
||
for row in rows[header_index + 1:]:
|
||
rec = dict(zip(header, row))
|
||
cabinet = rec.get("Name of cabinet") or ""
|
||
m = CABINET.search(cabinet.strip())
|
||
if not m or not rec.get("Date of meeting"):
|
||
continue
|
||
commissioner = m.group("name").strip()
|
||
if normalize(commissioner) not in wanted:
|
||
ctx.stats["other_commissioners"] += 1
|
||
continue
|
||
org_name = (rec.get("Name of interest representative") or "").strip()
|
||
held = str(rec["Date of meeting"])[:10]
|
||
subject = (rec.get("Subject of the meeting") or "").strip()
|
||
reps = rec.get("Name of EC representative") or ""
|
||
titles = rec.get("Title of EC representative") or ""
|
||
register_id = (rec.get("Transparency register ID") or "").strip()
|
||
event_key = fingerprint("ec-meeting", cabinet, held, org_name, subject)
|
||
yield Event(key=event_key, kind="meeting", title=f"{cabinet}: {org_name}", starts_at=held,
|
||
location=rec.get("Location"), url=url, description=subject,
|
||
body=OrgRef(slug="europaeische-kommission"))
|
||
yield Claim(
|
||
claim_type="meeting", person=PersonRef(name=commissioner), occurred_at=held,
|
||
counterpart_org=OrgRef(name=org_name, kind="lobby",
|
||
ids={"eu_transparency": register_id} if register_id else {}) if org_name else None,
|
||
summary_de=f"Treffen ({'Kommissar persönlich' if 'Commissioner' in titles else 'Kabinett'}) mit "
|
||
f"{org_name}: {excerpt(subject, 160)}",
|
||
summary_en=f"Meeting ({'Commissioner' if 'Commissioner' in titles else 'cabinet'}) with {org_name}: "
|
||
f"{excerpt(subject, 160)}",
|
||
data={"cabinet": cabinet, "representatives": reps, "titles": titles, "location": rec.get("Location"),
|
||
"register_id": register_id, "date_precision": "day"},
|
||
evidence=[Evidence(url=url, doc_type="dataset_row", title=f"EU-Kommission: Treffen {cabinet}",
|
||
published_at=held, quote=f"{held} | {org_name} | {subject}",
|
||
locator=f"{held}:{org_name}:{subject[:60]}")],
|
||
topics=("Europäische Union",),
|
||
facet=Participation(event_key=event_key, participation="host"),
|
||
)
|
||
|
||
|
||
@register
|
||
class EuropeanParliament(Adapter):
|
||
spec = EP_SPEC
|
||
name = "europarl.meps"
|
||
description = "Current Austrian MEPs (mandates) and their declarations"
|
||
schedule = "30 4 * * 2"
|
||
creates_persons = True
|
||
|
||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||
yield Request(url=f"{EP_API}/meps/show-current", kind="meps",
|
||
params={"country-of-representation": "AT", "format": "application/ld+json", "limit": 100})
|
||
|
||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||
data = response.json().get("data") or []
|
||
if response.request.kind == "meps":
|
||
for mep in data:
|
||
mep_id = str(mep.get("identifier"))
|
||
given, family = mep.get("givenName"), mep.get("familyName")
|
||
name = f"{given} {family}" if given and family else mep.get("label")
|
||
ref = PersonRef(ids={"ep": mep_id}, name=name)
|
||
yield PersonRecord(ref=ref, full_name=name, given_name=given, family_name=family,
|
||
aliases=(mep.get("label") or "",), match_by_name=True)
|
||
yield Request(url=f"{EP_API}/meps/{mep_id}", params={"format": "application/ld+json"}, kind="mep",
|
||
meta={"mep_id": mep_id, "name": name, "group": mep.get("api:political-group")})
|
||
yield Request(url=f"{EP_API}/meps-declarations", kind="declarations",
|
||
params={"person-id": mep_id, "format": "application/ld+json", "limit": 100},
|
||
meta={"mep_id": mep_id, "name": name})
|
||
elif response.request.kind == "mep":
|
||
yield from self.parse_mep(response, data[0] if data else {})
|
||
else:
|
||
yield from self.parse_declarations(response, data)
|
||
|
||
def parse_mep(self, response: Response, mep: dict) -> Iterator[Any]:
|
||
meta = response.meta
|
||
ref = PersonRef(ids={"ep": meta["mep_id"]})
|
||
if mep.get("img"):
|
||
yield PersonRecord(ref=ref, full_name=meta["name"], photo_url=mep["img"],
|
||
birth_date=date.fromisoformat(mep["bday"][:10]) if mep.get("bday") else None)
|
||
starts = [m.get("memberDuring", {}).get("startDate") for m in mep.get("hasMembership") or []
|
||
if "EU_PARLIAMENT" in str(m.get("membershipClassification", ""))]
|
||
starts = sorted(s for s in starts if s)
|
||
body = OrgRef(slug="europaeisches-parlament")
|
||
start = date.fromisoformat(starts[-1]) if starts else None
|
||
url = f"https://www.europarl.europa.eu/meps/de/{meta['mep_id']}"
|
||
yield Claim(
|
||
claim_type="role", person=ref, org=body, valid_from=start,
|
||
summary_de=f"Mitglied des Europäischen Parlaments (Fraktion {meta.get('group') or '?'})",
|
||
summary_en=f"Member of the European Parliament ({meta.get('group') or '?'})",
|
||
data={"ep_group": meta.get("group")},
|
||
evidence=[Evidence(url=url, doc_type="api_record", title=f"Europäisches Parlament: {meta['name']}")],
|
||
facet=Role(role="mp", level="eu", body=body, start=start),
|
||
)
|
||
|
||
def parse_declarations(self, response: Response, docs: list[dict]) -> Iterator[Claim]:
|
||
meta = response.meta
|
||
for doc in docs:
|
||
titles = []
|
||
pdf = None
|
||
for expression in doc.get("is_realized_by") or []:
|
||
title = (expression.get("title") or {})
|
||
titles.extend(v for v in title.values() if v)
|
||
for manifestation in expression.get("is_embodied_by") or []:
|
||
if "pdf" in str(manifestation.get("media_type", "")).lower() and not pdf:
|
||
pdf = manifestation.get("is_exemplified_by")
|
||
title = next((t for t in titles if t), doc.get("id"))
|
||
url = f"https://www.europarl.europa.eu/{pdf}" if pdf and not pdf.startswith("http") else (
|
||
pdf or f"https://data.europarl.europa.eu/{doc.get('id')}")
|
||
kind = str(doc.get("work_type", "")).rsplit("/", 1)[-1]
|
||
yield Claim(
|
||
claim_type="declaration", person=PersonRef(ids={"ep": meta["mep_id"]}),
|
||
key=("ep-declaration", doc.get("id")), occurred_at=doc.get("document_date"),
|
||
summary_de=f"Erklärung im Europäischen Parlament: {excerpt(title, 200)}",
|
||
summary_en=f"Declaration filed with the European Parliament: {excerpt(title, 200)}",
|
||
data={"work_type": kind},
|
||
evidence=[Evidence(url=url, doc_type="pdf", title=title, published_at=doc.get("document_date"))],
|
||
topics=("Transparenz und Korruption",),
|
||
)
|