Politrack: track Austrian politicians from public sources
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
583ccba31c
187 files changed
+39890
No files matched your search
@@ -0,0 +1,193 @@
|
||||
"""EU institutions: Commission lobby meetings and the European Parliament (Austrian MEPs, declarations).
|
||||
|
||||
* European Commission - meetings with interest representatives (open data XLSX per Commission term).
|
||||
Every row names the cabinet ("Cabinet of Commissioner Magnus Brunner"). Only meetings of Austrian
|
||||
Commissioners' cabinets are kept (names passed from start(): Commission roles in the DB + a fallback list).
|
||||
* European Parliament Open Data API v2 (CC BY 4.0): current Austrian MEPs (authoritative mandate data),
|
||||
and their declarations (financial interests, conflicts of interest, gifts, ...).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import (
|
||||
Claim,
|
||||
Event,
|
||||
Evidence,
|
||||
OrgRef,
|
||||
Participation,
|
||||
PersonRecord,
|
||||
PersonRef,
|
||||
Role,
|
||||
)
|
||||
from politrack.text import excerpt, fingerprint, normalize
|
||||
|
||||
EC_SPEC = SourceSpec(
|
||||
slug="ec-meetings", name="Europäische Kommission – Treffen mit Interessenvertretern", kind="official",
|
||||
homepage="https://commission.europa.eu/about/service-standards-and-principles/transparency_en",
|
||||
license="EU open data (reuse permitted, attribution)", access="bulk", trust=90, min_interval=2.0,
|
||||
tos_notes="Open data export of meetings of Commissioners and their cabinets with interest representatives.",
|
||||
)
|
||||
EP_SPEC = SourceSpec(
|
||||
slug="europarl", name="Europäisches Parlament Open Data", kind="official",
|
||||
homepage="https://data.europarl.europa.eu", license="CC BY 4.0", access="api", trust=95, min_interval=1.0,
|
||||
tos_notes="EP Open Data API v2 (limit 500 requests / 5 min / endpoint).",
|
||||
)
|
||||
EC_EXPORT = "https://ec.europa.eu/transparency-initiative/meetings/data/meetings/dataxlsx"
|
||||
EC_TERMS = ("meetingscommissionrepresentatives2429", "meetingscommissionrepresentatives1924")
|
||||
EP_API = "https://data.europarl.europa.eu/api/v2"
|
||||
# Austrian Commissioners in the terms covered by EC_TERMS (fallback if the DB has no Commission roles yet).
|
||||
AUSTRIAN_COMMISSIONERS = ("Johannes Hahn", "Magnus Brunner")
|
||||
CABINET = re.compile(r"Cabinet of (?:Executive Vice-President|Vice-President|Commissioner|President)\s+(?P<name>.+)$")
|
||||
|
||||
|
||||
@register
|
||||
class CommissionMeetings(Adapter):
|
||||
spec = EC_SPEC
|
||||
name = "eu.commission_meetings"
|
||||
description = "Meetings of (Austrian) EU Commissioners and their cabinets with interest representatives"
|
||||
schedule = "0 5 * * 1"
|
||||
queue = "bulk"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
names = set(AUSTRIAN_COMMISSIONERS)
|
||||
if ctx.conn is not None:
|
||||
names |= {row["full_name"] for row in ctx.conn.execute(
|
||||
"""SELECT DISTINCT p.full_name FROM role r JOIN person p ON p.id = r.person_id
|
||||
JOIN organization o ON o.id = r.organization_id WHERE o.slug = 'europaeische-kommission'""")}
|
||||
for term in EC_TERMS:
|
||||
yield Request(url=EC_EXPORT, params={"name": term}, kind="xlsx", meta={"commissioners": sorted(names)})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
import openpyxl
|
||||
|
||||
wb = openpyxl.load_workbook(io.BytesIO(response.body), read_only=True, data_only=True)
|
||||
rows = list(wb.worksheets[0].iter_rows(values_only=True))
|
||||
header_index = next((i for i, r in enumerate(rows) if r and r[0] == "Name of cabinet"), None)
|
||||
if header_index is None:
|
||||
ctx.stats["unexpected_layout"] += 1
|
||||
return
|
||||
header = [str(h) for h in rows[header_index]]
|
||||
url = f"{EC_EXPORT}?name={response.request.params['name']}"
|
||||
wanted = {normalize(n) for n in response.meta.get("commissioners") or AUSTRIAN_COMMISSIONERS}
|
||||
for row in rows[header_index + 1:]:
|
||||
rec = dict(zip(header, row))
|
||||
cabinet = rec.get("Name of cabinet") or ""
|
||||
m = CABINET.search(cabinet.strip())
|
||||
if not m or not rec.get("Date of meeting"):
|
||||
continue
|
||||
commissioner = m.group("name").strip()
|
||||
if normalize(commissioner) not in wanted:
|
||||
ctx.stats["other_commissioners"] += 1
|
||||
continue
|
||||
org_name = (rec.get("Name of interest representative") or "").strip()
|
||||
held = str(rec["Date of meeting"])[:10]
|
||||
subject = (rec.get("Subject of the meeting") or "").strip()
|
||||
reps = rec.get("Name of EC representative") or ""
|
||||
titles = rec.get("Title of EC representative") or ""
|
||||
register_id = (rec.get("Transparency register ID") or "").strip()
|
||||
event_key = fingerprint("ec-meeting", cabinet, held, org_name, subject)
|
||||
yield Event(key=event_key, kind="meeting", title=f"{cabinet}: {org_name}", starts_at=held,
|
||||
location=rec.get("Location"), url=url, description=subject,
|
||||
body=OrgRef(slug="europaeische-kommission"))
|
||||
yield Claim(
|
||||
claim_type="meeting", person=PersonRef(name=commissioner), occurred_at=held,
|
||||
counterpart_org=OrgRef(name=org_name, kind="lobby",
|
||||
ids={"eu_transparency": register_id} if register_id else {}) if org_name else None,
|
||||
summary_de=f"Treffen ({'Kommissar persönlich' if 'Commissioner' in titles else 'Kabinett'}) mit "
|
||||
f"{org_name}: {excerpt(subject, 160)}",
|
||||
summary_en=f"Meeting ({'Commissioner' if 'Commissioner' in titles else 'cabinet'}) with {org_name}: "
|
||||
f"{excerpt(subject, 160)}",
|
||||
data={"cabinet": cabinet, "representatives": reps, "titles": titles, "location": rec.get("Location"),
|
||||
"register_id": register_id, "date_precision": "day"},
|
||||
evidence=[Evidence(url=url, doc_type="dataset_row", title=f"EU-Kommission: Treffen {cabinet}",
|
||||
published_at=held, quote=f"{held} | {org_name} | {subject}",
|
||||
locator=f"{held}:{org_name}:{subject[:60]}")],
|
||||
topics=("Europäische Union",),
|
||||
facet=Participation(event_key=event_key, participation="host"),
|
||||
)
|
||||
|
||||
|
||||
@register
|
||||
class EuropeanParliament(Adapter):
|
||||
spec = EP_SPEC
|
||||
name = "europarl.meps"
|
||||
description = "Current Austrian MEPs (mandates) and their declarations"
|
||||
schedule = "30 4 * * 2"
|
||||
creates_persons = True
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
yield Request(url=f"{EP_API}/meps/show-current", kind="meps",
|
||||
params={"country-of-representation": "AT", "format": "application/ld+json", "limit": 100})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
data = response.json().get("data") or []
|
||||
if response.request.kind == "meps":
|
||||
for mep in data:
|
||||
mep_id = str(mep.get("identifier"))
|
||||
given, family = mep.get("givenName"), mep.get("familyName")
|
||||
name = f"{given} {family}" if given and family else mep.get("label")
|
||||
ref = PersonRef(ids={"ep": mep_id}, name=name)
|
||||
yield PersonRecord(ref=ref, full_name=name, given_name=given, family_name=family,
|
||||
aliases=(mep.get("label") or "",), match_by_name=True)
|
||||
yield Request(url=f"{EP_API}/meps/{mep_id}", params={"format": "application/ld+json"}, kind="mep",
|
||||
meta={"mep_id": mep_id, "name": name, "group": mep.get("api:political-group")})
|
||||
yield Request(url=f"{EP_API}/meps-declarations", kind="declarations",
|
||||
params={"person-id": mep_id, "format": "application/ld+json", "limit": 100},
|
||||
meta={"mep_id": mep_id, "name": name})
|
||||
elif response.request.kind == "mep":
|
||||
yield from self.parse_mep(response, data[0] if data else {})
|
||||
else:
|
||||
yield from self.parse_declarations(response, data)
|
||||
|
||||
def parse_mep(self, response: Response, mep: dict) -> Iterator[Any]:
|
||||
meta = response.meta
|
||||
ref = PersonRef(ids={"ep": meta["mep_id"]})
|
||||
if mep.get("img"):
|
||||
yield PersonRecord(ref=ref, full_name=meta["name"], photo_url=mep["img"],
|
||||
birth_date=date.fromisoformat(mep["bday"][:10]) if mep.get("bday") else None)
|
||||
starts = [m.get("memberDuring", {}).get("startDate") for m in mep.get("hasMembership") or []
|
||||
if "EU_PARLIAMENT" in str(m.get("membershipClassification", ""))]
|
||||
starts = sorted(s for s in starts if s)
|
||||
body = OrgRef(slug="europaeisches-parlament")
|
||||
start = date.fromisoformat(starts[-1]) if starts else None
|
||||
url = f"https://www.europarl.europa.eu/meps/de/{meta['mep_id']}"
|
||||
yield Claim(
|
||||
claim_type="role", person=ref, org=body, valid_from=start,
|
||||
summary_de=f"Mitglied des Europäischen Parlaments (Fraktion {meta.get('group') or '?'})",
|
||||
summary_en=f"Member of the European Parliament ({meta.get('group') or '?'})",
|
||||
data={"ep_group": meta.get("group")},
|
||||
evidence=[Evidence(url=url, doc_type="api_record", title=f"Europäisches Parlament: {meta['name']}")],
|
||||
facet=Role(role="mp", level="eu", body=body, start=start),
|
||||
)
|
||||
|
||||
def parse_declarations(self, response: Response, docs: list[dict]) -> Iterator[Claim]:
|
||||
meta = response.meta
|
||||
for doc in docs:
|
||||
titles = []
|
||||
pdf = None
|
||||
for expression in doc.get("is_realized_by") or []:
|
||||
title = (expression.get("title") or {})
|
||||
titles.extend(v for v in title.values() if v)
|
||||
for manifestation in expression.get("is_embodied_by") or []:
|
||||
if "pdf" in str(manifestation.get("media_type", "")).lower() and not pdf:
|
||||
pdf = manifestation.get("is_exemplified_by")
|
||||
title = next((t for t in titles if t), doc.get("id"))
|
||||
url = f"https://www.europarl.europa.eu/{pdf}" if pdf and not pdf.startswith("http") else (
|
||||
pdf or f"https://data.europarl.europa.eu/{doc.get('id')}")
|
||||
kind = str(doc.get("work_type", "")).rsplit("/", 1)[-1]
|
||||
yield Claim(
|
||||
claim_type="declaration", person=PersonRef(ids={"ep": meta["mep_id"]}),
|
||||
key=("ep-declaration", doc.get("id")), occurred_at=doc.get("document_date"),
|
||||
summary_de=f"Erklärung im Europäischen Parlament: {excerpt(title, 200)}",
|
||||
summary_en=f"Declaration filed with the European Parliament: {excerpt(title, 200)}",
|
||||
data={"work_type": kind},
|
||||
evidence=[Evidence(url=url, doc_type="pdf", title=title, published_at=doc.get("document_date"))],
|
||||
topics=("Transparenz und Korruption",),
|
||||
)
|
||||
Reference in new issue
Block a user