simonandClaude Opus 5.5
583ccba31c
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
240 lines
13 KiB
Python
240 lines
13 KiB
Python
"""Parliament: people, mandates, government functions, side-job disclosures, social links, speeches.
|
||
|
||
Flow: list 409 (all MPs since 1918) -> PersonRecord + mandate roles
|
||
-> detail /person/<id> for members of the configured periods
|
||
detail -> photo, social accounts, functions, side jobs (§6 Unv-Transparenz-G)
|
||
-> list 251 (speeches of that person)
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
from collections.abc import Iterator
|
||
from typing import Any
|
||
|
||
from politrack.config import get_settings
|
||
from politrack.core import Adapter, Request, Response, RunContext, register
|
||
from politrack.core.records import Claim, Evidence, OrgRef, PersonRecord, PersonRef, Role, SocialAccount, Statement
|
||
from politrack.sources.parlament.common import (
|
||
BASE,
|
||
CHAMBER_NAME_DE,
|
||
CHAMBER_SLUG,
|
||
GOVERNMENT_RE,
|
||
SPEC,
|
||
detail_request,
|
||
list_request,
|
||
list_rows,
|
||
parse_date,
|
||
walk_dicts,
|
||
)
|
||
from politrack.text import firmenbuch_number, html_to_text, strip_titles
|
||
|
||
SOCIAL_PLATFORMS = {"TWITTER": "twitter", "FACEBOOK": "facebook", "INSTAGRAM": "instagram", "TIKTOK": "tiktok",
|
||
"YOUTUBE": "youtube", "LINKEDIN": "linkedin", "WEBSITE": "website", "BLUESKY": "bluesky",
|
||
"MASTODON": "mastodon", "TELEGRAM": "telegram", "THREADS": "threads"}
|
||
|
||
|
||
FN_IN_TEXT = re.compile(r"\bFN\s*\d{1,6}\s?[a-zA-Z]\b")
|
||
POSTCODE_CITY = re.compile(r"(?:(?<=\d)|\b)\d{4}\s+[A-ZÄÖÜ][\wäöüß .\-]*$")
|
||
LEGAL_FORM_END = re.compile(r"(?i:gesmbh|gmbh|ges\.\s?m\.\s?b\.\s?h\.?|m\.\s?b\.\s?h\.?|aktiengesellschaft|\bag|\bkg|\bog|"
|
||
r"e\.\s?u\.|\bse|privatstiftung|genossenschaft|sparkasse)(?=[A-ZÄÖÜ\s,]|$)")
|
||
|
||
|
||
STREET_WORD = r"\S*(?:straße|strasse|str\.|gasse|platz|weg|kai|ring|allee|zeile|lände|markt|steig)"
|
||
|
||
|
||
def strip_street(text: str) -> str:
|
||
"""'Verein Musterhilfe Hauptstraße 12' -> 'Verein Musterhilfe' (greedy name, shortest street suffix)."""
|
||
m = (re.match(rf"^(.*\S)\s+{STREET_WORD}\s*\d+[a-z]?(?:/\d+)*\s*$", text, re.IGNORECASE)
|
||
or re.match(r"^(.*\S)\s+\S+\s*\d+[a-z]?(?:/\d+)*\s*$", text))
|
||
return m.group(1) if m else text
|
||
|
||
|
||
def employer(cell: str) -> tuple[str | None, str | None]:
|
||
"""Employer name without business address, plus its company register number.
|
||
|
||
'Hypo Tirol Bank AG, Meraner Straße 8, 6020 Innsbruck' -> ('Hypo Tirol Bank AG', None)
|
||
'25 PersonaldienstleistungsgesmbHFranz Josef Kai 95020 Salzburg FN306527a' (parliament's JSON glues
|
||
the lines together) -> ('25 PersonaldienstleistungsgesmbH', '306527a')
|
||
"""
|
||
cell = re.sub(r"^lit\.\s*\w\s*:\s*", "", cell).strip()
|
||
fn_match = FN_IN_TEXT.search(cell)
|
||
fn = firmenbuch_number(fn_match.group(0)) if fn_match else None
|
||
name = FN_IN_TEXT.sub("", cell).split(",")[0].strip(" *")
|
||
address = POSTCODE_CITY.search(name)
|
||
if address:
|
||
head = name[:address.start()]
|
||
ends = [m.end() for m in LEGAL_FORM_END.finditer(head)]
|
||
# cut after the last legal form; otherwise drop "<street> <number>" before the postcode
|
||
name = head[:ends[-1]] if ends else strip_street(head)
|
||
name = name.strip(" ,-*")
|
||
return (name if len(name) >= 3 and not re.fullmatch(r"[\d\s.€-]+", name) else None), fn
|
||
|
||
|
||
@register
|
||
class ParlamentPersons(Adapter):
|
||
spec = SPEC
|
||
name = "parlament.persons"
|
||
description = "Members of Nationalrat and Bundesrat since 1918, mandates, functions, side jobs, speeches"
|
||
schedule = "15 3 * * *"
|
||
creates_persons = True
|
||
parser_version = 3
|
||
|
||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||
yield list_request(409, {"NRBR": ["NR", "BR"]}, kind="persons")
|
||
|
||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||
kind = response.request.kind
|
||
if kind == "persons":
|
||
yield from self.parse_persons(ctx, response)
|
||
elif kind == "person":
|
||
yield from self.parse_person(ctx, response)
|
||
elif kind == "speeches":
|
||
yield from self.parse_speeches(ctx, response)
|
||
|
||
# -- persons & mandates -------------------------------------------------------------
|
||
|
||
def parse_persons(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||
periods = set(ctx.params.get("periods") or get_settings().parlament_periods)
|
||
for row in list_rows(response.json()):
|
||
attr = row.get("Attribute") or {}
|
||
if isinstance(attr, str):
|
||
attr = json.loads(attr)
|
||
m = re.search(r"/person/(\d+)", attr.get("uri") or "")
|
||
if not m:
|
||
continue
|
||
pad = m.group(1)
|
||
ref = PersonRef(ids={"parlament": pad})
|
||
display = attr.get("zit") or strip_titles(row.get("Name") or "")
|
||
yield PersonRecord(ref=ref, full_name=display, given_name=row.get("vorname"), family_name=row.get("nachname"),
|
||
gender={"M": "male", "W": "female"}.get(row.get("GESCHL_CODE") or attr.get("geschlecht")))
|
||
mandates = attr.get("mandate_detail") or []
|
||
for mandate in mandates:
|
||
claim = self.mandate_claim(pad, display, mandate)
|
||
if claim:
|
||
yield claim
|
||
if any(md.get("gp_code") in periods for md in mandates):
|
||
yield detail_request(f"/person/{pad}", kind="person", meta={"pad": pad, "name": display})
|
||
|
||
def mandate_claim(self, pad: str, name: str, m: dict) -> Claim | None:
|
||
code = m.get("mand_code") or "NR"
|
||
if code not in CHAMBER_SLUG:
|
||
return None
|
||
party_code = m.get("wahlpartei_code")
|
||
party_name = m.get("wahlpartei_full_txt") or m.get("wahlpartei_txt") or party_code
|
||
party = OrgRef(name=party_name, short_name=party_code, kind="party") if party_name else None
|
||
start, end = parse_date(m.get("mandat_von")), parse_date(m.get("mandat_bis"))
|
||
gp = m.get("gp_code") or ""
|
||
region = m.get("wahlkreis_bundesland")
|
||
label = party_code or party_name or "?"
|
||
body = OrgRef(slug=CHAMBER_SLUG[code])
|
||
return Claim(
|
||
claim_type="role",
|
||
person=PersonRef(ids={"parlament": pad}),
|
||
org=body,
|
||
counterpart_org=party,
|
||
valid_from=start,
|
||
valid_to=end,
|
||
summary_de=f"{name}: Mandat im {CHAMBER_NAME_DE[code]} ({gp}. GP) für {label}"
|
||
+ (f", Wahlkreis {region}" if region else ""),
|
||
summary_en=f"{name}: member of the {CHAMBER_NAME_DE[code]} ({gp} legislative period) for {label}",
|
||
data={"gp": gp, "party_code": party_code, "faction_code": m.get("fraktionscode"), "region": region},
|
||
evidence=[Evidence(url=f"{BASE}/person/{pad}", doc_type="api_record", title=f"Parlament: {name}",
|
||
locator=f"mandat:{code}:{gp}:{start}")],
|
||
facet=Role(role="mp", level="federal", body=body, party=party, region=region, start=start, end=end),
|
||
)
|
||
|
||
# -- person detail ----------------------------------------------------------------
|
||
|
||
def parse_person(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||
data = response.json()
|
||
content = data.get("content") or {}
|
||
pad, name = response.meta["pad"], response.meta["name"]
|
||
ref = PersonRef(ids={"parlament": pad})
|
||
url = f"{BASE}/person/{pad}"
|
||
photo = ((data.get("meta") or {}).get("openGraph") or {}).get("image")
|
||
if photo:
|
||
yield PersonRecord(ref=ref, full_name=name, photo_url=photo)
|
||
|
||
for account in (content.get("kontakt") or {}).get("socialMedia") or []:
|
||
platform = SOCIAL_PLATFORMS.get((account.get("type") or "").upper(), (account.get("type") or "website").lower())
|
||
yield SocialAccount(platform=platform, url=account["url"], person=ref, verified=True)
|
||
|
||
for fn in walk_dicts(content.get("biografie") or {}, "funktion_text"):
|
||
if fn.get("gremium") in CHAMBER_SLUG:
|
||
continue
|
||
text = fn.get("funktion_text") or fn.get("bez")
|
||
if not text:
|
||
continue
|
||
start, end = parse_date(fn.get("funktion_von")), parse_date(fn.get("funktion_bis"))
|
||
is_gov = bool(GOVERNMENT_RE.search(text))
|
||
body = OrgRef(slug="bundesregierung") if is_gov else None
|
||
yield Claim(
|
||
claim_type="role", person=ref, org=body, valid_from=start, valid_to=end,
|
||
summary_de=f"{name}: {text}",
|
||
evidence=[Evidence(url=url, doc_type="api_record", title=f"Parlament: {name}",
|
||
locator=f"funktion:{text}:{start}")],
|
||
facet=Role(role=text, level="federal" if is_gov else "other", body=body, start=start, end=end),
|
||
)
|
||
|
||
yield from self.side_jobs(ref, pad, name, content.get("unvtrans") or {})
|
||
yield list_request(251, {"PAD_INTERN": [int(pad)]}, kind="speeches", meta={"pad": pad, "name": name})
|
||
|
||
def side_jobs(self, ref: PersonRef, pad: str, name: str, unv: dict) -> Iterator[Claim]:
|
||
"""Disclosures under §6 Unv-Transparenz-G: leading positions, paid side jobs, honorary positions."""
|
||
tables = unv.get("unvtransData") or {}
|
||
pdf = next((d["link"] for group in unv.get("linkliste") or [] for d in group.get("documents") or []
|
||
if d.get("link", "").endswith(f"/person/{pad}/BezBegrBVGPar9.pdf")), None)
|
||
for key, table in tables.items():
|
||
m = re.fullmatch(r"(z\d+)Table", key)
|
||
if not m or not isinstance(table, dict):
|
||
continue
|
||
header = html_to_text(tables.get(f"{m.group(1)}Header")) or key
|
||
for row in (table.get("data") or {}).get("rows") or []:
|
||
cells = [c for c in (html_to_text(c) for c in row if isinstance(c, str)) if c]
|
||
if not cells:
|
||
continue
|
||
name_, fn = employer(cells[0]) if m.group(1) in ("z1", "z2", "z3") else (None, None)
|
||
if name_: # never keep the (business) address that parliament publishes with the employer
|
||
cells = [name_ + (f" (FN {fn})" if fn else ""), *cells[1:]]
|
||
text = " – ".join(cells)
|
||
yield Claim(
|
||
claim_type="side_job", person=ref, key=("unvtrans", pad, m.group(1), text),
|
||
counterpart_org=OrgRef(name=name_, kind="company" if LEGAL_FORM_END.search(name_) else "other",
|
||
ids={"firmenbuch": fn} if fn else {}) if name_ else None,
|
||
summary_de=f"{name} – {header.split(' - ')[0]}: {text}",
|
||
data={"category": header, "cells": cells, "unpaid": "*" in "".join(str(c) for c in row),
|
||
"employer": name_, "firmenbuch": fn},
|
||
evidence=[Evidence(url=f"{BASE}{pdf}" if pdf else f"{BASE}/person/{pad}", doc_type="register_entry",
|
||
title=f"Unvereinbarkeits-/Transparenzliste {name}", quote=text,
|
||
locator=f"unvtrans:{m.group(1)}")],
|
||
topics=("Transparenz und Korruption",),
|
||
)
|
||
|
||
# -- speeches -----------------------------------------------------------------------
|
||
|
||
def parse_speeches(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||
periods = set(ctx.params.get("periods") or get_settings().parlament_periods)
|
||
pad, name = response.meta["pad"], response.meta["name"]
|
||
for row in list_rows(response.json()):
|
||
if row.get("GP") not in periods and row.get("GREMIUM") != "B":
|
||
continue
|
||
html_cells = " ".join(str(v) for v in row.values() if isinstance(v, str))
|
||
protocol = re.search(r'href="(/dokument/[^"]+)"', html_cells)
|
||
session = html_to_text(row.get("Sitzung"))
|
||
topic = html_to_text(row.get("Rede"))
|
||
said_at = row.get("fromDate")
|
||
url = f"{BASE}{protocol.group(1)}" if protocol else f"{BASE}/person/{pad}"
|
||
chamber = "Bundesrat" if row.get("GREMIUM") == "B" else "Nationalrat"
|
||
yield Claim(
|
||
claim_type="statement", person=PersonRef(ids={"parlament": pad}),
|
||
key=("parl-speech", pad, row.get("Sortier"), topic), occurred_at=said_at,
|
||
summary_de=f"{name} sprach im {chamber} zu: {topic}",
|
||
summary_en=f"{name} spoke in the {chamber} on: {topic}",
|
||
data={"session": session, "gp": row.get("GP"), "item_title": topic},
|
||
evidence=[Evidence(url=url, doc_type="protocol", title=session, published_at=parse_date(said_at),
|
||
locator=url.split("#", 1)[-1] if "#" in url else "")],
|
||
facet=Statement(context="parliament_speech", url=url, said_at=said_at, medium=chamber, title=topic),
|
||
)
|