"""Parliament: people, mandates, government functions, side-job disclosures, social links, speeches. Flow: list 409 (all MPs since 1918) -> PersonRecord + mandate roles -> detail /person/ for members of the configured periods detail -> photo, social accounts, functions, side jobs (§6 Unv-Transparenz-G) -> list 251 (speeches of that person) """ from __future__ import annotations import json import re from collections.abc import Iterator from typing import Any from politrack.config import get_settings from politrack.core import Adapter, Request, Response, RunContext, register from politrack.core.records import Claim, Evidence, OrgRef, PersonRecord, PersonRef, Role, SocialAccount, Statement from politrack.sources.parlament.common import ( BASE, CHAMBER_NAME_DE, CHAMBER_SLUG, GOVERNMENT_RE, SPEC, detail_request, list_request, list_rows, parse_date, walk_dicts, ) from politrack.text import firmenbuch_number, html_to_text, strip_titles SOCIAL_PLATFORMS = {"TWITTER": "twitter", "FACEBOOK": "facebook", "INSTAGRAM": "instagram", "TIKTOK": "tiktok", "YOUTUBE": "youtube", "LINKEDIN": "linkedin", "WEBSITE": "website", "BLUESKY": "bluesky", "MASTODON": "mastodon", "TELEGRAM": "telegram", "THREADS": "threads"} FN_IN_TEXT = re.compile(r"\bFN\s*\d{1,6}\s?[a-zA-Z]\b") POSTCODE_CITY = re.compile(r"(?:(?<=\d)|\b)\d{4}\s+[A-ZÄÖÜ][\wäöüß .\-]*$") LEGAL_FORM_END = re.compile(r"(?i:gesmbh|gmbh|ges\.\s?m\.\s?b\.\s?h\.?|m\.\s?b\.\s?h\.?|aktiengesellschaft|\bag|\bkg|\bog|" r"e\.\s?u\.|\bse|privatstiftung|genossenschaft|sparkasse)(?=[A-ZÄÖÜ\s,]|$)") COMMITTEE_END = re.compile(r"\(\d{2}\.\d{2}\.\d{4}\s*-\s*(\d{2}\.\d{2}\.\d{4})\s*\)\s*$") STREET_WORD = r"\S*(?:straße|strasse|str\.|gasse|platz|weg|kai|ring|allee|zeile|lände|markt|steig)" def strip_street(text: str) -> str: """'Verein Musterhilfe Hauptstraße 12' -> 'Verein Musterhilfe' (greedy name, shortest street suffix).""" m = (re.match(rf"^(.*\S)\s+{STREET_WORD}\s*\d+[a-z]?(?:/\d+)*\s*$", text, re.IGNORECASE) or re.match(r"^(.*\S)\s+\S+\s*\d+[a-z]?(?:/\d+)*\s*$", text)) return m.group(1) if m else text def employer(cell: str) -> tuple[str | None, str | None]: """Employer name without business address, plus its company register number. 'Hypo Tirol Bank AG, Meraner Straße 8, 6020 Innsbruck' -> ('Hypo Tirol Bank AG', None) '25 PersonaldienstleistungsgesmbHFranz Josef Kai 95020 Salzburg FN306527a' (parliament's JSON glues the lines together) -> ('25 PersonaldienstleistungsgesmbH', '306527a') """ cell = re.sub(r"^lit\.\s*\w\s*:\s*", "", cell).strip() fn_match = FN_IN_TEXT.search(cell) fn = firmenbuch_number(fn_match.group(0)) if fn_match else None name = FN_IN_TEXT.sub("", cell).split(",")[0].strip(" *") address = POSTCODE_CITY.search(name) if address: head = name[:address.start()] ends = [m.end() for m in LEGAL_FORM_END.finditer(head)] # cut after the last legal form; otherwise drop " " before the postcode name = head[:ends[-1]] if ends else strip_street(head) name = name.strip(" ,-*") return (name if len(name) >= 3 and not re.fullmatch(r"[\d\s.€-]+", name) else None), fn @register class ParlamentPersons(Adapter): spec = SPEC name = "parlament.persons" description = "Members of Nationalrat and Bundesrat since 1918, mandates, functions, side jobs, speeches" schedule = "15 3 * * *" creates_persons = True parser_version = 4 def start(self, ctx: RunContext) -> Iterator[Request]: yield list_request(409, {"NRBR": ["NR", "BR"]}, kind="persons") def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: kind = response.request.kind if kind == "persons": yield from self.parse_persons(ctx, response) elif kind == "person": yield from self.parse_person(ctx, response) elif kind == "speeches": yield from self.parse_speeches(ctx, response) elif kind == "committees": yield from self.parse_committees(ctx, response) # -- persons & mandates ------------------------------------------------------------- def parse_persons(self, ctx: RunContext, response: Response) -> Iterator[Any]: periods = set(ctx.params.get("periods") or get_settings().parlament_periods) for row in list_rows(response.json()): attr = row.get("Attribute") or {} if isinstance(attr, str): attr = json.loads(attr) m = re.search(r"/person/(\d+)", attr.get("uri") or "") if not m: continue pad = m.group(1) ref = PersonRef(ids={"parlament": pad}) display = attr.get("zit") or strip_titles(row.get("Name") or "") yield PersonRecord(ref=ref, full_name=display, given_name=row.get("vorname"), family_name=row.get("nachname"), gender={"M": "male", "W": "female"}.get(row.get("GESCHL_CODE") or attr.get("geschlecht"))) mandates = attr.get("mandate_detail") or [] for mandate in mandates: claim = self.mandate_claim(pad, display, mandate) if claim: yield claim if any(md.get("gp_code") in periods for md in mandates): yield detail_request(f"/person/{pad}", kind="person", meta={"pad": pad, "name": display}) def mandate_claim(self, pad: str, name: str, m: dict) -> Claim | None: code = m.get("mand_code") or "NR" if code not in CHAMBER_SLUG: return None party_code = m.get("wahlpartei_code") party_name = m.get("wahlpartei_full_txt") or m.get("wahlpartei_txt") or party_code party = OrgRef(name=party_name, short_name=party_code, kind="party") if party_name else None start, end = parse_date(m.get("mandat_von")), parse_date(m.get("mandat_bis")) gp = m.get("gp_code") or "" region = m.get("wahlkreis_bundesland") label = party_code or party_name or "?" body = OrgRef(slug=CHAMBER_SLUG[code]) return Claim( claim_type="role", person=PersonRef(ids={"parlament": pad}), org=body, counterpart_org=party, valid_from=start, valid_to=end, summary_de=f"{name}: Mandat im {CHAMBER_NAME_DE[code]} ({gp}. GP) für {label}" + (f", Wahlkreis {region}" if region else ""), summary_en=f"{name}: member of the {CHAMBER_NAME_DE[code]} ({gp} legislative period) for {label}", data={"gp": gp, "party_code": party_code, "faction_code": m.get("fraktionscode"), "region": region}, evidence=[Evidence(url=f"{BASE}/person/{pad}", doc_type="api_record", title=f"Parlament: {name}", locator=f"mandat:{code}:{gp}:{start}")], facet=Role(role="mp", level="federal", body=body, party=party, region=region, start=start, end=end), ) # -- person detail ---------------------------------------------------------------- def parse_person(self, ctx: RunContext, response: Response) -> Iterator[Any]: data = response.json() content = data.get("content") or {} pad, name = response.meta["pad"], response.meta["name"] ref = PersonRef(ids={"parlament": pad}) url = f"{BASE}/person/{pad}" photo = ((data.get("meta") or {}).get("openGraph") or {}).get("image") if photo: yield PersonRecord(ref=ref, full_name=name, photo_url=photo) for account in (content.get("kontakt") or {}).get("socialMedia") or []: platform = SOCIAL_PLATFORMS.get((account.get("type") or "").upper(), (account.get("type") or "website").lower()) yield SocialAccount(platform=platform, url=account["url"], person=ref, verified=True) for fn in walk_dicts(content.get("biografie") or {}, "funktion_text"): if fn.get("gremium") in CHAMBER_SLUG: continue text = fn.get("funktion_text") or fn.get("bez") if not text: continue start, end = parse_date(fn.get("funktion_von")), parse_date(fn.get("funktion_bis")) is_gov = bool(GOVERNMENT_RE.search(text)) body = OrgRef(slug="bundesregierung") if is_gov else None yield Claim( claim_type="role", person=ref, org=body, valid_from=start, valid_to=end, summary_de=f"{name}: {text}", evidence=[Evidence(url=url, doc_type="api_record", title=f"Parlament: {name}", locator=f"funktion:{text}:{start}")], facet=Role(role=text, level="federal" if is_gov else "other", body=body, start=start, end=end), ) yield from self.side_jobs(ref, pad, name, content.get("unvtrans") or {}) yield list_request(251, {"PAD_INTERN": [int(pad)]}, kind="speeches", meta={"pad": pad, "name": name}) yield list_request(250, {"PAD_INTERN": [int(pad)]}, kind="committees", meta={"pad": pad, "name": name}) def parse_committees(self, ctx: RunContext, response: Response) -> Iterator[Claim]: """Committee memberships (list 250). Affiliations, not roles: a member has dozens of them per period, and they say nothing about whether the person currently holds a mandate.""" periods = set(ctx.params.get("periods") or get_settings().parlament_periods) pad, name = response.meta["pad"], response.meta["name"] for row in list_rows(response.json()): if row.get("GP") not in periods: continue labelled = row.get("Ausschuss") or "" committee = re.sub(r"\s*\([^()]*\)\s*$", "", labelled).strip() if not committee: ctx.stats["committee_without_name"] += 1 continue function = row.get("Funktion") or "Mitglied" start = parse_date(row.get("Beginn")) end = COMMITTEE_END.search(labelled) chamber = "Bundesrat" if row.get("nrbr") == "BR" else "Nationalrat" yield Claim( claim_type="affiliation", person=PersonRef(ids={"parlament": pad}), key=("parl-committee", pad, row.get("GP"), committee, function, start), valid_from=start, valid_to=parse_date(end.group(1)) if end else None, summary_de=f"{name}: {function}, {committee} ({chamber}, {row.get('GP')}. GP)", summary_en=f"{name}: {function}, {committee} ({chamber}, {row.get('GP')} legislative period)", data={"committee": committee, "function": function, "gp": row.get("GP"), "chamber": row.get("nrbr")}, evidence=[Evidence(url=f"{BASE}/person/{pad}", doc_type="api_record", title=f"Parlament: {name}", locator=f"ausschuss:{row.get('GP')}:{committee}:{function}:{start}")], ) def side_jobs(self, ref: PersonRef, pad: str, name: str, unv: dict) -> Iterator[Claim]: """Disclosures under §6 Unv-Transparenz-G: leading positions, paid side jobs, honorary positions.""" tables = unv.get("unvtransData") or {} pdf = next((d["link"] for group in unv.get("linkliste") or [] for d in group.get("documents") or [] if d.get("link", "").endswith(f"/person/{pad}/BezBegrBVGPar9.pdf")), None) for key, table in tables.items(): m = re.fullmatch(r"(z\d+)Table", key) if not m or not isinstance(table, dict): continue header = html_to_text(tables.get(f"{m.group(1)}Header")) or key for row in (table.get("data") or {}).get("rows") or []: cells = [c for c in (html_to_text(c) for c in row if isinstance(c, str)) if c] if not cells: continue name_, fn = employer(cells[0]) if m.group(1) in ("z1", "z2", "z3") else (None, None) if name_: # never keep the (business) address that parliament publishes with the employer cells = [name_ + (f" (FN {fn})" if fn else ""), *cells[1:]] text = " – ".join(cells) yield Claim( claim_type="side_job", person=ref, key=("unvtrans", pad, m.group(1), text), counterpart_org=OrgRef(name=name_, kind="company" if LEGAL_FORM_END.search(name_) else "other", ids={"firmenbuch": fn} if fn else {}) if name_ else None, summary_de=f"{name} – {header.split(' - ')[0]}: {text}", data={"category": header, "cells": cells, "unpaid": "*" in "".join(str(c) for c in row), "employer": name_, "firmenbuch": fn}, evidence=[Evidence(url=f"{BASE}{pdf}" if pdf else f"{BASE}/person/{pad}", doc_type="register_entry", title=f"Unvereinbarkeits-/Transparenzliste {name}", quote=text, locator=f"unvtrans:{m.group(1)}")], topics=("Transparenz und Korruption",), ) # -- speeches ----------------------------------------------------------------------- def parse_speeches(self, ctx: RunContext, response: Response) -> Iterator[Claim]: periods = set(ctx.params.get("periods") or get_settings().parlament_periods) pad, name = response.meta["pad"], response.meta["name"] for row in list_rows(response.json()): if row.get("GP") not in periods and row.get("GREMIUM") != "B": continue html_cells = " ".join(str(v) for v in row.values() if isinstance(v, str)) protocol = re.search(r'href="(/dokument/[^"]+)"', html_cells) session = html_to_text(row.get("Sitzung")) topic = html_to_text(row.get("Rede")) said_at = row.get("fromDate") url = f"{BASE}{protocol.group(1)}" if protocol else f"{BASE}/person/{pad}" chamber = "Bundesrat" if row.get("GREMIUM") == "B" else "Nationalrat" yield Claim( claim_type="statement", person=PersonRef(ids={"parlament": pad}), key=("parl-speech", pad, row.get("Sortier"), topic), occurred_at=said_at, summary_de=f"{name} sprach im {chamber} zu: {topic}", summary_en=f"{name} spoke in the {chamber} on: {topic}", data={"session": session, "gp": row.get("GP"), "item_title": topic}, evidence=[Evidence(url=url, doc_type="protocol", title=session, published_at=parse_date(said_at), locator=url.split("#", 1)[-1] if "#" in url else "")], facet=Statement(context="parliament_speech", url=url, said_at=said_at, medium=chamber, title=topic), )