"""Parliament items (Gegenstände) of the configured legislative periods. From one list export per period (list 101) we derive: * votes -> item detail -> VoteEvent + faction VotePositions roll calls -> stenographic protocol -> individual VotePositions * immunity requests ("Ersuchen um Zustimmung zur behördlichen Verfolgung") -> committee report -> Legal claim * written questions and answers (ITYP J / AB) -> parliamentary_question claims (directly from the list) * investigative-committee hearings ("öffentliche Befragung der Auskunftsperson ...") -> hearing Event + appearance claim for politicians who testified (other witnesses are not stored) """ from __future__ import annotations import json import re from collections.abc import Iterator from typing import Any from politrack.config import get_settings from politrack.core import Adapter, Request, Response, RunContext, register from politrack.core.records import ( Claim, Event, Evidence, Legal, OrgRef, Participation, PersonRef, VoteEvent, VotePosition, ) from politrack.sources.parlament.common import ( BASE, CHAMBER_SLUG, SPEC, detail_request, list_request, list_rows, parse_date, parse_immunity_report, parse_roll_call, parse_vote_text, walk_dicts, ) from politrack.text import fingerprint, html_to_text, strip_titles HEARING = re.compile(r"Befragung der Auskunftsperson\s+(?P.+)$") WITNESS_TITLES = re.compile(r"^(?:(?:Oberstaatsanwalt|Staatsanwalt|Sektionschef(?:in)?|MinR|Generalsekretär(?:in)?|" r"Bundesminister(?:in)?|Abgeordnete[rn]?|Klubobmann|Klubobfrau|Bundeskanzler|" r"Vizekanzler|Präsident(?:in)?)\s+)+") @register class ParlamentItems(Adapter): spec = SPEC name = "parlament.items" description = "Votes (faction + roll call), immunity requests, written questions, U-committee hearings" schedule = "45 3 * * *" parser_version = 2 def start(self, ctx: RunContext) -> Iterator[Request]: for gp in ctx.params.get("periods") or get_settings().parlament_periods: yield list_request(101, {"GP_CODE": [gp], "NRBR": ["NR"]}, kind="items", meta={"gp": gp}) def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: handler = { "items": self.parse_items, "votes": self.parse_votes, "rollcall": self.parse_rollcall, "immunity": self.parse_immunity_item, "immunity_report": self.parse_immunity_report, "hearing": self.parse_hearing_item, }[response.request.kind] yield from handler(ctx, response) # -- list --------------------------------------------------------------------------- def parse_items(self, ctx: RunContext, response: Response) -> Iterator[Any]: for row in list_rows(response.json()): path, subject, kind = row.get("HIS_URL"), row.get("Betreff") or "", row.get("ITYP") if not path: continue if row.get("Dafür") or row.get("Dagegen") or row.get("Abstimmungstext"): # The list row changes when new votes happen, so the detail is re-fetched only then # (conditional request + unchanged-body detection do the rest). yield detail_request(path, kind="votes", meta={"title": subject, "ref": row.get("Nummer")}) if subject.startswith("Ersuchen") and "behördlichen Verfolgung" in subject: yield detail_request(path, kind="immunity", meta={"title": subject, "date": row.get("Datum"), "status": row.get("Status")}) if kind in ("J", "AB"): yield from self.question_claims(row, path) if kind == "KOMM" and HEARING.search(subject): yield detail_request(path, kind="hearing", meta={"title": subject, "date": row.get("Datum")}) def question_claims(self, row: dict, path: str) -> Iterator[Claim]: pads = row.get("Personen") or "[]" pads = json.loads(pads) if isinstance(pads, str) else pads subject = row.get("Betreff") or "" is_question = row.get("ITYP") == "J" asked = parse_date(row.get("Datum")) topics = tuple(json.loads(row.get("THEMEN") or "[]")) if isinstance(row.get("THEMEN"), str) else () for pad in pads: if not str(pad).strip(): continue verb_de = "stellte die schriftliche Anfrage" if is_question else "beantwortete die Anfrage" verb_en = "submitted the written question" if is_question else "answered the question" yield Claim( claim_type="parliamentary_question", person=PersonRef(ids={"parlament": str(pad)}), key=("question", path, str(pad)), occurred_at=asked, summary_de=f"{verb_de[0].upper()}{verb_de[1:]} „{subject}“ ({row.get('Nummer')})", summary_en=f"{verb_en[0].upper()}{verb_en[1:]} \"{subject}\" ({row.get('Nummer')})", data={"kind": "question" if is_question else "answer", "number": row.get("Nummer")}, evidence=[Evidence(url=f"{BASE}{path}", doc_type="api_record", title=subject, published_at=asked)], topics=topics, ) # -- votes ---------------------------------------------------------------------------- def parse_votes(self, ctx: RunContext, response: Response) -> Iterator[Any]: content = response.json().get("content") or {} path = response.request.url.removeprefix(BASE) title = html_to_text(content.get("title")) or response.meta.get("title") or path item_ref = html_to_text(content.get("zitation")) or response.meta.get("ref") item_url = f"{BASE}{path}" topics = tuple(b.get("label") for b in ((content.get("topics") or {}).get("data") or {}).get("bubbles") or []) eurovoc = [b.get("label") for b in ((content.get("eurovoc") or {}).get("data") or {}).get("bubbles") or []] for stage in walk_dicts(content.get("phase") or content, "text"): text = html_to_text(stage.get("text")) if "angenommen" not in text and "abgelehnt" not in text: continue parsed = parse_vote_text(text) if not parsed or not (parsed["for"] or parsed["against"] or parsed["unanimous"] or parsed["split"] or parsed["roll_call"]): continue voted_on = parse_date(stage.get("date")) refs = stage.get("fsth") or [] protocol_url = f"{BASE}{refs[0]['url']}" if refs and refs[0].get("url") else None chamber = "BR" if (refs and refs[0].get("gp_code") == "BR") or "Bundesrat" in text else "NR" chamber_ref = OrgRef(slug=CHAMBER_SLUG[chamber]) event_key = fingerprint("vote", path, voted_on, parsed["stage"]) yield VoteEvent(key=event_key, chamber=chamber_ref, title=title, voted_on=voted_on, result=parsed["result"], stage=parsed["stage"], item_ref=item_ref, roll_call=parsed["roll_call"], url=item_url, protocol_url=protocol_url, raw_text=text) evidence = [Evidence(url=item_url, doc_type="api_record", title=title, published_at=voted_on, quote=text, locator=f"stage:{voted_on}:{parsed['stage'][:80]}")] if protocol_url: evidence.append(Evidence(url=protocol_url, doc_type="protocol", published_at=voted_on, title=refs[0].get("title") or "Stenographisches Protokoll")) outcome = "angenommen" if parsed["result"] == "accepted" else "abgelehnt" for party, position in [(p, "for") for p in parsed["for"]] + [(p, "against") for p in parsed["against"]]: verb_de, verb_en = ("stimmte für", "voted for") if position == "for" else ("stimmte gegen", "voted against") yield Claim( claim_type="vote", org=OrgRef(name=party, short_name=party, kind="parliamentary_group"), counterpart_org=chamber_ref, occurred_at=voted_on, summary_de=f"{party} {verb_de}: {title} ({parsed['stage']}) – {outcome}", summary_en=f"{party} {verb_en}: {title} ({parsed['stage']})", data={"position": position, "result": parsed["result"], "item_ref": item_ref, "chamber": chamber, "eurovoc": eurovoc}, evidence=evidence, topics=topics, facet=VotePosition(event_key=event_key, position=position), ) if parsed["roll_call"]: # Individual votes are listed in the protocol after one of the referenced anchors; the # result is usually behind the last one, sometimes in another fragment (session break). refs_list = [r["url"].split("#", 1) for r in reversed(refs) if r.get("url") and "#" in r["url"]] if refs_list: yield Request(url=f"{BASE}{refs_list[0][0]}", kind="rollcall", meta={"event_key": event_key, "refs": refs_list, "tried": [refs_list[0][0]], "title": title, "chamber": chamber, "date": str(voted_on), "item_url": item_url, "topics": list(topics)}) def parse_rollcall(self, ctx: RunContext, response: Response) -> Iterator[Any]: meta = response.meta doc = response.request.url.removeprefix(BASE) names: dict[str, list] = {} anchor_used = None for ref_doc, anchor in meta["refs"]: if ref_doc != doc: continue names = parse_roll_call(response.text, anchor) if names.get("for") or names.get("against"): anchor_used = anchor break if anchor_used is None: remaining = [d for d, _ in meta["refs"] if d not in meta["tried"]] if remaining: ctx.stats["rollcall_followup"] += 1 yield Request(url=f"{BASE}{remaining[0]}", kind="rollcall", meta={**meta, "tried": [*meta["tried"], remaining[0]]}) else: ctx.stats["rollcall_unparsed"] += 1 return voted_on = parse_date(meta["date"]) body = CHAMBER_SLUG[meta["chamber"]] source_url = f"{response.request.url}#{anchor_used}" evidence = [ Evidence(url=source_url, doc_type="protocol", published_at=voted_on, title="Stenographisches Protokoll – Ergebnis der namentlichen Abstimmung", locator=anchor_used), Evidence(url=meta["item_url"], doc_type="api_record", title=meta["title"], published_at=voted_on, locator=f"rollcall:{voted_on}"), ] for position, entries in names.items(): answer = "Ja" if position == "for" else "Nein" for label, pad in entries: ref = (PersonRef(ids={"parlament": str(pad)}) if pad is not None else PersonRef(name=strip_titles(label), hints={"surname_in_body": body, "on": voted_on})) yield Claim( claim_type="vote", person=ref, counterpart_org=OrgRef(slug=body), occurred_at=voted_on, summary_de=f"Stimmte in namentlicher Abstimmung mit „{answer}“: {meta['title']}", summary_en=f"Voted \"{'yes' if position == 'for' else 'no'}\" in a roll-call vote: {meta['title']}", data={"position": position, "roll_call": True, "chamber": meta["chamber"], "label": label}, evidence=evidence, topics=tuple(meta.get("topics") or ()), facet=VotePosition(event_key=meta["event_key"], position=position), ) # -- immunity ----------------------------------------------------------------------- def parse_immunity_item(self, ctx: RunContext, response: Response) -> Iterator[Request]: content = response.json().get("content") or {} report = next((d["link"] for group in content.get("documents") or [] for d in group.get("documents") or [] if d.get("type") == "HTML"), None) if report: yield Request(url=f"{BASE}{report}", kind="immunity_report", meta={**response.meta, "item": response.request.url.removeprefix(BASE), "title": html_to_text(content.get("title")) or response.meta.get("title")}) def parse_immunity_report(self, ctx: RunContext, response: Response) -> Iterator[Claim]: meta = response.meta text = html_to_text(response.text) details = parse_immunity_report(text) title = meta["title"] name = re.sub(r"^.*?(Abgeordneten zum Nationalrat|Mitglied(es)? des Bundesrates)\s*(Klubobmann|Klubobfrau)?", "", title).strip() decided = str(meta.get("status")) == "5" stage = {"consent": "immunity_waived" if decided else "immunity_waiver_requested", "refuse": "immunity_refused" if decided else "immunity_waiver_requested"}.get( details["recommendation"] or "", "immunity_waiver_requested") stage_de = {"immunity_waived": "Nationalrat stimmte der behördlichen Verfolgung zu (Immunität aufgehoben)", "immunity_refused": "Nationalrat verweigerte die Zustimmung zur behördlichen Verfolgung", "immunity_waiver_requested": "Ersuchen um Aufhebung der Immunität"}[stage] offense = details["offense"] requested = parse_date(meta.get("date")) yield Claim( claim_type="legal_proceeding", person=PersonRef(name=strip_titles(name)), key=("immunity", meta["item"]), occurred_at=requested, sensitive=True, summary_de=stage_de + (f" – Verdacht nach {offense}" if offense else "") + (f" (Ersuchen: {details['authority']})" if details["authority"] else ""), data={"political_connection": details["political_connection"], "recommendation": details["recommendation"], "item": meta["item"]}, evidence=[ Evidence(url=response.request.url, doc_type="register_entry", published_at=requested, title=f"Bericht des Immunitätsausschusses: {title}", quote=details["quote"], full_text=text), Evidence(url=f"{BASE}{meta['item']}", doc_type="api_record", title=title, published_at=requested), ], topics=("Inneres und Recht",), facet=Legal(stage=stage, offense=offense, authority=details["authority"], case_ref=details["case_ref"], stage_date=requested), ) # -- investigative-committee hearings ---------------------------------------------- def parse_hearing_item(self, ctx: RunContext, response: Response) -> Iterator[Any]: content = response.json().get("content") or {} title = html_to_text(content.get("title")) or response.meta["title"] m = HEARING.search(title) if not m: return witness = WITNESS_TITLES.sub("", strip_titles(m.group("name"))).strip() published = parse_date(response.meta.get("date")) protocol = next((d["link"] for group in content.get("documents") or [] for d in group.get("documents") or [] if d.get("type") == "HTML"), None) url = f"{BASE}{protocol}" if protocol else response.request.url.split("?")[0] event_key = fingerprint("hearing", response.request.url) yield Event(key=event_key, kind="hearing", title=f"Befragung im Untersuchungsausschuss: {witness}", starts_at=published, location="Parlament, Wien", country="AT", url=url, description=title, body=OrgRef(slug="nationalrat")) yield Claim( claim_type="appearance", person=PersonRef(name=witness), occurred_at=published, summary_de=f"Befragung als Auskunftsperson im Untersuchungsausschuss ({title})", summary_en="Questioned as a witness by a parliamentary investigative committee", data={"context": "investigative_committee"}, evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=published)], topics=("Transparenz und Korruption",), facet=Participation(event_key=event_key, participation="witness"), )