simonandClaude Opus 5.5
583ccba31c
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
286 lines
16 KiB
Python
286 lines
16 KiB
Python
"""Parliament items (Gegenstände) of the configured legislative periods.
|
||
|
||
From one list export per period (list 101) we derive:
|
||
* votes -> item detail -> VoteEvent + faction VotePositions
|
||
roll calls -> stenographic protocol -> individual VotePositions
|
||
* immunity requests ("Ersuchen um Zustimmung zur behördlichen Verfolgung") -> committee report -> Legal claim
|
||
* written questions and answers (ITYP J / AB) -> parliamentary_question claims (directly from the list)
|
||
* investigative-committee hearings ("öffentliche Befragung der Auskunftsperson ...") -> hearing Event +
|
||
appearance claim for politicians who testified (other witnesses are not stored)
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
from collections.abc import Iterator
|
||
from typing import Any
|
||
|
||
from politrack.config import get_settings
|
||
from politrack.core import Adapter, Request, Response, RunContext, register
|
||
from politrack.core.records import (
|
||
Claim,
|
||
Event,
|
||
Evidence,
|
||
Legal,
|
||
OrgRef,
|
||
Participation,
|
||
PersonRef,
|
||
VoteEvent,
|
||
VotePosition,
|
||
)
|
||
from politrack.sources.parlament.common import (
|
||
BASE,
|
||
CHAMBER_SLUG,
|
||
SPEC,
|
||
detail_request,
|
||
list_request,
|
||
list_rows,
|
||
parse_date,
|
||
parse_immunity_report,
|
||
parse_roll_call,
|
||
parse_vote_text,
|
||
walk_dicts,
|
||
)
|
||
from politrack.text import fingerprint, html_to_text, strip_titles
|
||
|
||
HEARING = re.compile(r"Befragung der Auskunftsperson\s+(?P<name>.+)$")
|
||
WITNESS_TITLES = re.compile(r"^(?:(?:Oberstaatsanwalt|Staatsanwalt|Sektionschef(?:in)?|MinR|Generalsekretär(?:in)?|"
|
||
r"Bundesminister(?:in)?|Abgeordnete[rn]?|Klubobmann|Klubobfrau|Bundeskanzler|"
|
||
r"Vizekanzler|Präsident(?:in)?)\s+)+")
|
||
|
||
|
||
@register
|
||
class ParlamentItems(Adapter):
|
||
spec = SPEC
|
||
name = "parlament.items"
|
||
description = "Votes (faction + roll call), immunity requests, written questions, U-committee hearings"
|
||
schedule = "45 3 * * *"
|
||
parser_version = 2
|
||
|
||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||
for gp in ctx.params.get("periods") or get_settings().parlament_periods:
|
||
yield list_request(101, {"GP_CODE": [gp], "NRBR": ["NR"]}, kind="items", meta={"gp": gp})
|
||
|
||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||
handler = {
|
||
"items": self.parse_items,
|
||
"votes": self.parse_votes,
|
||
"rollcall": self.parse_rollcall,
|
||
"immunity": self.parse_immunity_item,
|
||
"immunity_report": self.parse_immunity_report,
|
||
"hearing": self.parse_hearing_item,
|
||
}[response.request.kind]
|
||
yield from handler(ctx, response)
|
||
|
||
# -- list ---------------------------------------------------------------------------
|
||
|
||
def parse_items(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||
for row in list_rows(response.json()):
|
||
path, subject, kind = row.get("HIS_URL"), row.get("Betreff") or "", row.get("ITYP")
|
||
if not path:
|
||
continue
|
||
if row.get("Dafür") or row.get("Dagegen") or row.get("Abstimmungstext"):
|
||
# The list row changes when new votes happen, so the detail is re-fetched only then
|
||
# (conditional request + unchanged-body detection do the rest).
|
||
yield detail_request(path, kind="votes", meta={"title": subject, "ref": row.get("Nummer")})
|
||
if subject.startswith("Ersuchen") and "behördlichen Verfolgung" in subject:
|
||
yield detail_request(path, kind="immunity",
|
||
meta={"title": subject, "date": row.get("Datum"), "status": row.get("Status")})
|
||
if kind in ("J", "AB"):
|
||
yield from self.question_claims(row, path)
|
||
if kind == "KOMM" and HEARING.search(subject):
|
||
yield detail_request(path, kind="hearing", meta={"title": subject, "date": row.get("Datum")})
|
||
|
||
def question_claims(self, row: dict, path: str) -> Iterator[Claim]:
|
||
pads = row.get("Personen") or "[]"
|
||
pads = json.loads(pads) if isinstance(pads, str) else pads
|
||
subject = row.get("Betreff") or ""
|
||
is_question = row.get("ITYP") == "J"
|
||
asked = parse_date(row.get("Datum"))
|
||
topics = tuple(json.loads(row.get("THEMEN") or "[]")) if isinstance(row.get("THEMEN"), str) else ()
|
||
for pad in pads:
|
||
if not str(pad).strip():
|
||
continue
|
||
verb_de = "stellte die schriftliche Anfrage" if is_question else "beantwortete die Anfrage"
|
||
verb_en = "submitted the written question" if is_question else "answered the question"
|
||
yield Claim(
|
||
claim_type="parliamentary_question", person=PersonRef(ids={"parlament": str(pad)}),
|
||
key=("question", path, str(pad)), occurred_at=asked,
|
||
summary_de=f"{verb_de[0].upper()}{verb_de[1:]} „{subject}“ ({row.get('Nummer')})",
|
||
summary_en=f"{verb_en[0].upper()}{verb_en[1:]} \"{subject}\" ({row.get('Nummer')})",
|
||
data={"kind": "question" if is_question else "answer", "number": row.get("Nummer")},
|
||
evidence=[Evidence(url=f"{BASE}{path}", doc_type="api_record", title=subject, published_at=asked)],
|
||
topics=topics,
|
||
)
|
||
|
||
# -- votes ----------------------------------------------------------------------------
|
||
|
||
def parse_votes(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||
content = response.json().get("content") or {}
|
||
path = response.request.url.removeprefix(BASE)
|
||
title = html_to_text(content.get("title")) or response.meta.get("title") or path
|
||
item_ref = html_to_text(content.get("zitation")) or response.meta.get("ref")
|
||
item_url = f"{BASE}{path}"
|
||
topics = tuple(b.get("label") for b in ((content.get("topics") or {}).get("data") or {}).get("bubbles") or [])
|
||
eurovoc = [b.get("label") for b in ((content.get("eurovoc") or {}).get("data") or {}).get("bubbles") or []]
|
||
|
||
for stage in walk_dicts(content.get("phase") or content, "text"):
|
||
text = html_to_text(stage.get("text"))
|
||
if "angenommen" not in text and "abgelehnt" not in text:
|
||
continue
|
||
parsed = parse_vote_text(text)
|
||
if not parsed or not (parsed["for"] or parsed["against"] or parsed["unanimous"] or parsed["split"]
|
||
or parsed["roll_call"]):
|
||
continue
|
||
voted_on = parse_date(stage.get("date"))
|
||
refs = stage.get("fsth") or []
|
||
protocol_url = f"{BASE}{refs[0]['url']}" if refs and refs[0].get("url") else None
|
||
chamber = "BR" if (refs and refs[0].get("gp_code") == "BR") or "Bundesrat" in text else "NR"
|
||
chamber_ref = OrgRef(slug=CHAMBER_SLUG[chamber])
|
||
event_key = fingerprint("vote", path, voted_on, parsed["stage"])
|
||
yield VoteEvent(key=event_key, chamber=chamber_ref, title=title, voted_on=voted_on, result=parsed["result"],
|
||
stage=parsed["stage"], item_ref=item_ref, roll_call=parsed["roll_call"], url=item_url,
|
||
protocol_url=protocol_url, raw_text=text)
|
||
evidence = [Evidence(url=item_url, doc_type="api_record", title=title, published_at=voted_on, quote=text,
|
||
locator=f"stage:{voted_on}:{parsed['stage'][:80]}")]
|
||
if protocol_url:
|
||
evidence.append(Evidence(url=protocol_url, doc_type="protocol", published_at=voted_on,
|
||
title=refs[0].get("title") or "Stenographisches Protokoll"))
|
||
outcome = "angenommen" if parsed["result"] == "accepted" else "abgelehnt"
|
||
for party, position in [(p, "for") for p in parsed["for"]] + [(p, "against") for p in parsed["against"]]:
|
||
verb_de, verb_en = ("stimmte für", "voted for") if position == "for" else ("stimmte gegen", "voted against")
|
||
yield Claim(
|
||
claim_type="vote", org=OrgRef(name=party, short_name=party, kind="parliamentary_group"),
|
||
counterpart_org=chamber_ref, occurred_at=voted_on,
|
||
summary_de=f"{party} {verb_de}: {title} ({parsed['stage']}) – {outcome}",
|
||
summary_en=f"{party} {verb_en}: {title} ({parsed['stage']})",
|
||
data={"position": position, "result": parsed["result"], "item_ref": item_ref, "chamber": chamber,
|
||
"eurovoc": eurovoc},
|
||
evidence=evidence, topics=topics, facet=VotePosition(event_key=event_key, position=position),
|
||
)
|
||
if parsed["roll_call"]:
|
||
# Individual votes are listed in the protocol after one of the referenced anchors; the
|
||
# result is usually behind the last one, sometimes in another fragment (session break).
|
||
refs_list = [r["url"].split("#", 1) for r in reversed(refs) if r.get("url") and "#" in r["url"]]
|
||
if refs_list:
|
||
yield Request(url=f"{BASE}{refs_list[0][0]}", kind="rollcall",
|
||
meta={"event_key": event_key, "refs": refs_list, "tried": [refs_list[0][0]],
|
||
"title": title, "chamber": chamber, "date": str(voted_on),
|
||
"item_url": item_url, "topics": list(topics)})
|
||
|
||
def parse_rollcall(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||
meta = response.meta
|
||
doc = response.request.url.removeprefix(BASE)
|
||
names: dict[str, list] = {}
|
||
anchor_used = None
|
||
for ref_doc, anchor in meta["refs"]:
|
||
if ref_doc != doc:
|
||
continue
|
||
names = parse_roll_call(response.text, anchor)
|
||
if names.get("for") or names.get("against"):
|
||
anchor_used = anchor
|
||
break
|
||
if anchor_used is None:
|
||
remaining = [d for d, _ in meta["refs"] if d not in meta["tried"]]
|
||
if remaining:
|
||
ctx.stats["rollcall_followup"] += 1
|
||
yield Request(url=f"{BASE}{remaining[0]}", kind="rollcall",
|
||
meta={**meta, "tried": [*meta["tried"], remaining[0]]})
|
||
else:
|
||
ctx.stats["rollcall_unparsed"] += 1
|
||
return
|
||
voted_on = parse_date(meta["date"])
|
||
body = CHAMBER_SLUG[meta["chamber"]]
|
||
source_url = f"{response.request.url}#{anchor_used}"
|
||
evidence = [
|
||
Evidence(url=source_url, doc_type="protocol", published_at=voted_on,
|
||
title="Stenographisches Protokoll – Ergebnis der namentlichen Abstimmung", locator=anchor_used),
|
||
Evidence(url=meta["item_url"], doc_type="api_record", title=meta["title"], published_at=voted_on,
|
||
locator=f"rollcall:{voted_on}"),
|
||
]
|
||
for position, entries in names.items():
|
||
answer = "Ja" if position == "for" else "Nein"
|
||
for label, pad in entries:
|
||
ref = (PersonRef(ids={"parlament": str(pad)}) if pad is not None else
|
||
PersonRef(name=strip_titles(label), hints={"surname_in_body": body, "on": voted_on}))
|
||
yield Claim(
|
||
claim_type="vote", person=ref, counterpart_org=OrgRef(slug=body), occurred_at=voted_on,
|
||
summary_de=f"Stimmte in namentlicher Abstimmung mit „{answer}“: {meta['title']}",
|
||
summary_en=f"Voted \"{'yes' if position == 'for' else 'no'}\" in a roll-call vote: {meta['title']}",
|
||
data={"position": position, "roll_call": True, "chamber": meta["chamber"], "label": label},
|
||
evidence=evidence, topics=tuple(meta.get("topics") or ()),
|
||
facet=VotePosition(event_key=meta["event_key"], position=position),
|
||
)
|
||
|
||
# -- immunity -----------------------------------------------------------------------
|
||
|
||
def parse_immunity_item(self, ctx: RunContext, response: Response) -> Iterator[Request]:
|
||
content = response.json().get("content") or {}
|
||
report = next((d["link"] for group in content.get("documents") or [] for d in group.get("documents") or []
|
||
if d.get("type") == "HTML"), None)
|
||
if report:
|
||
yield Request(url=f"{BASE}{report}", kind="immunity_report",
|
||
meta={**response.meta, "item": response.request.url.removeprefix(BASE),
|
||
"title": html_to_text(content.get("title")) or response.meta.get("title")})
|
||
|
||
def parse_immunity_report(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||
meta = response.meta
|
||
text = html_to_text(response.text)
|
||
details = parse_immunity_report(text)
|
||
title = meta["title"]
|
||
name = re.sub(r"^.*?(Abgeordneten zum Nationalrat|Mitglied(es)? des Bundesrates)\s*(Klubobmann|Klubobfrau)?",
|
||
"", title).strip()
|
||
decided = str(meta.get("status")) == "5"
|
||
stage = {"consent": "immunity_waived" if decided else "immunity_waiver_requested",
|
||
"refuse": "immunity_refused" if decided else "immunity_waiver_requested"}.get(
|
||
details["recommendation"] or "", "immunity_waiver_requested")
|
||
stage_de = {"immunity_waived": "Nationalrat stimmte der behördlichen Verfolgung zu (Immunität aufgehoben)",
|
||
"immunity_refused": "Nationalrat verweigerte die Zustimmung zur behördlichen Verfolgung",
|
||
"immunity_waiver_requested": "Ersuchen um Aufhebung der Immunität"}[stage]
|
||
offense = details["offense"]
|
||
requested = parse_date(meta.get("date"))
|
||
yield Claim(
|
||
claim_type="legal_proceeding", person=PersonRef(name=strip_titles(name)),
|
||
key=("immunity", meta["item"]), occurred_at=requested, sensitive=True,
|
||
summary_de=stage_de + (f" – Verdacht nach {offense}" if offense else "")
|
||
+ (f" (Ersuchen: {details['authority']})" if details["authority"] else ""),
|
||
data={"political_connection": details["political_connection"],
|
||
"recommendation": details["recommendation"], "item": meta["item"]},
|
||
evidence=[
|
||
Evidence(url=response.request.url, doc_type="register_entry", published_at=requested,
|
||
title=f"Bericht des Immunitätsausschusses: {title}", quote=details["quote"], full_text=text),
|
||
Evidence(url=f"{BASE}{meta['item']}", doc_type="api_record", title=title, published_at=requested),
|
||
],
|
||
topics=("Inneres und Recht",),
|
||
facet=Legal(stage=stage, offense=offense, authority=details["authority"], case_ref=details["case_ref"],
|
||
stage_date=requested),
|
||
)
|
||
|
||
# -- investigative-committee hearings ----------------------------------------------
|
||
|
||
def parse_hearing_item(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||
content = response.json().get("content") or {}
|
||
title = html_to_text(content.get("title")) or response.meta["title"]
|
||
m = HEARING.search(title)
|
||
if not m:
|
||
return
|
||
witness = WITNESS_TITLES.sub("", strip_titles(m.group("name"))).strip()
|
||
published = parse_date(response.meta.get("date"))
|
||
protocol = next((d["link"] for group in content.get("documents") or [] for d in group.get("documents") or []
|
||
if d.get("type") == "HTML"), None)
|
||
url = f"{BASE}{protocol}" if protocol else response.request.url.split("?")[0]
|
||
event_key = fingerprint("hearing", response.request.url)
|
||
yield Event(key=event_key, kind="hearing", title=f"Befragung im Untersuchungsausschuss: {witness}",
|
||
starts_at=published, location="Parlament, Wien", country="AT", url=url,
|
||
description=title, body=OrgRef(slug="nationalrat"))
|
||
yield Claim(
|
||
claim_type="appearance", person=PersonRef(name=witness), occurred_at=published,
|
||
summary_de=f"Befragung als Auskunftsperson im Untersuchungsausschuss ({title})",
|
||
summary_en="Questioned as a witness by a parliamentary investigative committee",
|
||
data={"context": "investigative_committee"},
|
||
evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=published)],
|
||
topics=("Transparenz und Korruption",),
|
||
facet=Participation(event_key=event_key, participation="witness"),
|
||
)
|