Politrack: track Austrian politicians from public sources

Initial import of Politrack, a self-hosted service that tracks Austrian
politicians from federal to municipal level - roles, votes, money flows,
statements, meetings, social media and legal proceedings - with every claim
linked to its sources.

- backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse ->
  load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive
  for reparsing, and a claim/evidence model with corroboration
- sources: Parliament, Wikidata, EU Parliament/Commission, Council of
  Ministers, Rechnungshof party reports and donations, procurement, RTR media
  transparency, political ad libraries, WKStA, news feeds, Wikipedia edits,
  Bluesky, Mastodon, public Telegram channels, YouTube and X (official API)
- web: Next.js frontend in German and English with typed API client
- mcp: MCP server for AI agents and RAG
- docs: architecture, data model, sources, legal notes, Unraid deployment and
  contributor guides; Docker Compose setup for Unraid

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
simonandClaude Opus 5.5 committed 2026-10-28 20:02:01 +01:00
commit 583ccba31c
187 files changed
+39890

No files matched your search

@@ -0,0 +1,285 @@
"""Parliament items (Gegenstände) of the configured legislative periods.
From one list export per period (list 101) we derive:
* votes -> item detail -> VoteEvent + faction VotePositions
roll calls -> stenographic protocol -> individual VotePositions
* immunity requests ("Ersuchen um Zustimmung zur behördlichen Verfolgung") -> committee report -> Legal claim
* written questions and answers (ITYP J / AB) -> parliamentary_question claims (directly from the list)
* investigative-committee hearings ("öffentliche Befragung der Auskunftsperson ...") -> hearing Event +
appearance claim for politicians who testified (other witnesses are not stored)
"""
from __future__ import annotations
import json
import re
from collections.abc import Iterator
from typing import Any
from politrack.config import get_settings
from politrack.core import Adapter, Request, Response, RunContext, register
from politrack.core.records import (
Claim,
Event,
Evidence,
Legal,
OrgRef,
Participation,
PersonRef,
VoteEvent,
VotePosition,
)
from politrack.sources.parlament.common import (
BASE,
CHAMBER_SLUG,
SPEC,
detail_request,
list_request,
list_rows,
parse_date,
parse_immunity_report,
parse_roll_call,
parse_vote_text,
walk_dicts,
)
from politrack.text import fingerprint, html_to_text, strip_titles
HEARING = re.compile(r"Befragung der Auskunftsperson\s+(?P<name>.+)$")
WITNESS_TITLES = re.compile(r"^(?:(?:Oberstaatsanwalt|Staatsanwalt|Sektionschef(?:in)?|MinR|Generalsekretär(?:in)?|"
r"Bundesminister(?:in)?|Abgeordnete[rn]?|Klubobmann|Klubobfrau|Bundeskanzler|"
r"Vizekanzler|Präsident(?:in)?)\s+)+")
@register
class ParlamentItems(Adapter):
spec = SPEC
name = "parlament.items"
description = "Votes (faction + roll call), immunity requests, written questions, U-committee hearings"
schedule = "45 3 * * *"
parser_version = 2
def start(self, ctx: RunContext) -> Iterator[Request]:
for gp in ctx.params.get("periods") or get_settings().parlament_periods:
yield list_request(101, {"GP_CODE": [gp], "NRBR": ["NR"]}, kind="items", meta={"gp": gp})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
handler = {
"items": self.parse_items,
"votes": self.parse_votes,
"rollcall": self.parse_rollcall,
"immunity": self.parse_immunity_item,
"immunity_report": self.parse_immunity_report,
"hearing": self.parse_hearing_item,
}[response.request.kind]
yield from handler(ctx, response)
# -- list ---------------------------------------------------------------------------
def parse_items(self, ctx: RunContext, response: Response) -> Iterator[Any]:
for row in list_rows(response.json()):
path, subject, kind = row.get("HIS_URL"), row.get("Betreff") or "", row.get("ITYP")
if not path:
continue
if row.get("Dafür") or row.get("Dagegen") or row.get("Abstimmungstext"):
# The list row changes when new votes happen, so the detail is re-fetched only then
# (conditional request + unchanged-body detection do the rest).
yield detail_request(path, kind="votes", meta={"title": subject, "ref": row.get("Nummer")})
if subject.startswith("Ersuchen") and "behördlichen Verfolgung" in subject:
yield detail_request(path, kind="immunity",
meta={"title": subject, "date": row.get("Datum"), "status": row.get("Status")})
if kind in ("J", "AB"):
yield from self.question_claims(row, path)
if kind == "KOMM" and HEARING.search(subject):
yield detail_request(path, kind="hearing", meta={"title": subject, "date": row.get("Datum")})
def question_claims(self, row: dict, path: str) -> Iterator[Claim]:
pads = row.get("Personen") or "[]"
pads = json.loads(pads) if isinstance(pads, str) else pads
subject = row.get("Betreff") or ""
is_question = row.get("ITYP") == "J"
asked = parse_date(row.get("Datum"))
topics = tuple(json.loads(row.get("THEMEN") or "[]")) if isinstance(row.get("THEMEN"), str) else ()
for pad in pads:
if not str(pad).strip():
continue
verb_de = "stellte die schriftliche Anfrage" if is_question else "beantwortete die Anfrage"
verb_en = "submitted the written question" if is_question else "answered the question"
yield Claim(
claim_type="parliamentary_question", person=PersonRef(ids={"parlament": str(pad)}),
key=("question", path, str(pad)), occurred_at=asked,
summary_de=f"{verb_de[0].upper()}{verb_de[1:]} „{subject}“ ({row.get('Nummer')})",
summary_en=f"{verb_en[0].upper()}{verb_en[1:]} \"{subject}\" ({row.get('Nummer')})",
data={"kind": "question" if is_question else "answer", "number": row.get("Nummer")},
evidence=[Evidence(url=f"{BASE}{path}", doc_type="api_record", title=subject, published_at=asked)],
topics=topics,
)
# -- votes ----------------------------------------------------------------------------
def parse_votes(self, ctx: RunContext, response: Response) -> Iterator[Any]:
content = response.json().get("content") or {}
path = response.request.url.removeprefix(BASE)
title = html_to_text(content.get("title")) or response.meta.get("title") or path
item_ref = html_to_text(content.get("zitation")) or response.meta.get("ref")
item_url = f"{BASE}{path}"
topics = tuple(b.get("label") for b in ((content.get("topics") or {}).get("data") or {}).get("bubbles") or [])
eurovoc = [b.get("label") for b in ((content.get("eurovoc") or {}).get("data") or {}).get("bubbles") or []]
for stage in walk_dicts(content.get("phase") or content, "text"):
text = html_to_text(stage.get("text"))
if "angenommen" not in text and "abgelehnt" not in text:
continue
parsed = parse_vote_text(text)
if not parsed or not (parsed["for"] or parsed["against"] or parsed["unanimous"] or parsed["split"]
or parsed["roll_call"]):
continue
voted_on = parse_date(stage.get("date"))
refs = stage.get("fsth") or []
protocol_url = f"{BASE}{refs[0]['url']}" if refs and refs[0].get("url") else None
chamber = "BR" if (refs and refs[0].get("gp_code") == "BR") or "Bundesrat" in text else "NR"
chamber_ref = OrgRef(slug=CHAMBER_SLUG[chamber])
event_key = fingerprint("vote", path, voted_on, parsed["stage"])
yield VoteEvent(key=event_key, chamber=chamber_ref, title=title, voted_on=voted_on, result=parsed["result"],
stage=parsed["stage"], item_ref=item_ref, roll_call=parsed["roll_call"], url=item_url,
protocol_url=protocol_url, raw_text=text)
evidence = [Evidence(url=item_url, doc_type="api_record", title=title, published_at=voted_on, quote=text,
locator=f"stage:{voted_on}:{parsed['stage'][:80]}")]
if protocol_url:
evidence.append(Evidence(url=protocol_url, doc_type="protocol", published_at=voted_on,
title=refs[0].get("title") or "Stenographisches Protokoll"))
outcome = "angenommen" if parsed["result"] == "accepted" else "abgelehnt"
for party, position in [(p, "for") for p in parsed["for"]] + [(p, "against") for p in parsed["against"]]:
verb_de, verb_en = ("stimmte für", "voted for") if position == "for" else ("stimmte gegen", "voted against")
yield Claim(
claim_type="vote", org=OrgRef(name=party, short_name=party, kind="parliamentary_group"),
counterpart_org=chamber_ref, occurred_at=voted_on,
summary_de=f"{party} {verb_de}: {title} ({parsed['stage']}) – {outcome}",
summary_en=f"{party} {verb_en}: {title} ({parsed['stage']})",
data={"position": position, "result": parsed["result"], "item_ref": item_ref, "chamber": chamber,
"eurovoc": eurovoc},
evidence=evidence, topics=topics, facet=VotePosition(event_key=event_key, position=position),
)
if parsed["roll_call"]:
# Individual votes are listed in the protocol after one of the referenced anchors; the
# result is usually behind the last one, sometimes in another fragment (session break).
refs_list = [r["url"].split("#", 1) for r in reversed(refs) if r.get("url") and "#" in r["url"]]
if refs_list:
yield Request(url=f"{BASE}{refs_list[0][0]}", kind="rollcall",
meta={"event_key": event_key, "refs": refs_list, "tried": [refs_list[0][0]],
"title": title, "chamber": chamber, "date": str(voted_on),
"item_url": item_url, "topics": list(topics)})
def parse_rollcall(self, ctx: RunContext, response: Response) -> Iterator[Any]:
meta = response.meta
doc = response.request.url.removeprefix(BASE)
names: dict[str, list] = {}
anchor_used = None
for ref_doc, anchor in meta["refs"]:
if ref_doc != doc:
continue
names = parse_roll_call(response.text, anchor)
if names.get("for") or names.get("against"):
anchor_used = anchor
break
if anchor_used is None:
remaining = [d for d, _ in meta["refs"] if d not in meta["tried"]]
if remaining:
ctx.stats["rollcall_followup"] += 1
yield Request(url=f"{BASE}{remaining[0]}", kind="rollcall",
meta={**meta, "tried": [*meta["tried"], remaining[0]]})
else:
ctx.stats["rollcall_unparsed"] += 1
return
voted_on = parse_date(meta["date"])
body = CHAMBER_SLUG[meta["chamber"]]
source_url = f"{response.request.url}#{anchor_used}"
evidence = [
Evidence(url=source_url, doc_type="protocol", published_at=voted_on,
title="Stenographisches Protokoll – Ergebnis der namentlichen Abstimmung", locator=anchor_used),
Evidence(url=meta["item_url"], doc_type="api_record", title=meta["title"], published_at=voted_on,
locator=f"rollcall:{voted_on}"),
]
for position, entries in names.items():
answer = "Ja" if position == "for" else "Nein"
for label, pad in entries:
ref = (PersonRef(ids={"parlament": str(pad)}) if pad is not None else
PersonRef(name=strip_titles(label), hints={"surname_in_body": body, "on": voted_on}))
yield Claim(
claim_type="vote", person=ref, counterpart_org=OrgRef(slug=body), occurred_at=voted_on,
summary_de=f"Stimmte in namentlicher Abstimmung mit „{answer}“: {meta['title']}",
summary_en=f"Voted \"{'yes' if position == 'for' else 'no'}\" in a roll-call vote: {meta['title']}",
data={"position": position, "roll_call": True, "chamber": meta["chamber"], "label": label},
evidence=evidence, topics=tuple(meta.get("topics") or ()),
facet=VotePosition(event_key=meta["event_key"], position=position),
)
# -- immunity -----------------------------------------------------------------------
def parse_immunity_item(self, ctx: RunContext, response: Response) -> Iterator[Request]:
content = response.json().get("content") or {}
report = next((d["link"] for group in content.get("documents") or [] for d in group.get("documents") or []
if d.get("type") == "HTML"), None)
if report:
yield Request(url=f"{BASE}{report}", kind="immunity_report",
meta={**response.meta, "item": response.request.url.removeprefix(BASE),
"title": html_to_text(content.get("title")) or response.meta.get("title")})
def parse_immunity_report(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
meta = response.meta
text = html_to_text(response.text)
details = parse_immunity_report(text)
title = meta["title"]
name = re.sub(r"^.*?(Abgeordneten zum Nationalrat|Mitglied(es)? des Bundesrates)\s*(Klubobmann|Klubobfrau)?",
"", title).strip()
decided = str(meta.get("status")) == "5"
stage = {"consent": "immunity_waived" if decided else "immunity_waiver_requested",
"refuse": "immunity_refused" if decided else "immunity_waiver_requested"}.get(
details["recommendation"] or "", "immunity_waiver_requested")
stage_de = {"immunity_waived": "Nationalrat stimmte der behördlichen Verfolgung zu (Immunität aufgehoben)",
"immunity_refused": "Nationalrat verweigerte die Zustimmung zur behördlichen Verfolgung",
"immunity_waiver_requested": "Ersuchen um Aufhebung der Immunität"}[stage]
offense = details["offense"]
requested = parse_date(meta.get("date"))
yield Claim(
claim_type="legal_proceeding", person=PersonRef(name=strip_titles(name)),
key=("immunity", meta["item"]), occurred_at=requested, sensitive=True,
summary_de=stage_de + (f" – Verdacht nach {offense}" if offense else "")
+ (f" (Ersuchen: {details['authority']})" if details["authority"] else ""),
data={"political_connection": details["political_connection"],
"recommendation": details["recommendation"], "item": meta["item"]},
evidence=[
Evidence(url=response.request.url, doc_type="register_entry", published_at=requested,
title=f"Bericht des Immunitätsausschusses: {title}", quote=details["quote"], full_text=text),
Evidence(url=f"{BASE}{meta['item']}", doc_type="api_record", title=title, published_at=requested),
],
topics=("Inneres und Recht",),
facet=Legal(stage=stage, offense=offense, authority=details["authority"], case_ref=details["case_ref"],
stage_date=requested),
)
# -- investigative-committee hearings ----------------------------------------------
def parse_hearing_item(self, ctx: RunContext, response: Response) -> Iterator[Any]:
content = response.json().get("content") or {}
title = html_to_text(content.get("title")) or response.meta["title"]
m = HEARING.search(title)
if not m:
return
witness = WITNESS_TITLES.sub("", strip_titles(m.group("name"))).strip()
published = parse_date(response.meta.get("date"))
protocol = next((d["link"] for group in content.get("documents") or [] for d in group.get("documents") or []
if d.get("type") == "HTML"), None)
url = f"{BASE}{protocol}" if protocol else response.request.url.split("?")[0]
event_key = fingerprint("hearing", response.request.url)
yield Event(key=event_key, kind="hearing", title=f"Befragung im Untersuchungsausschuss: {witness}",
starts_at=published, location="Parlament, Wien", country="AT", url=url,
description=title, body=OrgRef(slug="nationalrat"))
yield Claim(
claim_type="appearance", person=PersonRef(name=witness), occurred_at=published,
summary_de=f"Befragung als Auskunftsperson im Untersuchungsausschuss ({title})",
summary_en="Questioned as a witness by a parliamentary investigative committee",
data={"context": "investigative_committee"},
evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=published)],
topics=("Transparenz und Korruption",),
facet=Participation(event_key=event_key, participation="witness"),
)