Politrack: track Austrian politicians from public sources

Initial import of Politrack, a self-hosted service that tracks Austrian
politicians from federal to municipal level - roles, votes, money flows,
statements, meetings, social media and legal proceedings - with every claim
linked to its sources.

- backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse ->
  load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive
  for reparsing, and a claim/evidence model with corroboration
- sources: Parliament, Wikidata, EU Parliament/Commission, Council of
  Ministers, Rechnungshof party reports and donations, procurement, RTR media
  transparency, political ad libraries, WKStA, news feeds, Wikipedia edits,
  Bluesky, Mastodon, public Telegram channels, YouTube and X (official API)
- web: Next.js frontend in German and English with typed API client
- mcp: MCP server for AI agents and RAG
- docs: architecture, data model, sources, legal notes, Unraid deployment and
  contributor guides; Docker Compose setup for Unraid

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
simonandClaude Opus 5.5 committed 2026-10-28 20:02:01 +01:00
commit 583ccba31c
187 files changed
+39890

No files matched your search

@@ -0,0 +1 @@
"""Austrian Parliament adapters: persons, items (votes, immunity, questions, hearings), press, calendar."""
@@ -0,0 +1,199 @@
"""Shared spec and pure parsing helpers for the Austrian Parliament adapters (CC BY 4.0).
API (https://www.parlament.gv.at/recherchieren/open-data/):
* list exports: POST /Filter/api/filter/data/<listeId>?js=eval&showAll=true (JSON body = filter)
409 persons since 1918 · 101 parliamentary items · 251 speeches of a person
110 Parlamentskorrespondenz (press releases) · 600 calendar (Termine)
* detail pages: <page URL>?json=TRUE
"""
from __future__ import annotations
import re
from datetime import date, datetime
from typing import Any
from politrack.core import Request, SourceSpec, register_spec
from politrack.text import html_to_text
BASE = "https://www.parlament.gv.at"
CHAMBER_SLUG = {"NR": "nationalrat", "BR": "bundesrat"}
CHAMBER_NAME_DE = {"NR": "Nationalrat", "BR": "Bundesrat"}
GOVERNMENT_RE = re.compile(r"Bundeskanzler|Vizekanzler|Bundesminister|Staatssekretär", re.IGNORECASE)
SPEC = register_spec(SourceSpec(
slug="parlament",
name="Parlament Österreich (Open Data)",
kind="official",
homepage="https://www.parlament.gv.at/recherchieren/open-data/",
license="CC BY 4.0",
access="api",
trust=95,
store_fulltext=True,
min_interval=1.0,
tos_notes="Official JSON API (?json=TRUE, /Filter/api/filter/data/<list>). Attribution required. The "
"Parlamentsdirektion may prohibit uses incompatible with the dignity of parliament.",
))
def list_request(list_id: int, body: dict, kind: str, meta: dict | None = None) -> Request:
return Request(url=f"{BASE}/Filter/api/filter/data/{list_id}?js=eval&showAll=true", method="POST", json=body,
kind=kind, meta=meta or {})
def detail_request(path: str, kind: str, meta: dict | None = None) -> Request:
return Request(url=f"{BASE}{path}", params={"json": "TRUE"}, kind=kind, meta=meta or {})
def list_rows(data: dict) -> list[dict[str, Any]]:
"""Turn a filter-API export into dicts keyed by column label (duplicates get a _<index> suffix)."""
labels: list[str] = []
for i, header in enumerate(data.get("header", [])):
label = header.get("label") or header.get("feld_name") or f"col{i}"
labels.append(label if label not in labels else f"{label}_{i}")
return [dict(zip(labels, row)) for row in data.get("rows", [])]
def parse_date(value: str | None) -> date | None:
if not value:
return None
for fmt in ("%d.%m.%Y", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d"):
try:
return datetime.strptime(value[:19], fmt).date()
except ValueError:
continue
return None
def walk_dicts(obj: Any, required_key: str):
"""Yield every nested dict that contains `required_key`."""
if isinstance(obj, dict):
if required_key in obj:
yield obj
for value in obj.values():
yield from walk_dicts(value, required_key)
elif isinstance(obj, list):
for value in obj:
yield from walk_dicts(value, required_key)
# -- faction votes ------------------------------------------------------------------------
PARTY_TOKEN = re.compile(r"^[A-ZÄÖÜ][A-Za-zÄÖÜäöüß.\- ]{0,30}$")
# Split votes ("teilweise", "bzw.", per-part results in parentheses) can't be attributed to factions.
SPLIT_VOTE = re.compile(r"\(|\bbzw\.|\bteilweise\b|\btlw\.|\bmehrstimmig\b|\bgetrennt", re.IGNORECASE)
def parse_party_list(value: str | None) -> list[str]:
if not value:
return []
parts = [p.strip(" .;") for p in re.split(r",|\bund\b", value)]
return [p for p in parts if PARTY_TOKEN.match(p)]
def parse_vote_text(text: str) -> dict[str, Any] | None:
"""'137. Sitzung …: Gesetzesvorschlag in dritter Lesung angenommen Dafür: ÖVP, SPÖ, dagegen: FPÖ'"""
m = re.search(r"\b(angenommen|abgelehnt)\b", text, re.IGNORECASE)
if not m:
return None
rest = text[m.end():]
roll_call = "namentliche abstimmung" in text.lower()
split = bool(SPLIT_VOTE.search(rest)) and not roll_call
pro = re.search(r"Dafür:\s*(.*?)(?:,?\s*dagegen:|$)", rest, re.IGNORECASE)
contra = re.search(r"dagegen:\s*(.*)$", rest, re.IGNORECASE)
return {
"stage": text[: m.start()].strip(" :,"),
"result": "accepted" if m.group(1).lower() == "angenommen" else "rejected",
"for": [] if split or roll_call else parse_party_list(pro.group(1) if pro else None),
"against": [] if split or roll_call else parse_party_list(contra.group(1) if contra else None),
"unanimous": "einstimmig" in text.lower(),
"split": split,
"roll_call": roll_call,
}
# -- roll-call votes ----------------------------------------------------------------------
ROLL_CALL_HEADER = re.compile(r"Mit\s*[„\"“]?\s*(Ja|Nein)\s*[“\"”]?\s*stimmten[^:]*:", re.IGNORECASE)
NEW_VOTE = re.compile(r"\b(gelangen|kommen)\s+(?:\w+\s+){0,2}zur\s+Abstimmung\b", re.IGNORECASE)
PERSON_LINK = re.compile(r"<a[^>]*href=\"[^\"]*/person/(\d+)\"[^>]*>.*?</a>", re.IGNORECASE | re.DOTALL)
def parse_roll_call(html_text: str, anchor: str | None) -> dict[str, list[tuple[str, int | None]]]:
"""Extract the "Mit „Ja“ stimmten …" / "Mit „Nein“ stimmten …" lists that follow `anchor`.
Returns {"for": [(label, pad|None)], "against": [...]}. Names are mostly surnames; where a surname is
ambiguous the protocol links the person, which gives us the parliament id directly.
"""
start = 0
if anchor:
m = re.search(r'id="%s"' % re.escape(anchor), html_text)
if not m:
return {}
start = m.start()
first = ROLL_CALL_HEADER.search(html_text, start)
if not first:
return {}
# The lists must belong to this anchor: they follow within a few numbered paragraphs and no new vote
# may be called in between.
between = html_text[start + 10 : first.start()]
if anchor and (len(re.findall(r'class="[^"]*randnummer', between)) > 3 or NEW_VOTE.search(html_to_text(between))):
return {}
end_match = re.compile(r"<hr|class=\"[^\"]*randnummer").search(html_text, first.end())
segment = html_text[first.start() : end_match.start() if end_match else len(html_text)]
segment = PERSON_LINK.sub(lambda m: f" @@{m.group(1)}@@ ", segment)
text = html_to_text(segment)
result: dict[str, list[tuple[str, int | None]]] = {"for": [], "against": []}
headers = list(ROLL_CALL_HEADER.finditer(text))
for i, header in enumerate(headers):
block = text[header.end() : headers[i + 1].start() if i + 1 < len(headers) else len(text)]
key = "for" if header.group(1).lower() == "ja" else "against"
for token in re.split(r"[,;]|\.\s*$", block):
token = token.strip(" .")
if not token:
continue
link = re.fullmatch(r"@@(\d+)@@", token)
result[key].append((token, int(link.group(1))) if link else (token, None))
return result
# -- immunity -----------------------------------------------------------------------------
STGB_OFFENSES = {
"111": "Üble Nachrede", "115": "Beleidigung", "146": "Betrug", "153": "Untreue", "165": "Geldwäscherei",
"201": "Vergewaltigung", "283": "Verhetzung", "288": "Falsche Beweisaussage", "293": "Fälschung eines Beweismittels",
"295": "Unterdrückung eines Beweismittels", "297": "Verleumdung", "302": "Missbrauch der Amtsgewalt",
"304": "Bestechlichkeit", "305": "Vorteilsannahme", "306": "Vorteilsannahme zur Beeinflussung",
"307": "Bestechung", "308": "Verbotene Intervention", "310": "Verletzung des Amtsgeheimnisses",
}
def parse_immunity_report(text: str) -> dict[str, Any]:
authority = re.search(r"Ersuchen (?:der|des) (.+?),\s*(?:do\.|GZ|Zl)", text)
case_ref = re.search(r"\bGZ\.?\s*([^,]+?),", text)
# "wegen des Verdachtes einer strafbaren Handlung nach § 283 Abs. 1 Z 1 und Z 2, Abs. 2 StGB"
offense = re.search(r"wegen[^§]{0,90}?(§§? ?\d+.{0,120}?\b(?:StGB|VerbotsG|FinStrG|[A-Z][A-Za-z]*G)\b)", text)
offense_text = offense.group(1).strip() if offense else None
if offense_text:
section = re.search(r"§ ?(\d+)", offense_text)
if section and section.group(1) in STGB_OFFENSES and "StGB" in offense_text:
offense_text = f"{offense_text} ({STGB_OFFENSES[section.group(1)]})"
decision = re.search(r"(?:wird|werde)\s+(nicht\s+)?zugestimmt|(nicht\s+)?zuzustimmen", text)
recommendation = None
if decision:
recommendation = "refuse" if (decision.group(1) or decision.group(2)) else "consent"
connection = None
if re.search(r"\bkein Zusammenhang\b", text):
connection = False
elif re.search(r"\bein Zusammenhang\b", text):
connection = True
quote = re.search(r"Als Ergebnis seiner Beratungen.*?(?:zugestimmt|nicht zugestimmt)\s*\.", text)
return {
"authority": authority.group(1).strip() if authority else None,
"case_ref": case_ref.group(1).strip() if case_ref else None,
"offense": offense_text,
"recommendation": recommendation,
"political_connection": connection,
"quote": quote.group(0) if quote else None,
}
@@ -0,0 +1,285 @@
"""Parliament items (Gegenstände) of the configured legislative periods.
From one list export per period (list 101) we derive:
* votes -> item detail -> VoteEvent + faction VotePositions
roll calls -> stenographic protocol -> individual VotePositions
* immunity requests ("Ersuchen um Zustimmung zur behördlichen Verfolgung") -> committee report -> Legal claim
* written questions and answers (ITYP J / AB) -> parliamentary_question claims (directly from the list)
* investigative-committee hearings ("öffentliche Befragung der Auskunftsperson ...") -> hearing Event +
appearance claim for politicians who testified (other witnesses are not stored)
"""
from __future__ import annotations
import json
import re
from collections.abc import Iterator
from typing import Any
from politrack.config import get_settings
from politrack.core import Adapter, Request, Response, RunContext, register
from politrack.core.records import (
Claim,
Event,
Evidence,
Legal,
OrgRef,
Participation,
PersonRef,
VoteEvent,
VotePosition,
)
from politrack.sources.parlament.common import (
BASE,
CHAMBER_SLUG,
SPEC,
detail_request,
list_request,
list_rows,
parse_date,
parse_immunity_report,
parse_roll_call,
parse_vote_text,
walk_dicts,
)
from politrack.text import fingerprint, html_to_text, strip_titles
HEARING = re.compile(r"Befragung der Auskunftsperson\s+(?P<name>.+)$")
WITNESS_TITLES = re.compile(r"^(?:(?:Oberstaatsanwalt|Staatsanwalt|Sektionschef(?:in)?|MinR|Generalsekretär(?:in)?|"
r"Bundesminister(?:in)?|Abgeordnete[rn]?|Klubobmann|Klubobfrau|Bundeskanzler|"
r"Vizekanzler|Präsident(?:in)?)\s+)+")
@register
class ParlamentItems(Adapter):
spec = SPEC
name = "parlament.items"
description = "Votes (faction + roll call), immunity requests, written questions, U-committee hearings"
schedule = "45 3 * * *"
parser_version = 2
def start(self, ctx: RunContext) -> Iterator[Request]:
for gp in ctx.params.get("periods") or get_settings().parlament_periods:
yield list_request(101, {"GP_CODE": [gp], "NRBR": ["NR"]}, kind="items", meta={"gp": gp})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
handler = {
"items": self.parse_items,
"votes": self.parse_votes,
"rollcall": self.parse_rollcall,
"immunity": self.parse_immunity_item,
"immunity_report": self.parse_immunity_report,
"hearing": self.parse_hearing_item,
}[response.request.kind]
yield from handler(ctx, response)
# -- list ---------------------------------------------------------------------------
def parse_items(self, ctx: RunContext, response: Response) -> Iterator[Any]:
for row in list_rows(response.json()):
path, subject, kind = row.get("HIS_URL"), row.get("Betreff") or "", row.get("ITYP")
if not path:
continue
if row.get("Dafür") or row.get("Dagegen") or row.get("Abstimmungstext"):
# The list row changes when new votes happen, so the detail is re-fetched only then
# (conditional request + unchanged-body detection do the rest).
yield detail_request(path, kind="votes", meta={"title": subject, "ref": row.get("Nummer")})
if subject.startswith("Ersuchen") and "behördlichen Verfolgung" in subject:
yield detail_request(path, kind="immunity",
meta={"title": subject, "date": row.get("Datum"), "status": row.get("Status")})
if kind in ("J", "AB"):
yield from self.question_claims(row, path)
if kind == "KOMM" and HEARING.search(subject):
yield detail_request(path, kind="hearing", meta={"title": subject, "date": row.get("Datum")})
def question_claims(self, row: dict, path: str) -> Iterator[Claim]:
pads = row.get("Personen") or "[]"
pads = json.loads(pads) if isinstance(pads, str) else pads
subject = row.get("Betreff") or ""
is_question = row.get("ITYP") == "J"
asked = parse_date(row.get("Datum"))
topics = tuple(json.loads(row.get("THEMEN") or "[]")) if isinstance(row.get("THEMEN"), str) else ()
for pad in pads:
if not str(pad).strip():
continue
verb_de = "stellte die schriftliche Anfrage" if is_question else "beantwortete die Anfrage"
verb_en = "submitted the written question" if is_question else "answered the question"
yield Claim(
claim_type="parliamentary_question", person=PersonRef(ids={"parlament": str(pad)}),
key=("question", path, str(pad)), occurred_at=asked,
summary_de=f"{verb_de[0].upper()}{verb_de[1:]} „{subject}“ ({row.get('Nummer')})",
summary_en=f"{verb_en[0].upper()}{verb_en[1:]} \"{subject}\" ({row.get('Nummer')})",
data={"kind": "question" if is_question else "answer", "number": row.get("Nummer")},
evidence=[Evidence(url=f"{BASE}{path}", doc_type="api_record", title=subject, published_at=asked)],
topics=topics,
)
# -- votes ----------------------------------------------------------------------------
def parse_votes(self, ctx: RunContext, response: Response) -> Iterator[Any]:
content = response.json().get("content") or {}
path = response.request.url.removeprefix(BASE)
title = html_to_text(content.get("title")) or response.meta.get("title") or path
item_ref = html_to_text(content.get("zitation")) or response.meta.get("ref")
item_url = f"{BASE}{path}"
topics = tuple(b.get("label") for b in ((content.get("topics") or {}).get("data") or {}).get("bubbles") or [])
eurovoc = [b.get("label") for b in ((content.get("eurovoc") or {}).get("data") or {}).get("bubbles") or []]
for stage in walk_dicts(content.get("phase") or content, "text"):
text = html_to_text(stage.get("text"))
if "angenommen" not in text and "abgelehnt" not in text:
continue
parsed = parse_vote_text(text)
if not parsed or not (parsed["for"] or parsed["against"] or parsed["unanimous"] or parsed["split"]
or parsed["roll_call"]):
continue
voted_on = parse_date(stage.get("date"))
refs = stage.get("fsth") or []
protocol_url = f"{BASE}{refs[0]['url']}" if refs and refs[0].get("url") else None
chamber = "BR" if (refs and refs[0].get("gp_code") == "BR") or "Bundesrat" in text else "NR"
chamber_ref = OrgRef(slug=CHAMBER_SLUG[chamber])
event_key = fingerprint("vote", path, voted_on, parsed["stage"])
yield VoteEvent(key=event_key, chamber=chamber_ref, title=title, voted_on=voted_on, result=parsed["result"],
stage=parsed["stage"], item_ref=item_ref, roll_call=parsed["roll_call"], url=item_url,
protocol_url=protocol_url, raw_text=text)
evidence = [Evidence(url=item_url, doc_type="api_record", title=title, published_at=voted_on, quote=text,
locator=f"stage:{voted_on}:{parsed['stage'][:80]}")]
if protocol_url:
evidence.append(Evidence(url=protocol_url, doc_type="protocol", published_at=voted_on,
title=refs[0].get("title") or "Stenographisches Protokoll"))
outcome = "angenommen" if parsed["result"] == "accepted" else "abgelehnt"
for party, position in [(p, "for") for p in parsed["for"]] + [(p, "against") for p in parsed["against"]]:
verb_de, verb_en = ("stimmte für", "voted for") if position == "for" else ("stimmte gegen", "voted against")
yield Claim(
claim_type="vote", org=OrgRef(name=party, short_name=party, kind="parliamentary_group"),
counterpart_org=chamber_ref, occurred_at=voted_on,
summary_de=f"{party} {verb_de}: {title} ({parsed['stage']}) – {outcome}",
summary_en=f"{party} {verb_en}: {title} ({parsed['stage']})",
data={"position": position, "result": parsed["result"], "item_ref": item_ref, "chamber": chamber,
"eurovoc": eurovoc},
evidence=evidence, topics=topics, facet=VotePosition(event_key=event_key, position=position),
)
if parsed["roll_call"]:
# Individual votes are listed in the protocol after one of the referenced anchors; the
# result is usually behind the last one, sometimes in another fragment (session break).
refs_list = [r["url"].split("#", 1) for r in reversed(refs) if r.get("url") and "#" in r["url"]]
if refs_list:
yield Request(url=f"{BASE}{refs_list[0][0]}", kind="rollcall",
meta={"event_key": event_key, "refs": refs_list, "tried": [refs_list[0][0]],
"title": title, "chamber": chamber, "date": str(voted_on),
"item_url": item_url, "topics": list(topics)})
def parse_rollcall(self, ctx: RunContext, response: Response) -> Iterator[Any]:
meta = response.meta
doc = response.request.url.removeprefix(BASE)
names: dict[str, list] = {}
anchor_used = None
for ref_doc, anchor in meta["refs"]:
if ref_doc != doc:
continue
names = parse_roll_call(response.text, anchor)
if names.get("for") or names.get("against"):
anchor_used = anchor
break
if anchor_used is None:
remaining = [d for d, _ in meta["refs"] if d not in meta["tried"]]
if remaining:
ctx.stats["rollcall_followup"] += 1
yield Request(url=f"{BASE}{remaining[0]}", kind="rollcall",
meta={**meta, "tried": [*meta["tried"], remaining[0]]})
else:
ctx.stats["rollcall_unparsed"] += 1
return
voted_on = parse_date(meta["date"])
body = CHAMBER_SLUG[meta["chamber"]]
source_url = f"{response.request.url}#{anchor_used}"
evidence = [
Evidence(url=source_url, doc_type="protocol", published_at=voted_on,
title="Stenographisches Protokoll – Ergebnis der namentlichen Abstimmung", locator=anchor_used),
Evidence(url=meta["item_url"], doc_type="api_record", title=meta["title"], published_at=voted_on,
locator=f"rollcall:{voted_on}"),
]
for position, entries in names.items():
answer = "Ja" if position == "for" else "Nein"
for label, pad in entries:
ref = (PersonRef(ids={"parlament": str(pad)}) if pad is not None else
PersonRef(name=strip_titles(label), hints={"surname_in_body": body, "on": voted_on}))
yield Claim(
claim_type="vote", person=ref, counterpart_org=OrgRef(slug=body), occurred_at=voted_on,
summary_de=f"Stimmte in namentlicher Abstimmung mit „{answer}“: {meta['title']}",
summary_en=f"Voted \"{'yes' if position == 'for' else 'no'}\" in a roll-call vote: {meta['title']}",
data={"position": position, "roll_call": True, "chamber": meta["chamber"], "label": label},
evidence=evidence, topics=tuple(meta.get("topics") or ()),
facet=VotePosition(event_key=meta["event_key"], position=position),
)
# -- immunity -----------------------------------------------------------------------
def parse_immunity_item(self, ctx: RunContext, response: Response) -> Iterator[Request]:
content = response.json().get("content") or {}
report = next((d["link"] for group in content.get("documents") or [] for d in group.get("documents") or []
if d.get("type") == "HTML"), None)
if report:
yield Request(url=f"{BASE}{report}", kind="immunity_report",
meta={**response.meta, "item": response.request.url.removeprefix(BASE),
"title": html_to_text(content.get("title")) or response.meta.get("title")})
def parse_immunity_report(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
meta = response.meta
text = html_to_text(response.text)
details = parse_immunity_report(text)
title = meta["title"]
name = re.sub(r"^.*?(Abgeordneten zum Nationalrat|Mitglied(es)? des Bundesrates)\s*(Klubobmann|Klubobfrau)?",
"", title).strip()
decided = str(meta.get("status")) == "5"
stage = {"consent": "immunity_waived" if decided else "immunity_waiver_requested",
"refuse": "immunity_refused" if decided else "immunity_waiver_requested"}.get(
details["recommendation"] or "", "immunity_waiver_requested")
stage_de = {"immunity_waived": "Nationalrat stimmte der behördlichen Verfolgung zu (Immunität aufgehoben)",
"immunity_refused": "Nationalrat verweigerte die Zustimmung zur behördlichen Verfolgung",
"immunity_waiver_requested": "Ersuchen um Aufhebung der Immunität"}[stage]
offense = details["offense"]
requested = parse_date(meta.get("date"))
yield Claim(
claim_type="legal_proceeding", person=PersonRef(name=strip_titles(name)),
key=("immunity", meta["item"]), occurred_at=requested, sensitive=True,
summary_de=stage_de + (f" – Verdacht nach {offense}" if offense else "")
+ (f" (Ersuchen: {details['authority']})" if details["authority"] else ""),
data={"political_connection": details["political_connection"],
"recommendation": details["recommendation"], "item": meta["item"]},
evidence=[
Evidence(url=response.request.url, doc_type="register_entry", published_at=requested,
title=f"Bericht des Immunitätsausschusses: {title}", quote=details["quote"], full_text=text),
Evidence(url=f"{BASE}{meta['item']}", doc_type="api_record", title=title, published_at=requested),
],
topics=("Inneres und Recht",),
facet=Legal(stage=stage, offense=offense, authority=details["authority"], case_ref=details["case_ref"],
stage_date=requested),
)
# -- investigative-committee hearings ----------------------------------------------
def parse_hearing_item(self, ctx: RunContext, response: Response) -> Iterator[Any]:
content = response.json().get("content") or {}
title = html_to_text(content.get("title")) or response.meta["title"]
m = HEARING.search(title)
if not m:
return
witness = WITNESS_TITLES.sub("", strip_titles(m.group("name"))).strip()
published = parse_date(response.meta.get("date"))
protocol = next((d["link"] for group in content.get("documents") or [] for d in group.get("documents") or []
if d.get("type") == "HTML"), None)
url = f"{BASE}{protocol}" if protocol else response.request.url.split("?")[0]
event_key = fingerprint("hearing", response.request.url)
yield Event(key=event_key, kind="hearing", title=f"Befragung im Untersuchungsausschuss: {witness}",
starts_at=published, location="Parlament, Wien", country="AT", url=url,
description=title, body=OrgRef(slug="nationalrat"))
yield Claim(
claim_type="appearance", person=PersonRef(name=witness), occurred_at=published,
summary_de=f"Befragung als Auskunftsperson im Untersuchungsausschuss ({title})",
summary_en="Questioned as a witness by a parliamentary investigative committee",
data={"context": "investigative_committee"},
evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=published)],
topics=("Transparenz und Korruption",),
facet=Participation(event_key=event_key, participation="witness"),
)
+124
View File
@@ -0,0 +1,124 @@
"""Parliament press releases (Parlamentskorrespondenz, list 110) and calendar (Termine, list 600). CC BY 4.0.
Press releases are stored with full text; the people they link (/person/<id>) get a mention claim.
The event extractor later turns visits, receptions and trips described in them into meeting/travel
claims. The calendar fills the calendar view with sessions, committee meetings, hearings and events.
"""
from __future__ import annotations
import json
import re
from collections.abc import Iterator
from datetime import date, timedelta
from typing import Any
from politrack.core import Adapter, Request, Response, RunContext, register
from politrack.core.records import Claim, Document, Event, Evidence, OrgRef, PersonRef
from politrack.sources.parlament.common import BASE, CHAMBER_SLUG, SPEC, list_request, list_rows, parse_date
from politrack.text import excerpt, fingerprint, html_to_text
ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}")
SKIP_TERMIN = re.compile(r"Guided Tour|Führung|Galeriebesuch|Besuch einer Plenarsitzung|Demokratiebildung", re.I)
TERMIN_KIND = [
(re.compile(r"Plenarsitzung|Bundesversammlung|Fest-|Gedenksitzung", re.I), "parliament_session"),
(re.compile(r"Untersuchungsausschuss", re.I), "hearing"),
(re.compile(r"Ausschuss|Enquete", re.I), "committee"),
(re.compile(r"Pressekonferenz", re.I), "press_conference"),
(re.compile(r"Internationales|Konferenz", re.I), "meeting"),
]
def release_body(content: list) -> str:
"""The detail JSON is a list of render blocks; the article is the one wrapped in <div class=pk>."""
for block in content:
text = (block.get("data") or {}).get("text") if isinstance(block, dict) else None
if isinstance(text, str) and re.search(r"class=\"?pk\b", text):
return text
return ""
@register
class ParlamentPress(Adapter):
spec = SPEC
name = "parlament.press"
description = "Parlamentskorrespondenz: official press releases with linked people"
schedule = "5 */2 * * *"
def start(self, ctx: RunContext) -> Iterator[Request]:
last = ctx.state.get("last_date")
backfill = int(ctx.params.get("backfill_days", 365))
since = (date.fromisoformat(last) - timedelta(days=2)) if last else date.today() - timedelta(days=backfill)
yield list_request(110, {"DATERANGE": [since.isoformat(), date.today().isoformat()]}, kind="list",
meta={"since": since.isoformat()})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if response.request.kind == "list":
newest = response.meta["since"]
for row in list_rows(response.json()):
if not row.get("Link"):
continue
published = parse_date(row.get("Datum")) # "23.10.2026" (German) in list 110
if published:
newest = max(newest, published.isoformat())
yield Request(url=f"{BASE}{row['Link']}", params={"json": "TRUE"}, kind="release",
meta={k: row.get(k) for k in ("Titel", "Untertitel", "Datum", "Nr.", "Abothemen",
"Stichworte")})
if ctx.mode != "test":
ctx.state.set("last_date", newest)
return
yield from self.parse_release(response)
def parse_release(self, response: Response) -> Iterator[Any]:
meta = response.meta
url = response.request.url
body_html = release_body(response.json().get("content") or [])
pads = sorted({p for p in re.findall(r"/person/(\d+)", body_html)})
text = html_to_text(body_html)
title, subtitle = meta.get("Titel") or "", meta.get("Untertitel") or ""
topics_raw = meta.get("Abothemen")
topics = tuple(json.loads(topics_raw)) if isinstance(topics_raw, str) and topics_raw.startswith("[") else ()
published = parse_date(meta.get("Datum"))
yield Document(Evidence(url=url, doc_type="press_release", title=title, published_at=published,
excerpt=subtitle or excerpt(text, 300), full_text=f"{title}\n{subtitle}\n{text}",
meta={"pk_number": meta.get("Nr."), "person_pads": pads, "topics": list(topics)}))
for pad in pads:
yield Claim(
claim_type="media_mention", person=PersonRef(ids={"parlament": pad}), occurred_at=published,
summary_de=f"Parlamentskorrespondenz: {title}",
data={"context": "parlamentskorrespondenz"},
evidence=[Evidence(url=url, doc_type="press_release", title=title, published_at=published,
quote=subtitle or None, stance="mentions")],
topics=topics,
)
@register
class ParlamentCalendar(Adapter):
spec = SPEC
name = "parlament.calendar"
description = "Parliament calendar: plenary sessions, committees, hearings, events (tours skipped)"
schedule = "30 2 * * *"
def start(self, ctx: RunContext) -> Iterator[Request]:
yield list_request(600, {}, kind="list")
def parse(self, ctx: RunContext, response: Response) -> Iterator[Event]:
since = str(ctx.params.get("since_year", date.today().year - 2))
for row in list_rows(response.json()):
kind_label = (row.get("Terminart") or "").replace("<BR>", " ")
starts = next((v for k, v in row.items() if k.startswith("Datum") and k != "Datum Bis"
and isinstance(v, str) and ISO_DATE.match(v)), None)
if not starts or starts[:4] < since or SKIP_TERMIN.search(f"{kind_label} {row.get('Bezeichnung') or ''}"):
continue
title = row.get("Bezeichnung") or kind_label
kind = next((k for pattern, k in TERMIN_KIND if pattern.search(f"{kind_label} {title}")), "public_event")
chamber = {"Nationalrat": "NR", "Bundesrat": "BR"}.get(row.get("Gremium") or "")
ends = next((v for k, v in row.items() if k.startswith("Datum Bis") and isinstance(v, str)
and ISO_DATE.match(v)), None)
yield Event(
key=fingerprint("termin", row.get("WENTRY_ID") or row.get("Link")), kind=kind, title=title,
starts_at=starts, ends_at=ends, location=(row.get("Ort") or "").replace(" | ", ", ") or "Parlament, Wien",
country="AT", url=f"{BASE}{row['Link']}" if row.get("Link") else None, description=kind_label,
body=OrgRef(slug=CHAMBER_SLUG[chamber]) if chamber else None,
)
@@ -0,0 +1,239 @@
"""Parliament: people, mandates, government functions, side-job disclosures, social links, speeches.
Flow: list 409 (all MPs since 1918) -> PersonRecord + mandate roles
-> detail /person/<id> for members of the configured periods
detail -> photo, social accounts, functions, side jobs (§6 Unv-Transparenz-G)
-> list 251 (speeches of that person)
"""
from __future__ import annotations
import json
import re
from collections.abc import Iterator
from typing import Any
from politrack.config import get_settings
from politrack.core import Adapter, Request, Response, RunContext, register
from politrack.core.records import Claim, Evidence, OrgRef, PersonRecord, PersonRef, Role, SocialAccount, Statement
from politrack.sources.parlament.common import (
BASE,
CHAMBER_NAME_DE,
CHAMBER_SLUG,
GOVERNMENT_RE,
SPEC,
detail_request,
list_request,
list_rows,
parse_date,
walk_dicts,
)
from politrack.text import firmenbuch_number, html_to_text, strip_titles
SOCIAL_PLATFORMS = {"TWITTER": "twitter", "FACEBOOK": "facebook", "INSTAGRAM": "instagram", "TIKTOK": "tiktok",
"YOUTUBE": "youtube", "LINKEDIN": "linkedin", "WEBSITE": "website", "BLUESKY": "bluesky",
"MASTODON": "mastodon", "TELEGRAM": "telegram", "THREADS": "threads"}
FN_IN_TEXT = re.compile(r"\bFN\s*\d{1,6}\s?[a-zA-Z]\b")
POSTCODE_CITY = re.compile(r"(?:(?<=\d)|\b)\d{4}\s+[A-ZÄÖÜ][\wäöüß .\-]*$")
LEGAL_FORM_END = re.compile(r"(?i:gesmbh|gmbh|ges\.\s?m\.\s?b\.\s?h\.?|m\.\s?b\.\s?h\.?|aktiengesellschaft|\bag|\bkg|\bog|"
r"e\.\s?u\.|\bse|privatstiftung|genossenschaft|sparkasse)(?=[A-ZÄÖÜ\s,]|$)")
STREET_WORD = r"\S*(?:straße|strasse|str\.|gasse|platz|weg|kai|ring|allee|zeile|lände|markt|steig)"
def strip_street(text: str) -> str:
"""'Verein Musterhilfe Hauptstraße 12' -> 'Verein Musterhilfe' (greedy name, shortest street suffix)."""
m = (re.match(rf"^(.*\S)\s+{STREET_WORD}\s*\d+[a-z]?(?:/\d+)*\s*$", text, re.IGNORECASE)
or re.match(r"^(.*\S)\s+\S+\s*\d+[a-z]?(?:/\d+)*\s*$", text))
return m.group(1) if m else text
def employer(cell: str) -> tuple[str | None, str | None]:
"""Employer name without business address, plus its company register number.
'Hypo Tirol Bank AG, Meraner Straße 8, 6020 Innsbruck' -> ('Hypo Tirol Bank AG', None)
'25 PersonaldienstleistungsgesmbHFranz Josef Kai 95020 Salzburg FN306527a' (parliament's JSON glues
the lines together) -> ('25 PersonaldienstleistungsgesmbH', '306527a')
"""
cell = re.sub(r"^lit\.\s*\w\s*:\s*", "", cell).strip()
fn_match = FN_IN_TEXT.search(cell)
fn = firmenbuch_number(fn_match.group(0)) if fn_match else None
name = FN_IN_TEXT.sub("", cell).split(",")[0].strip(" *")
address = POSTCODE_CITY.search(name)
if address:
head = name[:address.start()]
ends = [m.end() for m in LEGAL_FORM_END.finditer(head)]
# cut after the last legal form; otherwise drop "<street> <number>" before the postcode
name = head[:ends[-1]] if ends else strip_street(head)
name = name.strip(" ,-*")
return (name if len(name) >= 3 and not re.fullmatch(r"[\d\s.€-]+", name) else None), fn
@register
class ParlamentPersons(Adapter):
spec = SPEC
name = "parlament.persons"
description = "Members of Nationalrat and Bundesrat since 1918, mandates, functions, side jobs, speeches"
schedule = "15 3 * * *"
creates_persons = True
parser_version = 3
def start(self, ctx: RunContext) -> Iterator[Request]:
yield list_request(409, {"NRBR": ["NR", "BR"]}, kind="persons")
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
kind = response.request.kind
if kind == "persons":
yield from self.parse_persons(ctx, response)
elif kind == "person":
yield from self.parse_person(ctx, response)
elif kind == "speeches":
yield from self.parse_speeches(ctx, response)
# -- persons & mandates -------------------------------------------------------------
def parse_persons(self, ctx: RunContext, response: Response) -> Iterator[Any]:
periods = set(ctx.params.get("periods") or get_settings().parlament_periods)
for row in list_rows(response.json()):
attr = row.get("Attribute") or {}
if isinstance(attr, str):
attr = json.loads(attr)
m = re.search(r"/person/(\d+)", attr.get("uri") or "")
if not m:
continue
pad = m.group(1)
ref = PersonRef(ids={"parlament": pad})
display = attr.get("zit") or strip_titles(row.get("Name") or "")
yield PersonRecord(ref=ref, full_name=display, given_name=row.get("vorname"), family_name=row.get("nachname"),
gender={"M": "male", "W": "female"}.get(row.get("GESCHL_CODE") or attr.get("geschlecht")))
mandates = attr.get("mandate_detail") or []
for mandate in mandates:
claim = self.mandate_claim(pad, display, mandate)
if claim:
yield claim
if any(md.get("gp_code") in periods for md in mandates):
yield detail_request(f"/person/{pad}", kind="person", meta={"pad": pad, "name": display})
def mandate_claim(self, pad: str, name: str, m: dict) -> Claim | None:
code = m.get("mand_code") or "NR"
if code not in CHAMBER_SLUG:
return None
party_code = m.get("wahlpartei_code")
party_name = m.get("wahlpartei_full_txt") or m.get("wahlpartei_txt") or party_code
party = OrgRef(name=party_name, short_name=party_code, kind="party") if party_name else None
start, end = parse_date(m.get("mandat_von")), parse_date(m.get("mandat_bis"))
gp = m.get("gp_code") or ""
region = m.get("wahlkreis_bundesland")
label = party_code or party_name or "?"
body = OrgRef(slug=CHAMBER_SLUG[code])
return Claim(
claim_type="role",
person=PersonRef(ids={"parlament": pad}),
org=body,
counterpart_org=party,
valid_from=start,
valid_to=end,
summary_de=f"{name}: Mandat im {CHAMBER_NAME_DE[code]} ({gp}. GP) für {label}"
+ (f", Wahlkreis {region}" if region else ""),
summary_en=f"{name}: member of the {CHAMBER_NAME_DE[code]} ({gp} legislative period) for {label}",
data={"gp": gp, "party_code": party_code, "faction_code": m.get("fraktionscode"), "region": region},
evidence=[Evidence(url=f"{BASE}/person/{pad}", doc_type="api_record", title=f"Parlament: {name}",
locator=f"mandat:{code}:{gp}:{start}")],
facet=Role(role="mp", level="federal", body=body, party=party, region=region, start=start, end=end),
)
# -- person detail ----------------------------------------------------------------
def parse_person(self, ctx: RunContext, response: Response) -> Iterator[Any]:
data = response.json()
content = data.get("content") or {}
pad, name = response.meta["pad"], response.meta["name"]
ref = PersonRef(ids={"parlament": pad})
url = f"{BASE}/person/{pad}"
photo = ((data.get("meta") or {}).get("openGraph") or {}).get("image")
if photo:
yield PersonRecord(ref=ref, full_name=name, photo_url=photo)
for account in (content.get("kontakt") or {}).get("socialMedia") or []:
platform = SOCIAL_PLATFORMS.get((account.get("type") or "").upper(), (account.get("type") or "website").lower())
yield SocialAccount(platform=platform, url=account["url"], person=ref, verified=True)
for fn in walk_dicts(content.get("biografie") or {}, "funktion_text"):
if fn.get("gremium") in CHAMBER_SLUG:
continue
text = fn.get("funktion_text") or fn.get("bez")
if not text:
continue
start, end = parse_date(fn.get("funktion_von")), parse_date(fn.get("funktion_bis"))
is_gov = bool(GOVERNMENT_RE.search(text))
body = OrgRef(slug="bundesregierung") if is_gov else None
yield Claim(
claim_type="role", person=ref, org=body, valid_from=start, valid_to=end,
summary_de=f"{name}: {text}",
evidence=[Evidence(url=url, doc_type="api_record", title=f"Parlament: {name}",
locator=f"funktion:{text}:{start}")],
facet=Role(role=text, level="federal" if is_gov else "other", body=body, start=start, end=end),
)
yield from self.side_jobs(ref, pad, name, content.get("unvtrans") or {})
yield list_request(251, {"PAD_INTERN": [int(pad)]}, kind="speeches", meta={"pad": pad, "name": name})
def side_jobs(self, ref: PersonRef, pad: str, name: str, unv: dict) -> Iterator[Claim]:
"""Disclosures under §6 Unv-Transparenz-G: leading positions, paid side jobs, honorary positions."""
tables = unv.get("unvtransData") or {}
pdf = next((d["link"] for group in unv.get("linkliste") or [] for d in group.get("documents") or []
if d.get("link", "").endswith(f"/person/{pad}/BezBegrBVGPar9.pdf")), None)
for key, table in tables.items():
m = re.fullmatch(r"(z\d+)Table", key)
if not m or not isinstance(table, dict):
continue
header = html_to_text(tables.get(f"{m.group(1)}Header")) or key
for row in (table.get("data") or {}).get("rows") or []:
cells = [c for c in (html_to_text(c) for c in row if isinstance(c, str)) if c]
if not cells:
continue
name_, fn = employer(cells[0]) if m.group(1) in ("z1", "z2", "z3") else (None, None)
if name_: # never keep the (business) address that parliament publishes with the employer
cells = [name_ + (f" (FN {fn})" if fn else ""), *cells[1:]]
text = " – ".join(cells)
yield Claim(
claim_type="side_job", person=ref, key=("unvtrans", pad, m.group(1), text),
counterpart_org=OrgRef(name=name_, kind="company" if LEGAL_FORM_END.search(name_) else "other",
ids={"firmenbuch": fn} if fn else {}) if name_ else None,
summary_de=f"{name} – {header.split(' - ')[0]}: {text}",
data={"category": header, "cells": cells, "unpaid": "*" in "".join(str(c) for c in row),
"employer": name_, "firmenbuch": fn},
evidence=[Evidence(url=f"{BASE}{pdf}" if pdf else f"{BASE}/person/{pad}", doc_type="register_entry",
title=f"Unvereinbarkeits-/Transparenzliste {name}", quote=text,
locator=f"unvtrans:{m.group(1)}")],
topics=("Transparenz und Korruption",),
)
# -- speeches -----------------------------------------------------------------------
def parse_speeches(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
periods = set(ctx.params.get("periods") or get_settings().parlament_periods)
pad, name = response.meta["pad"], response.meta["name"]
for row in list_rows(response.json()):
if row.get("GP") not in periods and row.get("GREMIUM") != "B":
continue
html_cells = " ".join(str(v) for v in row.values() if isinstance(v, str))
protocol = re.search(r'href="(/dokument/[^"]+)"', html_cells)
session = html_to_text(row.get("Sitzung"))
topic = html_to_text(row.get("Rede"))
said_at = row.get("fromDate")
url = f"{BASE}{protocol.group(1)}" if protocol else f"{BASE}/person/{pad}"
chamber = "Bundesrat" if row.get("GREMIUM") == "B" else "Nationalrat"
yield Claim(
claim_type="statement", person=PersonRef(ids={"parlament": pad}),
key=("parl-speech", pad, row.get("Sortier"), topic), occurred_at=said_at,
summary_de=f"{name} sprach im {chamber} zu: {topic}",
summary_en=f"{name} spoke in the {chamber} on: {topic}",
data={"session": session, "gp": row.get("GP"), "item_title": topic},
evidence=[Evidence(url=url, doc_type="protocol", title=session, published_at=parse_date(said_at),
locator=url.split("#", 1)[-1] if "#" in url else "")],
facet=Statement(context="parliament_speech", url=url, said_at=said_at, medium=chamber, title=topic),
)