Politrack: track Austrian politicians from public sources
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
583ccba31c
187 files changed
+39890
No files matched your search
@@ -0,0 +1 @@
|
||||
"""Austrian Parliament adapters: persons, items (votes, immunity, questions, hearings), press, calendar."""
|
||||
@@ -0,0 +1,199 @@
|
||||
"""Shared spec and pure parsing helpers for the Austrian Parliament adapters (CC BY 4.0).
|
||||
|
||||
API (https://www.parlament.gv.at/recherchieren/open-data/):
|
||||
* list exports: POST /Filter/api/filter/data/<listeId>?js=eval&showAll=true (JSON body = filter)
|
||||
409 persons since 1918 · 101 parliamentary items · 251 speeches of a person
|
||||
110 Parlamentskorrespondenz (press releases) · 600 calendar (Termine)
|
||||
* detail pages: <page URL>?json=TRUE
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import date, datetime
|
||||
from typing import Any
|
||||
|
||||
from politrack.core import Request, SourceSpec, register_spec
|
||||
from politrack.text import html_to_text
|
||||
|
||||
BASE = "https://www.parlament.gv.at"
|
||||
CHAMBER_SLUG = {"NR": "nationalrat", "BR": "bundesrat"}
|
||||
CHAMBER_NAME_DE = {"NR": "Nationalrat", "BR": "Bundesrat"}
|
||||
GOVERNMENT_RE = re.compile(r"Bundeskanzler|Vizekanzler|Bundesminister|Staatssekretär", re.IGNORECASE)
|
||||
|
||||
SPEC = register_spec(SourceSpec(
|
||||
slug="parlament",
|
||||
name="Parlament Österreich (Open Data)",
|
||||
kind="official",
|
||||
homepage="https://www.parlament.gv.at/recherchieren/open-data/",
|
||||
license="CC BY 4.0",
|
||||
access="api",
|
||||
trust=95,
|
||||
store_fulltext=True,
|
||||
min_interval=1.0,
|
||||
tos_notes="Official JSON API (?json=TRUE, /Filter/api/filter/data/<list>). Attribution required. The "
|
||||
"Parlamentsdirektion may prohibit uses incompatible with the dignity of parliament.",
|
||||
))
|
||||
|
||||
|
||||
def list_request(list_id: int, body: dict, kind: str, meta: dict | None = None) -> Request:
|
||||
return Request(url=f"{BASE}/Filter/api/filter/data/{list_id}?js=eval&showAll=true", method="POST", json=body,
|
||||
kind=kind, meta=meta or {})
|
||||
|
||||
|
||||
def detail_request(path: str, kind: str, meta: dict | None = None) -> Request:
|
||||
return Request(url=f"{BASE}{path}", params={"json": "TRUE"}, kind=kind, meta=meta or {})
|
||||
|
||||
|
||||
def list_rows(data: dict) -> list[dict[str, Any]]:
|
||||
"""Turn a filter-API export into dicts keyed by column label (duplicates get a _<index> suffix)."""
|
||||
labels: list[str] = []
|
||||
for i, header in enumerate(data.get("header", [])):
|
||||
label = header.get("label") or header.get("feld_name") or f"col{i}"
|
||||
labels.append(label if label not in labels else f"{label}_{i}")
|
||||
return [dict(zip(labels, row)) for row in data.get("rows", [])]
|
||||
|
||||
|
||||
def parse_date(value: str | None) -> date | None:
|
||||
if not value:
|
||||
return None
|
||||
for fmt in ("%d.%m.%Y", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d"):
|
||||
try:
|
||||
return datetime.strptime(value[:19], fmt).date()
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def walk_dicts(obj: Any, required_key: str):
|
||||
"""Yield every nested dict that contains `required_key`."""
|
||||
if isinstance(obj, dict):
|
||||
if required_key in obj:
|
||||
yield obj
|
||||
for value in obj.values():
|
||||
yield from walk_dicts(value, required_key)
|
||||
elif isinstance(obj, list):
|
||||
for value in obj:
|
||||
yield from walk_dicts(value, required_key)
|
||||
|
||||
|
||||
# -- faction votes ------------------------------------------------------------------------
|
||||
|
||||
PARTY_TOKEN = re.compile(r"^[A-ZÄÖÜ][A-Za-zÄÖÜäöüß.\- ]{0,30}$")
|
||||
# Split votes ("teilweise", "bzw.", per-part results in parentheses) can't be attributed to factions.
|
||||
SPLIT_VOTE = re.compile(r"\(|\bbzw\.|\bteilweise\b|\btlw\.|\bmehrstimmig\b|\bgetrennt", re.IGNORECASE)
|
||||
|
||||
|
||||
def parse_party_list(value: str | None) -> list[str]:
|
||||
if not value:
|
||||
return []
|
||||
parts = [p.strip(" .;") for p in re.split(r",|\bund\b", value)]
|
||||
return [p for p in parts if PARTY_TOKEN.match(p)]
|
||||
|
||||
|
||||
def parse_vote_text(text: str) -> dict[str, Any] | None:
|
||||
"""'137. Sitzung …: Gesetzesvorschlag in dritter Lesung angenommen Dafür: ÖVP, SPÖ, dagegen: FPÖ'"""
|
||||
m = re.search(r"\b(angenommen|abgelehnt)\b", text, re.IGNORECASE)
|
||||
if not m:
|
||||
return None
|
||||
rest = text[m.end():]
|
||||
roll_call = "namentliche abstimmung" in text.lower()
|
||||
split = bool(SPLIT_VOTE.search(rest)) and not roll_call
|
||||
pro = re.search(r"Dafür:\s*(.*?)(?:,?\s*dagegen:|$)", rest, re.IGNORECASE)
|
||||
contra = re.search(r"dagegen:\s*(.*)$", rest, re.IGNORECASE)
|
||||
return {
|
||||
"stage": text[: m.start()].strip(" :,"),
|
||||
"result": "accepted" if m.group(1).lower() == "angenommen" else "rejected",
|
||||
"for": [] if split or roll_call else parse_party_list(pro.group(1) if pro else None),
|
||||
"against": [] if split or roll_call else parse_party_list(contra.group(1) if contra else None),
|
||||
"unanimous": "einstimmig" in text.lower(),
|
||||
"split": split,
|
||||
"roll_call": roll_call,
|
||||
}
|
||||
|
||||
|
||||
# -- roll-call votes ----------------------------------------------------------------------
|
||||
|
||||
ROLL_CALL_HEADER = re.compile(r"Mit\s*[„\"“]?\s*(Ja|Nein)\s*[“\"”]?\s*stimmten[^:]*:", re.IGNORECASE)
|
||||
NEW_VOTE = re.compile(r"\b(gelangen|kommen)\s+(?:\w+\s+){0,2}zur\s+Abstimmung\b", re.IGNORECASE)
|
||||
PERSON_LINK = re.compile(r"<a[^>]*href=\"[^\"]*/person/(\d+)\"[^>]*>.*?</a>", re.IGNORECASE | re.DOTALL)
|
||||
|
||||
|
||||
def parse_roll_call(html_text: str, anchor: str | None) -> dict[str, list[tuple[str, int | None]]]:
|
||||
"""Extract the "Mit „Ja“ stimmten …" / "Mit „Nein“ stimmten …" lists that follow `anchor`.
|
||||
|
||||
Returns {"for": [(label, pad|None)], "against": [...]}. Names are mostly surnames; where a surname is
|
||||
ambiguous the protocol links the person, which gives us the parliament id directly.
|
||||
"""
|
||||
start = 0
|
||||
if anchor:
|
||||
m = re.search(r'id="%s"' % re.escape(anchor), html_text)
|
||||
if not m:
|
||||
return {}
|
||||
start = m.start()
|
||||
first = ROLL_CALL_HEADER.search(html_text, start)
|
||||
if not first:
|
||||
return {}
|
||||
# The lists must belong to this anchor: they follow within a few numbered paragraphs and no new vote
|
||||
# may be called in between.
|
||||
between = html_text[start + 10 : first.start()]
|
||||
if anchor and (len(re.findall(r'class="[^"]*randnummer', between)) > 3 or NEW_VOTE.search(html_to_text(between))):
|
||||
return {}
|
||||
end_match = re.compile(r"<hr|class=\"[^\"]*randnummer").search(html_text, first.end())
|
||||
segment = html_text[first.start() : end_match.start() if end_match else len(html_text)]
|
||||
segment = PERSON_LINK.sub(lambda m: f" @@{m.group(1)}@@ ", segment)
|
||||
text = html_to_text(segment)
|
||||
|
||||
result: dict[str, list[tuple[str, int | None]]] = {"for": [], "against": []}
|
||||
headers = list(ROLL_CALL_HEADER.finditer(text))
|
||||
for i, header in enumerate(headers):
|
||||
block = text[header.end() : headers[i + 1].start() if i + 1 < len(headers) else len(text)]
|
||||
key = "for" if header.group(1).lower() == "ja" else "against"
|
||||
for token in re.split(r"[,;]|\.\s*$", block):
|
||||
token = token.strip(" .")
|
||||
if not token:
|
||||
continue
|
||||
link = re.fullmatch(r"@@(\d+)@@", token)
|
||||
result[key].append((token, int(link.group(1))) if link else (token, None))
|
||||
return result
|
||||
|
||||
|
||||
# -- immunity -----------------------------------------------------------------------------
|
||||
|
||||
STGB_OFFENSES = {
|
||||
"111": "Üble Nachrede", "115": "Beleidigung", "146": "Betrug", "153": "Untreue", "165": "Geldwäscherei",
|
||||
"201": "Vergewaltigung", "283": "Verhetzung", "288": "Falsche Beweisaussage", "293": "Fälschung eines Beweismittels",
|
||||
"295": "Unterdrückung eines Beweismittels", "297": "Verleumdung", "302": "Missbrauch der Amtsgewalt",
|
||||
"304": "Bestechlichkeit", "305": "Vorteilsannahme", "306": "Vorteilsannahme zur Beeinflussung",
|
||||
"307": "Bestechung", "308": "Verbotene Intervention", "310": "Verletzung des Amtsgeheimnisses",
|
||||
}
|
||||
|
||||
|
||||
def parse_immunity_report(text: str) -> dict[str, Any]:
|
||||
authority = re.search(r"Ersuchen (?:der|des) (.+?),\s*(?:do\.|GZ|Zl)", text)
|
||||
case_ref = re.search(r"\bGZ\.?\s*([^,]+?),", text)
|
||||
# "wegen des Verdachtes einer strafbaren Handlung nach § 283 Abs. 1 Z 1 und Z 2, Abs. 2 StGB"
|
||||
offense = re.search(r"wegen[^§]{0,90}?(§§? ?\d+.{0,120}?\b(?:StGB|VerbotsG|FinStrG|[A-Z][A-Za-z]*G)\b)", text)
|
||||
offense_text = offense.group(1).strip() if offense else None
|
||||
if offense_text:
|
||||
section = re.search(r"§ ?(\d+)", offense_text)
|
||||
if section and section.group(1) in STGB_OFFENSES and "StGB" in offense_text:
|
||||
offense_text = f"{offense_text} ({STGB_OFFENSES[section.group(1)]})"
|
||||
decision = re.search(r"(?:wird|werde)\s+(nicht\s+)?zugestimmt|(nicht\s+)?zuzustimmen", text)
|
||||
recommendation = None
|
||||
if decision:
|
||||
recommendation = "refuse" if (decision.group(1) or decision.group(2)) else "consent"
|
||||
connection = None
|
||||
if re.search(r"\bkein Zusammenhang\b", text):
|
||||
connection = False
|
||||
elif re.search(r"\bein Zusammenhang\b", text):
|
||||
connection = True
|
||||
quote = re.search(r"Als Ergebnis seiner Beratungen.*?(?:zugestimmt|nicht zugestimmt)\s*\.", text)
|
||||
return {
|
||||
"authority": authority.group(1).strip() if authority else None,
|
||||
"case_ref": case_ref.group(1).strip() if case_ref else None,
|
||||
"offense": offense_text,
|
||||
"recommendation": recommendation,
|
||||
"political_connection": connection,
|
||||
"quote": quote.group(0) if quote else None,
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
"""Parliament items (Gegenstände) of the configured legislative periods.
|
||||
|
||||
From one list export per period (list 101) we derive:
|
||||
* votes -> item detail -> VoteEvent + faction VotePositions
|
||||
roll calls -> stenographic protocol -> individual VotePositions
|
||||
* immunity requests ("Ersuchen um Zustimmung zur behördlichen Verfolgung") -> committee report -> Legal claim
|
||||
* written questions and answers (ITYP J / AB) -> parliamentary_question claims (directly from the list)
|
||||
* investigative-committee hearings ("öffentliche Befragung der Auskunftsperson ...") -> hearing Event +
|
||||
appearance claim for politicians who testified (other witnesses are not stored)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Adapter, Request, Response, RunContext, register
|
||||
from politrack.core.records import (
|
||||
Claim,
|
||||
Event,
|
||||
Evidence,
|
||||
Legal,
|
||||
OrgRef,
|
||||
Participation,
|
||||
PersonRef,
|
||||
VoteEvent,
|
||||
VotePosition,
|
||||
)
|
||||
from politrack.sources.parlament.common import (
|
||||
BASE,
|
||||
CHAMBER_SLUG,
|
||||
SPEC,
|
||||
detail_request,
|
||||
list_request,
|
||||
list_rows,
|
||||
parse_date,
|
||||
parse_immunity_report,
|
||||
parse_roll_call,
|
||||
parse_vote_text,
|
||||
walk_dicts,
|
||||
)
|
||||
from politrack.text import fingerprint, html_to_text, strip_titles
|
||||
|
||||
HEARING = re.compile(r"Befragung der Auskunftsperson\s+(?P<name>.+)$")
|
||||
WITNESS_TITLES = re.compile(r"^(?:(?:Oberstaatsanwalt|Staatsanwalt|Sektionschef(?:in)?|MinR|Generalsekretär(?:in)?|"
|
||||
r"Bundesminister(?:in)?|Abgeordnete[rn]?|Klubobmann|Klubobfrau|Bundeskanzler|"
|
||||
r"Vizekanzler|Präsident(?:in)?)\s+)+")
|
||||
|
||||
|
||||
@register
|
||||
class ParlamentItems(Adapter):
|
||||
spec = SPEC
|
||||
name = "parlament.items"
|
||||
description = "Votes (faction + roll call), immunity requests, written questions, U-committee hearings"
|
||||
schedule = "45 3 * * *"
|
||||
parser_version = 2
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
for gp in ctx.params.get("periods") or get_settings().parlament_periods:
|
||||
yield list_request(101, {"GP_CODE": [gp], "NRBR": ["NR"]}, kind="items", meta={"gp": gp})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
handler = {
|
||||
"items": self.parse_items,
|
||||
"votes": self.parse_votes,
|
||||
"rollcall": self.parse_rollcall,
|
||||
"immunity": self.parse_immunity_item,
|
||||
"immunity_report": self.parse_immunity_report,
|
||||
"hearing": self.parse_hearing_item,
|
||||
}[response.request.kind]
|
||||
yield from handler(ctx, response)
|
||||
|
||||
# -- list ---------------------------------------------------------------------------
|
||||
|
||||
def parse_items(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
for row in list_rows(response.json()):
|
||||
path, subject, kind = row.get("HIS_URL"), row.get("Betreff") or "", row.get("ITYP")
|
||||
if not path:
|
||||
continue
|
||||
if row.get("Dafür") or row.get("Dagegen") or row.get("Abstimmungstext"):
|
||||
# The list row changes when new votes happen, so the detail is re-fetched only then
|
||||
# (conditional request + unchanged-body detection do the rest).
|
||||
yield detail_request(path, kind="votes", meta={"title": subject, "ref": row.get("Nummer")})
|
||||
if subject.startswith("Ersuchen") and "behördlichen Verfolgung" in subject:
|
||||
yield detail_request(path, kind="immunity",
|
||||
meta={"title": subject, "date": row.get("Datum"), "status": row.get("Status")})
|
||||
if kind in ("J", "AB"):
|
||||
yield from self.question_claims(row, path)
|
||||
if kind == "KOMM" and HEARING.search(subject):
|
||||
yield detail_request(path, kind="hearing", meta={"title": subject, "date": row.get("Datum")})
|
||||
|
||||
def question_claims(self, row: dict, path: str) -> Iterator[Claim]:
|
||||
pads = row.get("Personen") or "[]"
|
||||
pads = json.loads(pads) if isinstance(pads, str) else pads
|
||||
subject = row.get("Betreff") or ""
|
||||
is_question = row.get("ITYP") == "J"
|
||||
asked = parse_date(row.get("Datum"))
|
||||
topics = tuple(json.loads(row.get("THEMEN") or "[]")) if isinstance(row.get("THEMEN"), str) else ()
|
||||
for pad in pads:
|
||||
if not str(pad).strip():
|
||||
continue
|
||||
verb_de = "stellte die schriftliche Anfrage" if is_question else "beantwortete die Anfrage"
|
||||
verb_en = "submitted the written question" if is_question else "answered the question"
|
||||
yield Claim(
|
||||
claim_type="parliamentary_question", person=PersonRef(ids={"parlament": str(pad)}),
|
||||
key=("question", path, str(pad)), occurred_at=asked,
|
||||
summary_de=f"{verb_de[0].upper()}{verb_de[1:]} „{subject}“ ({row.get('Nummer')})",
|
||||
summary_en=f"{verb_en[0].upper()}{verb_en[1:]} \"{subject}\" ({row.get('Nummer')})",
|
||||
data={"kind": "question" if is_question else "answer", "number": row.get("Nummer")},
|
||||
evidence=[Evidence(url=f"{BASE}{path}", doc_type="api_record", title=subject, published_at=asked)],
|
||||
topics=topics,
|
||||
)
|
||||
|
||||
# -- votes ----------------------------------------------------------------------------
|
||||
|
||||
def parse_votes(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
content = response.json().get("content") or {}
|
||||
path = response.request.url.removeprefix(BASE)
|
||||
title = html_to_text(content.get("title")) or response.meta.get("title") or path
|
||||
item_ref = html_to_text(content.get("zitation")) or response.meta.get("ref")
|
||||
item_url = f"{BASE}{path}"
|
||||
topics = tuple(b.get("label") for b in ((content.get("topics") or {}).get("data") or {}).get("bubbles") or [])
|
||||
eurovoc = [b.get("label") for b in ((content.get("eurovoc") or {}).get("data") or {}).get("bubbles") or []]
|
||||
|
||||
for stage in walk_dicts(content.get("phase") or content, "text"):
|
||||
text = html_to_text(stage.get("text"))
|
||||
if "angenommen" not in text and "abgelehnt" not in text:
|
||||
continue
|
||||
parsed = parse_vote_text(text)
|
||||
if not parsed or not (parsed["for"] or parsed["against"] or parsed["unanimous"] or parsed["split"]
|
||||
or parsed["roll_call"]):
|
||||
continue
|
||||
voted_on = parse_date(stage.get("date"))
|
||||
refs = stage.get("fsth") or []
|
||||
protocol_url = f"{BASE}{refs[0]['url']}" if refs and refs[0].get("url") else None
|
||||
chamber = "BR" if (refs and refs[0].get("gp_code") == "BR") or "Bundesrat" in text else "NR"
|
||||
chamber_ref = OrgRef(slug=CHAMBER_SLUG[chamber])
|
||||
event_key = fingerprint("vote", path, voted_on, parsed["stage"])
|
||||
yield VoteEvent(key=event_key, chamber=chamber_ref, title=title, voted_on=voted_on, result=parsed["result"],
|
||||
stage=parsed["stage"], item_ref=item_ref, roll_call=parsed["roll_call"], url=item_url,
|
||||
protocol_url=protocol_url, raw_text=text)
|
||||
evidence = [Evidence(url=item_url, doc_type="api_record", title=title, published_at=voted_on, quote=text,
|
||||
locator=f"stage:{voted_on}:{parsed['stage'][:80]}")]
|
||||
if protocol_url:
|
||||
evidence.append(Evidence(url=protocol_url, doc_type="protocol", published_at=voted_on,
|
||||
title=refs[0].get("title") or "Stenographisches Protokoll"))
|
||||
outcome = "angenommen" if parsed["result"] == "accepted" else "abgelehnt"
|
||||
for party, position in [(p, "for") for p in parsed["for"]] + [(p, "against") for p in parsed["against"]]:
|
||||
verb_de, verb_en = ("stimmte für", "voted for") if position == "for" else ("stimmte gegen", "voted against")
|
||||
yield Claim(
|
||||
claim_type="vote", org=OrgRef(name=party, short_name=party, kind="parliamentary_group"),
|
||||
counterpart_org=chamber_ref, occurred_at=voted_on,
|
||||
summary_de=f"{party} {verb_de}: {title} ({parsed['stage']}) – {outcome}",
|
||||
summary_en=f"{party} {verb_en}: {title} ({parsed['stage']})",
|
||||
data={"position": position, "result": parsed["result"], "item_ref": item_ref, "chamber": chamber,
|
||||
"eurovoc": eurovoc},
|
||||
evidence=evidence, topics=topics, facet=VotePosition(event_key=event_key, position=position),
|
||||
)
|
||||
if parsed["roll_call"]:
|
||||
# Individual votes are listed in the protocol after one of the referenced anchors; the
|
||||
# result is usually behind the last one, sometimes in another fragment (session break).
|
||||
refs_list = [r["url"].split("#", 1) for r in reversed(refs) if r.get("url") and "#" in r["url"]]
|
||||
if refs_list:
|
||||
yield Request(url=f"{BASE}{refs_list[0][0]}", kind="rollcall",
|
||||
meta={"event_key": event_key, "refs": refs_list, "tried": [refs_list[0][0]],
|
||||
"title": title, "chamber": chamber, "date": str(voted_on),
|
||||
"item_url": item_url, "topics": list(topics)})
|
||||
|
||||
def parse_rollcall(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
meta = response.meta
|
||||
doc = response.request.url.removeprefix(BASE)
|
||||
names: dict[str, list] = {}
|
||||
anchor_used = None
|
||||
for ref_doc, anchor in meta["refs"]:
|
||||
if ref_doc != doc:
|
||||
continue
|
||||
names = parse_roll_call(response.text, anchor)
|
||||
if names.get("for") or names.get("against"):
|
||||
anchor_used = anchor
|
||||
break
|
||||
if anchor_used is None:
|
||||
remaining = [d for d, _ in meta["refs"] if d not in meta["tried"]]
|
||||
if remaining:
|
||||
ctx.stats["rollcall_followup"] += 1
|
||||
yield Request(url=f"{BASE}{remaining[0]}", kind="rollcall",
|
||||
meta={**meta, "tried": [*meta["tried"], remaining[0]]})
|
||||
else:
|
||||
ctx.stats["rollcall_unparsed"] += 1
|
||||
return
|
||||
voted_on = parse_date(meta["date"])
|
||||
body = CHAMBER_SLUG[meta["chamber"]]
|
||||
source_url = f"{response.request.url}#{anchor_used}"
|
||||
evidence = [
|
||||
Evidence(url=source_url, doc_type="protocol", published_at=voted_on,
|
||||
title="Stenographisches Protokoll – Ergebnis der namentlichen Abstimmung", locator=anchor_used),
|
||||
Evidence(url=meta["item_url"], doc_type="api_record", title=meta["title"], published_at=voted_on,
|
||||
locator=f"rollcall:{voted_on}"),
|
||||
]
|
||||
for position, entries in names.items():
|
||||
answer = "Ja" if position == "for" else "Nein"
|
||||
for label, pad in entries:
|
||||
ref = (PersonRef(ids={"parlament": str(pad)}) if pad is not None else
|
||||
PersonRef(name=strip_titles(label), hints={"surname_in_body": body, "on": voted_on}))
|
||||
yield Claim(
|
||||
claim_type="vote", person=ref, counterpart_org=OrgRef(slug=body), occurred_at=voted_on,
|
||||
summary_de=f"Stimmte in namentlicher Abstimmung mit „{answer}“: {meta['title']}",
|
||||
summary_en=f"Voted \"{'yes' if position == 'for' else 'no'}\" in a roll-call vote: {meta['title']}",
|
||||
data={"position": position, "roll_call": True, "chamber": meta["chamber"], "label": label},
|
||||
evidence=evidence, topics=tuple(meta.get("topics") or ()),
|
||||
facet=VotePosition(event_key=meta["event_key"], position=position),
|
||||
)
|
||||
|
||||
# -- immunity -----------------------------------------------------------------------
|
||||
|
||||
def parse_immunity_item(self, ctx: RunContext, response: Response) -> Iterator[Request]:
|
||||
content = response.json().get("content") or {}
|
||||
report = next((d["link"] for group in content.get("documents") or [] for d in group.get("documents") or []
|
||||
if d.get("type") == "HTML"), None)
|
||||
if report:
|
||||
yield Request(url=f"{BASE}{report}", kind="immunity_report",
|
||||
meta={**response.meta, "item": response.request.url.removeprefix(BASE),
|
||||
"title": html_to_text(content.get("title")) or response.meta.get("title")})
|
||||
|
||||
def parse_immunity_report(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||||
meta = response.meta
|
||||
text = html_to_text(response.text)
|
||||
details = parse_immunity_report(text)
|
||||
title = meta["title"]
|
||||
name = re.sub(r"^.*?(Abgeordneten zum Nationalrat|Mitglied(es)? des Bundesrates)\s*(Klubobmann|Klubobfrau)?",
|
||||
"", title).strip()
|
||||
decided = str(meta.get("status")) == "5"
|
||||
stage = {"consent": "immunity_waived" if decided else "immunity_waiver_requested",
|
||||
"refuse": "immunity_refused" if decided else "immunity_waiver_requested"}.get(
|
||||
details["recommendation"] or "", "immunity_waiver_requested")
|
||||
stage_de = {"immunity_waived": "Nationalrat stimmte der behördlichen Verfolgung zu (Immunität aufgehoben)",
|
||||
"immunity_refused": "Nationalrat verweigerte die Zustimmung zur behördlichen Verfolgung",
|
||||
"immunity_waiver_requested": "Ersuchen um Aufhebung der Immunität"}[stage]
|
||||
offense = details["offense"]
|
||||
requested = parse_date(meta.get("date"))
|
||||
yield Claim(
|
||||
claim_type="legal_proceeding", person=PersonRef(name=strip_titles(name)),
|
||||
key=("immunity", meta["item"]), occurred_at=requested, sensitive=True,
|
||||
summary_de=stage_de + (f" – Verdacht nach {offense}" if offense else "")
|
||||
+ (f" (Ersuchen: {details['authority']})" if details["authority"] else ""),
|
||||
data={"political_connection": details["political_connection"],
|
||||
"recommendation": details["recommendation"], "item": meta["item"]},
|
||||
evidence=[
|
||||
Evidence(url=response.request.url, doc_type="register_entry", published_at=requested,
|
||||
title=f"Bericht des Immunitätsausschusses: {title}", quote=details["quote"], full_text=text),
|
||||
Evidence(url=f"{BASE}{meta['item']}", doc_type="api_record", title=title, published_at=requested),
|
||||
],
|
||||
topics=("Inneres und Recht",),
|
||||
facet=Legal(stage=stage, offense=offense, authority=details["authority"], case_ref=details["case_ref"],
|
||||
stage_date=requested),
|
||||
)
|
||||
|
||||
# -- investigative-committee hearings ----------------------------------------------
|
||||
|
||||
def parse_hearing_item(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
content = response.json().get("content") or {}
|
||||
title = html_to_text(content.get("title")) or response.meta["title"]
|
||||
m = HEARING.search(title)
|
||||
if not m:
|
||||
return
|
||||
witness = WITNESS_TITLES.sub("", strip_titles(m.group("name"))).strip()
|
||||
published = parse_date(response.meta.get("date"))
|
||||
protocol = next((d["link"] for group in content.get("documents") or [] for d in group.get("documents") or []
|
||||
if d.get("type") == "HTML"), None)
|
||||
url = f"{BASE}{protocol}" if protocol else response.request.url.split("?")[0]
|
||||
event_key = fingerprint("hearing", response.request.url)
|
||||
yield Event(key=event_key, kind="hearing", title=f"Befragung im Untersuchungsausschuss: {witness}",
|
||||
starts_at=published, location="Parlament, Wien", country="AT", url=url,
|
||||
description=title, body=OrgRef(slug="nationalrat"))
|
||||
yield Claim(
|
||||
claim_type="appearance", person=PersonRef(name=witness), occurred_at=published,
|
||||
summary_de=f"Befragung als Auskunftsperson im Untersuchungsausschuss ({title})",
|
||||
summary_en="Questioned as a witness by a parliamentary investigative committee",
|
||||
data={"context": "investigative_committee"},
|
||||
evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=published)],
|
||||
topics=("Transparenz und Korruption",),
|
||||
facet=Participation(event_key=event_key, participation="witness"),
|
||||
)
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Parliament press releases (Parlamentskorrespondenz, list 110) and calendar (Termine, list 600). CC BY 4.0.
|
||||
|
||||
Press releases are stored with full text; the people they link (/person/<id>) get a mention claim.
|
||||
The event extractor later turns visits, receptions and trips described in them into meeting/travel
|
||||
claims. The calendar fills the calendar view with sessions, committee meetings, hearings and events.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import date, timedelta
|
||||
from typing import Any
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, register
|
||||
from politrack.core.records import Claim, Document, Event, Evidence, OrgRef, PersonRef
|
||||
from politrack.sources.parlament.common import BASE, CHAMBER_SLUG, SPEC, list_request, list_rows, parse_date
|
||||
from politrack.text import excerpt, fingerprint, html_to_text
|
||||
|
||||
ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}")
|
||||
SKIP_TERMIN = re.compile(r"Guided Tour|Führung|Galeriebesuch|Besuch einer Plenarsitzung|Demokratiebildung", re.I)
|
||||
TERMIN_KIND = [
|
||||
(re.compile(r"Plenarsitzung|Bundesversammlung|Fest-|Gedenksitzung", re.I), "parliament_session"),
|
||||
(re.compile(r"Untersuchungsausschuss", re.I), "hearing"),
|
||||
(re.compile(r"Ausschuss|Enquete", re.I), "committee"),
|
||||
(re.compile(r"Pressekonferenz", re.I), "press_conference"),
|
||||
(re.compile(r"Internationales|Konferenz", re.I), "meeting"),
|
||||
]
|
||||
|
||||
|
||||
def release_body(content: list) -> str:
|
||||
"""The detail JSON is a list of render blocks; the article is the one wrapped in <div class=pk>."""
|
||||
for block in content:
|
||||
text = (block.get("data") or {}).get("text") if isinstance(block, dict) else None
|
||||
if isinstance(text, str) and re.search(r"class=\"?pk\b", text):
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
@register
|
||||
class ParlamentPress(Adapter):
|
||||
spec = SPEC
|
||||
name = "parlament.press"
|
||||
description = "Parlamentskorrespondenz: official press releases with linked people"
|
||||
schedule = "5 */2 * * *"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
last = ctx.state.get("last_date")
|
||||
backfill = int(ctx.params.get("backfill_days", 365))
|
||||
since = (date.fromisoformat(last) - timedelta(days=2)) if last else date.today() - timedelta(days=backfill)
|
||||
yield list_request(110, {"DATERANGE": [since.isoformat(), date.today().isoformat()]}, kind="list",
|
||||
meta={"since": since.isoformat()})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
if response.request.kind == "list":
|
||||
newest = response.meta["since"]
|
||||
for row in list_rows(response.json()):
|
||||
if not row.get("Link"):
|
||||
continue
|
||||
published = parse_date(row.get("Datum")) # "23.10.2026" (German) in list 110
|
||||
if published:
|
||||
newest = max(newest, published.isoformat())
|
||||
yield Request(url=f"{BASE}{row['Link']}", params={"json": "TRUE"}, kind="release",
|
||||
meta={k: row.get(k) for k in ("Titel", "Untertitel", "Datum", "Nr.", "Abothemen",
|
||||
"Stichworte")})
|
||||
if ctx.mode != "test":
|
||||
ctx.state.set("last_date", newest)
|
||||
return
|
||||
yield from self.parse_release(response)
|
||||
|
||||
def parse_release(self, response: Response) -> Iterator[Any]:
|
||||
meta = response.meta
|
||||
url = response.request.url
|
||||
body_html = release_body(response.json().get("content") or [])
|
||||
pads = sorted({p for p in re.findall(r"/person/(\d+)", body_html)})
|
||||
text = html_to_text(body_html)
|
||||
title, subtitle = meta.get("Titel") or "", meta.get("Untertitel") or ""
|
||||
topics_raw = meta.get("Abothemen")
|
||||
topics = tuple(json.loads(topics_raw)) if isinstance(topics_raw, str) and topics_raw.startswith("[") else ()
|
||||
published = parse_date(meta.get("Datum"))
|
||||
yield Document(Evidence(url=url, doc_type="press_release", title=title, published_at=published,
|
||||
excerpt=subtitle or excerpt(text, 300), full_text=f"{title}\n{subtitle}\n{text}",
|
||||
meta={"pk_number": meta.get("Nr."), "person_pads": pads, "topics": list(topics)}))
|
||||
for pad in pads:
|
||||
yield Claim(
|
||||
claim_type="media_mention", person=PersonRef(ids={"parlament": pad}), occurred_at=published,
|
||||
summary_de=f"Parlamentskorrespondenz: {title}",
|
||||
data={"context": "parlamentskorrespondenz"},
|
||||
evidence=[Evidence(url=url, doc_type="press_release", title=title, published_at=published,
|
||||
quote=subtitle or None, stance="mentions")],
|
||||
topics=topics,
|
||||
)
|
||||
|
||||
|
||||
@register
|
||||
class ParlamentCalendar(Adapter):
|
||||
spec = SPEC
|
||||
name = "parlament.calendar"
|
||||
description = "Parliament calendar: plenary sessions, committees, hearings, events (tours skipped)"
|
||||
schedule = "30 2 * * *"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
yield list_request(600, {}, kind="list")
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Event]:
|
||||
since = str(ctx.params.get("since_year", date.today().year - 2))
|
||||
for row in list_rows(response.json()):
|
||||
kind_label = (row.get("Terminart") or "").replace("<BR>", " ")
|
||||
starts = next((v for k, v in row.items() if k.startswith("Datum") and k != "Datum Bis"
|
||||
and isinstance(v, str) and ISO_DATE.match(v)), None)
|
||||
if not starts or starts[:4] < since or SKIP_TERMIN.search(f"{kind_label} {row.get('Bezeichnung') or ''}"):
|
||||
continue
|
||||
title = row.get("Bezeichnung") or kind_label
|
||||
kind = next((k for pattern, k in TERMIN_KIND if pattern.search(f"{kind_label} {title}")), "public_event")
|
||||
chamber = {"Nationalrat": "NR", "Bundesrat": "BR"}.get(row.get("Gremium") or "")
|
||||
ends = next((v for k, v in row.items() if k.startswith("Datum Bis") and isinstance(v, str)
|
||||
and ISO_DATE.match(v)), None)
|
||||
yield Event(
|
||||
key=fingerprint("termin", row.get("WENTRY_ID") or row.get("Link")), kind=kind, title=title,
|
||||
starts_at=starts, ends_at=ends, location=(row.get("Ort") or "").replace(" | ", ", ") or "Parlament, Wien",
|
||||
country="AT", url=f"{BASE}{row['Link']}" if row.get("Link") else None, description=kind_label,
|
||||
body=OrgRef(slug=CHAMBER_SLUG[chamber]) if chamber else None,
|
||||
)
|
||||
@@ -0,0 +1,239 @@
|
||||
"""Parliament: people, mandates, government functions, side-job disclosures, social links, speeches.
|
||||
|
||||
Flow: list 409 (all MPs since 1918) -> PersonRecord + mandate roles
|
||||
-> detail /person/<id> for members of the configured periods
|
||||
detail -> photo, social accounts, functions, side jobs (§6 Unv-Transparenz-G)
|
||||
-> list 251 (speeches of that person)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Adapter, Request, Response, RunContext, register
|
||||
from politrack.core.records import Claim, Evidence, OrgRef, PersonRecord, PersonRef, Role, SocialAccount, Statement
|
||||
from politrack.sources.parlament.common import (
|
||||
BASE,
|
||||
CHAMBER_NAME_DE,
|
||||
CHAMBER_SLUG,
|
||||
GOVERNMENT_RE,
|
||||
SPEC,
|
||||
detail_request,
|
||||
list_request,
|
||||
list_rows,
|
||||
parse_date,
|
||||
walk_dicts,
|
||||
)
|
||||
from politrack.text import firmenbuch_number, html_to_text, strip_titles
|
||||
|
||||
SOCIAL_PLATFORMS = {"TWITTER": "twitter", "FACEBOOK": "facebook", "INSTAGRAM": "instagram", "TIKTOK": "tiktok",
|
||||
"YOUTUBE": "youtube", "LINKEDIN": "linkedin", "WEBSITE": "website", "BLUESKY": "bluesky",
|
||||
"MASTODON": "mastodon", "TELEGRAM": "telegram", "THREADS": "threads"}
|
||||
|
||||
|
||||
FN_IN_TEXT = re.compile(r"\bFN\s*\d{1,6}\s?[a-zA-Z]\b")
|
||||
POSTCODE_CITY = re.compile(r"(?:(?<=\d)|\b)\d{4}\s+[A-ZÄÖÜ][\wäöüß .\-]*$")
|
||||
LEGAL_FORM_END = re.compile(r"(?i:gesmbh|gmbh|ges\.\s?m\.\s?b\.\s?h\.?|m\.\s?b\.\s?h\.?|aktiengesellschaft|\bag|\bkg|\bog|"
|
||||
r"e\.\s?u\.|\bse|privatstiftung|genossenschaft|sparkasse)(?=[A-ZÄÖÜ\s,]|$)")
|
||||
|
||||
|
||||
STREET_WORD = r"\S*(?:straße|strasse|str\.|gasse|platz|weg|kai|ring|allee|zeile|lände|markt|steig)"
|
||||
|
||||
|
||||
def strip_street(text: str) -> str:
|
||||
"""'Verein Musterhilfe Hauptstraße 12' -> 'Verein Musterhilfe' (greedy name, shortest street suffix)."""
|
||||
m = (re.match(rf"^(.*\S)\s+{STREET_WORD}\s*\d+[a-z]?(?:/\d+)*\s*$", text, re.IGNORECASE)
|
||||
or re.match(r"^(.*\S)\s+\S+\s*\d+[a-z]?(?:/\d+)*\s*$", text))
|
||||
return m.group(1) if m else text
|
||||
|
||||
|
||||
def employer(cell: str) -> tuple[str | None, str | None]:
|
||||
"""Employer name without business address, plus its company register number.
|
||||
|
||||
'Hypo Tirol Bank AG, Meraner Straße 8, 6020 Innsbruck' -> ('Hypo Tirol Bank AG', None)
|
||||
'25 PersonaldienstleistungsgesmbHFranz Josef Kai 95020 Salzburg FN306527a' (parliament's JSON glues
|
||||
the lines together) -> ('25 PersonaldienstleistungsgesmbH', '306527a')
|
||||
"""
|
||||
cell = re.sub(r"^lit\.\s*\w\s*:\s*", "", cell).strip()
|
||||
fn_match = FN_IN_TEXT.search(cell)
|
||||
fn = firmenbuch_number(fn_match.group(0)) if fn_match else None
|
||||
name = FN_IN_TEXT.sub("", cell).split(",")[0].strip(" *")
|
||||
address = POSTCODE_CITY.search(name)
|
||||
if address:
|
||||
head = name[:address.start()]
|
||||
ends = [m.end() for m in LEGAL_FORM_END.finditer(head)]
|
||||
# cut after the last legal form; otherwise drop "<street> <number>" before the postcode
|
||||
name = head[:ends[-1]] if ends else strip_street(head)
|
||||
name = name.strip(" ,-*")
|
||||
return (name if len(name) >= 3 and not re.fullmatch(r"[\d\s.€-]+", name) else None), fn
|
||||
|
||||
|
||||
@register
|
||||
class ParlamentPersons(Adapter):
|
||||
spec = SPEC
|
||||
name = "parlament.persons"
|
||||
description = "Members of Nationalrat and Bundesrat since 1918, mandates, functions, side jobs, speeches"
|
||||
schedule = "15 3 * * *"
|
||||
creates_persons = True
|
||||
parser_version = 3
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
yield list_request(409, {"NRBR": ["NR", "BR"]}, kind="persons")
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
kind = response.request.kind
|
||||
if kind == "persons":
|
||||
yield from self.parse_persons(ctx, response)
|
||||
elif kind == "person":
|
||||
yield from self.parse_person(ctx, response)
|
||||
elif kind == "speeches":
|
||||
yield from self.parse_speeches(ctx, response)
|
||||
|
||||
# -- persons & mandates -------------------------------------------------------------
|
||||
|
||||
def parse_persons(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
periods = set(ctx.params.get("periods") or get_settings().parlament_periods)
|
||||
for row in list_rows(response.json()):
|
||||
attr = row.get("Attribute") or {}
|
||||
if isinstance(attr, str):
|
||||
attr = json.loads(attr)
|
||||
m = re.search(r"/person/(\d+)", attr.get("uri") or "")
|
||||
if not m:
|
||||
continue
|
||||
pad = m.group(1)
|
||||
ref = PersonRef(ids={"parlament": pad})
|
||||
display = attr.get("zit") or strip_titles(row.get("Name") or "")
|
||||
yield PersonRecord(ref=ref, full_name=display, given_name=row.get("vorname"), family_name=row.get("nachname"),
|
||||
gender={"M": "male", "W": "female"}.get(row.get("GESCHL_CODE") or attr.get("geschlecht")))
|
||||
mandates = attr.get("mandate_detail") or []
|
||||
for mandate in mandates:
|
||||
claim = self.mandate_claim(pad, display, mandate)
|
||||
if claim:
|
||||
yield claim
|
||||
if any(md.get("gp_code") in periods for md in mandates):
|
||||
yield detail_request(f"/person/{pad}", kind="person", meta={"pad": pad, "name": display})
|
||||
|
||||
def mandate_claim(self, pad: str, name: str, m: dict) -> Claim | None:
|
||||
code = m.get("mand_code") or "NR"
|
||||
if code not in CHAMBER_SLUG:
|
||||
return None
|
||||
party_code = m.get("wahlpartei_code")
|
||||
party_name = m.get("wahlpartei_full_txt") or m.get("wahlpartei_txt") or party_code
|
||||
party = OrgRef(name=party_name, short_name=party_code, kind="party") if party_name else None
|
||||
start, end = parse_date(m.get("mandat_von")), parse_date(m.get("mandat_bis"))
|
||||
gp = m.get("gp_code") or ""
|
||||
region = m.get("wahlkreis_bundesland")
|
||||
label = party_code or party_name or "?"
|
||||
body = OrgRef(slug=CHAMBER_SLUG[code])
|
||||
return Claim(
|
||||
claim_type="role",
|
||||
person=PersonRef(ids={"parlament": pad}),
|
||||
org=body,
|
||||
counterpart_org=party,
|
||||
valid_from=start,
|
||||
valid_to=end,
|
||||
summary_de=f"{name}: Mandat im {CHAMBER_NAME_DE[code]} ({gp}. GP) für {label}"
|
||||
+ (f", Wahlkreis {region}" if region else ""),
|
||||
summary_en=f"{name}: member of the {CHAMBER_NAME_DE[code]} ({gp} legislative period) for {label}",
|
||||
data={"gp": gp, "party_code": party_code, "faction_code": m.get("fraktionscode"), "region": region},
|
||||
evidence=[Evidence(url=f"{BASE}/person/{pad}", doc_type="api_record", title=f"Parlament: {name}",
|
||||
locator=f"mandat:{code}:{gp}:{start}")],
|
||||
facet=Role(role="mp", level="federal", body=body, party=party, region=region, start=start, end=end),
|
||||
)
|
||||
|
||||
# -- person detail ----------------------------------------------------------------
|
||||
|
||||
def parse_person(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
data = response.json()
|
||||
content = data.get("content") or {}
|
||||
pad, name = response.meta["pad"], response.meta["name"]
|
||||
ref = PersonRef(ids={"parlament": pad})
|
||||
url = f"{BASE}/person/{pad}"
|
||||
photo = ((data.get("meta") or {}).get("openGraph") or {}).get("image")
|
||||
if photo:
|
||||
yield PersonRecord(ref=ref, full_name=name, photo_url=photo)
|
||||
|
||||
for account in (content.get("kontakt") or {}).get("socialMedia") or []:
|
||||
platform = SOCIAL_PLATFORMS.get((account.get("type") or "").upper(), (account.get("type") or "website").lower())
|
||||
yield SocialAccount(platform=platform, url=account["url"], person=ref, verified=True)
|
||||
|
||||
for fn in walk_dicts(content.get("biografie") or {}, "funktion_text"):
|
||||
if fn.get("gremium") in CHAMBER_SLUG:
|
||||
continue
|
||||
text = fn.get("funktion_text") or fn.get("bez")
|
||||
if not text:
|
||||
continue
|
||||
start, end = parse_date(fn.get("funktion_von")), parse_date(fn.get("funktion_bis"))
|
||||
is_gov = bool(GOVERNMENT_RE.search(text))
|
||||
body = OrgRef(slug="bundesregierung") if is_gov else None
|
||||
yield Claim(
|
||||
claim_type="role", person=ref, org=body, valid_from=start, valid_to=end,
|
||||
summary_de=f"{name}: {text}",
|
||||
evidence=[Evidence(url=url, doc_type="api_record", title=f"Parlament: {name}",
|
||||
locator=f"funktion:{text}:{start}")],
|
||||
facet=Role(role=text, level="federal" if is_gov else "other", body=body, start=start, end=end),
|
||||
)
|
||||
|
||||
yield from self.side_jobs(ref, pad, name, content.get("unvtrans") or {})
|
||||
yield list_request(251, {"PAD_INTERN": [int(pad)]}, kind="speeches", meta={"pad": pad, "name": name})
|
||||
|
||||
def side_jobs(self, ref: PersonRef, pad: str, name: str, unv: dict) -> Iterator[Claim]:
|
||||
"""Disclosures under §6 Unv-Transparenz-G: leading positions, paid side jobs, honorary positions."""
|
||||
tables = unv.get("unvtransData") or {}
|
||||
pdf = next((d["link"] for group in unv.get("linkliste") or [] for d in group.get("documents") or []
|
||||
if d.get("link", "").endswith(f"/person/{pad}/BezBegrBVGPar9.pdf")), None)
|
||||
for key, table in tables.items():
|
||||
m = re.fullmatch(r"(z\d+)Table", key)
|
||||
if not m or not isinstance(table, dict):
|
||||
continue
|
||||
header = html_to_text(tables.get(f"{m.group(1)}Header")) or key
|
||||
for row in (table.get("data") or {}).get("rows") or []:
|
||||
cells = [c for c in (html_to_text(c) for c in row if isinstance(c, str)) if c]
|
||||
if not cells:
|
||||
continue
|
||||
name_, fn = employer(cells[0]) if m.group(1) in ("z1", "z2", "z3") else (None, None)
|
||||
if name_: # never keep the (business) address that parliament publishes with the employer
|
||||
cells = [name_ + (f" (FN {fn})" if fn else ""), *cells[1:]]
|
||||
text = " – ".join(cells)
|
||||
yield Claim(
|
||||
claim_type="side_job", person=ref, key=("unvtrans", pad, m.group(1), text),
|
||||
counterpart_org=OrgRef(name=name_, kind="company" if LEGAL_FORM_END.search(name_) else "other",
|
||||
ids={"firmenbuch": fn} if fn else {}) if name_ else None,
|
||||
summary_de=f"{name} – {header.split(' - ')[0]}: {text}",
|
||||
data={"category": header, "cells": cells, "unpaid": "*" in "".join(str(c) for c in row),
|
||||
"employer": name_, "firmenbuch": fn},
|
||||
evidence=[Evidence(url=f"{BASE}{pdf}" if pdf else f"{BASE}/person/{pad}", doc_type="register_entry",
|
||||
title=f"Unvereinbarkeits-/Transparenzliste {name}", quote=text,
|
||||
locator=f"unvtrans:{m.group(1)}")],
|
||||
topics=("Transparenz und Korruption",),
|
||||
)
|
||||
|
||||
# -- speeches -----------------------------------------------------------------------
|
||||
|
||||
def parse_speeches(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||||
periods = set(ctx.params.get("periods") or get_settings().parlament_periods)
|
||||
pad, name = response.meta["pad"], response.meta["name"]
|
||||
for row in list_rows(response.json()):
|
||||
if row.get("GP") not in periods and row.get("GREMIUM") != "B":
|
||||
continue
|
||||
html_cells = " ".join(str(v) for v in row.values() if isinstance(v, str))
|
||||
protocol = re.search(r'href="(/dokument/[^"]+)"', html_cells)
|
||||
session = html_to_text(row.get("Sitzung"))
|
||||
topic = html_to_text(row.get("Rede"))
|
||||
said_at = row.get("fromDate")
|
||||
url = f"{BASE}{protocol.group(1)}" if protocol else f"{BASE}/person/{pad}"
|
||||
chamber = "Bundesrat" if row.get("GREMIUM") == "B" else "Nationalrat"
|
||||
yield Claim(
|
||||
claim_type="statement", person=PersonRef(ids={"parlament": pad}),
|
||||
key=("parl-speech", pad, row.get("Sortier"), topic), occurred_at=said_at,
|
||||
summary_de=f"{name} sprach im {chamber} zu: {topic}",
|
||||
summary_en=f"{name} spoke in the {chamber} on: {topic}",
|
||||
data={"session": session, "gp": row.get("GP"), "item_title": topic},
|
||||
evidence=[Evidence(url=url, doc_type="protocol", title=session, published_at=parse_date(said_at),
|
||||
locator=url.split("#", 1)[-1] if "#" in url else "")],
|
||||
facet=Statement(context="parliament_speech", url=url, said_at=said_at, medium=chamber, title=topic),
|
||||
)
|
||||
Reference in new issue
Block a user