Politrack: track Austrian politicians from public sources
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
583ccba31c
187 files changed
+39890
No files matched your search
@@ -0,0 +1,183 @@
|
||||
"""Bundeskanzleramt: government news archive and Council of Ministers minutes (Ministerratsprotokolle).
|
||||
|
||||
Official works (§7 UrhG); robots.txt only blocks /401.
|
||||
|
||||
* News ("Nachrichten der Bundesregierung", one index per year): stored with full text; the event
|
||||
extractor turns summits, trips abroad and bilateral meetings into travel/meeting claims.
|
||||
* Council of Ministers minutes (Beschlussprotokolle): every agenda item names the reporting office(s)
|
||||
("Bericht der Bundesministerin für Justiz, Zahl ..., betreffend ..."). Each becomes a
|
||||
government_decision claim for the office holder on that date (resolved via the office hint).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Document, Event, Evidence, OrgRef, Participation, PersonRef
|
||||
from politrack.text import excerpt, fingerprint
|
||||
|
||||
BASE = "https://www.bundeskanzleramt.gv.at"
|
||||
NEWS_INDEX = BASE + "/bundeskanzleramt/nachrichten-der-bundesregierung/{year}.html"
|
||||
MR_INDEX = BASE + "/bundeskanzleramt/die-bundesregierung/ministerratsprotokolle.html"
|
||||
SPEC = SourceSpec(
|
||||
slug="bundesregierung",
|
||||
name="Bundeskanzleramt – Bundesregierung",
|
||||
kind="official",
|
||||
homepage=BASE,
|
||||
license="Amtliches Werk (§7 UrhG)",
|
||||
access="scrape",
|
||||
trust=90,
|
||||
min_interval=2.0,
|
||||
tos_notes="Government news archive and Council of Ministers minutes. robots.txt only blocks /401.",
|
||||
)
|
||||
|
||||
MONTHS = {m: i for i, m in enumerate(["jänner", "februar", "märz", "april", "mai", "juni", "juli", "august",
|
||||
"september", "oktober", "november", "dezember"], start=1)}
|
||||
MONTHS["januar"] = 1
|
||||
GERMAN_DATE = re.compile(r"(\d{1,2})\.\s*([A-Za-zäÄ]+)\s+(\d{4})")
|
||||
# Office names may contain commas ("für Wohnen, Kunst, Kultur, Medien und Sport"); file numbers sometimes use
|
||||
# non-breaking hyphens or spaces ("2026 0.040.056"); subjects span lines (joined with " | ") and end at the
|
||||
# decision ("Der Ministerrat beschließt ...") or at the next item.
|
||||
ITEM = re.compile(r"(?:(?:Gemeinsamer\s+)?Bericht|Vortrag|Antrag)\s+(?P<offices>de[sr]\s+[^|]+?),?\s+"
|
||||
r"Zahl\s+(?P<zahl>\d[\d./\-\u2010-\u2015 ]*?)\s*,\s*betreffend\s+(?P<subject>.+?)"
|
||||
r"(?=\s*\|?\s*Der Ministerrat\b"
|
||||
r"|\s*\|\s*(?:\d+(?:\.\d+)*\.?\s+)?(?:Gemeinsamer\s+)?(?:Bericht|Vortrag|Antrag)\s+de[sr]\s|$)")
|
||||
OFFICE_SPLIT = re.compile(r",\s*(?=de[sr]\s)|\s+und\s+(?=de[sr]\s)")
|
||||
GENITIVE = [(r"\bBundeskanzlers\b", "Bundeskanzler"), (r"\bVizekanzlers\b", "Vizekanzler"),
|
||||
(r"\bBundesministers\b", "Bundesminister"), (r"\bStaatssekretärs\b", "Staatssekretär")]
|
||||
|
||||
|
||||
def german_date(text: str) -> date | None:
|
||||
m = GERMAN_DATE.search(text)
|
||||
if not m or m.group(2).lower() not in MONTHS:
|
||||
return None
|
||||
return date(int(m.group(3)), MONTHS[m.group(2).lower()], int(m.group(1)))
|
||||
|
||||
|
||||
def period_start(path: str) -> int:
|
||||
"""'.../ministerratsprotokolle-der-xxvi-regierungsperiode-2017-2018.html' -> 2018 (latest year named)."""
|
||||
return max((int(y) for y in re.findall(r"(20\d\d)", path)), default=0)
|
||||
|
||||
|
||||
def offices(text: str) -> list[str]:
|
||||
"""'des Bundeskanzlers, des Bundesministers für Wohnen, Kunst, Kultur, Medien und Sport und der
|
||||
Bundesministerin für Justiz' -> ['Bundeskanzler', 'Bundesminister für Wohnen, ...', 'Bundesministerin für Justiz']."""
|
||||
result = []
|
||||
for part in OFFICE_SPLIT.split(text):
|
||||
part = re.sub(r"^\s*de[sr]\s+", "", part).strip(" ,")
|
||||
for pattern, repl in GENITIVE:
|
||||
part = re.sub(pattern, repl, part)
|
||||
if part:
|
||||
result.append(part)
|
||||
return result
|
||||
|
||||
|
||||
def main_text(response: Response) -> tuple[str, str]:
|
||||
soup = response.soup
|
||||
main = soup.find("main") or soup
|
||||
title_el = main.find(class_="title") or main.find("h1")
|
||||
title = title_el.get_text(" ", strip=True) if title_el else (soup.title.string if soup.title else response.url)
|
||||
for noise in main.find_all(["nav", "aside", "script", "style", "form", "footer"]):
|
||||
noise.decompose()
|
||||
parts = [p.get_text(" ", strip=True) for p in main.find_all(["p", "li", "h2", "h3"])]
|
||||
return title, "\n".join(p for p in parts if p)
|
||||
|
||||
|
||||
@register
|
||||
class GovernmentNews(Adapter):
|
||||
spec = SPEC
|
||||
name = "bundesregierung.news"
|
||||
description = "Government news archive (summits, trips, meetings of government members)"
|
||||
schedule = "10 */3 * * *"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
first = int(ctx.params.get("since_year", date.today().year - 1))
|
||||
for year in range(first, date.today().year + 1):
|
||||
yield Request(url=NEWS_INDEX.format(year=year), kind="index", meta={"year": year})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
if response.request.kind == "index":
|
||||
year = response.meta["year"]
|
||||
for path in sorted(set(re.findall(
|
||||
rf'href="(/bundeskanzleramt/nachrichten-der-bundesregierung/{year}/[^"]+\.html)"', response.text))):
|
||||
yield Request(url=BASE + path, kind="article")
|
||||
return
|
||||
title, text = main_text(response)
|
||||
time_el = response.soup.find("time")
|
||||
yield Document(Evidence(url=response.request.url, doc_type="press_release", title=title,
|
||||
published_at=time_el.get("datetime") if time_el else None,
|
||||
excerpt=excerpt(text, 300), full_text=f"{title}\n{text}"))
|
||||
|
||||
|
||||
@register
|
||||
class CouncilOfMinisters(Adapter):
|
||||
spec = SPEC
|
||||
name = "bundesregierung.ministerrat"
|
||||
description = "Council of Ministers minutes: decisions per reporting minister"
|
||||
schedule = "20 6 * * 3,4"
|
||||
parser_version = 2
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
yield Request(url=MR_INDEX, kind="index")
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
kind = response.request.kind
|
||||
if kind == "index":
|
||||
periods = sorted(set(re.findall(r'href="(/bundeskanzleramt/die-bundesregierung/ministerratsprotokolle/'
|
||||
r'[^"?]+\.html)"', response.text)), key=period_start)
|
||||
# Period pages are named after the year a government took office ("...-xxviii-2025") and list all
|
||||
# its sessions, so the latest period that started before `since_year` may still contain newer ones.
|
||||
since = int(ctx.params.get("since_year", date.today().year - 1))
|
||||
started_before = [period_start(p) for p in periods if period_start(p) < since]
|
||||
keep_from = max(started_before, default=0)
|
||||
for path in reversed([p for p in periods if period_start(p) >= keep_from]): # newest first
|
||||
yield Request(url=BASE + path, kind="period")
|
||||
elif kind == "period":
|
||||
base = response.request.url.split("?")[0]
|
||||
for href in sorted(set(re.findall(r'href="([^"]+)"', response.text))):
|
||||
url = urljoin(response.url, href)
|
||||
if url.startswith(base.removesuffix(".html") + "/"):
|
||||
yield Request(url=url, kind="protocol")
|
||||
elif url.startswith(base + "?currentPage="):
|
||||
yield Request(url=url, kind="period")
|
||||
else:
|
||||
yield from self.parse_protocol(response)
|
||||
|
||||
def parse_protocol(self, response: Response) -> Iterator[Any]:
|
||||
title, text = main_text(response)
|
||||
held_on = german_date(title) or german_date(text[:300])
|
||||
url = response.request.url
|
||||
yield Document(Evidence(url=url, doc_type="protocol", title=title, published_at=held_on,
|
||||
excerpt=excerpt(text, 300), full_text=f"{title}\n{text}"))
|
||||
event_key = fingerprint("ministerrat", url)
|
||||
yield Event(key=event_key, kind="government_meeting", title=title, starts_at=held_on, location="Wien",
|
||||
country="AT", url=url, body=OrgRef(slug="bundesregierung"))
|
||||
flat = " | ".join(text.split("\n"))
|
||||
for n, m in enumerate(ITEM.finditer(flat)):
|
||||
subject = re.sub(r"\s*\|\s*", " ", m.group("subject")).strip(" .")
|
||||
zahl = re.sub(r"[\s\u2010-\u2015]+", "-", m.group("zahl").strip())
|
||||
for office in offices(m.group("offices")):
|
||||
yield Claim(
|
||||
claim_type="government_decision",
|
||||
person=PersonRef(name=None, hints={"office": office, "on": held_on}),
|
||||
key=("ministerrat", url, zahl, office), occurred_at=held_on,
|
||||
summary_de=f"Bericht im Ministerrat ({office}): {excerpt(subject, 220)}",
|
||||
summary_en=f"Report to the Council of Ministers ({office}): {excerpt(subject, 220)}",
|
||||
data={"office": office, "zahl": zahl, "item": n + 1},
|
||||
evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=held_on,
|
||||
quote=excerpt(re.sub(r"\s*\|\s*", " ", m.group(0)), 500),
|
||||
locator=f"item:{zahl}")],
|
||||
)
|
||||
# Participation of the reporting ministers in this session (calendar)
|
||||
for office in offices(m.group("offices")):
|
||||
yield Claim(
|
||||
claim_type="appearance", person=PersonRef(hints={"office": office, "on": held_on}),
|
||||
occurred_at=held_on, summary_de=f"Teilnahme am Ministerrat ({title})",
|
||||
evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=held_on)],
|
||||
facet=Participation(event_key=event_key, participation="reporting_minister"),
|
||||
)
|
||||
Reference in new issue
Block a user