simonandClaude Opus 5.5
583ccba31c
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
184 lines
9.6 KiB
Python
184 lines
9.6 KiB
Python
"""Bundeskanzleramt: government news archive and Council of Ministers minutes (Ministerratsprotokolle).
|
||
|
||
Official works (§7 UrhG); robots.txt only blocks /401.
|
||
|
||
* News ("Nachrichten der Bundesregierung", one index per year): stored with full text; the event
|
||
extractor turns summits, trips abroad and bilateral meetings into travel/meeting claims.
|
||
* Council of Ministers minutes (Beschlussprotokolle): every agenda item names the reporting office(s)
|
||
("Bericht der Bundesministerin für Justiz, Zahl ..., betreffend ..."). Each becomes a
|
||
government_decision claim for the office holder on that date (resolved via the office hint).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
from collections.abc import Iterator
|
||
from datetime import date
|
||
from typing import Any
|
||
from urllib.parse import urljoin
|
||
|
||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||
from politrack.core.records import Claim, Document, Event, Evidence, OrgRef, Participation, PersonRef
|
||
from politrack.text import excerpt, fingerprint
|
||
|
||
BASE = "https://www.bundeskanzleramt.gv.at"
|
||
NEWS_INDEX = BASE + "/bundeskanzleramt/nachrichten-der-bundesregierung/{year}.html"
|
||
MR_INDEX = BASE + "/bundeskanzleramt/die-bundesregierung/ministerratsprotokolle.html"
|
||
SPEC = SourceSpec(
|
||
slug="bundesregierung",
|
||
name="Bundeskanzleramt – Bundesregierung",
|
||
kind="official",
|
||
homepage=BASE,
|
||
license="Amtliches Werk (§7 UrhG)",
|
||
access="scrape",
|
||
trust=90,
|
||
min_interval=2.0,
|
||
tos_notes="Government news archive and Council of Ministers minutes. robots.txt only blocks /401.",
|
||
)
|
||
|
||
MONTHS = {m: i for i, m in enumerate(["jänner", "februar", "märz", "april", "mai", "juni", "juli", "august",
|
||
"september", "oktober", "november", "dezember"], start=1)}
|
||
MONTHS["januar"] = 1
|
||
GERMAN_DATE = re.compile(r"(\d{1,2})\.\s*([A-Za-zäÄ]+)\s+(\d{4})")
|
||
# Office names may contain commas ("für Wohnen, Kunst, Kultur, Medien und Sport"); file numbers sometimes use
|
||
# non-breaking hyphens or spaces ("2026 0.040.056"); subjects span lines (joined with " | ") and end at the
|
||
# decision ("Der Ministerrat beschließt ...") or at the next item.
|
||
ITEM = re.compile(r"(?:(?:Gemeinsamer\s+)?Bericht|Vortrag|Antrag)\s+(?P<offices>de[sr]\s+[^|]+?),?\s+"
|
||
r"Zahl\s+(?P<zahl>\d[\d./\-\u2010-\u2015 ]*?)\s*,\s*betreffend\s+(?P<subject>.+?)"
|
||
r"(?=\s*\|?\s*Der Ministerrat\b"
|
||
r"|\s*\|\s*(?:\d+(?:\.\d+)*\.?\s+)?(?:Gemeinsamer\s+)?(?:Bericht|Vortrag|Antrag)\s+de[sr]\s|$)")
|
||
OFFICE_SPLIT = re.compile(r",\s*(?=de[sr]\s)|\s+und\s+(?=de[sr]\s)")
|
||
GENITIVE = [(r"\bBundeskanzlers\b", "Bundeskanzler"), (r"\bVizekanzlers\b", "Vizekanzler"),
|
||
(r"\bBundesministers\b", "Bundesminister"), (r"\bStaatssekretärs\b", "Staatssekretär")]
|
||
|
||
|
||
def german_date(text: str) -> date | None:
|
||
m = GERMAN_DATE.search(text)
|
||
if not m or m.group(2).lower() not in MONTHS:
|
||
return None
|
||
return date(int(m.group(3)), MONTHS[m.group(2).lower()], int(m.group(1)))
|
||
|
||
|
||
def period_start(path: str) -> int:
|
||
"""'.../ministerratsprotokolle-der-xxvi-regierungsperiode-2017-2018.html' -> 2018 (latest year named)."""
|
||
return max((int(y) for y in re.findall(r"(20\d\d)", path)), default=0)
|
||
|
||
|
||
def offices(text: str) -> list[str]:
|
||
"""'des Bundeskanzlers, des Bundesministers für Wohnen, Kunst, Kultur, Medien und Sport und der
|
||
Bundesministerin für Justiz' -> ['Bundeskanzler', 'Bundesminister für Wohnen, ...', 'Bundesministerin für Justiz']."""
|
||
result = []
|
||
for part in OFFICE_SPLIT.split(text):
|
||
part = re.sub(r"^\s*de[sr]\s+", "", part).strip(" ,")
|
||
for pattern, repl in GENITIVE:
|
||
part = re.sub(pattern, repl, part)
|
||
if part:
|
||
result.append(part)
|
||
return result
|
||
|
||
|
||
def main_text(response: Response) -> tuple[str, str]:
|
||
soup = response.soup
|
||
main = soup.find("main") or soup
|
||
title_el = main.find(class_="title") or main.find("h1")
|
||
title = title_el.get_text(" ", strip=True) if title_el else (soup.title.string if soup.title else response.url)
|
||
for noise in main.find_all(["nav", "aside", "script", "style", "form", "footer"]):
|
||
noise.decompose()
|
||
parts = [p.get_text(" ", strip=True) for p in main.find_all(["p", "li", "h2", "h3"])]
|
||
return title, "\n".join(p for p in parts if p)
|
||
|
||
|
||
@register
|
||
class GovernmentNews(Adapter):
|
||
spec = SPEC
|
||
name = "bundesregierung.news"
|
||
description = "Government news archive (summits, trips, meetings of government members)"
|
||
schedule = "10 */3 * * *"
|
||
|
||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||
first = int(ctx.params.get("since_year", date.today().year - 1))
|
||
for year in range(first, date.today().year + 1):
|
||
yield Request(url=NEWS_INDEX.format(year=year), kind="index", meta={"year": year})
|
||
|
||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||
if response.request.kind == "index":
|
||
year = response.meta["year"]
|
||
for path in sorted(set(re.findall(
|
||
rf'href="(/bundeskanzleramt/nachrichten-der-bundesregierung/{year}/[^"]+\.html)"', response.text))):
|
||
yield Request(url=BASE + path, kind="article")
|
||
return
|
||
title, text = main_text(response)
|
||
time_el = response.soup.find("time")
|
||
yield Document(Evidence(url=response.request.url, doc_type="press_release", title=title,
|
||
published_at=time_el.get("datetime") if time_el else None,
|
||
excerpt=excerpt(text, 300), full_text=f"{title}\n{text}"))
|
||
|
||
|
||
@register
|
||
class CouncilOfMinisters(Adapter):
|
||
spec = SPEC
|
||
name = "bundesregierung.ministerrat"
|
||
description = "Council of Ministers minutes: decisions per reporting minister"
|
||
schedule = "20 6 * * 3,4"
|
||
parser_version = 2
|
||
|
||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||
yield Request(url=MR_INDEX, kind="index")
|
||
|
||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||
kind = response.request.kind
|
||
if kind == "index":
|
||
periods = sorted(set(re.findall(r'href="(/bundeskanzleramt/die-bundesregierung/ministerratsprotokolle/'
|
||
r'[^"?]+\.html)"', response.text)), key=period_start)
|
||
# Period pages are named after the year a government took office ("...-xxviii-2025") and list all
|
||
# its sessions, so the latest period that started before `since_year` may still contain newer ones.
|
||
since = int(ctx.params.get("since_year", date.today().year - 1))
|
||
started_before = [period_start(p) for p in periods if period_start(p) < since]
|
||
keep_from = max(started_before, default=0)
|
||
for path in reversed([p for p in periods if period_start(p) >= keep_from]): # newest first
|
||
yield Request(url=BASE + path, kind="period")
|
||
elif kind == "period":
|
||
base = response.request.url.split("?")[0]
|
||
for href in sorted(set(re.findall(r'href="([^"]+)"', response.text))):
|
||
url = urljoin(response.url, href)
|
||
if url.startswith(base.removesuffix(".html") + "/"):
|
||
yield Request(url=url, kind="protocol")
|
||
elif url.startswith(base + "?currentPage="):
|
||
yield Request(url=url, kind="period")
|
||
else:
|
||
yield from self.parse_protocol(response)
|
||
|
||
def parse_protocol(self, response: Response) -> Iterator[Any]:
|
||
title, text = main_text(response)
|
||
held_on = german_date(title) or german_date(text[:300])
|
||
url = response.request.url
|
||
yield Document(Evidence(url=url, doc_type="protocol", title=title, published_at=held_on,
|
||
excerpt=excerpt(text, 300), full_text=f"{title}\n{text}"))
|
||
event_key = fingerprint("ministerrat", url)
|
||
yield Event(key=event_key, kind="government_meeting", title=title, starts_at=held_on, location="Wien",
|
||
country="AT", url=url, body=OrgRef(slug="bundesregierung"))
|
||
flat = " | ".join(text.split("\n"))
|
||
for n, m in enumerate(ITEM.finditer(flat)):
|
||
subject = re.sub(r"\s*\|\s*", " ", m.group("subject")).strip(" .")
|
||
zahl = re.sub(r"[\s\u2010-\u2015]+", "-", m.group("zahl").strip())
|
||
for office in offices(m.group("offices")):
|
||
yield Claim(
|
||
claim_type="government_decision",
|
||
person=PersonRef(name=None, hints={"office": office, "on": held_on}),
|
||
key=("ministerrat", url, zahl, office), occurred_at=held_on,
|
||
summary_de=f"Bericht im Ministerrat ({office}): {excerpt(subject, 220)}",
|
||
summary_en=f"Report to the Council of Ministers ({office}): {excerpt(subject, 220)}",
|
||
data={"office": office, "zahl": zahl, "item": n + 1},
|
||
evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=held_on,
|
||
quote=excerpt(re.sub(r"\s*\|\s*", " ", m.group(0)), 500),
|
||
locator=f"item:{zahl}")],
|
||
)
|
||
# Participation of the reporting ministers in this session (calendar)
|
||
for office in offices(m.group("offices")):
|
||
yield Claim(
|
||
claim_type="appearance", person=PersonRef(hints={"office": office, "on": held_on}),
|
||
occurred_at=held_on, summary_de=f"Teilnahme am Ministerrat ({title})",
|
||
evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=held_on)],
|
||
facet=Participation(event_key=event_key, participation="reporting_minister"),
|
||
)
|