"""Bundeskanzleramt: government news archive and Council of Ministers minutes (Ministerratsprotokolle). Official works (§7 UrhG); robots.txt only blocks /401. * News ("Nachrichten der Bundesregierung", one index per year): stored with full text; the event extractor turns summits, trips abroad and bilateral meetings into travel/meeting claims. * Council of Ministers minutes (Beschlussprotokolle): every agenda item names the reporting office(s) ("Bericht der Bundesministerin für Justiz, Zahl ..., betreffend ..."). Each becomes a government_decision claim for the office holder on that date (resolved via the office hint). """ from __future__ import annotations import re from collections.abc import Iterator from datetime import date from typing import Any from urllib.parse import urljoin from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register from politrack.core.records import Claim, Document, Event, Evidence, OrgRef, Participation, PersonRef from politrack.text import excerpt, fingerprint BASE = "https://www.bundeskanzleramt.gv.at" NEWS_INDEX = BASE + "/bundeskanzleramt/nachrichten-der-bundesregierung/{year}.html" MR_INDEX = BASE + "/bundeskanzleramt/die-bundesregierung/ministerratsprotokolle.html" SPEC = SourceSpec( slug="bundesregierung", name="Bundeskanzleramt – Bundesregierung", kind="official", homepage=BASE, license="Amtliches Werk (§7 UrhG)", access="scrape", trust=90, min_interval=2.0, tos_notes="Government news archive and Council of Ministers minutes. robots.txt only blocks /401.", ) MONTHS = {m: i for i, m in enumerate(["jänner", "februar", "märz", "april", "mai", "juni", "juli", "august", "september", "oktober", "november", "dezember"], start=1)} MONTHS["januar"] = 1 GERMAN_DATE = re.compile(r"(\d{1,2})\.\s*([A-Za-zäÄ]+)\s+(\d{4})") # Office names may contain commas ("für Wohnen, Kunst, Kultur, Medien und Sport"); file numbers sometimes use # non-breaking hyphens or spaces ("2026 0.040.056"); subjects span lines (joined with " | ") and end at the # decision ("Der Ministerrat beschließt ...") or at the next item. ITEM = re.compile(r"(?:(?:Gemeinsamer\s+)?Bericht|Vortrag|Antrag)\s+(?Pde[sr]\s+[^|]+?),?\s+" r"Zahl\s+(?P\d[\d./\-\u2010-\u2015 ]*?)\s*,\s*betreffend\s+(?P.+?)" r"(?=\s*\|?\s*Der Ministerrat\b" r"|\s*\|\s*(?:\d+(?:\.\d+)*\.?\s+)?(?:Gemeinsamer\s+)?(?:Bericht|Vortrag|Antrag)\s+de[sr]\s|$)") OFFICE_SPLIT = re.compile(r",\s*(?=de[sr]\s)|\s+und\s+(?=de[sr]\s)") GENITIVE = [(r"\bBundeskanzlers\b", "Bundeskanzler"), (r"\bVizekanzlers\b", "Vizekanzler"), (r"\bBundesministers\b", "Bundesminister"), (r"\bStaatssekretärs\b", "Staatssekretär")] def german_date(text: str) -> date | None: m = GERMAN_DATE.search(text) if not m or m.group(2).lower() not in MONTHS: return None return date(int(m.group(3)), MONTHS[m.group(2).lower()], int(m.group(1))) def period_start(path: str) -> int: """'.../ministerratsprotokolle-der-xxvi-regierungsperiode-2017-2018.html' -> 2018 (latest year named).""" return max((int(y) for y in re.findall(r"(20\d\d)", path)), default=0) def offices(text: str) -> list[str]: """'des Bundeskanzlers, des Bundesministers für Wohnen, Kunst, Kultur, Medien und Sport und der Bundesministerin für Justiz' -> ['Bundeskanzler', 'Bundesminister für Wohnen, ...', 'Bundesministerin für Justiz'].""" result = [] for part in OFFICE_SPLIT.split(text): part = re.sub(r"^\s*de[sr]\s+", "", part).strip(" ,") for pattern, repl in GENITIVE: part = re.sub(pattern, repl, part) if part: result.append(part) return result def main_text(response: Response) -> tuple[str, str]: soup = response.soup main = soup.find("main") or soup title_el = main.find(class_="title") or main.find("h1") title = title_el.get_text(" ", strip=True) if title_el else (soup.title.string if soup.title else response.url) for noise in main.find_all(["nav", "aside", "script", "style", "form", "footer"]): noise.decompose() parts = [p.get_text(" ", strip=True) for p in main.find_all(["p", "li", "h2", "h3"])] return title, "\n".join(p for p in parts if p) @register class GovernmentNews(Adapter): spec = SPEC name = "bundesregierung.news" description = "Government news archive (summits, trips, meetings of government members)" schedule = "10 */3 * * *" def start(self, ctx: RunContext) -> Iterator[Request]: first = int(ctx.params.get("since_year", date.today().year - 1)) for year in range(first, date.today().year + 1): yield Request(url=NEWS_INDEX.format(year=year), kind="index", meta={"year": year}) def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: if response.request.kind == "index": year = response.meta["year"] for path in sorted(set(re.findall( rf'href="(/bundeskanzleramt/nachrichten-der-bundesregierung/{year}/[^"]+\.html)"', response.text))): yield Request(url=BASE + path, kind="article") return title, text = main_text(response) time_el = response.soup.find("time") yield Document(Evidence(url=response.request.url, doc_type="press_release", title=title, published_at=time_el.get("datetime") if time_el else None, excerpt=excerpt(text, 300), full_text=f"{title}\n{text}")) @register class CouncilOfMinisters(Adapter): spec = SPEC name = "bundesregierung.ministerrat" description = "Council of Ministers minutes: decisions per reporting minister" schedule = "20 6 * * 3,4" parser_version = 2 def start(self, ctx: RunContext) -> Iterator[Request]: yield Request(url=MR_INDEX, kind="index") def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: kind = response.request.kind if kind == "index": periods = sorted(set(re.findall(r'href="(/bundeskanzleramt/die-bundesregierung/ministerratsprotokolle/' r'[^"?]+\.html)"', response.text)), key=period_start) # Period pages are named after the year a government took office ("...-xxviii-2025") and list all # its sessions, so the latest period that started before `since_year` may still contain newer ones. since = int(ctx.params.get("since_year", date.today().year - 1)) started_before = [period_start(p) for p in periods if period_start(p) < since] keep_from = max(started_before, default=0) for path in reversed([p for p in periods if period_start(p) >= keep_from]): # newest first yield Request(url=BASE + path, kind="period") elif kind == "period": base = response.request.url.split("?")[0] for href in sorted(set(re.findall(r'href="([^"]+)"', response.text))): url = urljoin(response.url, href) if url.startswith(base.removesuffix(".html") + "/"): yield Request(url=url, kind="protocol") elif url.startswith(base + "?currentPage="): yield Request(url=url, kind="period") else: yield from self.parse_protocol(response) def parse_protocol(self, response: Response) -> Iterator[Any]: title, text = main_text(response) held_on = german_date(title) or german_date(text[:300]) url = response.request.url yield Document(Evidence(url=url, doc_type="protocol", title=title, published_at=held_on, excerpt=excerpt(text, 300), full_text=f"{title}\n{text}")) event_key = fingerprint("ministerrat", url) yield Event(key=event_key, kind="government_meeting", title=title, starts_at=held_on, location="Wien", country="AT", url=url, body=OrgRef(slug="bundesregierung")) flat = " | ".join(text.split("\n")) for n, m in enumerate(ITEM.finditer(flat)): subject = re.sub(r"\s*\|\s*", " ", m.group("subject")).strip(" .") zahl = re.sub(r"[\s\u2010-\u2015]+", "-", m.group("zahl").strip()) for office in offices(m.group("offices")): yield Claim( claim_type="government_decision", person=PersonRef(name=None, hints={"office": office, "on": held_on}), key=("ministerrat", url, zahl, office), occurred_at=held_on, summary_de=f"Bericht im Ministerrat ({office}): {excerpt(subject, 220)}", summary_en=f"Report to the Council of Ministers ({office}): {excerpt(subject, 220)}", data={"office": office, "zahl": zahl, "item": n + 1}, evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=held_on, quote=excerpt(re.sub(r"\s*\|\s*", " ", m.group(0)), 500), locator=f"item:{zahl}")], ) # Participation of the reporting ministers in this session (calendar) for office in offices(m.group("offices")): yield Claim( claim_type="appearance", person=PersonRef(hints={"office": office, "on": held_on}), occurred_at=held_on, summary_de=f"Teilnahme am Ministerrat ({title})", evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=held_on)], facet=Participation(event_key=event_key, participation="reporting_minister"), )