"""Press releases of the Wirtschafts- und Korruptionsstaatsanwaltschaft (WKStA, anti-corruption prosecutors). Official statements on high-profile proceedings. They usually anonymise suspects ("ein ehemaliger Bundesminister"), so we store them as official documents (full text, searchable, RAG) and let the legal-matter detector propose claims - which, as always for legal matters, need human review. """ from __future__ import annotations import re from collections.abc import Iterator from datetime import date from typing import Any from urllib.parse import urljoin from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register from politrack.core.records import Document, Evidence from politrack.sources.bundesregierung import german_date from politrack.text import excerpt BASE = "https://www.justiz.gv.at" START = BASE + "/wksta/wirtschafts-und-korruptionsstaatsanwaltschaft.312.de.html" SPEC = SourceSpec( slug="wksta", name="WKStA – Pressemitteilungen", kind="official", homepage=START, license="Amtliches Werk (§7 UrhG)", access="scrape", trust=90, min_interval=3.0, tos_notes="Press releases of the anti-corruption prosecutors. robots.txt allows all.", ) RELEASE = re.compile(r"/medienstelle/pressemitteilungen/pressemitteilungen-(\d{4})/") LISTING = re.compile(r"/medienstelle(?:/pressemitteilungen(?:/pressemitteilungen-\d{4})?)?\.[0-9a-f]+\.de\.html") def clean(url: str) -> str: return re.sub(r";jsessionid=[^?#]*", "", url) @register class WkstaPress(Adapter): spec = SPEC name = "justiz.wksta" description = "WKStA press releases (official documents for search and reviewed legal matters)" schedule = "40 8 * * *" def start(self, ctx: RunContext) -> Iterator[Request]: yield Request(url=START, kind="listing", meta={"depth": 0}) def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: if response.request.kind == "listing": since = int(ctx.params.get("since_year", date.today().year - 2)) for a in response.soup.find_all("a", href=True): url = clean(urljoin(response.url, a["href"])) year = RELEASE.search(url) if year and int(year.group(1)) >= since: yield Request(url=url, kind="release") elif LISTING.search(url) and response.meta["depth"] < 2: yield Request(url=url, kind="listing", meta={"depth": response.meta["depth"] + 1}) return main = response.soup.find("main") or response.soup.find(id="content") or response.soup title_el = main.find("h1") title = title_el.get_text(" ", strip=True) if title_el else response.url text = "\n".join(p.get_text(" ", strip=True) for p in main.find_all(["p", "li"]) if p.get_text(strip=True)) yield Document(Evidence(url=clean(response.request.url), doc_type="press_release", title=title, published_at=german_date(text[:400]) or german_date(title), excerpt=excerpt(text, 300), full_text=f"{title}\n{text}"))