simonandClaude Opus 5.5
583ccba31c
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
64 lines
3.0 KiB
Python
64 lines
3.0 KiB
Python
"""Press releases of the Wirtschafts- und Korruptionsstaatsanwaltschaft (WKStA, anti-corruption prosecutors).
|
||
|
||
Official statements on high-profile proceedings. They usually anonymise suspects ("ein ehemaliger
|
||
Bundesminister"), so we store them as official documents (full text, searchable, RAG) and let the
|
||
legal-matter detector propose claims - which, as always for legal matters, need human review.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
from collections.abc import Iterator
|
||
from datetime import date
|
||
from typing import Any
|
||
from urllib.parse import urljoin
|
||
|
||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||
from politrack.core.records import Document, Evidence
|
||
from politrack.sources.bundesregierung import german_date
|
||
from politrack.text import excerpt
|
||
|
||
BASE = "https://www.justiz.gv.at"
|
||
START = BASE + "/wksta/wirtschafts-und-korruptionsstaatsanwaltschaft.312.de.html"
|
||
SPEC = SourceSpec(
|
||
slug="wksta", name="WKStA – Pressemitteilungen", kind="official", homepage=START,
|
||
license="Amtliches Werk (§7 UrhG)", access="scrape", trust=90, min_interval=3.0,
|
||
tos_notes="Press releases of the anti-corruption prosecutors. robots.txt allows all.",
|
||
)
|
||
RELEASE = re.compile(r"/medienstelle/pressemitteilungen/pressemitteilungen-(\d{4})/")
|
||
LISTING = re.compile(r"/medienstelle(?:/pressemitteilungen(?:/pressemitteilungen-\d{4})?)?\.[0-9a-f]+\.de\.html")
|
||
|
||
|
||
def clean(url: str) -> str:
|
||
return re.sub(r";jsessionid=[^?#]*", "", url)
|
||
|
||
|
||
@register
|
||
class WkstaPress(Adapter):
|
||
spec = SPEC
|
||
name = "justiz.wksta"
|
||
description = "WKStA press releases (official documents for search and reviewed legal matters)"
|
||
schedule = "40 8 * * *"
|
||
|
||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||
yield Request(url=START, kind="listing", meta={"depth": 0})
|
||
|
||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||
if response.request.kind == "listing":
|
||
since = int(ctx.params.get("since_year", date.today().year - 2))
|
||
for a in response.soup.find_all("a", href=True):
|
||
url = clean(urljoin(response.url, a["href"]))
|
||
year = RELEASE.search(url)
|
||
if year and int(year.group(1)) >= since:
|
||
yield Request(url=url, kind="release")
|
||
elif LISTING.search(url) and response.meta["depth"] < 2:
|
||
yield Request(url=url, kind="listing", meta={"depth": response.meta["depth"] + 1})
|
||
return
|
||
main = response.soup.find("main") or response.soup.find(id="content") or response.soup
|
||
title_el = main.find("h1")
|
||
title = title_el.get_text(" ", strip=True) if title_el else response.url
|
||
text = "\n".join(p.get_text(" ", strip=True) for p in main.find_all(["p", "li"]) if p.get_text(strip=True))
|
||
yield Document(Evidence(url=clean(response.request.url), doc_type="press_release", title=title,
|
||
published_at=german_date(text[:400]) or german_date(title), excerpt=excerpt(text, 300),
|
||
full_text=f"{title}\n{text}"))
|