Politrack: track Austrian politicians from public sources
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
9da5a1dc48
187 files changed
+39890
No files matched your search
@@ -0,0 +1,124 @@
|
||||
"""Parliament press releases (Parlamentskorrespondenz, list 110) and calendar (Termine, list 600). CC BY 4.0.
|
||||
|
||||
Press releases are stored with full text; the people they link (/person/<id>) get a mention claim.
|
||||
The event extractor later turns visits, receptions and trips described in them into meeting/travel
|
||||
claims. The calendar fills the calendar view with sessions, committee meetings, hearings and events.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import date, timedelta
|
||||
from typing import Any
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, register
|
||||
from politrack.core.records import Claim, Document, Event, Evidence, OrgRef, PersonRef
|
||||
from politrack.sources.parlament.common import BASE, CHAMBER_SLUG, SPEC, list_request, list_rows, parse_date
|
||||
from politrack.text import excerpt, fingerprint, html_to_text
|
||||
|
||||
ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}")
|
||||
SKIP_TERMIN = re.compile(r"Guided Tour|Führung|Galeriebesuch|Besuch einer Plenarsitzung|Demokratiebildung", re.I)
|
||||
TERMIN_KIND = [
|
||||
(re.compile(r"Plenarsitzung|Bundesversammlung|Fest-|Gedenksitzung", re.I), "parliament_session"),
|
||||
(re.compile(r"Untersuchungsausschuss", re.I), "hearing"),
|
||||
(re.compile(r"Ausschuss|Enquete", re.I), "committee"),
|
||||
(re.compile(r"Pressekonferenz", re.I), "press_conference"),
|
||||
(re.compile(r"Internationales|Konferenz", re.I), "meeting"),
|
||||
]
|
||||
|
||||
|
||||
def release_body(content: list) -> str:
|
||||
"""The detail JSON is a list of render blocks; the article is the one wrapped in <div class=pk>."""
|
||||
for block in content:
|
||||
text = (block.get("data") or {}).get("text") if isinstance(block, dict) else None
|
||||
if isinstance(text, str) and re.search(r"class=\"?pk\b", text):
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
@register
|
||||
class ParlamentPress(Adapter):
|
||||
spec = SPEC
|
||||
name = "parlament.press"
|
||||
description = "Parlamentskorrespondenz: official press releases with linked people"
|
||||
schedule = "5 */2 * * *"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
last = ctx.state.get("last_date")
|
||||
backfill = int(ctx.params.get("backfill_days", 365))
|
||||
since = (date.fromisoformat(last) - timedelta(days=2)) if last else date.today() - timedelta(days=backfill)
|
||||
yield list_request(110, {"DATERANGE": [since.isoformat(), date.today().isoformat()]}, kind="list",
|
||||
meta={"since": since.isoformat()})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
if response.request.kind == "list":
|
||||
newest = response.meta["since"]
|
||||
for row in list_rows(response.json()):
|
||||
if not row.get("Link"):
|
||||
continue
|
||||
published = parse_date(row.get("Datum")) # "23.10.2026" (German) in list 110
|
||||
if published:
|
||||
newest = max(newest, published.isoformat())
|
||||
yield Request(url=f"{BASE}{row['Link']}", params={"json": "TRUE"}, kind="release",
|
||||
meta={k: row.get(k) for k in ("Titel", "Untertitel", "Datum", "Nr.", "Abothemen",
|
||||
"Stichworte")})
|
||||
if ctx.mode != "test":
|
||||
ctx.state.set("last_date", newest)
|
||||
return
|
||||
yield from self.parse_release(response)
|
||||
|
||||
def parse_release(self, response: Response) -> Iterator[Any]:
|
||||
meta = response.meta
|
||||
url = response.request.url
|
||||
body_html = release_body(response.json().get("content") or [])
|
||||
pads = sorted({p for p in re.findall(r"/person/(\d+)", body_html)})
|
||||
text = html_to_text(body_html)
|
||||
title, subtitle = meta.get("Titel") or "", meta.get("Untertitel") or ""
|
||||
topics_raw = meta.get("Abothemen")
|
||||
topics = tuple(json.loads(topics_raw)) if isinstance(topics_raw, str) and topics_raw.startswith("[") else ()
|
||||
published = parse_date(meta.get("Datum"))
|
||||
yield Document(Evidence(url=url, doc_type="press_release", title=title, published_at=published,
|
||||
excerpt=subtitle or excerpt(text, 300), full_text=f"{title}\n{subtitle}\n{text}",
|
||||
meta={"pk_number": meta.get("Nr."), "person_pads": pads, "topics": list(topics)}))
|
||||
for pad in pads:
|
||||
yield Claim(
|
||||
claim_type="media_mention", person=PersonRef(ids={"parlament": pad}), occurred_at=published,
|
||||
summary_de=f"Parlamentskorrespondenz: {title}",
|
||||
data={"context": "parlamentskorrespondenz"},
|
||||
evidence=[Evidence(url=url, doc_type="press_release", title=title, published_at=published,
|
||||
quote=subtitle or None, stance="mentions")],
|
||||
topics=topics,
|
||||
)
|
||||
|
||||
|
||||
@register
|
||||
class ParlamentCalendar(Adapter):
|
||||
spec = SPEC
|
||||
name = "parlament.calendar"
|
||||
description = "Parliament calendar: plenary sessions, committees, hearings, events (tours skipped)"
|
||||
schedule = "30 2 * * *"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
yield list_request(600, {}, kind="list")
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Event]:
|
||||
since = str(ctx.params.get("since_year", date.today().year - 2))
|
||||
for row in list_rows(response.json()):
|
||||
kind_label = (row.get("Terminart") or "").replace("<BR>", " ")
|
||||
starts = next((v for k, v in row.items() if k.startswith("Datum") and k != "Datum Bis"
|
||||
and isinstance(v, str) and ISO_DATE.match(v)), None)
|
||||
if not starts or starts[:4] < since or SKIP_TERMIN.search(f"{kind_label} {row.get('Bezeichnung') or ''}"):
|
||||
continue
|
||||
title = row.get("Bezeichnung") or kind_label
|
||||
kind = next((k for pattern, k in TERMIN_KIND if pattern.search(f"{kind_label} {title}")), "public_event")
|
||||
chamber = {"Nationalrat": "NR", "Bundesrat": "BR"}.get(row.get("Gremium") or "")
|
||||
ends = next((v for k, v in row.items() if k.startswith("Datum Bis") and isinstance(v, str)
|
||||
and ISO_DATE.match(v)), None)
|
||||
yield Event(
|
||||
key=fingerprint("termin", row.get("WENTRY_ID") or row.get("Link")), kind=kind, title=title,
|
||||
starts_at=starts, ends_at=ends, location=(row.get("Ort") or "").replace(" | ", ", ") or "Parlament, Wien",
|
||||
country="AT", url=f"{BASE}{row['Link']}" if row.get("Link") else None, description=kind_label,
|
||||
body=OrgRef(slug=CHAMBER_SLUG[chamber]) if chamber else None,
|
||||
)
|
||||
Reference in new issue
Block a user