Politrack: track Austrian politicians from public sources
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
9da5a1dc48
187 files changed
+39890
No files matched your search
@@ -0,0 +1,106 @@
|
||||
"""RTR / KommAustria media transparency (MedKF-TG): advertising and media subsidies paid by public bodies.
|
||||
|
||||
Open data API (no usage restrictions, attribution): https://data.rtr.at/api/v1/tables/<table>.json
|
||||
* mt-bekanntgabe2024 (half-yearly since 2024): payer, medium, media owner, campaign, amount, ad sujet link
|
||||
* mt-bekanntgabe (quarterly until 2023): payer, medium/owner, amount
|
||||
Each row -> money_flow (kind government_ad for advertising, subsidy for media funding) from the public body
|
||||
(ministry, state government, municipality, public company...) to the media owner.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Evidence, Money, OrgRef
|
||||
from politrack.text import format_eur_de
|
||||
|
||||
API = "https://data.rtr.at/api/v1/tables"
|
||||
PAGE_SIZE = 5000
|
||||
SPEC = SourceSpec(
|
||||
slug="rtr-medientransparenz", name="RTR/KommAustria Medientransparenz", kind="official",
|
||||
homepage="https://www.rtr.at/medien/was_wir_tun/medientransparenz/",
|
||||
license="Amtliches Werk (§7 UrhG); attribution \"KommAustria – data.rtr.at\"", access="api",
|
||||
trust=95, min_interval=2.0,
|
||||
# data.rtr.at/robots.txt disallows everything for crawlers, but RTR documents this API as "der Endpunkt zur
|
||||
# automatischen Abfrage der Daten" (rtr.at/rtr/service/opendata/OD_Schnittstellenbeschreibung.de.html).
|
||||
robots_exempt=(API + "/",),
|
||||
tos_notes="Government advertising and media funding reported under MedKF-TG. The REST API is documented by "
|
||||
"RTR for automated queries (OD_Schnittstellenbeschreibung); robots.txt on data.rtr.at blocks "
|
||||
"crawlers only. Terms (OD_Nutzungsbedingungen): free works under §7 UrhG, attribution requested, "
|
||||
"no use for defamatory/discriminatory purposes.",
|
||||
)
|
||||
KIND = {2: "government_ad", 4: "subsidy"}
|
||||
|
||||
|
||||
def half_years(since_year: int) -> list[str]:
|
||||
today = date.today()
|
||||
return [f"{y}{h}" for y in range(since_year, today.year + 1) for h in (1, 2)
|
||||
if (y, h) <= (today.year, 1 if today.month <= 6 else 2)]
|
||||
|
||||
|
||||
def quarters(since_year: int, until_year: int = 2023) -> list[str]:
|
||||
return [f"{y}{q}" for y in range(since_year, until_year + 1) for q in (1, 2, 3, 4)]
|
||||
|
||||
|
||||
@register
|
||||
class MediaTransparency(Adapter):
|
||||
spec = SPEC
|
||||
name = "rtr.medientransparenz"
|
||||
description = "Government advertising and media subsidies per public body and media owner"
|
||||
schedule = "0 4 * * 2"
|
||||
queue = "bulk"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
for period in half_years(int(ctx.params.get("since_year", 2024))):
|
||||
yield Request(url=f"{API}/mt-bekanntgabe2024.json", params={"halbjahr": period, "size": PAGE_SIZE, "page": 0},
|
||||
kind="halbjahr", meta={"period": period})
|
||||
for period in quarters(int(ctx.params.get("legacy_since_year", 2021))):
|
||||
yield Request(url=f"{API}/mt-bekanntgabe.json", params={"quartal": period, "size": PAGE_SIZE, "page": 0},
|
||||
kind="quartal", meta={"period": period})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
rows = response.json().get("data") or []
|
||||
params = response.request.params or {}
|
||||
if len(rows) >= PAGE_SIZE:
|
||||
yield Request(url=response.request.url, params={**params, "page": params.get("page", 0) + 1},
|
||||
kind=response.request.kind, meta=response.meta)
|
||||
period = response.meta["period"]
|
||||
label = f"{period[:4]}-H{period[4:]}" if response.request.kind == "halbjahr" else f"{period[:4]}-Q{period[4:]}"
|
||||
min_euro = float(ctx.params.get("min_euro", 0))
|
||||
for row in rows:
|
||||
amount = row.get("euro")
|
||||
if row.get("leermeldung") or amount is None or float(amount) < min_euro:
|
||||
continue
|
||||
kind = KIND.get(row.get("bekanntgabe"), "other")
|
||||
payer_name = (row.get("rechtstraeger") or "").strip().strip('"')
|
||||
owner = (row.get("medieninhaber") or row.get("mediumMedieninhaber") or "").strip()
|
||||
medium = (row.get("medium") or row.get("mediumMedieninhaber") or "").strip()
|
||||
if not payer_name or not owner:
|
||||
continue
|
||||
payer = OrgRef(name=payer_name, kind="public_body",
|
||||
ids={"rtr": str(row["rechtstraeger_id"])} if row.get("rechtstraeger_id") else {})
|
||||
payee = OrgRef(name=owner, kind="media")
|
||||
campaign = row.get("kampagne")
|
||||
what = "Werbung" if kind == "government_ad" else "Förderung"
|
||||
evidence_url = (f"{API}/{'mt-bekanntgabe2024' if response.request.kind == 'halbjahr' else 'mt-bekanntgabe'}"
|
||||
f".json?{'halbjahr' if response.request.kind == 'halbjahr' else 'quartal'}={period}"
|
||||
+ (f"&rechtstraeger_id={row['rechtstraeger_id']}" if row.get("rechtstraeger_id") else ""))
|
||||
yield Claim(
|
||||
claim_type="money_flow", org=payer, counterpart_org=payee,
|
||||
summary_de=f"{payer_name}: {what} in {medium} ({owner}), {format_eur_de(amount)} € ({label})"
|
||||
+ (f" – Kampagne „{campaign}“" if campaign else ""),
|
||||
summary_en=f"{payer_name}: {'advertising' if kind == 'government_ad' else 'funding'} in {medium} "
|
||||
f"({owner}), EUR {amount:,.2f} ({label})",
|
||||
data={"medium": medium, "category": row.get("kategorie"), "subcategory": row.get("subkategorie"),
|
||||
"campaign": campaign, "sujet": row.get("sujet"), "period": label},
|
||||
evidence=[Evidence(url=evidence_url, doc_type="dataset_row",
|
||||
title=f"Medientransparenz {label}: {payer_name}",
|
||||
quote=f"{payer_name} → {medium} ({owner}): {amount} €")],
|
||||
topics=("Information und Medien",),
|
||||
facet=Money(kind=kind, amount=amount, payer=payer, payer_name=payer_name, payee=payee, payee_name=owner,
|
||||
period=label, reference=f"{campaign or ''}|{medium}|{row.get('sujet_dateiname') or ''}",
|
||||
category=row.get("kategorie")),
|
||||
)
|
||||
Reference in new issue
Block a user