simonandClaude Opus 5.5
583ccba31c
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
107 lines
6.2 KiB
Python
107 lines
6.2 KiB
Python
"""RTR / KommAustria media transparency (MedKF-TG): advertising and media subsidies paid by public bodies.
|
||
|
||
Open data API (no usage restrictions, attribution): https://data.rtr.at/api/v1/tables/<table>.json
|
||
* mt-bekanntgabe2024 (half-yearly since 2024): payer, medium, media owner, campaign, amount, ad sujet link
|
||
* mt-bekanntgabe (quarterly until 2023): payer, medium/owner, amount
|
||
Each row -> money_flow (kind government_ad for advertising, subsidy for media funding) from the public body
|
||
(ministry, state government, municipality, public company...) to the media owner.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from collections.abc import Iterator
|
||
from datetime import date
|
||
from typing import Any
|
||
|
||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||
from politrack.core.records import Claim, Evidence, Money, OrgRef
|
||
from politrack.text import format_eur_de
|
||
|
||
API = "https://data.rtr.at/api/v1/tables"
|
||
PAGE_SIZE = 5000
|
||
SPEC = SourceSpec(
|
||
slug="rtr-medientransparenz", name="RTR/KommAustria Medientransparenz", kind="official",
|
||
homepage="https://www.rtr.at/medien/was_wir_tun/medientransparenz/",
|
||
license="Amtliches Werk (§7 UrhG); attribution \"KommAustria – data.rtr.at\"", access="api",
|
||
trust=95, min_interval=2.0,
|
||
# data.rtr.at/robots.txt disallows everything for crawlers, but RTR documents this API as "der Endpunkt zur
|
||
# automatischen Abfrage der Daten" (rtr.at/rtr/service/opendata/OD_Schnittstellenbeschreibung.de.html).
|
||
robots_exempt=(API + "/",),
|
||
tos_notes="Government advertising and media funding reported under MedKF-TG. The REST API is documented by "
|
||
"RTR for automated queries (OD_Schnittstellenbeschreibung); robots.txt on data.rtr.at blocks "
|
||
"crawlers only. Terms (OD_Nutzungsbedingungen): free works under §7 UrhG, attribution requested, "
|
||
"no use for defamatory/discriminatory purposes.",
|
||
)
|
||
KIND = {2: "government_ad", 4: "subsidy"}
|
||
|
||
|
||
def half_years(since_year: int) -> list[str]:
|
||
today = date.today()
|
||
return [f"{y}{h}" for y in range(since_year, today.year + 1) for h in (1, 2)
|
||
if (y, h) <= (today.year, 1 if today.month <= 6 else 2)]
|
||
|
||
|
||
def quarters(since_year: int, until_year: int = 2023) -> list[str]:
|
||
return [f"{y}{q}" for y in range(since_year, until_year + 1) for q in (1, 2, 3, 4)]
|
||
|
||
|
||
@register
|
||
class MediaTransparency(Adapter):
|
||
spec = SPEC
|
||
name = "rtr.medientransparenz"
|
||
description = "Government advertising and media subsidies per public body and media owner"
|
||
schedule = "0 4 * * 2"
|
||
queue = "bulk"
|
||
|
||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||
for period in half_years(int(ctx.params.get("since_year", 2024))):
|
||
yield Request(url=f"{API}/mt-bekanntgabe2024.json", params={"halbjahr": period, "size": PAGE_SIZE, "page": 0},
|
||
kind="halbjahr", meta={"period": period})
|
||
for period in quarters(int(ctx.params.get("legacy_since_year", 2021))):
|
||
yield Request(url=f"{API}/mt-bekanntgabe.json", params={"quartal": period, "size": PAGE_SIZE, "page": 0},
|
||
kind="quartal", meta={"period": period})
|
||
|
||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||
rows = response.json().get("data") or []
|
||
params = response.request.params or {}
|
||
if len(rows) >= PAGE_SIZE:
|
||
yield Request(url=response.request.url, params={**params, "page": params.get("page", 0) + 1},
|
||
kind=response.request.kind, meta=response.meta)
|
||
period = response.meta["period"]
|
||
label = f"{period[:4]}-H{period[4:]}" if response.request.kind == "halbjahr" else f"{period[:4]}-Q{period[4:]}"
|
||
min_euro = float(ctx.params.get("min_euro", 0))
|
||
for row in rows:
|
||
amount = row.get("euro")
|
||
if row.get("leermeldung") or amount is None or float(amount) < min_euro:
|
||
continue
|
||
kind = KIND.get(row.get("bekanntgabe"), "other")
|
||
payer_name = (row.get("rechtstraeger") or "").strip().strip('"')
|
||
owner = (row.get("medieninhaber") or row.get("mediumMedieninhaber") or "").strip()
|
||
medium = (row.get("medium") or row.get("mediumMedieninhaber") or "").strip()
|
||
if not payer_name or not owner:
|
||
continue
|
||
payer = OrgRef(name=payer_name, kind="public_body",
|
||
ids={"rtr": str(row["rechtstraeger_id"])} if row.get("rechtstraeger_id") else {})
|
||
payee = OrgRef(name=owner, kind="media")
|
||
campaign = row.get("kampagne")
|
||
what = "Werbung" if kind == "government_ad" else "Förderung"
|
||
evidence_url = (f"{API}/{'mt-bekanntgabe2024' if response.request.kind == 'halbjahr' else 'mt-bekanntgabe'}"
|
||
f".json?{'halbjahr' if response.request.kind == 'halbjahr' else 'quartal'}={period}"
|
||
+ (f"&rechtstraeger_id={row['rechtstraeger_id']}" if row.get("rechtstraeger_id") else ""))
|
||
yield Claim(
|
||
claim_type="money_flow", org=payer, counterpart_org=payee,
|
||
summary_de=f"{payer_name}: {what} in {medium} ({owner}), {format_eur_de(amount)} € ({label})"
|
||
+ (f" – Kampagne „{campaign}“" if campaign else ""),
|
||
summary_en=f"{payer_name}: {'advertising' if kind == 'government_ad' else 'funding'} in {medium} "
|
||
f"({owner}), EUR {amount:,.2f} ({label})",
|
||
data={"medium": medium, "category": row.get("kategorie"), "subcategory": row.get("subkategorie"),
|
||
"campaign": campaign, "sujet": row.get("sujet"), "period": label},
|
||
evidence=[Evidence(url=evidence_url, doc_type="dataset_row",
|
||
title=f"Medientransparenz {label}: {payer_name}",
|
||
quote=f"{payer_name} → {medium} ({owner}): {amount} €")],
|
||
topics=("Information und Medien",),
|
||
facet=Money(kind=kind, amount=amount, payer=payer, payer_name=payer_name, payee=payee, payee_name=owner,
|
||
period=label, reference=f"{campaign or ''}|{medium}|{row.get('sujet_dateiname') or ''}",
|
||
category=row.get("kategorie")),
|
||
)
|