Politrack: track Austrian politicians from public sources

Initial import of Politrack, a self-hosted service that tracks Austrian
politicians from federal to municipal level - roles, votes, money flows,
statements, meetings, social media and legal proceedings - with every claim
linked to its sources.

- backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse ->
  load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive
  for reparsing, and a claim/evidence model with corroboration
- sources: Parliament, Wikidata, EU Parliament/Commission, Council of
  Ministers, Rechnungshof party reports and donations, procurement, RTR media
  transparency, political ad libraries, WKStA, news feeds, Wikipedia edits,
  Bluesky, Mastodon, public Telegram channels, YouTube and X (official API)
- web: Next.js frontend in German and English with typed API client
- mcp: MCP server for AI agents and RAG
- docs: architecture, data model, sources, legal notes, Unraid deployment and
  contributor guides; Docker Compose setup for Unraid

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
simonandClaude Opus 5.5 committed 2026-10-28 20:02:01 +01:00
commit 583ccba31c
187 files changed
+39890

No files matched your search

+5
View File
@@ -0,0 +1,5 @@
"""Built-in source adapters. Every module in this package (and sub-packages) is imported by
politrack.core.registry.load_builtin(); modules register their adapters with @register.
To add a source: create one module here (see docs/contributing/adapters.md).
"""
@@ -0,0 +1,183 @@
"""Bundeskanzleramt: government news archive and Council of Ministers minutes (Ministerratsprotokolle).
Official works (§7 UrhG); robots.txt only blocks /401.
* News ("Nachrichten der Bundesregierung", one index per year): stored with full text; the event
extractor turns summits, trips abroad and bilateral meetings into travel/meeting claims.
* Council of Ministers minutes (Beschlussprotokolle): every agenda item names the reporting office(s)
("Bericht der Bundesministerin für Justiz, Zahl ..., betreffend ..."). Each becomes a
government_decision claim for the office holder on that date (resolved via the office hint).
"""
from __future__ import annotations
import re
from collections.abc import Iterator
from datetime import date
from typing import Any
from urllib.parse import urljoin
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Document, Event, Evidence, OrgRef, Participation, PersonRef
from politrack.text import excerpt, fingerprint
BASE = "https://www.bundeskanzleramt.gv.at"
NEWS_INDEX = BASE + "/bundeskanzleramt/nachrichten-der-bundesregierung/{year}.html"
MR_INDEX = BASE + "/bundeskanzleramt/die-bundesregierung/ministerratsprotokolle.html"
SPEC = SourceSpec(
slug="bundesregierung",
name="Bundeskanzleramt – Bundesregierung",
kind="official",
homepage=BASE,
license="Amtliches Werk (§7 UrhG)",
access="scrape",
trust=90,
min_interval=2.0,
tos_notes="Government news archive and Council of Ministers minutes. robots.txt only blocks /401.",
)
MONTHS = {m: i for i, m in enumerate(["jänner", "februar", "märz", "april", "mai", "juni", "juli", "august",
"september", "oktober", "november", "dezember"], start=1)}
MONTHS["januar"] = 1
GERMAN_DATE = re.compile(r"(\d{1,2})\.\s*([A-Za-zäÄ]+)\s+(\d{4})")
# Office names may contain commas ("für Wohnen, Kunst, Kultur, Medien und Sport"); file numbers sometimes use
# non-breaking hyphens or spaces ("2026 0.040.056"); subjects span lines (joined with " | ") and end at the
# decision ("Der Ministerrat beschließt ...") or at the next item.
ITEM = re.compile(r"(?:(?:Gemeinsamer\s+)?Bericht|Vortrag|Antrag)\s+(?P<offices>de[sr]\s+[^|]+?),?\s+"
r"Zahl\s+(?P<zahl>\d[\d./\-\u2010-\u2015 ]*?)\s*,\s*betreffend\s+(?P<subject>.+?)"
r"(?=\s*\|?\s*Der Ministerrat\b"
r"|\s*\|\s*(?:\d+(?:\.\d+)*\.?\s+)?(?:Gemeinsamer\s+)?(?:Bericht|Vortrag|Antrag)\s+de[sr]\s|$)")
OFFICE_SPLIT = re.compile(r",\s*(?=de[sr]\s)|\s+und\s+(?=de[sr]\s)")
GENITIVE = [(r"\bBundeskanzlers\b", "Bundeskanzler"), (r"\bVizekanzlers\b", "Vizekanzler"),
(r"\bBundesministers\b", "Bundesminister"), (r"\bStaatssekretärs\b", "Staatssekretär")]
def german_date(text: str) -> date | None:
m = GERMAN_DATE.search(text)
if not m or m.group(2).lower() not in MONTHS:
return None
return date(int(m.group(3)), MONTHS[m.group(2).lower()], int(m.group(1)))
def period_start(path: str) -> int:
"""'.../ministerratsprotokolle-der-xxvi-regierungsperiode-2017-2018.html' -> 2018 (latest year named)."""
return max((int(y) for y in re.findall(r"(20\d\d)", path)), default=0)
def offices(text: str) -> list[str]:
"""'des Bundeskanzlers, des Bundesministers für Wohnen, Kunst, Kultur, Medien und Sport und der
Bundesministerin für Justiz' -> ['Bundeskanzler', 'Bundesminister für Wohnen, ...', 'Bundesministerin für Justiz']."""
result = []
for part in OFFICE_SPLIT.split(text):
part = re.sub(r"^\s*de[sr]\s+", "", part).strip(" ,")
for pattern, repl in GENITIVE:
part = re.sub(pattern, repl, part)
if part:
result.append(part)
return result
def main_text(response: Response) -> tuple[str, str]:
soup = response.soup
main = soup.find("main") or soup
title_el = main.find(class_="title") or main.find("h1")
title = title_el.get_text(" ", strip=True) if title_el else (soup.title.string if soup.title else response.url)
for noise in main.find_all(["nav", "aside", "script", "style", "form", "footer"]):
noise.decompose()
parts = [p.get_text(" ", strip=True) for p in main.find_all(["p", "li", "h2", "h3"])]
return title, "\n".join(p for p in parts if p)
@register
class GovernmentNews(Adapter):
spec = SPEC
name = "bundesregierung.news"
description = "Government news archive (summits, trips, meetings of government members)"
schedule = "10 */3 * * *"
def start(self, ctx: RunContext) -> Iterator[Request]:
first = int(ctx.params.get("since_year", date.today().year - 1))
for year in range(first, date.today().year + 1):
yield Request(url=NEWS_INDEX.format(year=year), kind="index", meta={"year": year})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if response.request.kind == "index":
year = response.meta["year"]
for path in sorted(set(re.findall(
rf'href="(/bundeskanzleramt/nachrichten-der-bundesregierung/{year}/[^"]+\.html)"', response.text))):
yield Request(url=BASE + path, kind="article")
return
title, text = main_text(response)
time_el = response.soup.find("time")
yield Document(Evidence(url=response.request.url, doc_type="press_release", title=title,
published_at=time_el.get("datetime") if time_el else None,
excerpt=excerpt(text, 300), full_text=f"{title}\n{text}"))
@register
class CouncilOfMinisters(Adapter):
spec = SPEC
name = "bundesregierung.ministerrat"
description = "Council of Ministers minutes: decisions per reporting minister"
schedule = "20 6 * * 3,4"
parser_version = 2
def start(self, ctx: RunContext) -> Iterator[Request]:
yield Request(url=MR_INDEX, kind="index")
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
kind = response.request.kind
if kind == "index":
periods = sorted(set(re.findall(r'href="(/bundeskanzleramt/die-bundesregierung/ministerratsprotokolle/'
r'[^"?]+\.html)"', response.text)), key=period_start)
# Period pages are named after the year a government took office ("...-xxviii-2025") and list all
# its sessions, so the latest period that started before `since_year` may still contain newer ones.
since = int(ctx.params.get("since_year", date.today().year - 1))
started_before = [period_start(p) for p in periods if period_start(p) < since]
keep_from = max(started_before, default=0)
for path in reversed([p for p in periods if period_start(p) >= keep_from]): # newest first
yield Request(url=BASE + path, kind="period")
elif kind == "period":
base = response.request.url.split("?")[0]
for href in sorted(set(re.findall(r'href="([^"]+)"', response.text))):
url = urljoin(response.url, href)
if url.startswith(base.removesuffix(".html") + "/"):
yield Request(url=url, kind="protocol")
elif url.startswith(base + "?currentPage="):
yield Request(url=url, kind="period")
else:
yield from self.parse_protocol(response)
def parse_protocol(self, response: Response) -> Iterator[Any]:
title, text = main_text(response)
held_on = german_date(title) or german_date(text[:300])
url = response.request.url
yield Document(Evidence(url=url, doc_type="protocol", title=title, published_at=held_on,
excerpt=excerpt(text, 300), full_text=f"{title}\n{text}"))
event_key = fingerprint("ministerrat", url)
yield Event(key=event_key, kind="government_meeting", title=title, starts_at=held_on, location="Wien",
country="AT", url=url, body=OrgRef(slug="bundesregierung"))
flat = " | ".join(text.split("\n"))
for n, m in enumerate(ITEM.finditer(flat)):
subject = re.sub(r"\s*\|\s*", " ", m.group("subject")).strip(" .")
zahl = re.sub(r"[\s\u2010-\u2015]+", "-", m.group("zahl").strip())
for office in offices(m.group("offices")):
yield Claim(
claim_type="government_decision",
person=PersonRef(name=None, hints={"office": office, "on": held_on}),
key=("ministerrat", url, zahl, office), occurred_at=held_on,
summary_de=f"Bericht im Ministerrat ({office}): {excerpt(subject, 220)}",
summary_en=f"Report to the Council of Ministers ({office}): {excerpt(subject, 220)}",
data={"office": office, "zahl": zahl, "item": n + 1},
evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=held_on,
quote=excerpt(re.sub(r"\s*\|\s*", " ", m.group(0)), 500),
locator=f"item:{zahl}")],
)
# Participation of the reporting ministers in this session (calendar)
for office in offices(m.group("offices")):
yield Claim(
claim_type="appearance", person=PersonRef(hints={"office": office, "on": held_on}),
occurred_at=held_on, summary_de=f"Teilnahme am Ministerrat ({title})",
evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=held_on)],
facet=Participation(event_key=event_key, participation="reporting_minister"),
)
+193
View File
@@ -0,0 +1,193 @@
"""EU institutions: Commission lobby meetings and the European Parliament (Austrian MEPs, declarations).
* European Commission - meetings with interest representatives (open data XLSX per Commission term).
Every row names the cabinet ("Cabinet of Commissioner Magnus Brunner"). Only meetings of Austrian
Commissioners' cabinets are kept (names passed from start(): Commission roles in the DB + a fallback list).
* European Parliament Open Data API v2 (CC BY 4.0): current Austrian MEPs (authoritative mandate data),
and their declarations (financial interests, conflicts of interest, gifts, ...).
"""
from __future__ import annotations
import io
import re
from collections.abc import Iterator
from datetime import date
from typing import Any
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import (
Claim,
Event,
Evidence,
OrgRef,
Participation,
PersonRecord,
PersonRef,
Role,
)
from politrack.text import excerpt, fingerprint, normalize
EC_SPEC = SourceSpec(
slug="ec-meetings", name="Europäische Kommission – Treffen mit Interessenvertretern", kind="official",
homepage="https://commission.europa.eu/about/service-standards-and-principles/transparency_en",
license="EU open data (reuse permitted, attribution)", access="bulk", trust=90, min_interval=2.0,
tos_notes="Open data export of meetings of Commissioners and their cabinets with interest representatives.",
)
EP_SPEC = SourceSpec(
slug="europarl", name="Europäisches Parlament Open Data", kind="official",
homepage="https://data.europarl.europa.eu", license="CC BY 4.0", access="api", trust=95, min_interval=1.0,
tos_notes="EP Open Data API v2 (limit 500 requests / 5 min / endpoint).",
)
EC_EXPORT = "https://ec.europa.eu/transparency-initiative/meetings/data/meetings/dataxlsx"
EC_TERMS = ("meetingscommissionrepresentatives2429", "meetingscommissionrepresentatives1924")
EP_API = "https://data.europarl.europa.eu/api/v2"
# Austrian Commissioners in the terms covered by EC_TERMS (fallback if the DB has no Commission roles yet).
AUSTRIAN_COMMISSIONERS = ("Johannes Hahn", "Magnus Brunner")
CABINET = re.compile(r"Cabinet of (?:Executive Vice-President|Vice-President|Commissioner|President)\s+(?P<name>.+)$")
@register
class CommissionMeetings(Adapter):
spec = EC_SPEC
name = "eu.commission_meetings"
description = "Meetings of (Austrian) EU Commissioners and their cabinets with interest representatives"
schedule = "0 5 * * 1"
queue = "bulk"
def start(self, ctx: RunContext) -> Iterator[Request]:
names = set(AUSTRIAN_COMMISSIONERS)
if ctx.conn is not None:
names |= {row["full_name"] for row in ctx.conn.execute(
"""SELECT DISTINCT p.full_name FROM role r JOIN person p ON p.id = r.person_id
JOIN organization o ON o.id = r.organization_id WHERE o.slug = 'europaeische-kommission'""")}
for term in EC_TERMS:
yield Request(url=EC_EXPORT, params={"name": term}, kind="xlsx", meta={"commissioners": sorted(names)})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
import openpyxl
wb = openpyxl.load_workbook(io.BytesIO(response.body), read_only=True, data_only=True)
rows = list(wb.worksheets[0].iter_rows(values_only=True))
header_index = next((i for i, r in enumerate(rows) if r and r[0] == "Name of cabinet"), None)
if header_index is None:
ctx.stats["unexpected_layout"] += 1
return
header = [str(h) for h in rows[header_index]]
url = f"{EC_EXPORT}?name={response.request.params['name']}"
wanted = {normalize(n) for n in response.meta.get("commissioners") or AUSTRIAN_COMMISSIONERS}
for row in rows[header_index + 1:]:
rec = dict(zip(header, row))
cabinet = rec.get("Name of cabinet") or ""
m = CABINET.search(cabinet.strip())
if not m or not rec.get("Date of meeting"):
continue
commissioner = m.group("name").strip()
if normalize(commissioner) not in wanted:
ctx.stats["other_commissioners"] += 1
continue
org_name = (rec.get("Name of interest representative") or "").strip()
held = str(rec["Date of meeting"])[:10]
subject = (rec.get("Subject of the meeting") or "").strip()
reps = rec.get("Name of EC representative") or ""
titles = rec.get("Title of EC representative") or ""
register_id = (rec.get("Transparency register ID") or "").strip()
event_key = fingerprint("ec-meeting", cabinet, held, org_name, subject)
yield Event(key=event_key, kind="meeting", title=f"{cabinet}: {org_name}", starts_at=held,
location=rec.get("Location"), url=url, description=subject,
body=OrgRef(slug="europaeische-kommission"))
yield Claim(
claim_type="meeting", person=PersonRef(name=commissioner), occurred_at=held,
counterpart_org=OrgRef(name=org_name, kind="lobby",
ids={"eu_transparency": register_id} if register_id else {}) if org_name else None,
summary_de=f"Treffen ({'Kommissar persönlich' if 'Commissioner' in titles else 'Kabinett'}) mit "
f"{org_name}: {excerpt(subject, 160)}",
summary_en=f"Meeting ({'Commissioner' if 'Commissioner' in titles else 'cabinet'}) with {org_name}: "
f"{excerpt(subject, 160)}",
data={"cabinet": cabinet, "representatives": reps, "titles": titles, "location": rec.get("Location"),
"register_id": register_id, "date_precision": "day"},
evidence=[Evidence(url=url, doc_type="dataset_row", title=f"EU-Kommission: Treffen {cabinet}",
published_at=held, quote=f"{held} | {org_name} | {subject}",
locator=f"{held}:{org_name}:{subject[:60]}")],
topics=("Europäische Union",),
facet=Participation(event_key=event_key, participation="host"),
)
@register
class EuropeanParliament(Adapter):
spec = EP_SPEC
name = "europarl.meps"
description = "Current Austrian MEPs (mandates) and their declarations"
schedule = "30 4 * * 2"
creates_persons = True
def start(self, ctx: RunContext) -> Iterator[Request]:
yield Request(url=f"{EP_API}/meps/show-current", kind="meps",
params={"country-of-representation": "AT", "format": "application/ld+json", "limit": 100})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
data = response.json().get("data") or []
if response.request.kind == "meps":
for mep in data:
mep_id = str(mep.get("identifier"))
given, family = mep.get("givenName"), mep.get("familyName")
name = f"{given} {family}" if given and family else mep.get("label")
ref = PersonRef(ids={"ep": mep_id}, name=name)
yield PersonRecord(ref=ref, full_name=name, given_name=given, family_name=family,
aliases=(mep.get("label") or "",), match_by_name=True)
yield Request(url=f"{EP_API}/meps/{mep_id}", params={"format": "application/ld+json"}, kind="mep",
meta={"mep_id": mep_id, "name": name, "group": mep.get("api:political-group")})
yield Request(url=f"{EP_API}/meps-declarations", kind="declarations",
params={"person-id": mep_id, "format": "application/ld+json", "limit": 100},
meta={"mep_id": mep_id, "name": name})
elif response.request.kind == "mep":
yield from self.parse_mep(response, data[0] if data else {})
else:
yield from self.parse_declarations(response, data)
def parse_mep(self, response: Response, mep: dict) -> Iterator[Any]:
meta = response.meta
ref = PersonRef(ids={"ep": meta["mep_id"]})
if mep.get("img"):
yield PersonRecord(ref=ref, full_name=meta["name"], photo_url=mep["img"],
birth_date=date.fromisoformat(mep["bday"][:10]) if mep.get("bday") else None)
starts = [m.get("memberDuring", {}).get("startDate") for m in mep.get("hasMembership") or []
if "EU_PARLIAMENT" in str(m.get("membershipClassification", ""))]
starts = sorted(s for s in starts if s)
body = OrgRef(slug="europaeisches-parlament")
start = date.fromisoformat(starts[-1]) if starts else None
url = f"https://www.europarl.europa.eu/meps/de/{meta['mep_id']}"
yield Claim(
claim_type="role", person=ref, org=body, valid_from=start,
summary_de=f"Mitglied des Europäischen Parlaments (Fraktion {meta.get('group') or '?'})",
summary_en=f"Member of the European Parliament ({meta.get('group') or '?'})",
data={"ep_group": meta.get("group")},
evidence=[Evidence(url=url, doc_type="api_record", title=f"Europäisches Parlament: {meta['name']}")],
facet=Role(role="mp", level="eu", body=body, start=start),
)
def parse_declarations(self, response: Response, docs: list[dict]) -> Iterator[Claim]:
meta = response.meta
for doc in docs:
titles = []
pdf = None
for expression in doc.get("is_realized_by") or []:
title = (expression.get("title") or {})
titles.extend(v for v in title.values() if v)
for manifestation in expression.get("is_embodied_by") or []:
if "pdf" in str(manifestation.get("media_type", "")).lower() and not pdf:
pdf = manifestation.get("is_exemplified_by")
title = next((t for t in titles if t), doc.get("id"))
url = f"https://www.europarl.europa.eu/{pdf}" if pdf and not pdf.startswith("http") else (
pdf or f"https://data.europarl.europa.eu/{doc.get('id')}")
kind = str(doc.get("work_type", "")).rsplit("/", 1)[-1]
yield Claim(
claim_type="declaration", person=PersonRef(ids={"ep": meta["mep_id"]}),
key=("ep-declaration", doc.get("id")), occurred_at=doc.get("document_date"),
summary_de=f"Erklärung im Europäischen Parlament: {excerpt(title, 200)}",
summary_en=f"Declaration filed with the European Parliament: {excerpt(title, 200)}",
data={"work_type": kind},
evidence=[Evidence(url=url, doc_type="pdf", title=title, published_at=doc.get("document_date"))],
topics=("Transparenz und Korruption",),
)
+70
View File
@@ -0,0 +1,70 @@
"""GDELT DOC 2.0 API: news coverage of politicians across many more outlets (metadata only).
Free, open, max. one request every 5 s (we use 6 s and back off on the plain-text throttle notice).
Coverage of German-language Austrian media is thin, so this is a supplement. Queries: active
federal/EU politicians daily, former politicians in a weekly rotation ("what are they up to?").
"""
from __future__ import annotations
import json
from collections.abc import Iterator
from datetime import date
from typing import Any
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Document, Evidence
API = "https://api.gdeltproject.org/api/v2/doc/doc"
SPEC = SourceSpec(
slug="gdelt", name="Weitere Medien (via GDELT)", kind="aggregator", homepage="https://www.gdeltproject.org",
license="GDELT open data (citation required)", access="api", trust=55, store_fulltext=False, keep_raw=False,
min_interval=6.0, tos_notes="GDELT DOC 2.0 API: article metadata (URL, title, date). Max 1 request / 5 s.",
)
DOMAIN_SOURCES = {
"orf.at": "orf", "derstandard.at": "derstandard", "diepresse.com": "diepresse", "kurier.at": "kurier",
"profil.at": "profil", "falter.at": "falter", "krone.at": "krone", "sn.at": "sn", "kleinezeitung.at": "kleinezeitung",
"ots.at": "apa-ots", "vol.at": "vol", "vienna.at": "vol",
}
@register
class Gdelt(Adapter):
spec = SPEC
name = "gdelt.coverage"
description = "Supplementary news coverage via GDELT (active and former politicians)"
schedule = "0 1 * * *"
queue = "bulk"
def start(self, ctx: RunContext) -> Iterator[Request]:
if ctx.conn is None:
return
rows = ctx.conn.execute(
"""SELECT p.id, p.given_name, p.family_name, s.status FROM person p
JOIN person_status s ON s.person_id = p.id
WHERE p.given_name IS NOT NULL AND p.family_name IS NOT NULL AND (
(s.status = 'active' AND EXISTS (SELECT 1 FROM role r WHERE r.person_id = p.id
AND r.level IN ('federal', 'eu') AND r.end_date IS NULL))
OR (s.status IN ('former', 'party_member') AND s.last_office_end > current_date - interval '25 years'
AND p.id %% 7 = %s))
ORDER BY s.status, p.id LIMIT %s""",
(date.today().weekday(), int(ctx.params.get("max_queries", 400))),
).fetchall()
for row in rows:
name = f"{row['given_name']} {row['family_name']}"
yield Request(url=API, kind="artlist", meta={"person_id": row["id"]}, conditional=False,
params={"query": f'"{name}" sourcecountry:AU', "mode": "artlist", "format": "json",
"maxrecords": 50, "timespan": "7d" if row["status"] == "active" else "1m",
"sort": "datedesc"})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if not response.text.strip().startswith("{"):
ctx.stats["throttled"] += 1 # plain-text rate-limit notice
return
for article in json.loads(response.text, strict=False).get("articles", []):
domain = (article.get("domain") or "").removeprefix("www.")
seen = article.get("seendate") or ""
published = f"{seen[:4]}-{seen[4:6]}-{seen[6:8]}T{seen[9:11]}:{seen[11:13]}:00Z" if len(seen) >= 13 else None
yield Document(Evidence(url=article["url"], source=DOMAIN_SOURCES.get(domain, "gdelt"),
title=article.get("title"), doc_type="article", published_at=published,
meta={"via": "gdelt", "domain": domain}))
+63
View File
@@ -0,0 +1,63 @@
"""Press releases of the Wirtschafts- und Korruptionsstaatsanwaltschaft (WKStA, anti-corruption prosecutors).
Official statements on high-profile proceedings. They usually anonymise suspects ("ein ehemaliger
Bundesminister"), so we store them as official documents (full text, searchable, RAG) and let the
legal-matter detector propose claims - which, as always for legal matters, need human review.
"""
from __future__ import annotations
import re
from collections.abc import Iterator
from datetime import date
from typing import Any
from urllib.parse import urljoin
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Document, Evidence
from politrack.sources.bundesregierung import german_date
from politrack.text import excerpt
BASE = "https://www.justiz.gv.at"
START = BASE + "/wksta/wirtschafts-und-korruptionsstaatsanwaltschaft.312.de.html"
SPEC = SourceSpec(
slug="wksta", name="WKStA – Pressemitteilungen", kind="official", homepage=START,
license="Amtliches Werk (§7 UrhG)", access="scrape", trust=90, min_interval=3.0,
tos_notes="Press releases of the anti-corruption prosecutors. robots.txt allows all.",
)
RELEASE = re.compile(r"/medienstelle/pressemitteilungen/pressemitteilungen-(\d{4})/")
LISTING = re.compile(r"/medienstelle(?:/pressemitteilungen(?:/pressemitteilungen-\d{4})?)?\.[0-9a-f]+\.de\.html")
def clean(url: str) -> str:
return re.sub(r";jsessionid=[^?#]*", "", url)
@register
class WkstaPress(Adapter):
spec = SPEC
name = "justiz.wksta"
description = "WKStA press releases (official documents for search and reviewed legal matters)"
schedule = "40 8 * * *"
def start(self, ctx: RunContext) -> Iterator[Request]:
yield Request(url=START, kind="listing", meta={"depth": 0})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if response.request.kind == "listing":
since = int(ctx.params.get("since_year", date.today().year - 2))
for a in response.soup.find_all("a", href=True):
url = clean(urljoin(response.url, a["href"]))
year = RELEASE.search(url)
if year and int(year.group(1)) >= since:
yield Request(url=url, kind="release")
elif LISTING.search(url) and response.meta["depth"] < 2:
yield Request(url=url, kind="listing", meta={"depth": response.meta["depth"] + 1})
return
main = response.soup.find("main") or response.soup.find(id="content") or response.soup
title_el = main.find("h1")
title = title_el.get_text(" ", strip=True) if title_el else response.url
text = "\n".join(p.get_text(" ", strip=True) for p in main.find_all(["p", "li"]) if p.get_text(strip=True))
yield Document(Evidence(url=clean(response.request.url), doc_type="press_release", title=title,
published_at=german_date(text[:400]) or german_date(title), excerpt=excerpt(text, 300),
full_text=f"{title}\n{text}"))
+86
View File
@@ -0,0 +1,86 @@
"""Headline monitoring via publishers' RSS feeds and Google-News sitemaps (listed in robots.txt).
Policy (docs/sources.md): only feeds/sitemaps publishers offer for automated discovery; store headline,
teaser/keywords and link - never article text (store_fulltext=False, keep_raw=False). If a site declares
a TDM reservation, the Loader drops even the teaser and linking uses the headline only.
The feed list lives in config/feeds.yaml (user-editable); every `source` there needs a SourceSpec below.
"""
from __future__ import annotations
import xml.etree.ElementTree as ET
from collections.abc import Iterator
from datetime import datetime, timezone
from time import mktime
from typing import Any
import feedparser
import yaml
from politrack.config import get_settings
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register, register_spec
from politrack.core.records import Document, Evidence
from politrack.text import html_to_text
def media(slug: str, name: str, homepage: str, trust: int, notes: str = "RSS/news sitemap only; headline + link.",
license: str = "All rights reserved") -> SourceSpec:
return register_spec(SourceSpec(slug=slug, name=name, kind="media", homepage=homepage, license=license,
access="rss", trust=trust, store_fulltext=False, keep_raw=False,
min_interval=2.0, tos_notes=notes))
ORF = media("orf", "ORF.at", "https://orf.at", 75, "Public RSS feeds (rss.orf.at). Headline, teaser, link.")
media("derstandard", "DER STANDARD", "https://www.derstandard.at", 75,
"RSS + news sitemap. robots.txt blocks AI crawlers by name; we only use feed/sitemap and link out.")
media("diepresse", "Die Presse", "https://www.diepresse.com", 75)
media("kurier", "KURIER", "https://kurier.at", 70)
media("profil", "profil", "https://www.profil.at", 75)
media("falter", "FALTER", "https://www.falter.at", 75)
media("krone", "Kronen Zeitung", "https://www.krone.at", 55)
media("sn", "Salzburger Nachrichten", "https://www.sn.at", 75)
media("kleinezeitung", "Kleine Zeitung", "https://www.kleinezeitung.at", 75)
media("vol", "Vorarlberg Online", "https://www.vol.at", 65)
media("apa-ots", "APA-OTS Presseaussendungen", "https://www.ots.at", 60,
"tdmrep.json declares tdm-reservation=1 for the whole site: headline + link only.")
NS = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9", "news": "http://www.google.com/schemas/sitemap-news/0.9"}
@register
class NewsFeeds(Adapter):
spec = ORF
name = "news.feeds"
description = "Headlines from outlets' RSS feeds and Google-News sitemaps (config/feeds.yaml)"
schedule = "*/20 * * * *"
def start(self, ctx: RunContext) -> Iterator[Request]:
config = yaml.safe_load((get_settings().config_dir / "feeds.yaml").read_text(encoding="utf-8"))
for feed in config.get("feeds") or []:
yield Request(url=feed["url"], kind=feed.get("type", "rss"), meta={"source": feed["source"]},
skip_unchanged=True)
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
source = response.meta["source"]
doc_type = "press_release" if source == "apa-ots" else "article"
if response.request.kind == "sitemap":
root = ET.fromstring(response.body)
for node in root.findall("sm:url", NS):
link, title = node.findtext("sm:loc", namespaces=NS), node.findtext("news:news/news:title", namespaces=NS)
if not link or not title:
continue
keywords = node.findtext("news:news/news:keywords", namespaces=NS)
yield Document(Evidence(url=link.strip(), source=source, title=html_to_text(title), doc_type=doc_type,
published_at=node.findtext("news:news/news:publication_date", namespaces=NS),
excerpt=f"Schlagworte: {keywords}" if keywords else None,
meta={"sitemap": response.request.url}))
return
for entry in feedparser.parse(response.body).entries:
if not entry.get("link"):
continue
published = (datetime.fromtimestamp(mktime(entry.published_parsed), tz=timezone.utc)
if entry.get("published_parsed") else None)
yield Document(Evidence(url=entry["link"], source=source, title=html_to_text(entry.get("title")),
doc_type=doc_type, published_at=published, excerpt=html_to_text(entry.get("summary")),
meta={"feed": response.request.url}))
@@ -0,0 +1 @@
"""Austrian Parliament adapters: persons, items (votes, immunity, questions, hearings), press, calendar."""
@@ -0,0 +1,199 @@
"""Shared spec and pure parsing helpers for the Austrian Parliament adapters (CC BY 4.0).
API (https://www.parlament.gv.at/recherchieren/open-data/):
* list exports: POST /Filter/api/filter/data/<listeId>?js=eval&showAll=true (JSON body = filter)
409 persons since 1918 · 101 parliamentary items · 251 speeches of a person
110 Parlamentskorrespondenz (press releases) · 600 calendar (Termine)
* detail pages: <page URL>?json=TRUE
"""
from __future__ import annotations
import re
from datetime import date, datetime
from typing import Any
from politrack.core import Request, SourceSpec, register_spec
from politrack.text import html_to_text
BASE = "https://www.parlament.gv.at"
CHAMBER_SLUG = {"NR": "nationalrat", "BR": "bundesrat"}
CHAMBER_NAME_DE = {"NR": "Nationalrat", "BR": "Bundesrat"}
GOVERNMENT_RE = re.compile(r"Bundeskanzler|Vizekanzler|Bundesminister|Staatssekretär", re.IGNORECASE)
SPEC = register_spec(SourceSpec(
slug="parlament",
name="Parlament Österreich (Open Data)",
kind="official",
homepage="https://www.parlament.gv.at/recherchieren/open-data/",
license="CC BY 4.0",
access="api",
trust=95,
store_fulltext=True,
min_interval=1.0,
tos_notes="Official JSON API (?json=TRUE, /Filter/api/filter/data/<list>). Attribution required. The "
"Parlamentsdirektion may prohibit uses incompatible with the dignity of parliament.",
))
def list_request(list_id: int, body: dict, kind: str, meta: dict | None = None) -> Request:
return Request(url=f"{BASE}/Filter/api/filter/data/{list_id}?js=eval&showAll=true", method="POST", json=body,
kind=kind, meta=meta or {})
def detail_request(path: str, kind: str, meta: dict | None = None) -> Request:
return Request(url=f"{BASE}{path}", params={"json": "TRUE"}, kind=kind, meta=meta or {})
def list_rows(data: dict) -> list[dict[str, Any]]:
"""Turn a filter-API export into dicts keyed by column label (duplicates get a _<index> suffix)."""
labels: list[str] = []
for i, header in enumerate(data.get("header", [])):
label = header.get("label") or header.get("feld_name") or f"col{i}"
labels.append(label if label not in labels else f"{label}_{i}")
return [dict(zip(labels, row)) for row in data.get("rows", [])]
def parse_date(value: str | None) -> date | None:
if not value:
return None
for fmt in ("%d.%m.%Y", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d"):
try:
return datetime.strptime(value[:19], fmt).date()
except ValueError:
continue
return None
def walk_dicts(obj: Any, required_key: str):
"""Yield every nested dict that contains `required_key`."""
if isinstance(obj, dict):
if required_key in obj:
yield obj
for value in obj.values():
yield from walk_dicts(value, required_key)
elif isinstance(obj, list):
for value in obj:
yield from walk_dicts(value, required_key)
# -- faction votes ------------------------------------------------------------------------
PARTY_TOKEN = re.compile(r"^[A-ZÄÖÜ][A-Za-zÄÖÜäöüß.\- ]{0,30}$")
# Split votes ("teilweise", "bzw.", per-part results in parentheses) can't be attributed to factions.
SPLIT_VOTE = re.compile(r"\(|\bbzw\.|\bteilweise\b|\btlw\.|\bmehrstimmig\b|\bgetrennt", re.IGNORECASE)
def parse_party_list(value: str | None) -> list[str]:
if not value:
return []
parts = [p.strip(" .;") for p in re.split(r",|\bund\b", value)]
return [p for p in parts if PARTY_TOKEN.match(p)]
def parse_vote_text(text: str) -> dict[str, Any] | None:
"""'137. Sitzung …: Gesetzesvorschlag in dritter Lesung angenommen Dafür: ÖVP, SPÖ, dagegen: FPÖ'"""
m = re.search(r"\b(angenommen|abgelehnt)\b", text, re.IGNORECASE)
if not m:
return None
rest = text[m.end():]
roll_call = "namentliche abstimmung" in text.lower()
split = bool(SPLIT_VOTE.search(rest)) and not roll_call
pro = re.search(r"Dafür:\s*(.*?)(?:,?\s*dagegen:|$)", rest, re.IGNORECASE)
contra = re.search(r"dagegen:\s*(.*)$", rest, re.IGNORECASE)
return {
"stage": text[: m.start()].strip(" :,"),
"result": "accepted" if m.group(1).lower() == "angenommen" else "rejected",
"for": [] if split or roll_call else parse_party_list(pro.group(1) if pro else None),
"against": [] if split or roll_call else parse_party_list(contra.group(1) if contra else None),
"unanimous": "einstimmig" in text.lower(),
"split": split,
"roll_call": roll_call,
}
# -- roll-call votes ----------------------------------------------------------------------
ROLL_CALL_HEADER = re.compile(r"Mit\s*[„\"“]?\s*(Ja|Nein)\s*[“\"”]?\s*stimmten[^:]*:", re.IGNORECASE)
NEW_VOTE = re.compile(r"\b(gelangen|kommen)\s+(?:\w+\s+){0,2}zur\s+Abstimmung\b", re.IGNORECASE)
PERSON_LINK = re.compile(r"<a[^>]*href=\"[^\"]*/person/(\d+)\"[^>]*>.*?</a>", re.IGNORECASE | re.DOTALL)
def parse_roll_call(html_text: str, anchor: str | None) -> dict[str, list[tuple[str, int | None]]]:
"""Extract the "Mit „Ja“ stimmten …" / "Mit „Nein“ stimmten …" lists that follow `anchor`.
Returns {"for": [(label, pad|None)], "against": [...]}. Names are mostly surnames; where a surname is
ambiguous the protocol links the person, which gives us the parliament id directly.
"""
start = 0
if anchor:
m = re.search(r'id="%s"' % re.escape(anchor), html_text)
if not m:
return {}
start = m.start()
first = ROLL_CALL_HEADER.search(html_text, start)
if not first:
return {}
# The lists must belong to this anchor: they follow within a few numbered paragraphs and no new vote
# may be called in between.
between = html_text[start + 10 : first.start()]
if anchor and (len(re.findall(r'class="[^"]*randnummer', between)) > 3 or NEW_VOTE.search(html_to_text(between))):
return {}
end_match = re.compile(r"<hr|class=\"[^\"]*randnummer").search(html_text, first.end())
segment = html_text[first.start() : end_match.start() if end_match else len(html_text)]
segment = PERSON_LINK.sub(lambda m: f" @@{m.group(1)}@@ ", segment)
text = html_to_text(segment)
result: dict[str, list[tuple[str, int | None]]] = {"for": [], "against": []}
headers = list(ROLL_CALL_HEADER.finditer(text))
for i, header in enumerate(headers):
block = text[header.end() : headers[i + 1].start() if i + 1 < len(headers) else len(text)]
key = "for" if header.group(1).lower() == "ja" else "against"
for token in re.split(r"[,;]|\.\s*$", block):
token = token.strip(" .")
if not token:
continue
link = re.fullmatch(r"@@(\d+)@@", token)
result[key].append((token, int(link.group(1))) if link else (token, None))
return result
# -- immunity -----------------------------------------------------------------------------
STGB_OFFENSES = {
"111": "Üble Nachrede", "115": "Beleidigung", "146": "Betrug", "153": "Untreue", "165": "Geldwäscherei",
"201": "Vergewaltigung", "283": "Verhetzung", "288": "Falsche Beweisaussage", "293": "Fälschung eines Beweismittels",
"295": "Unterdrückung eines Beweismittels", "297": "Verleumdung", "302": "Missbrauch der Amtsgewalt",
"304": "Bestechlichkeit", "305": "Vorteilsannahme", "306": "Vorteilsannahme zur Beeinflussung",
"307": "Bestechung", "308": "Verbotene Intervention", "310": "Verletzung des Amtsgeheimnisses",
}
def parse_immunity_report(text: str) -> dict[str, Any]:
authority = re.search(r"Ersuchen (?:der|des) (.+?),\s*(?:do\.|GZ|Zl)", text)
case_ref = re.search(r"\bGZ\.?\s*([^,]+?),", text)
# "wegen des Verdachtes einer strafbaren Handlung nach § 283 Abs. 1 Z 1 und Z 2, Abs. 2 StGB"
offense = re.search(r"wegen[^§]{0,90}?(§§? ?\d+.{0,120}?\b(?:StGB|VerbotsG|FinStrG|[A-Z][A-Za-z]*G)\b)", text)
offense_text = offense.group(1).strip() if offense else None
if offense_text:
section = re.search(r"§ ?(\d+)", offense_text)
if section and section.group(1) in STGB_OFFENSES and "StGB" in offense_text:
offense_text = f"{offense_text} ({STGB_OFFENSES[section.group(1)]})"
decision = re.search(r"(?:wird|werde)\s+(nicht\s+)?zugestimmt|(nicht\s+)?zuzustimmen", text)
recommendation = None
if decision:
recommendation = "refuse" if (decision.group(1) or decision.group(2)) else "consent"
connection = None
if re.search(r"\bkein Zusammenhang\b", text):
connection = False
elif re.search(r"\bein Zusammenhang\b", text):
connection = True
quote = re.search(r"Als Ergebnis seiner Beratungen.*?(?:zugestimmt|nicht zugestimmt)\s*\.", text)
return {
"authority": authority.group(1).strip() if authority else None,
"case_ref": case_ref.group(1).strip() if case_ref else None,
"offense": offense_text,
"recommendation": recommendation,
"political_connection": connection,
"quote": quote.group(0) if quote else None,
}
@@ -0,0 +1,285 @@
"""Parliament items (Gegenstände) of the configured legislative periods.
From one list export per period (list 101) we derive:
* votes -> item detail -> VoteEvent + faction VotePositions
roll calls -> stenographic protocol -> individual VotePositions
* immunity requests ("Ersuchen um Zustimmung zur behördlichen Verfolgung") -> committee report -> Legal claim
* written questions and answers (ITYP J / AB) -> parliamentary_question claims (directly from the list)
* investigative-committee hearings ("öffentliche Befragung der Auskunftsperson ...") -> hearing Event +
appearance claim for politicians who testified (other witnesses are not stored)
"""
from __future__ import annotations
import json
import re
from collections.abc import Iterator
from typing import Any
from politrack.config import get_settings
from politrack.core import Adapter, Request, Response, RunContext, register
from politrack.core.records import (
Claim,
Event,
Evidence,
Legal,
OrgRef,
Participation,
PersonRef,
VoteEvent,
VotePosition,
)
from politrack.sources.parlament.common import (
BASE,
CHAMBER_SLUG,
SPEC,
detail_request,
list_request,
list_rows,
parse_date,
parse_immunity_report,
parse_roll_call,
parse_vote_text,
walk_dicts,
)
from politrack.text import fingerprint, html_to_text, strip_titles
HEARING = re.compile(r"Befragung der Auskunftsperson\s+(?P<name>.+)$")
WITNESS_TITLES = re.compile(r"^(?:(?:Oberstaatsanwalt|Staatsanwalt|Sektionschef(?:in)?|MinR|Generalsekretär(?:in)?|"
r"Bundesminister(?:in)?|Abgeordnete[rn]?|Klubobmann|Klubobfrau|Bundeskanzler|"
r"Vizekanzler|Präsident(?:in)?)\s+)+")
@register
class ParlamentItems(Adapter):
spec = SPEC
name = "parlament.items"
description = "Votes (faction + roll call), immunity requests, written questions, U-committee hearings"
schedule = "45 3 * * *"
parser_version = 2
def start(self, ctx: RunContext) -> Iterator[Request]:
for gp in ctx.params.get("periods") or get_settings().parlament_periods:
yield list_request(101, {"GP_CODE": [gp], "NRBR": ["NR"]}, kind="items", meta={"gp": gp})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
handler = {
"items": self.parse_items,
"votes": self.parse_votes,
"rollcall": self.parse_rollcall,
"immunity": self.parse_immunity_item,
"immunity_report": self.parse_immunity_report,
"hearing": self.parse_hearing_item,
}[response.request.kind]
yield from handler(ctx, response)
# -- list ---------------------------------------------------------------------------
def parse_items(self, ctx: RunContext, response: Response) -> Iterator[Any]:
for row in list_rows(response.json()):
path, subject, kind = row.get("HIS_URL"), row.get("Betreff") or "", row.get("ITYP")
if not path:
continue
if row.get("Dafür") or row.get("Dagegen") or row.get("Abstimmungstext"):
# The list row changes when new votes happen, so the detail is re-fetched only then
# (conditional request + unchanged-body detection do the rest).
yield detail_request(path, kind="votes", meta={"title": subject, "ref": row.get("Nummer")})
if subject.startswith("Ersuchen") and "behördlichen Verfolgung" in subject:
yield detail_request(path, kind="immunity",
meta={"title": subject, "date": row.get("Datum"), "status": row.get("Status")})
if kind in ("J", "AB"):
yield from self.question_claims(row, path)
if kind == "KOMM" and HEARING.search(subject):
yield detail_request(path, kind="hearing", meta={"title": subject, "date": row.get("Datum")})
def question_claims(self, row: dict, path: str) -> Iterator[Claim]:
pads = row.get("Personen") or "[]"
pads = json.loads(pads) if isinstance(pads, str) else pads
subject = row.get("Betreff") or ""
is_question = row.get("ITYP") == "J"
asked = parse_date(row.get("Datum"))
topics = tuple(json.loads(row.get("THEMEN") or "[]")) if isinstance(row.get("THEMEN"), str) else ()
for pad in pads:
if not str(pad).strip():
continue
verb_de = "stellte die schriftliche Anfrage" if is_question else "beantwortete die Anfrage"
verb_en = "submitted the written question" if is_question else "answered the question"
yield Claim(
claim_type="parliamentary_question", person=PersonRef(ids={"parlament": str(pad)}),
key=("question", path, str(pad)), occurred_at=asked,
summary_de=f"{verb_de[0].upper()}{verb_de[1:]} „{subject}“ ({row.get('Nummer')})",
summary_en=f"{verb_en[0].upper()}{verb_en[1:]} \"{subject}\" ({row.get('Nummer')})",
data={"kind": "question" if is_question else "answer", "number": row.get("Nummer")},
evidence=[Evidence(url=f"{BASE}{path}", doc_type="api_record", title=subject, published_at=asked)],
topics=topics,
)
# -- votes ----------------------------------------------------------------------------
def parse_votes(self, ctx: RunContext, response: Response) -> Iterator[Any]:
content = response.json().get("content") or {}
path = response.request.url.removeprefix(BASE)
title = html_to_text(content.get("title")) or response.meta.get("title") or path
item_ref = html_to_text(content.get("zitation")) or response.meta.get("ref")
item_url = f"{BASE}{path}"
topics = tuple(b.get("label") for b in ((content.get("topics") or {}).get("data") or {}).get("bubbles") or [])
eurovoc = [b.get("label") for b in ((content.get("eurovoc") or {}).get("data") or {}).get("bubbles") or []]
for stage in walk_dicts(content.get("phase") or content, "text"):
text = html_to_text(stage.get("text"))
if "angenommen" not in text and "abgelehnt" not in text:
continue
parsed = parse_vote_text(text)
if not parsed or not (parsed["for"] or parsed["against"] or parsed["unanimous"] or parsed["split"]
or parsed["roll_call"]):
continue
voted_on = parse_date(stage.get("date"))
refs = stage.get("fsth") or []
protocol_url = f"{BASE}{refs[0]['url']}" if refs and refs[0].get("url") else None
chamber = "BR" if (refs and refs[0].get("gp_code") == "BR") or "Bundesrat" in text else "NR"
chamber_ref = OrgRef(slug=CHAMBER_SLUG[chamber])
event_key = fingerprint("vote", path, voted_on, parsed["stage"])
yield VoteEvent(key=event_key, chamber=chamber_ref, title=title, voted_on=voted_on, result=parsed["result"],
stage=parsed["stage"], item_ref=item_ref, roll_call=parsed["roll_call"], url=item_url,
protocol_url=protocol_url, raw_text=text)
evidence = [Evidence(url=item_url, doc_type="api_record", title=title, published_at=voted_on, quote=text,
locator=f"stage:{voted_on}:{parsed['stage'][:80]}")]
if protocol_url:
evidence.append(Evidence(url=protocol_url, doc_type="protocol", published_at=voted_on,
title=refs[0].get("title") or "Stenographisches Protokoll"))
outcome = "angenommen" if parsed["result"] == "accepted" else "abgelehnt"
for party, position in [(p, "for") for p in parsed["for"]] + [(p, "against") for p in parsed["against"]]:
verb_de, verb_en = ("stimmte für", "voted for") if position == "for" else ("stimmte gegen", "voted against")
yield Claim(
claim_type="vote", org=OrgRef(name=party, short_name=party, kind="parliamentary_group"),
counterpart_org=chamber_ref, occurred_at=voted_on,
summary_de=f"{party} {verb_de}: {title} ({parsed['stage']}) – {outcome}",
summary_en=f"{party} {verb_en}: {title} ({parsed['stage']})",
data={"position": position, "result": parsed["result"], "item_ref": item_ref, "chamber": chamber,
"eurovoc": eurovoc},
evidence=evidence, topics=topics, facet=VotePosition(event_key=event_key, position=position),
)
if parsed["roll_call"]:
# Individual votes are listed in the protocol after one of the referenced anchors; the
# result is usually behind the last one, sometimes in another fragment (session break).
refs_list = [r["url"].split("#", 1) for r in reversed(refs) if r.get("url") and "#" in r["url"]]
if refs_list:
yield Request(url=f"{BASE}{refs_list[0][0]}", kind="rollcall",
meta={"event_key": event_key, "refs": refs_list, "tried": [refs_list[0][0]],
"title": title, "chamber": chamber, "date": str(voted_on),
"item_url": item_url, "topics": list(topics)})
def parse_rollcall(self, ctx: RunContext, response: Response) -> Iterator[Any]:
meta = response.meta
doc = response.request.url.removeprefix(BASE)
names: dict[str, list] = {}
anchor_used = None
for ref_doc, anchor in meta["refs"]:
if ref_doc != doc:
continue
names = parse_roll_call(response.text, anchor)
if names.get("for") or names.get("against"):
anchor_used = anchor
break
if anchor_used is None:
remaining = [d for d, _ in meta["refs"] if d not in meta["tried"]]
if remaining:
ctx.stats["rollcall_followup"] += 1
yield Request(url=f"{BASE}{remaining[0]}", kind="rollcall",
meta={**meta, "tried": [*meta["tried"], remaining[0]]})
else:
ctx.stats["rollcall_unparsed"] += 1
return
voted_on = parse_date(meta["date"])
body = CHAMBER_SLUG[meta["chamber"]]
source_url = f"{response.request.url}#{anchor_used}"
evidence = [
Evidence(url=source_url, doc_type="protocol", published_at=voted_on,
title="Stenographisches Protokoll – Ergebnis der namentlichen Abstimmung", locator=anchor_used),
Evidence(url=meta["item_url"], doc_type="api_record", title=meta["title"], published_at=voted_on,
locator=f"rollcall:{voted_on}"),
]
for position, entries in names.items():
answer = "Ja" if position == "for" else "Nein"
for label, pad in entries:
ref = (PersonRef(ids={"parlament": str(pad)}) if pad is not None else
PersonRef(name=strip_titles(label), hints={"surname_in_body": body, "on": voted_on}))
yield Claim(
claim_type="vote", person=ref, counterpart_org=OrgRef(slug=body), occurred_at=voted_on,
summary_de=f"Stimmte in namentlicher Abstimmung mit „{answer}“: {meta['title']}",
summary_en=f"Voted \"{'yes' if position == 'for' else 'no'}\" in a roll-call vote: {meta['title']}",
data={"position": position, "roll_call": True, "chamber": meta["chamber"], "label": label},
evidence=evidence, topics=tuple(meta.get("topics") or ()),
facet=VotePosition(event_key=meta["event_key"], position=position),
)
# -- immunity -----------------------------------------------------------------------
def parse_immunity_item(self, ctx: RunContext, response: Response) -> Iterator[Request]:
content = response.json().get("content") or {}
report = next((d["link"] for group in content.get("documents") or [] for d in group.get("documents") or []
if d.get("type") == "HTML"), None)
if report:
yield Request(url=f"{BASE}{report}", kind="immunity_report",
meta={**response.meta, "item": response.request.url.removeprefix(BASE),
"title": html_to_text(content.get("title")) or response.meta.get("title")})
def parse_immunity_report(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
meta = response.meta
text = html_to_text(response.text)
details = parse_immunity_report(text)
title = meta["title"]
name = re.sub(r"^.*?(Abgeordneten zum Nationalrat|Mitglied(es)? des Bundesrates)\s*(Klubobmann|Klubobfrau)?",
"", title).strip()
decided = str(meta.get("status")) == "5"
stage = {"consent": "immunity_waived" if decided else "immunity_waiver_requested",
"refuse": "immunity_refused" if decided else "immunity_waiver_requested"}.get(
details["recommendation"] or "", "immunity_waiver_requested")
stage_de = {"immunity_waived": "Nationalrat stimmte der behördlichen Verfolgung zu (Immunität aufgehoben)",
"immunity_refused": "Nationalrat verweigerte die Zustimmung zur behördlichen Verfolgung",
"immunity_waiver_requested": "Ersuchen um Aufhebung der Immunität"}[stage]
offense = details["offense"]
requested = parse_date(meta.get("date"))
yield Claim(
claim_type="legal_proceeding", person=PersonRef(name=strip_titles(name)),
key=("immunity", meta["item"]), occurred_at=requested, sensitive=True,
summary_de=stage_de + (f" – Verdacht nach {offense}" if offense else "")
+ (f" (Ersuchen: {details['authority']})" if details["authority"] else ""),
data={"political_connection": details["political_connection"],
"recommendation": details["recommendation"], "item": meta["item"]},
evidence=[
Evidence(url=response.request.url, doc_type="register_entry", published_at=requested,
title=f"Bericht des Immunitätsausschusses: {title}", quote=details["quote"], full_text=text),
Evidence(url=f"{BASE}{meta['item']}", doc_type="api_record", title=title, published_at=requested),
],
topics=("Inneres und Recht",),
facet=Legal(stage=stage, offense=offense, authority=details["authority"], case_ref=details["case_ref"],
stage_date=requested),
)
# -- investigative-committee hearings ----------------------------------------------
def parse_hearing_item(self, ctx: RunContext, response: Response) -> Iterator[Any]:
content = response.json().get("content") or {}
title = html_to_text(content.get("title")) or response.meta["title"]
m = HEARING.search(title)
if not m:
return
witness = WITNESS_TITLES.sub("", strip_titles(m.group("name"))).strip()
published = parse_date(response.meta.get("date"))
protocol = next((d["link"] for group in content.get("documents") or [] for d in group.get("documents") or []
if d.get("type") == "HTML"), None)
url = f"{BASE}{protocol}" if protocol else response.request.url.split("?")[0]
event_key = fingerprint("hearing", response.request.url)
yield Event(key=event_key, kind="hearing", title=f"Befragung im Untersuchungsausschuss: {witness}",
starts_at=published, location="Parlament, Wien", country="AT", url=url,
description=title, body=OrgRef(slug="nationalrat"))
yield Claim(
claim_type="appearance", person=PersonRef(name=witness), occurred_at=published,
summary_de=f"Befragung als Auskunftsperson im Untersuchungsausschuss ({title})",
summary_en="Questioned as a witness by a parliamentary investigative committee",
data={"context": "investigative_committee"},
evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=published)],
topics=("Transparenz und Korruption",),
facet=Participation(event_key=event_key, participation="witness"),
)
+124
View File
@@ -0,0 +1,124 @@
"""Parliament press releases (Parlamentskorrespondenz, list 110) and calendar (Termine, list 600). CC BY 4.0.
Press releases are stored with full text; the people they link (/person/<id>) get a mention claim.
The event extractor later turns visits, receptions and trips described in them into meeting/travel
claims. The calendar fills the calendar view with sessions, committee meetings, hearings and events.
"""
from __future__ import annotations
import json
import re
from collections.abc import Iterator
from datetime import date, timedelta
from typing import Any
from politrack.core import Adapter, Request, Response, RunContext, register
from politrack.core.records import Claim, Document, Event, Evidence, OrgRef, PersonRef
from politrack.sources.parlament.common import BASE, CHAMBER_SLUG, SPEC, list_request, list_rows, parse_date
from politrack.text import excerpt, fingerprint, html_to_text
ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}")
SKIP_TERMIN = re.compile(r"Guided Tour|Führung|Galeriebesuch|Besuch einer Plenarsitzung|Demokratiebildung", re.I)
TERMIN_KIND = [
(re.compile(r"Plenarsitzung|Bundesversammlung|Fest-|Gedenksitzung", re.I), "parliament_session"),
(re.compile(r"Untersuchungsausschuss", re.I), "hearing"),
(re.compile(r"Ausschuss|Enquete", re.I), "committee"),
(re.compile(r"Pressekonferenz", re.I), "press_conference"),
(re.compile(r"Internationales|Konferenz", re.I), "meeting"),
]
def release_body(content: list) -> str:
"""The detail JSON is a list of render blocks; the article is the one wrapped in <div class=pk>."""
for block in content:
text = (block.get("data") or {}).get("text") if isinstance(block, dict) else None
if isinstance(text, str) and re.search(r"class=\"?pk\b", text):
return text
return ""
@register
class ParlamentPress(Adapter):
spec = SPEC
name = "parlament.press"
description = "Parlamentskorrespondenz: official press releases with linked people"
schedule = "5 */2 * * *"
def start(self, ctx: RunContext) -> Iterator[Request]:
last = ctx.state.get("last_date")
backfill = int(ctx.params.get("backfill_days", 365))
since = (date.fromisoformat(last) - timedelta(days=2)) if last else date.today() - timedelta(days=backfill)
yield list_request(110, {"DATERANGE": [since.isoformat(), date.today().isoformat()]}, kind="list",
meta={"since": since.isoformat()})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if response.request.kind == "list":
newest = response.meta["since"]
for row in list_rows(response.json()):
if not row.get("Link"):
continue
published = parse_date(row.get("Datum")) # "23.10.2026" (German) in list 110
if published:
newest = max(newest, published.isoformat())
yield Request(url=f"{BASE}{row['Link']}", params={"json": "TRUE"}, kind="release",
meta={k: row.get(k) for k in ("Titel", "Untertitel", "Datum", "Nr.", "Abothemen",
"Stichworte")})
if ctx.mode != "test":
ctx.state.set("last_date", newest)
return
yield from self.parse_release(response)
def parse_release(self, response: Response) -> Iterator[Any]:
meta = response.meta
url = response.request.url
body_html = release_body(response.json().get("content") or [])
pads = sorted({p for p in re.findall(r"/person/(\d+)", body_html)})
text = html_to_text(body_html)
title, subtitle = meta.get("Titel") or "", meta.get("Untertitel") or ""
topics_raw = meta.get("Abothemen")
topics = tuple(json.loads(topics_raw)) if isinstance(topics_raw, str) and topics_raw.startswith("[") else ()
published = parse_date(meta.get("Datum"))
yield Document(Evidence(url=url, doc_type="press_release", title=title, published_at=published,
excerpt=subtitle or excerpt(text, 300), full_text=f"{title}\n{subtitle}\n{text}",
meta={"pk_number": meta.get("Nr."), "person_pads": pads, "topics": list(topics)}))
for pad in pads:
yield Claim(
claim_type="media_mention", person=PersonRef(ids={"parlament": pad}), occurred_at=published,
summary_de=f"Parlamentskorrespondenz: {title}",
data={"context": "parlamentskorrespondenz"},
evidence=[Evidence(url=url, doc_type="press_release", title=title, published_at=published,
quote=subtitle or None, stance="mentions")],
topics=topics,
)
@register
class ParlamentCalendar(Adapter):
spec = SPEC
name = "parlament.calendar"
description = "Parliament calendar: plenary sessions, committees, hearings, events (tours skipped)"
schedule = "30 2 * * *"
def start(self, ctx: RunContext) -> Iterator[Request]:
yield list_request(600, {}, kind="list")
def parse(self, ctx: RunContext, response: Response) -> Iterator[Event]:
since = str(ctx.params.get("since_year", date.today().year - 2))
for row in list_rows(response.json()):
kind_label = (row.get("Terminart") or "").replace("<BR>", " ")
starts = next((v for k, v in row.items() if k.startswith("Datum") and k != "Datum Bis"
and isinstance(v, str) and ISO_DATE.match(v)), None)
if not starts or starts[:4] < since or SKIP_TERMIN.search(f"{kind_label} {row.get('Bezeichnung') or ''}"):
continue
title = row.get("Bezeichnung") or kind_label
kind = next((k for pattern, k in TERMIN_KIND if pattern.search(f"{kind_label} {title}")), "public_event")
chamber = {"Nationalrat": "NR", "Bundesrat": "BR"}.get(row.get("Gremium") or "")
ends = next((v for k, v in row.items() if k.startswith("Datum Bis") and isinstance(v, str)
and ISO_DATE.match(v)), None)
yield Event(
key=fingerprint("termin", row.get("WENTRY_ID") or row.get("Link")), kind=kind, title=title,
starts_at=starts, ends_at=ends, location=(row.get("Ort") or "").replace(" | ", ", ") or "Parlament, Wien",
country="AT", url=f"{BASE}{row['Link']}" if row.get("Link") else None, description=kind_label,
body=OrgRef(slug=CHAMBER_SLUG[chamber]) if chamber else None,
)
@@ -0,0 +1,239 @@
"""Parliament: people, mandates, government functions, side-job disclosures, social links, speeches.
Flow: list 409 (all MPs since 1918) -> PersonRecord + mandate roles
-> detail /person/<id> for members of the configured periods
detail -> photo, social accounts, functions, side jobs (§6 Unv-Transparenz-G)
-> list 251 (speeches of that person)
"""
from __future__ import annotations
import json
import re
from collections.abc import Iterator
from typing import Any
from politrack.config import get_settings
from politrack.core import Adapter, Request, Response, RunContext, register
from politrack.core.records import Claim, Evidence, OrgRef, PersonRecord, PersonRef, Role, SocialAccount, Statement
from politrack.sources.parlament.common import (
BASE,
CHAMBER_NAME_DE,
CHAMBER_SLUG,
GOVERNMENT_RE,
SPEC,
detail_request,
list_request,
list_rows,
parse_date,
walk_dicts,
)
from politrack.text import firmenbuch_number, html_to_text, strip_titles
SOCIAL_PLATFORMS = {"TWITTER": "twitter", "FACEBOOK": "facebook", "INSTAGRAM": "instagram", "TIKTOK": "tiktok",
"YOUTUBE": "youtube", "LINKEDIN": "linkedin", "WEBSITE": "website", "BLUESKY": "bluesky",
"MASTODON": "mastodon", "TELEGRAM": "telegram", "THREADS": "threads"}
FN_IN_TEXT = re.compile(r"\bFN\s*\d{1,6}\s?[a-zA-Z]\b")
POSTCODE_CITY = re.compile(r"(?:(?<=\d)|\b)\d{4}\s+[A-ZÄÖÜ][\wäöüß .\-]*$")
LEGAL_FORM_END = re.compile(r"(?i:gesmbh|gmbh|ges\.\s?m\.\s?b\.\s?h\.?|m\.\s?b\.\s?h\.?|aktiengesellschaft|\bag|\bkg|\bog|"
r"e\.\s?u\.|\bse|privatstiftung|genossenschaft|sparkasse)(?=[A-ZÄÖÜ\s,]|$)")
STREET_WORD = r"\S*(?:straße|strasse|str\.|gasse|platz|weg|kai|ring|allee|zeile|lände|markt|steig)"
def strip_street(text: str) -> str:
"""'Verein Musterhilfe Hauptstraße 12' -> 'Verein Musterhilfe' (greedy name, shortest street suffix)."""
m = (re.match(rf"^(.*\S)\s+{STREET_WORD}\s*\d+[a-z]?(?:/\d+)*\s*$", text, re.IGNORECASE)
or re.match(r"^(.*\S)\s+\S+\s*\d+[a-z]?(?:/\d+)*\s*$", text))
return m.group(1) if m else text
def employer(cell: str) -> tuple[str | None, str | None]:
"""Employer name without business address, plus its company register number.
'Hypo Tirol Bank AG, Meraner Straße 8, 6020 Innsbruck' -> ('Hypo Tirol Bank AG', None)
'25 PersonaldienstleistungsgesmbHFranz Josef Kai 95020 Salzburg FN306527a' (parliament's JSON glues
the lines together) -> ('25 PersonaldienstleistungsgesmbH', '306527a')
"""
cell = re.sub(r"^lit\.\s*\w\s*:\s*", "", cell).strip()
fn_match = FN_IN_TEXT.search(cell)
fn = firmenbuch_number(fn_match.group(0)) if fn_match else None
name = FN_IN_TEXT.sub("", cell).split(",")[0].strip(" *")
address = POSTCODE_CITY.search(name)
if address:
head = name[:address.start()]
ends = [m.end() for m in LEGAL_FORM_END.finditer(head)]
# cut after the last legal form; otherwise drop "<street> <number>" before the postcode
name = head[:ends[-1]] if ends else strip_street(head)
name = name.strip(" ,-*")
return (name if len(name) >= 3 and not re.fullmatch(r"[\d\s.€-]+", name) else None), fn
@register
class ParlamentPersons(Adapter):
spec = SPEC
name = "parlament.persons"
description = "Members of Nationalrat and Bundesrat since 1918, mandates, functions, side jobs, speeches"
schedule = "15 3 * * *"
creates_persons = True
parser_version = 3
def start(self, ctx: RunContext) -> Iterator[Request]:
yield list_request(409, {"NRBR": ["NR", "BR"]}, kind="persons")
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
kind = response.request.kind
if kind == "persons":
yield from self.parse_persons(ctx, response)
elif kind == "person":
yield from self.parse_person(ctx, response)
elif kind == "speeches":
yield from self.parse_speeches(ctx, response)
# -- persons & mandates -------------------------------------------------------------
def parse_persons(self, ctx: RunContext, response: Response) -> Iterator[Any]:
periods = set(ctx.params.get("periods") or get_settings().parlament_periods)
for row in list_rows(response.json()):
attr = row.get("Attribute") or {}
if isinstance(attr, str):
attr = json.loads(attr)
m = re.search(r"/person/(\d+)", attr.get("uri") or "")
if not m:
continue
pad = m.group(1)
ref = PersonRef(ids={"parlament": pad})
display = attr.get("zit") or strip_titles(row.get("Name") or "")
yield PersonRecord(ref=ref, full_name=display, given_name=row.get("vorname"), family_name=row.get("nachname"),
gender={"M": "male", "W": "female"}.get(row.get("GESCHL_CODE") or attr.get("geschlecht")))
mandates = attr.get("mandate_detail") or []
for mandate in mandates:
claim = self.mandate_claim(pad, display, mandate)
if claim:
yield claim
if any(md.get("gp_code") in periods for md in mandates):
yield detail_request(f"/person/{pad}", kind="person", meta={"pad": pad, "name": display})
def mandate_claim(self, pad: str, name: str, m: dict) -> Claim | None:
code = m.get("mand_code") or "NR"
if code not in CHAMBER_SLUG:
return None
party_code = m.get("wahlpartei_code")
party_name = m.get("wahlpartei_full_txt") or m.get("wahlpartei_txt") or party_code
party = OrgRef(name=party_name, short_name=party_code, kind="party") if party_name else None
start, end = parse_date(m.get("mandat_von")), parse_date(m.get("mandat_bis"))
gp = m.get("gp_code") or ""
region = m.get("wahlkreis_bundesland")
label = party_code or party_name or "?"
body = OrgRef(slug=CHAMBER_SLUG[code])
return Claim(
claim_type="role",
person=PersonRef(ids={"parlament": pad}),
org=body,
counterpart_org=party,
valid_from=start,
valid_to=end,
summary_de=f"{name}: Mandat im {CHAMBER_NAME_DE[code]} ({gp}. GP) für {label}"
+ (f", Wahlkreis {region}" if region else ""),
summary_en=f"{name}: member of the {CHAMBER_NAME_DE[code]} ({gp} legislative period) for {label}",
data={"gp": gp, "party_code": party_code, "faction_code": m.get("fraktionscode"), "region": region},
evidence=[Evidence(url=f"{BASE}/person/{pad}", doc_type="api_record", title=f"Parlament: {name}",
locator=f"mandat:{code}:{gp}:{start}")],
facet=Role(role="mp", level="federal", body=body, party=party, region=region, start=start, end=end),
)
# -- person detail ----------------------------------------------------------------
def parse_person(self, ctx: RunContext, response: Response) -> Iterator[Any]:
data = response.json()
content = data.get("content") or {}
pad, name = response.meta["pad"], response.meta["name"]
ref = PersonRef(ids={"parlament": pad})
url = f"{BASE}/person/{pad}"
photo = ((data.get("meta") or {}).get("openGraph") or {}).get("image")
if photo:
yield PersonRecord(ref=ref, full_name=name, photo_url=photo)
for account in (content.get("kontakt") or {}).get("socialMedia") or []:
platform = SOCIAL_PLATFORMS.get((account.get("type") or "").upper(), (account.get("type") or "website").lower())
yield SocialAccount(platform=platform, url=account["url"], person=ref, verified=True)
for fn in walk_dicts(content.get("biografie") or {}, "funktion_text"):
if fn.get("gremium") in CHAMBER_SLUG:
continue
text = fn.get("funktion_text") or fn.get("bez")
if not text:
continue
start, end = parse_date(fn.get("funktion_von")), parse_date(fn.get("funktion_bis"))
is_gov = bool(GOVERNMENT_RE.search(text))
body = OrgRef(slug="bundesregierung") if is_gov else None
yield Claim(
claim_type="role", person=ref, org=body, valid_from=start, valid_to=end,
summary_de=f"{name}: {text}",
evidence=[Evidence(url=url, doc_type="api_record", title=f"Parlament: {name}",
locator=f"funktion:{text}:{start}")],
facet=Role(role=text, level="federal" if is_gov else "other", body=body, start=start, end=end),
)
yield from self.side_jobs(ref, pad, name, content.get("unvtrans") or {})
yield list_request(251, {"PAD_INTERN": [int(pad)]}, kind="speeches", meta={"pad": pad, "name": name})
def side_jobs(self, ref: PersonRef, pad: str, name: str, unv: dict) -> Iterator[Claim]:
"""Disclosures under §6 Unv-Transparenz-G: leading positions, paid side jobs, honorary positions."""
tables = unv.get("unvtransData") or {}
pdf = next((d["link"] for group in unv.get("linkliste") or [] for d in group.get("documents") or []
if d.get("link", "").endswith(f"/person/{pad}/BezBegrBVGPar9.pdf")), None)
for key, table in tables.items():
m = re.fullmatch(r"(z\d+)Table", key)
if not m or not isinstance(table, dict):
continue
header = html_to_text(tables.get(f"{m.group(1)}Header")) or key
for row in (table.get("data") or {}).get("rows") or []:
cells = [c for c in (html_to_text(c) for c in row if isinstance(c, str)) if c]
if not cells:
continue
name_, fn = employer(cells[0]) if m.group(1) in ("z1", "z2", "z3") else (None, None)
if name_: # never keep the (business) address that parliament publishes with the employer
cells = [name_ + (f" (FN {fn})" if fn else ""), *cells[1:]]
text = " – ".join(cells)
yield Claim(
claim_type="side_job", person=ref, key=("unvtrans", pad, m.group(1), text),
counterpart_org=OrgRef(name=name_, kind="company" if LEGAL_FORM_END.search(name_) else "other",
ids={"firmenbuch": fn} if fn else {}) if name_ else None,
summary_de=f"{name} – {header.split(' - ')[0]}: {text}",
data={"category": header, "cells": cells, "unpaid": "*" in "".join(str(c) for c in row),
"employer": name_, "firmenbuch": fn},
evidence=[Evidence(url=f"{BASE}{pdf}" if pdf else f"{BASE}/person/{pad}", doc_type="register_entry",
title=f"Unvereinbarkeits-/Transparenzliste {name}", quote=text,
locator=f"unvtrans:{m.group(1)}")],
topics=("Transparenz und Korruption",),
)
# -- speeches -----------------------------------------------------------------------
def parse_speeches(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
periods = set(ctx.params.get("periods") or get_settings().parlament_periods)
pad, name = response.meta["pad"], response.meta["name"]
for row in list_rows(response.json()):
if row.get("GP") not in periods and row.get("GREMIUM") != "B":
continue
html_cells = " ".join(str(v) for v in row.values() if isinstance(v, str))
protocol = re.search(r'href="(/dokument/[^"]+)"', html_cells)
session = html_to_text(row.get("Sitzung"))
topic = html_to_text(row.get("Rede"))
said_at = row.get("fromDate")
url = f"{BASE}{protocol.group(1)}" if protocol else f"{BASE}/person/{pad}"
chamber = "Bundesrat" if row.get("GREMIUM") == "B" else "Nationalrat"
yield Claim(
claim_type="statement", person=PersonRef(ids={"parlament": pad}),
key=("parl-speech", pad, row.get("Sortier"), topic), occurred_at=said_at,
summary_de=f"{name} sprach im {chamber} zu: {topic}",
summary_en=f"{name} spoke in the {chamber} on: {topic}",
data={"session": session, "gp": row.get("GP"), "item_title": topic},
evidence=[Evidence(url=url, doc_type="protocol", title=session, published_at=parse_date(said_at),
locator=url.split("#", 1)[-1] if "#" in url else "")],
facet=Statement(context="parliament_speech", url=url, said_at=said_at, medium=chamber, title=topic),
)
+145
View File
@@ -0,0 +1,145 @@
"""Political advertising on online platforms.
* Google Political Ads Transparency Report - public download bundle (zip with CSVs, ~300 MB). We keep
ads served in Austria (Regions contains "AT"): advertiser, period, impressions bracket, spend range.
Note: Google stopped serving political ads in the EU in October 2025 (EU TTPA regulation), so this
is historical data (2019-2025).
* Meta Ad Library API (political and issue ads reaching Austria) - requires an identity-verified developer
token (POLITRACK_META_AD_LIBRARY_TOKEN) and page ids in config/meta_pages.yaml; skipped otherwise.
Advertisers are resolved to parties/organisations by name where possible (e.g. "FPÖ", "SPÖ Wien").
"""
from __future__ import annotations
import csv
import io
import zipfile
from collections.abc import Iterator
from datetime import date
from typing import Any
import yaml
from politrack.config import get_settings
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Evidence, Money, OrgRef
from politrack.text import format_eur_de
GOOGLE_BUNDLE = "https://storage.googleapis.com/political-csv/google-political-ads-transparency-bundle.zip"
GOOGLE_SPEC = SourceSpec(
slug="google-political-ads", name="Google Political Ads Transparency Report", kind="aggregator",
homepage="https://adstransparency.google.com/political", license="Google transparency data (public download)",
access="bulk", trust=85, keep_raw=False, min_interval=5.0,
tos_notes="Public CSV bundle of political ads. ~300 MB; fetched monthly with conditional requests.",
)
META_SPEC = SourceSpec(
slug="meta-adlibrary", name="Meta Ad Library", kind="social", homepage="https://www.facebook.com/ads/library/api",
license="Meta Platform Terms", access="api", trust=80, min_interval=2.0,
tos_notes="Official Ad Library API for political and issue ads (identity-verified developer token required).",
)
META_API = "https://graph.facebook.com/v21.0/ads_archive"
def col(row: dict[str, str], *names: str) -> str | None:
lowered = {k.lower(): v for k, v in row.items()}
for name in names:
if lowered.get(name.lower()) not in (None, ""):
return lowered[name.lower()]
return None
@register
class GooglePoliticalAds(Adapter):
spec = GOOGLE_SPEC
name = "ads.google"
description = "Political ads served in Austria on Google/YouTube (historical, until 10/2025)"
schedule = "0 3 1 * *"
queue = "bulk"
def start(self, ctx: RunContext) -> Iterator[Request]:
yield Request(url=GOOGLE_BUNDLE, kind="bundle")
def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
since = ctx.params.get("since") or get_settings().ad_archive_since
with zipfile.ZipFile(io.BytesIO(response.body)) as bundle:
name = next((n for n in bundle.namelist() if n.endswith("creative-stats.csv")), None)
if name is None:
ctx.stats["creative_stats_missing"] += 1
return
with bundle.open(name) as fh:
for row in csv.DictReader(io.TextIOWrapper(fh, encoding="utf-8", errors="replace")):
regions = (col(row, "Regions") or "").replace(" ", "").split(",")
if "AT" not in regions:
continue
start = col(row, "Date_Range_Start") or ""
if start < since:
continue
advertiser = col(row, "Advertiser_Name") or "?"
low, high = col(row, "Spend_Range_Min_EUR"), col(row, "Spend_Range_Max_EUR")
ad_url = col(row, "Ad_URL") or GOOGLE_BUNDLE
payer = OrgRef(name=advertiser, kind="other", ids={"google_advertiser": col(row, "Advertiser_ID") or ""})
spend = f"{format_eur_de(low)}–{format_eur_de(high)} €" if low and high else None
yield Claim(
claim_type="money_flow", org=payer, occurred_at=start,
summary_de=f"Politische Google-Werbung von {advertiser} ({start} – {col(row, 'Date_Range_End')})"
+ (f", Ausgaben {spend}" if spend else ""),
summary_en=f"Political Google ad by {advertiser} ({start} – {col(row, 'Date_Range_End')})",
data={"impressions": col(row, "Impressions"), "ad_type": col(row, "Ad_Type"),
"age_targeting": col(row, "Age_Targeting"), "gender_targeting": col(row, "Gender_Targeting"),
"geo_included": col(row, "Geo_Targeting_Included")},
evidence=[Evidence(url=ad_url, doc_type="dataset_row", title=f"Google Ads Transparency: {advertiser}",
published_at=start)],
facet=Money(kind="political_ad", payer=payer, payer_name=advertiser,
payee=OrgRef(name="Google", kind="company"), payee_name="Google",
paid_on=date.fromisoformat(start[:10]) if start else None,
amount_category=spend, reference=col(row, "Ad_ID"), category=col(row, "Ad_Type")),
)
@register
class MetaPoliticalAds(Adapter):
spec = META_SPEC
name = "ads.meta"
description = "Political/issue ads reaching Austria from Meta's Ad Library (token + page list required)"
schedule = "0 4 * * 4"
queue = "bulk"
FIELDS = ("id,page_id,page_name,bylines,ad_delivery_start_time,ad_delivery_stop_time,spend,impressions,"
"currency,ad_snapshot_url,ad_creative_bodies")
def start(self, ctx: RunContext) -> Iterator[Request]:
token = get_settings().meta_ad_library_token
path = get_settings().config_dir / "meta_pages.yaml"
if not token or not path.exists():
ctx.stats["skipped_not_configured"] = 1
return
pages = [str(p["page_id"]) for p in (yaml.safe_load(path.read_text(encoding="utf-8")) or {}).get("pages", [])]
for i in range(0, len(pages), 10):
yield Request(url=META_API, kind="ads", conditional=False, params={
"access_token": token, "ad_reached_countries": '["AT"]', "ad_type": "POLITICAL_AND_ISSUE_ADS",
"search_page_ids": ",".join(pages[i:i + 10]), "fields": self.FIELDS, "limit": 250,
"ad_delivery_date_min": ctx.params.get("since") or get_settings().ad_archive_since})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
data = response.json()
next_url = (data.get("paging") or {}).get("next")
if next_url:
yield Request(url=next_url, kind="ads", conditional=False)
for ad in data.get("data") or []:
spend = ad.get("spend") or {}
payer_name = ad.get("bylines") or ad.get("page_name") or "?"
payer = OrgRef(name=ad.get("page_name") or payer_name, kind="other", ids={"facebook_page": ad.get("page_id")})
rng = f"{spend.get('lower_bound')}–{spend.get('upper_bound')} {ad.get('currency') or 'EUR'}" if spend else None
start = (ad.get("ad_delivery_start_time") or "")[:10]
yield Claim(
claim_type="money_flow", org=payer, occurred_at=start or None,
summary_de=f"Politische Meta-Werbung von {payer_name} ab {start}" + (f", Ausgaben {rng}" if rng else ""),
data={"impressions": ad.get("impressions"), "bodies": (ad.get("ad_creative_bodies") or [])[:2]},
evidence=[Evidence(url=ad.get("ad_snapshot_url") or f"https://www.facebook.com/ads/library/?id={ad['id']}",
doc_type="dataset_row", title=f"Meta Ad Library: {payer_name}", published_at=start or None)],
facet=Money(kind="political_ad", payer=payer, payer_name=payer_name,
payee=OrgRef(name="Meta Platforms", kind="company"), payee_name="Meta",
paid_on=date.fromisoformat(start) if start else None, amount_category=rng,
reference=ad.get("id")),
)
+254
View File
@@ -0,0 +1,254 @@
"""Public procurement: contract award notices published as open "core data" (Kerndaten, §62/64 BVergG 2018).
Every contracting authority (or the platform it uses) publishes a "Kerndatenquelle" (KDQ): an XML index
<kdq><item id lastmod><url>…</url></item>…</kdq> listed as a dataset on data.gv.at. Each item is one
notice; since 2023/24 notices use the EU eForms UBL schema. We keep ContractAwardNotice documents
(who bought what from whom, for how much) and skip calls for tenders.
Correlations this enables: suppliers that are also party donors, sponsors or side-job employers of
politicians; contracts by ministries led by a given minister (organisation pages).
Volume: tens of thousands of notices; first run backfills from POLITRACK_PROCUREMENT_SINCE, later runs only
fetch items whose lastmod changed. Runs on the `bulk` queue.
"""
from __future__ import annotations
import xml.etree.ElementTree as ET
from collections.abc import Iterator
from datetime import date
from typing import Any
from politrack.config import get_settings
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Evidence, Money, OrgRef
from politrack.text import firmenbuch_number, format_eur_de
HUB = "https://www.data.gv.at/api/hub/search/search"
SPEC = SourceSpec(
slug="vergabe", name="Öffentliche Aufträge (Kerndaten, data.gv.at)", kind="official",
homepage="https://www.data.gv.at", license="Open data (free licence required by §64 BVergG 2018)", access="bulk",
trust=90, min_interval=1.0,
tos_notes="Contracting authorities must publish award core data as open, machine-readable data under a free "
"licence (BVergG 2018). We read each publisher's Kerndatenquelle (KDQ) XML and its eForms notices.",
)
NS = {
"cbc": "urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2",
"cac": "urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2",
"efac": "http://data.europa.eu/p27/eforms-ubl-extension-aggregate-components/1",
"efbc": "http://data.europa.eu/p27/eforms-ubl-extension-basic-components/1",
"efext": "http://data.europa.eu/p27/eforms-ubl-extensions/1",
"ext": "urn:oasis:names:specification:ubl:schema:xsd:CommonExtensionComponents-2",
}
CATALOG_QUERIES = ("Kerndaten", "Kerndatenquelle", "KDQ BVergG")
def local(tag: str) -> str:
return tag.rsplit("}", 1)[-1]
def text(node: ET.Element | None, path: str) -> str | None:
if node is None:
return None
found = node.find(path, NS)
return found.text.strip() if found is not None and found.text else None
def parse_kerndaten(root: ET.Element) -> dict[str, Any] | None:
"""Austrian core-data format (BVergG 2018, schema brz.gv.at/eproc/kd): KD_8_2_* = awarded contracts.
Still used by many publishers (e-shops, below-threshold awards). Contractor NATIONALID is usually the
company register number ("183286p"), occasionally the VAT id.
"""
if not local(root.tag).startswith("KD_8_2"):
return None
def t(node: ET.Element | None, path: str) -> str | None:
found = node.find(path) if node is not None else None
return found.text.strip() if found is not None and found.text else None
buyer = root.find("{*}CONTRACTING_BODY/{*}ADDRESS_CONTRACTING_BODY")
title = " ".join(p.text.strip() for p in root.iterfind("{*}OBJECT_CONTRACT/{*}TITLE/{*}P") if p.text)
cpv = root.find("{*}OBJECT_CONTRACT/{*}CPV_MAIN/{*}CPV_CODE")
contracts = []
for awarded in root.iterfind(".//{*}AWARD_CONTRACT/{*}AWARDED_CONTRACT"):
contracts.append({
"id": None,
"title": None,
"awarded": t(awarded, "{*}DATE_CONCLUSION_CONTRACT"),
"amount": t(awarded, "{*}VAL_TOTAL"),
"winners": [{"name": t(c, "{*}ADDRESS_CONTRACTOR/{*}OFFICIALNAME"),
"company_id": t(c, "{*}ADDRESS_CONTRACTOR/{*}NATIONALID")}
for c in awarded.iterfind("{*}CONTRACTOR")],
})
return {
"notice_id": t(root, "{*}EXTENSION[@KEY='Referenz']") or t(root, "{*}OBJECT_CONTRACT/{*}REFERENCE_NUMBER"),
"issued": None,
"title": title or None,
"cpv": cpv.get("CODE") if cpv is not None else None,
"total": None,
"buyers": [{"name": t(buyer, "{*}OFFICIALNAME"), "company_id": None}] if buyer is not None else [],
"contracts": contracts,
}
def parse_award_notice(xml: bytes) -> dict[str, Any] | None:
"""Extract buyer, winners, amounts, title, CPV and dates from an award notice: eForms
ContractAwardNotice, or the Austrian core-data format (parse_kerndaten)."""
root = ET.fromstring(xml)
if local(root.tag).startswith("KD_"):
return parse_kerndaten(root)
if local(root.tag) != "ContractAwardNotice":
return None
orgs: dict[str, dict[str, Any]] = {}
for org in root.iterfind(".//efac:Organizations/efac:Organization/efac:Company", NS):
org_id = text(org, "cac:PartyIdentification/cbc:ID")
if org_id:
orgs[org_id] = {"name": text(org, "cac:PartyName/cbc:Name"),
"company_id": text(org, "cac:PartyLegalEntity/cbc:CompanyID"),
"city": text(org, "cac:PostalAddress/cbc:CityName")}
buyer_ids = [e.text for e in root.iterfind("cac:ContractingParty/cac:Party/cac:PartyIdentification/cbc:ID", NS)
if e.text]
result = root.find(".//efac:NoticeResult", NS)
tender_amounts = {text(t, "cbc:ID"): text(t, "cac:LegalMonetaryTotal/cbc:PayableAmount")
for t in (result.iterfind("efac:LotTender", NS) if result is not None else [])}
tendering_parties = {text(tp, "cbc:ID"): [e.text for e in tp.iterfind("efac:Tenderer/cbc:ID", NS) if e.text]
for tp in (result.iterfind("efac:TenderingParty", NS) if result is not None else [])}
lot_tender_party = {text(t, "cbc:ID"): text(t, "efac:TenderingParty/cbc:ID")
for t in (result.iterfind("efac:LotTender", NS) if result is not None else [])}
contracts = []
for contract in (result.iterfind("efac:SettledContract", NS) if result is not None else []):
tender_id = text(contract, "efac:LotTender/cbc:ID")
winners = tendering_parties.get(lot_tender_party.get(tender_id), [])
contracts.append({
"id": text(contract, "cbc:ID"),
"title": text(contract, "cbc:Title"),
"awarded": text(contract, "cbc:AwardDate") or text(contract, "cbc:IssueDate"),
"amount": tender_amounts.get(tender_id),
"winners": [orgs.get(w, {"name": None}) | {"ref": w} for w in winners],
})
project_title = text(root, "cac:ProcurementProject/cbc:Name")
return {
"notice_id": text(root, "cbc:ID"),
"issued": text(root, "cbc:IssueDate"),
"title": project_title,
"cpv": text(root, "cac:ProcurementProject/cac:MainCommodityClassification/cbc:ItemClassificationCode"),
"total": text(root, ".//efac:NoticeResult/cbc:TotalAmount"),
"buyers": [orgs.get(b, {"name": None}) | {"ref": b} for b in buyer_ids],
"contracts": contracts,
}
def org_ref(org: dict[str, Any], kind: str) -> OrgRef | None:
if not org.get("name"):
return None
company_id = (org.get("company_id") or "").replace(" ", "")
ids = {}
if company_id.upper().startswith("ATU"):
ids["uid"] = company_id.upper()
elif fn := firmenbuch_number(company_id):
ids["firmenbuch"] = fn
return OrgRef(name=org["name"], kind=kind, ids=ids)
def as_date(value: str | None) -> date | None:
if not value:
return None
try:
return date.fromisoformat(value[:10])
except ValueError:
return None
@register
class PublicContracts(Adapter):
spec = SPEC
name = "vergabe.awards"
description = "Contract awards of Austrian public bodies (eForms and BVergG core data via data.gv.at)"
schedule = "0 2 * * *"
queue = "bulk"
parser_version = 2
def start(self, ctx: RunContext) -> Iterator[Request]:
for query in CATALOG_QUERIES:
yield Request(url=HUB, params={"q": query, "limit": 100, "page": 0}, kind="catalog", conditional=False)
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
kind = response.request.kind
if kind == "catalog":
result = response.json().get("result") or {}
for dataset in result.get("results") or []:
for dist in dataset.get("distributions") or []:
url = (dist.get("access_url") or dist.get("download_url") or [None])
url = url[0] if isinstance(url, list) else url
if url and url.startswith("http") and "data.gv.at/katalog" not in url:
if ctx.params.get("only_url") and ctx.params["only_url"] not in url:
continue # targeted runs: -p only_url=bbg.gv.at
title = (dataset.get("title") or {}).get("de") or ""
yield Request(url=url, kind="kdq", meta={"publisher": title.split("von", 1)[-1].strip(" '")})
if result.get("count", 0) > (response.request.params["page"] + 1) * 100:
yield Request(url=HUB, params={**response.request.params, "page": response.request.params["page"] + 1},
kind="catalog", conditional=False)
elif kind == "kdq":
yield from self.parse_kdq(ctx, response)
elif kind == "notice":
yield from self.parse_notice(ctx, response)
def parse_kdq(self, ctx: RunContext, response: Response) -> Iterator[Request]:
try:
root = ET.fromstring(response.body)
except ET.ParseError:
ctx.stats["kdq_not_xml"] += 1
return
state_key = f"kdq:{response.request.url}"
since = ctx.params.get("since") or ctx.state.get(state_key) or get_settings().procurement_since
limit = int(ctx.params.get("max_notices_per_source", 3000))
items = [(item.get("lastmod") or "", item.findtext("{*}url") or item.findtext("url"))
for item in root.iter() if local(item.tag) == "item"]
# Oldest first so an interrupted backfill resumes where it stopped (cursor = newest lastmod handled).
items = sorted(i for i in items if i[1] and i[0] > since)[:limit]
for lastmod, url in items:
# lastmod in meta (not in the URL) + conditional requests: unchanged notices cost a 304 at most.
# The cursor advances only when a notice has been parsed (see advance_cursor), so an interrupted
# run resumes with the first notice it did not get to.
yield Request(url=url.strip(), kind="notice", meta={"publisher": response.meta.get("publisher"),
"lastmod": lastmod, "state_key": state_key})
@staticmethod
def advance_cursor(ctx: RunContext, response: Response) -> None:
key, lastmod = response.meta.get("state_key"), response.meta.get("lastmod")
if key and lastmod and lastmod > (ctx.state.get(key) or ""):
ctx.state.set(key, lastmod)
def parse_notice(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
self.advance_cursor(ctx, response) # notices arrive oldest-first per publisher
try:
notice = parse_award_notice(response.body)
except ET.ParseError:
ctx.stats["notice_not_xml"] += 1
return
if notice is None:
ctx.stats["not_an_award"] += 1
return
buyer = next((org_ref(b, "public_body") for b in notice["buyers"] if b.get("name")), None)
if buyer is None:
return
for contract in notice["contracts"]:
for winner in contract["winners"]:
supplier = org_ref(winner, "company")
if supplier is None:
continue
amount = contract["amount"] or (notice["total"] if len(notice["contracts"]) == 1 else None)
awarded = as_date(contract["awarded"]) or as_date(notice["issued"])
title = contract["title"] or notice["title"] or "Auftrag"
yield Claim(
claim_type="money_flow", org=buyer, counterpart_org=supplier, occurred_at=awarded,
summary_de=f"Auftrag „{title}“ an {supplier.name}" + (f" – {format_eur_de(amount)} €" if amount else ""),
summary_en=f"Contract \"{title}\" awarded to {supplier.name}" + (f" – EUR {amount}" if amount else ""),
data={"notice_id": notice["notice_id"], "cpv": notice["cpv"], "publisher": response.meta.get("publisher")},
evidence=[Evidence(url=response.request.url, doc_type="register_entry", title=f"Vergabe: {title}",
published_at=awarded, quote=f"{buyer.name} → {supplier.name}: {title}")],
facet=Money(kind="contract", amount=amount, payer=buyer, payer_name=buyer.name, payee=supplier,
payee_name=supplier.name, paid_on=awarded,
reference=f"{notice['notice_id']}/{contract['id']}", category=notice["cpv"]),
)
+551
View File
@@ -0,0 +1,551 @@
"""Rechnungshof (Court of Audit): published party donations and parties' annual financial reports.
Official works (§7 UrhG); no robots restrictions.
Donations: HTML pages + one CSV per year/election. The CSVs come in UTF-8, Windows-1252 and Mac Roman,
with or without title rows, and one older file without header (date; amount; donor; address; party).
Annual reports (Rechenschaftsberichte, 2023+ as XLSX): income statement per organisation level
(public funding "Fördermittel", membership fees, donations, sponsoring, ad income...) and named lists:
donations > EUR 500, sponsors, advertisers in party media, party-owned companies, loans, affiliated
organisations.
Data minimisation: donor addresses and postcodes are never mapped; private donors are stored as names
only (no person entity is created); company donors become organisation entities (contract/donor correlation).
"""
from __future__ import annotations
import csv
import io
import re
from collections.abc import Iterator
from datetime import date, datetime
from decimal import Decimal, InvalidOperation
from typing import Any
from urllib.parse import urljoin
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Document, Evidence, Money, OrgRef
from politrack.text import firmenbuch_number, format_eur_de, normalize, normalize_org_name
BASE = "https://www.rechnungshof.gv.at"
SPEC = SourceSpec(
slug="rechnungshof",
name="Rechnungshof Österreich",
kind="official",
homepage=BASE,
license="Amtliches Werk (§7 UrhG)",
access="scrape",
trust=95,
min_interval=2.0,
tos_notes="Party donation lists and audited annual party reports. Official works; no robots restrictions.",
)
DONATION_INDEX = [
"/rh/home/was-wir-tun/was-wir-tun_5/Kontrolle_der_Parteien.html",
"/rh/home/was-wir-tun/was-wir-tun_5/was-wir-tun_5/Parteispenden/Parteispenden_2024.html",
]
REPORTS_INDEX = "/rh/home/was-wir-tun/was-wir-tun_5/was-wir-tun_6/Kontrolle_der_Parteien.html"
SECTION_PREFIX = "/rh/home/was-wir-tun/"
DONATION_LINK = re.compile(r"spende", re.IGNORECASE)
PARTY_PREFIXES = {"ovp": "oevp", "osterreichische volkspartei": "oevp", "die neue volkspartei": "oevp",
"spo": "spoe", "sozialdemokratische": "spoe", "fpo": "fpoe", "freiheitliche": "fpoe",
"die freiheitliche": "fpoe", "neos": "neos", "die grunen": "gruene", "grune": "gruene",
"kpo": "kpoe", "kommunistische": "kpoe", "bier": "bier", "die bierpartei": "bier", "mfg": "mfg"}
COMPANY = re.compile(r"\b(GmbH|AG|KG|OG|e\.U\.|GesmbH|Ges\.m\.b\.H|Gesellschaft|Verein|Stiftung|Bank|Holding|"
r"Genossenschaft|Verband|Kammer|Sparkasse|Versicherung|Privatstiftung)\b", re.IGNORECASE)
# -- shared helpers -----------------------------------------------------------------------
def decode(raw: bytes) -> str:
"""UTF-8, Windows-1252 or Mac Roman: pick the decoding that yields real umlauts."""
try:
return raw.decode("utf-8-sig")
except UnicodeDecodeError:
pass
candidates = [raw.decode(enc, errors="replace") for enc in ("cp1252", "mac_roman")]
return max(candidates, key=lambda t: sum(t.count(c) for c in "äöüÄÖÜß"))
def parse_amount(value: Any) -> Decimal | None:
if isinstance(value, (int, float, Decimal)):
return Decimal(str(value)).quantize(Decimal("0.01"))
value = (value or "").strip().replace(".", "").replace(",", ".").replace("€", "").strip()
try:
return Decimal(value)
except InvalidOperation:
return None
def parse_de_date(value: str) -> date | None:
for fmt in ("%d.%m.%Y", "%d.%m.%y"):
try:
return datetime.strptime((value or "").strip(), fmt).date()
except ValueError:
continue
return None
def format_de(amount: Decimal) -> str:
return format_eur_de(amount)
POSTAL_ADDRESS = re.compile(r"^\d{4}\s")
def clean_name(value: str) -> str:
"""Guard against malformed cells ('"2522 Ort Straße 4;Partei"') - an address never ends up in a name."""
value = (value or "").split(";")[-1].strip()
return "" if POSTAL_ADDRESS.match(value) else value
def party_ref(label: str) -> OrgRef | None:
if not label:
return None
norm = normalize(label)
for prefix, slug in PARTY_PREFIXES.items():
if norm.startswith(prefix):
return OrgRef(slug=slug)
return OrgRef(name=label.strip(), kind="party")
def donor_ref(name: str) -> OrgRef | None:
"""Organisations become entities (for correlations); private persons stay a plain name."""
return OrgRef(name=name, kind="company") if COMPANY.search(name) else None
def column(header: list[str], *needles: str) -> int | None:
for i, h in enumerate(header):
h = normalize(h)
if any(n in h for n in needles):
return i
return None
DATE_CELL = re.compile(r"^\d{1,2}\.\d{1,2}\.\d{2,4}$")
AMOUNT_CELL = re.compile(r"^[\d.]+(,\d{1,2})?$")
def find_columns(rows: list[list[str]]) -> tuple[int, dict[str, int | None]]:
"""Locate the header row (older files have a title row above it) and map the columns we use.
Address/postcode columns are never mapped."""
for i, row in enumerate(rows[:5]):
norm = [normalize(c) for c in row]
if any("betrag" in c for c in norm) and any("name" in c or "spender" in c for c in norm):
return i + 1, {
"party": column(row, "partei"),
"date": column(row, "eingang") if column(row, "eingang") is not None else column(row, "datum"),
"donor": column(row, "name", "spender"),
"amount": column(row, "betrag"),
"currency": column(row, "wahrung", "waehrung"),
"recipient": column(row, "empfang", "empfaeng"),
"note": column(row, "anmerk"),
}
first = next((r for r in rows if any(c.strip() for c in r)), [])
date_i = next((i for i, c in enumerate(first) if DATE_CELL.match(c.strip())), None)
amount_i = next((i for i, c in enumerate(first) if AMOUNT_CELL.match(c.strip()) and i != date_i), None)
if date_i is None or amount_i is None:
return 0, {}
return 0, {"party": len(first) - 1, "date": date_i, "donor": amount_i + 1, "amount": amount_i,
"currency": None, "recipient": len(first) - 1, "note": None}
def read_rows(text: str) -> tuple[dict[str, int | None], list[list[str]]]:
"""Parse the semicolon CSV; repairs rows broken by stray line breaks."""
reader = list(csv.reader(io.StringIO(text), delimiter=";"))
if not reader:
return {}, []
start, idx = find_columns(reader)
if not idx:
return {}, []
width = max(i for i in idx.values() if i is not None) + 1
rows, pending = [], []
for row in reader[start:]:
if pending:
pending[-1] += " " + (row[0] if row else "")
pending.extend(row[1:])
else:
pending = list(row)
if len(pending) >= width - 1:
rows.append(pending)
pending = []
return idx, [r for r in rows if any(c.strip() for c in r)]
# -- donations ----------------------------------------------------------------------------
@register
class RechnungshofDonations(Adapter):
spec = SPEC
name = "rechnungshof.donations"
description = "Published party donations (CSV per year / election)"
schedule = "30 5 * * *"
def start(self, ctx: RunContext) -> Iterator[Request]:
for path in DONATION_INDEX:
yield Request(url=urljoin(BASE, path), kind="page", meta={"depth": 0})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if response.request.kind == "page":
depth = response.meta.get("depth", 0)
for a in response.soup.find_all("a", href=True):
href = urljoin(response.url, a["href"])
path = href.removeprefix(BASE)
if href.lower().endswith(".csv") and DONATION_LINK.search(href):
yield Request(url=href, kind="csv", meta={"page": response.url})
elif (depth < 2 and path.startswith(SECTION_PREFIX) and path.endswith(".html")
and DONATION_LINK.search(path)):
yield Request(url=href, kind="page", meta={"depth": depth + 1})
return
yield from self.parse_csv(ctx, response)
def parse_csv(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
idx, rows = read_rows(decode(response.body))
if idx.get("donor") is None or idx.get("amount") is None:
ctx.stats["unparsed_files"] += 1
return
def cell(row: list[str], key: str) -> str:
i = idx[key]
value = row[i].strip() if i is not None and i < len(row) else ""
return clean_name(value) if key in ("party", "donor", "recipient") else value
for n, row in enumerate(rows):
party_label, donor, recipient = cell(row, "party"), cell(row, "donor"), cell(row, "recipient")
amount, received = parse_amount(cell(row, "amount")), parse_de_date(cell(row, "date"))
if not donor or amount is None:
continue
currency = cell(row, "currency") or "EUR"
currency = "EUR" if currency.lower().startswith("eur") else currency
party = party_ref(party_label) or party_ref(recipient)
line = " | ".join(c for c in (cell(row, "date"), party_label, donor, cell(row, "amount"), currency,
recipient) if c)
yield Claim(
claim_type="donation", org=party, occurred_at=received,
summary_de=f"{recipient or party_label} erhielt {format_de(amount)} {currency} von {donor}",
summary_en=f"{recipient or party_label} received {currency} {amount:,.2f} from {donor}",
data={"party_label": party_label, "recipient": recipient, "note": cell(row, "note")},
evidence=[
Evidence(url=response.meta["page"], doc_type="register_entry",
title="Rechnungshof: veröffentlichte Parteispenden", quote=line, locator=f"row:{line}"),
Evidence(url=response.request.url, doc_type="register_entry",
title="Rechnungshof: Parteispenden (CSV)", locator=f"row:{n + 1}"),
],
topics=("Transparenz und Korruption",),
facet=Money(kind="donation", amount=amount, currency=currency, payer=donor_ref(donor), payer_name=donor,
payee=party, payee_name=recipient or party_label, paid_on=received),
)
# -- annual party reports -------------------------------------------------------------------
# Lines of the income statement (§ 5 Abs. 4/5 PartG), matched on the normalised label. Order matters:
# the first matching pattern wins ("Inserate und Werbeeinschaltungen" is an expense, "Inserate" income).
FINANCE_LINES = [
(re.compile(r"^(gesamt)?summe (der )?ertrage"), "total_income"),
(re.compile(r"^(gesamt)?summe (der )?aufwendungen"), "total_expenses"),
(re.compile(r"^fordermittel"), "public_funding"),
(re.compile(r"^mitgliedsbeitrage"), "membership_fees"),
(re.compile(r"^beitrage der .*angehorenden mandatare"), "mandate_holder_levies"),
(re.compile(r"^geldspenden"), "donations"),
(re.compile(r"^sponsoring"), "sponsoring"),
(re.compile(r"^inserate und werbeeinschaltungen"), "ad_spending"),
(re.compile(r"^inserate\b"), "ad_income"),
(re.compile(r"^ertrage aus nahestehenden organisationen"), "from_affiliates"),
(re.compile(r"^personalaufwand"), "staff_costs"),
(re.compile(r"^au(ss|ß)enwerbung"), "outdoor_advertising"),
(re.compile(r"^direktwerbung"), "direct_marketing"),
]
def finance_lines(rows: list[list[Any]], year: str) -> dict[str, Decimal]:
"""Income/expense lines of one sheet, summed over all blocks in it.
Some parties stack one block per organisation in a sheet (e.g. every state below the other); each block
has its own header row with the years. The value is taken from the reporting-year column of the latest
header (columns are "2024, 2023" in some files and "2023, 2024" in others); without a header, the first
number after the label.
"""
total: dict[str, Decimal] = {}
block: dict[str, Decimal] = {}
year_col: int | None = None
for row in rows:
years = [i for i, c in enumerate(row) if str(c).strip() in (year, f"{year}.0")]
if years and sum(1 for c in row if re.fullmatch(r"(19|20)\d\d(\.0)?", str(c).strip() if c is not None else "")) >= 1:
year_col = years[0]
for i, cell in enumerate(row):
if not isinstance(cell, str) or len(cell.strip()) < 4:
continue
label = re.sub(r"\s+", " ", normalize(cell))
key = next((k for pattern, k in FINANCE_LINES if pattern.search(label)), None)
if key:
value = number(row[year_col]) if year_col is not None and year_col > i and year_col < len(row) else None
if value is None:
value = next((number(c) for c in row[i + 1:] if number(c) is not None), None)
if value is not None:
if key in block: # the same line again: a new block (next organisation) starts
for k, v in block.items():
total[k] = total.get(k, Decimal(0)) + v
block = {}
block[key] = value
break # only the first text cell of a row is its label
for k, v in block.items():
total[k] = total.get(k, Decimal(0)) + v
return total
def sum_levels(levels: dict[str, dict[str, Decimal]]) -> tuple[dict[str, Decimal], dict[str, dict[str, Decimal]]]:
"""Sum all organisation levels (federal, states, capitals, districts ...) -> (totals, levels used).
Overview sheets that add up other levels are left out: titles listing several levels ("BO&LOs&LH-St.&NTG")
and sheets whose public funding equals the sum of all the others. Totals (income, expenses) are only
reported when every level states them - a partial sum would be misleading.
"""
levels = {t: v for t, v in levels.items() if "&" not in t}
funding = {title: v.get("public_funding", Decimal(0)) for title, v in levels.items()}
for title, value in funding.items():
others = sum(v for t, v in funding.items() if t != title)
if value > 0 and len(funding) > 2 and abs(value - others) <= Decimal(1):
levels = {t: v for t, v in levels.items() if t != title}
break
totals: dict[str, Decimal] = {}
for values in levels.values():
for key, value in values.items():
totals[key] = totals.get(key, Decimal(0)) + value
for key in ("total_income", "total_expenses"):
if key in totals and not all(key in v for v in levels.values()):
del totals[key]
return totals, levels
AFFILIATE_SKIP = re.compile(
r"^(anlage|liste|ertrag|gem\.|gemass|\(?§|name$|zvr|reg\.? ?nr|organisationseinheit|nahestehende|"
r"bundesorganisation|landesorganisation|zweigvereine:?$|[a-z]\)|[a-z]\. anlage|keine|leermeldung|rechtsform|zweck)")
ZVR = re.compile(r"\(?ZVR(?:-Zahl)?:?\s*(\d{6,10})\)?", re.IGNORECASE)
def affiliate_from_row(row: list[Any]) -> tuple[str, str | None] | None:
"""Name (and association-register number) of an affiliated organisation from one row of the list."""
cells = [str(c).strip() for c in row if c is not None and str(c).strip()]
number_cell = next((c for c in cells if re.fullmatch(r"\d{6,10}", c)), None)
for cell in cells:
if re.fullmatch(r"[\d\s./-]+", cell) or cell.lower() in ("partei", "verein", "gmbh", "stiftung"):
continue
if AFFILIATE_SKIP.search(normalize(cell)) or len(cell) <= 3:
return None
zvr = ZVR.search(cell)
name = ZVR.sub("", cell)
name = re.sub(r'"?,?\s*Kurzbezeichnung:?\s*"?([^"]+)"?\s*$', r" (\1)", name).strip(' "„“,;-')
return (name, (zvr.group(1) if zvr else number_cell)) if len(name) > 3 else None
return None
def transliterate(text: str) -> str:
"""'Kaernten Koefer Gruenen' -> 'Kärnten Köfer Grünen' (file names; not after vowels: 'Neue', 'Steuer')."""
for plain, umlaut in (("ae", "ä"), ("oe", "ö"), ("ue", "ü")):
text = re.sub(rf"(?<![aeiouqAEIOUQ]){plain}", umlaut, text)
text = re.sub(rf"(?<![aeiouqAEIOUQ]){plain.capitalize()}", umlaut.upper(), text)
return text
def report_party_name(sheets: dict[str, list[list[Any]]], url: str) -> str:
"""Cover sheet, else the 'Territoriale Gliederung ... : <party>' line, else the file name."""
cover = [str(c).strip(" -–") for row in sheets.get("Deckblatt", []) for c in row if c and str(c).strip(" -–")]
names = [c for c in cover if len(c) > 2 and not re.search(
r"(?i)rechenschaftsbericht|bericht|kalenderjahr|jahr\b|partg|gemäß|name der|politische partei$|^\d", c)]
if names:
return names[0]
for rows in sheets.values():
for row in rows:
for cell in row:
if isinstance(cell, str) and (m := re.search(r"Territoriale Gliederung[^:]*:\s*(.+)$", cell.strip())):
return m.group(1).strip()
stem = url.rsplit("/", 1)[-1].rsplit(".", 1)[0]
stem = re.sub(r"(?i)^Rechenschaftsbericht[_-]?\d{4}[_-]?|[_-](gepr(ue|ü)ft|Anlagen|Verm(oe|ö)gensaufstellung)$", "", stem)
return transliterate(re.sub(r"[_-]+", " ", stem)).strip()
def sheets_titled(sheets: dict[str, list[list[Any]]], *keywords: str) -> list[list[Any]]:
"""All rows of every sheet whose (normalised) title contains one of the keywords."""
return [row for title, rows in sheets.items() if any(k in normalize(title) for k in keywords) for row in rows]
def number(value: Any) -> Decimal | None:
if value is None or isinstance(value, bool):
return None
if isinstance(value, (int, float, Decimal)):
return Decimal(str(value)).quantize(Decimal("0.01"))
text = str(value).strip()
if re.fullmatch(r"-?\d+\.\d{1,2}", text): # '301561.19' - plain decimal point (some exports)
return Decimal(text).quantize(Decimal("0.01"))
return parse_amount(text) if re.fullmatch(r"[\d.,\s€-]+", text or "x") else None
def sheet_lists(rows: list[list[Any]], header_needles: tuple[str, ...]) -> Iterator[dict[str, Any]]:
"""Yield dict rows of every table in a sheet whose header contains all needles (case-insensitive)."""
header: list[str] | None = None
for row in rows:
cells = ["" if c is None else str(c).strip() for c in row]
lowered = [normalize(c) for c in cells]
if all(any(n in c for c in lowered) for n in header_needles):
header = lowered
continue
if header is None or not any(cells):
continue
if any("leermeldung" in c or "keine" in c for c in lowered[:2]):
header = None
continue
record = {header[i]: row[i] for i in range(min(len(header), len(row))) if header[i]}
amount = next((number(row[i]) for i in range(len(row) - 1, -1, -1) if number(row[i]) is not None), None)
if amount is not None and any(isinstance(v, str) and v.strip() for v in record.values()):
record["__amount"] = amount
yield record
def pick(record: dict[str, Any], *needles: str) -> str:
for key, value in record.items():
if any(n in key for n in needles) and isinstance(value, str) and value.strip():
return value.strip()
return ""
@register
class RechnungshofPartyReports(Adapter):
spec = SPEC
name = "rechnungshof.party_reports"
description = "Parties' annual financial reports: funding, income, sponsors, advertisers, companies, loans"
schedule = "0 6 * * 1"
parser_version = 3
def start(self, ctx: RunContext) -> Iterator[Request]:
yield Request(url=f"{BASE}{REPORTS_INDEX}", kind="index")
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if response.request.kind == "index":
for a in response.soup.find_all("a", href=True):
href = urljoin(response.url, a["href"])
if re.search(r"Rechenschaftsbericht[_-]?\d{4}.*\.xlsx$", href, re.I):
yield Request(url=href, kind="xlsx")
elif re.search(r"(Ergebnis[_-]der[_-]Pruefung|Rechenschaftsbericht).*\.pdf$", href, re.I):
yield Document(Evidence(url=href, doc_type="pdf", title=a.get_text(" ", strip=True) or href.rsplit("/", 1)[-1],
meta={"kind": "party_report_pdf"}))
return
yield from self.parse_report(ctx, response)
def parse_report(self, ctx: RunContext, response: Response) -> Iterator[Any]:
import openpyxl
wb = openpyxl.load_workbook(io.BytesIO(response.body), data_only=True, read_only=True)
sheets = {ws.title.strip(): [list(r) for r in ws.iter_rows(values_only=True)] for ws in wb.worksheets}
url = response.request.url
year_match = re.search(r"Rechenschaftsbericht[_-]?(\d{4})", url) or re.search(r"(\d{4})", url)
year = year_match.group(1) if year_match else "?"
audited = bool(re.search(r"gepr(ue|ü)ft", url, re.IGNORECASE))
party_name = report_party_name(sheets, url)
party = party_ref(party_name)
version_de = "vom Rechnungshof geprüft" if audited else "eingereicht"
evidence = [Evidence(url=url, doc_type="register_entry",
title=f"Rechenschaftsbericht {year} ({version_de}): {party_name}")]
# Income statement per organisation level (federal, states, capitals, districts...), then summed.
levels = {title: lines for title, rows in sheets.items()
if (lines := finance_lines(rows, year)) and ("public_funding" in lines or "total_income" in lines)}
totals, levels = sum_levels(levels)
if totals:
income = totals.get("total_income")
yield Claim(
claim_type="party_finance", org=party,
key=("rh-report", normalize_org_name(party_name), year, "audited" if audited else "submitted"),
valid_from=date(int(year), 1, 1) if year.isdigit() else None,
valid_to=date(int(year), 12, 31) if year.isdigit() else None,
summary_de=f"Rechenschaftsbericht {year} ({version_de}): "
+ (f"Erträge {format_de(income)} €, " if income is not None else "")
+ f"davon Fördermittel {format_de(totals.get('public_funding', Decimal(0)))} €"
+ " (Summe aller ausgewiesenen Gliederungen)",
summary_en=f"Annual report {year} ({'audited' if audited else 'submitted'}): "
+ (f"income EUR {income:,.2f}, " if income is not None else "")
+ f"public funding EUR {totals.get('public_funding', 0):,.2f}",
data={"year": year, "audited": audited, **{k: str(v) for k, v in totals.items()},
"levels": {t: {k: str(v) for k, v in lines.items()} for t, lines in levels.items()}},
evidence=evidence, topics=("Transparenz und Korruption",),
)
if totals.get("public_funding"):
yield Claim(
claim_type="money_flow", org=party, valid_from=date(int(year), 1, 1) if year.isdigit() else None,
summary_de=f"{party_name} erhielt {year} öffentliche Fördermittel von {format_de(totals['public_funding'])} €",
summary_en=f"{party_name} received EUR {totals['public_funding']:,.2f} in public funding in {year}",
evidence=evidence, topics=("Transparenz und Korruption",),
facet=Money(kind="party_funding", amount=totals["public_funding"], payee=party,
payee_name=party_name, payer_name="Öffentliche Parteienförderung", period=year),
)
else:
ctx.stats["reports_without_income_statement"] += 1
def money_claims(rows: list[list[Any]], needles: tuple[str, ...], kind: str, who_keys: tuple[str, ...],
label_de: str) -> Iterator[Claim]:
for rec in sheet_lists(rows, needles):
who = clean_name(pick(rec, *who_keys))
if not who:
continue
recipient = pick(rec, "empfanger", "empfager")
medium = pick(rec, "medium")
amount = rec["__amount"]
yield Claim(
claim_type="donation" if kind == "donation" else "money_flow", org=party,
valid_from=date(int(year), 1, 1) if year.isdigit() else None,
summary_de=f"{label_de}: {who} → {recipient or party_name}"
+ (f" ({medium})" if medium else "") + f", {format_de(amount)} € ({year})",
data={"year": year, "recipient": recipient, "medium": medium},
evidence=[Evidence(url=url, doc_type="register_entry", title=evidence[0].title,
quote=f"{who} | {recipient} | {medium} | {amount}", locator=f"{kind}:{who}:{amount}")],
topics=("Transparenz und Korruption",),
facet=Money(kind=kind, amount=amount, payer=donor_ref(who), payer_name=who, payee=party,
payee_name=recipient or party_name, period=year, category=medium or None),
)
# List sheets are named differently by every party ("Spenden", "Spendenliste (§ 6 PartG)", "Spenden,
# Sponsoring, Inserate" ...): select by title keyword, find the tables by their header.
yield from money_claims(sheets_titled(sheets, "spende"), ("empfanger", "name", "plz"), "donation", ("name",),
"Spende laut Rechenschaftsbericht")
yield from money_claims(sheets_titled(sheets, "sponsor"), ("sponsor",), "sponsorship", ("sponsor",), "Sponsoring")
yield from money_claims(sheets_titled(sheets, "inserat"), ("inserent",), "party_ad", ("inserent",),
"Inserat in Parteimedium")
for rec in sheet_lists(sheets_titled(sheets, "kredit", "darlehen", "verbindlichkeit"), ("kreditgeber",)):
lender = clean_name(pick(rec, "kreditgeber"))
if lender:
yield Claim(
claim_type="money_flow", org=party,
summary_de=f"Kredit/Darlehen von {lender} an {party_name}: {format_de(rec['__amount'])} € ({year})",
evidence=evidence, facet=Money(kind="loan", amount=rec["__amount"], payer=OrgRef(name=lender, kind="company"),
payer_name=lender, payee=party, payee_name=party_name, period=year),
)
for row in sheets_titled(sheets, "beteiligung"):
cells = [str(c).strip() for c in row if c is not None and str(c).strip()]
fn = next((f for f in (firmenbuch_number(c) for c in cells[1:3]) if f), None)
if fn:
company = re.sub(r"\s*\d\)$", "", cells[0]) # footnote markers: "Werbeagentur 1)"
yield Claim(
claim_type="ownership", org=party,
counterpart_org=OrgRef(name=company, kind="company", ids={"firmenbuch": fn}),
key=("rh-ownership", normalize_org_name(party_name), year, fn),
summary_de=f"{party_name} hält eine Beteiligung an {company} (FN {fn}) – laut Rechenschaftsbericht {year}",
evidence=evidence, topics=("Transparenz und Korruption",),
)
for title, rows in sheets.items():
# list of affiliated organisations (not the sheet of income received from them)
if "nahestehend" not in normalize(title) or "ertr" in normalize(title):
continue
for row in rows:
affiliate = affiliate_from_row(row)
if affiliate:
name, zvr = affiliate
yield Claim(
claim_type="affiliation", org=party,
counterpart_org=OrgRef(name=name, kind="ngo", ids={"zvr": zvr} if zvr else {}),
key=("rh-affiliate", normalize_org_name(party_name), year, normalize_org_name(name)),
summary_de=f"Nahestehende Organisation von {party_name}: {name} ({year})",
evidence=evidence,
)
ctx.stats["reports"] += 1
+106
View File
@@ -0,0 +1,106 @@
"""RTR / KommAustria media transparency (MedKF-TG): advertising and media subsidies paid by public bodies.
Open data API (no usage restrictions, attribution): https://data.rtr.at/api/v1/tables/<table>.json
* mt-bekanntgabe2024 (half-yearly since 2024): payer, medium, media owner, campaign, amount, ad sujet link
* mt-bekanntgabe (quarterly until 2023): payer, medium/owner, amount
Each row -> money_flow (kind government_ad for advertising, subsidy for media funding) from the public body
(ministry, state government, municipality, public company...) to the media owner.
"""
from __future__ import annotations
from collections.abc import Iterator
from datetime import date
from typing import Any
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Evidence, Money, OrgRef
from politrack.text import format_eur_de
API = "https://data.rtr.at/api/v1/tables"
PAGE_SIZE = 5000
SPEC = SourceSpec(
slug="rtr-medientransparenz", name="RTR/KommAustria Medientransparenz", kind="official",
homepage="https://www.rtr.at/medien/was_wir_tun/medientransparenz/",
license="Amtliches Werk (§7 UrhG); attribution \"KommAustria – data.rtr.at\"", access="api",
trust=95, min_interval=2.0,
# data.rtr.at/robots.txt disallows everything for crawlers, but RTR documents this API as "der Endpunkt zur
# automatischen Abfrage der Daten" (rtr.at/rtr/service/opendata/OD_Schnittstellenbeschreibung.de.html).
robots_exempt=(API + "/",),
tos_notes="Government advertising and media funding reported under MedKF-TG. The REST API is documented by "
"RTR for automated queries (OD_Schnittstellenbeschreibung); robots.txt on data.rtr.at blocks "
"crawlers only. Terms (OD_Nutzungsbedingungen): free works under §7 UrhG, attribution requested, "
"no use for defamatory/discriminatory purposes.",
)
KIND = {2: "government_ad", 4: "subsidy"}
def half_years(since_year: int) -> list[str]:
today = date.today()
return [f"{y}{h}" for y in range(since_year, today.year + 1) for h in (1, 2)
if (y, h) <= (today.year, 1 if today.month <= 6 else 2)]
def quarters(since_year: int, until_year: int = 2023) -> list[str]:
return [f"{y}{q}" for y in range(since_year, until_year + 1) for q in (1, 2, 3, 4)]
@register
class MediaTransparency(Adapter):
spec = SPEC
name = "rtr.medientransparenz"
description = "Government advertising and media subsidies per public body and media owner"
schedule = "0 4 * * 2"
queue = "bulk"
def start(self, ctx: RunContext) -> Iterator[Request]:
for period in half_years(int(ctx.params.get("since_year", 2024))):
yield Request(url=f"{API}/mt-bekanntgabe2024.json", params={"halbjahr": period, "size": PAGE_SIZE, "page": 0},
kind="halbjahr", meta={"period": period})
for period in quarters(int(ctx.params.get("legacy_since_year", 2021))):
yield Request(url=f"{API}/mt-bekanntgabe.json", params={"quartal": period, "size": PAGE_SIZE, "page": 0},
kind="quartal", meta={"period": period})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
rows = response.json().get("data") or []
params = response.request.params or {}
if len(rows) >= PAGE_SIZE:
yield Request(url=response.request.url, params={**params, "page": params.get("page", 0) + 1},
kind=response.request.kind, meta=response.meta)
period = response.meta["period"]
label = f"{period[:4]}-H{period[4:]}" if response.request.kind == "halbjahr" else f"{period[:4]}-Q{period[4:]}"
min_euro = float(ctx.params.get("min_euro", 0))
for row in rows:
amount = row.get("euro")
if row.get("leermeldung") or amount is None or float(amount) < min_euro:
continue
kind = KIND.get(row.get("bekanntgabe"), "other")
payer_name = (row.get("rechtstraeger") or "").strip().strip('"')
owner = (row.get("medieninhaber") or row.get("mediumMedieninhaber") or "").strip()
medium = (row.get("medium") or row.get("mediumMedieninhaber") or "").strip()
if not payer_name or not owner:
continue
payer = OrgRef(name=payer_name, kind="public_body",
ids={"rtr": str(row["rechtstraeger_id"])} if row.get("rechtstraeger_id") else {})
payee = OrgRef(name=owner, kind="media")
campaign = row.get("kampagne")
what = "Werbung" if kind == "government_ad" else "Förderung"
evidence_url = (f"{API}/{'mt-bekanntgabe2024' if response.request.kind == 'halbjahr' else 'mt-bekanntgabe'}"
f".json?{'halbjahr' if response.request.kind == 'halbjahr' else 'quartal'}={period}"
+ (f"&rechtstraeger_id={row['rechtstraeger_id']}" if row.get("rechtstraeger_id") else ""))
yield Claim(
claim_type="money_flow", org=payer, counterpart_org=payee,
summary_de=f"{payer_name}: {what} in {medium} ({owner}), {format_eur_de(amount)} € ({label})"
+ (f" – Kampagne „{campaign}“" if campaign else ""),
summary_en=f"{payer_name}: {'advertising' if kind == 'government_ad' else 'funding'} in {medium} "
f"({owner}), EUR {amount:,.2f} ({label})",
data={"medium": medium, "category": row.get("kategorie"), "subcategory": row.get("subkategorie"),
"campaign": campaign, "sujet": row.get("sujet"), "period": label},
evidence=[Evidence(url=evidence_url, doc_type="dataset_row",
title=f"Medientransparenz {label}: {payer_name}",
quote=f"{payer_name} → {medium} ({owner}): {amount} €")],
topics=("Information und Medien",),
facet=Money(kind=kind, amount=amount, payer=payer, payer_name=payer_name, payee=payee, payee_name=owner,
period=label, reference=f"{campaign or ''}|{medium}|{row.get('sujet_dateiname') or ''}",
category=row.get("kategorie")),
)
+279
View File
@@ -0,0 +1,279 @@
"""Public posts of politicians on open platforms: Bluesky, Mastodon, Telegram (public channels), YouTube,
and X via the official (paid) API when a token is configured.
Accounts come from the `social_account` table: parliament profiles (verified), Wikidata
P2002/P12361/P4033/P3789/P2397 (unverified until confirmed) and the curated config/social_accounts.yaml
(enricher `config.social_accounts`). Accounts belong to a person or to an organisation (party channels).
Not collected (see docs/sources.md): X without the official API (ToS forbid scraping and proxies such as
Nitter), Facebook/Instagram (ToS), WhatsApp channels (no public read access, ToS forbid automated
collection), Signal (private by design, no public channels), private Telegram groups.
"""
from __future__ import annotations
import re
from collections.abc import Iterator
from typing import Any
from politrack.config import get_settings
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Evidence, OrgRef, PersonRef, Statement
from politrack.text import excerpt, html_to_text
def social_spec(slug: str, name: str, homepage: str, notes: str, robots_exempt: tuple[str, ...] = ()) -> SourceSpec:
return SourceSpec(slug=slug, name=name, kind="social", homepage=homepage, license="Per-post copyright of the author",
access="api", trust=40, store_fulltext=True, keep_raw=True, min_interval=2.0,
robots_exempt=robots_exempt, tos_notes=notes)
BLUESKY = social_spec("bluesky", "Bluesky", "https://bsky.app",
"Public AT Protocol AppView API, no auth for public reads. Posts by the politicians themselves.")
MASTODON = social_spec("mastodon", "Mastodon / Fediverse", "https://joinmastodon.org",
"Public REST API of the account's instance (no auth for public statuses). Respect instance rules.")
TELEGRAM = social_spec("telegram", "Telegram (öffentliche Kanäle)", "https://telegram.org",
"Public web preview of *public broadcast channels* (t.me/s/<channel>); no robots.txt restrictions. "
"Private groups/chats are never accessed. Low frequency, latest posts only.")
YOUTUBE = social_spec("youtube", "YouTube", "https://www.youtube.com",
"YouTube Data API v3 (free quota, API key required via POLITRACK_YOUTUBE_API_KEY).")
X_API = "https://api.x.com/2/"
X = SourceSpec(slug="x", name="X (Twitter)", kind="social", homepage="https://x.com",
license="Per-post copyright of the author; X Developer Agreement", access="api", trust=40,
store_fulltext=False, keep_raw=False, min_interval=2.0, robots_exempt=(X_API,),
tos_notes="Official X API v2 only (paid tier, POLITRACK_X_BEARER_TOKEN). Scraping x.com or using "
"mirrors/proxies (Nitter etc.) violates the ToS and is not done. The Developer Agreement requires "
"removing posts that were deleted on X; no raw archive or full text is kept, but the "
"deletion sync is still open (docs/roadmap.md).")
PLATFORM_LABELS = {"bluesky": "Bluesky", "mastodon": "Mastodon", "telegram": "Telegram", "youtube": "YouTube",
"x": "X"}
def accounts(ctx: RunContext, platform: str, url_pattern: str | None = None) -> list[dict[str, Any]]:
"""Accounts of persons or organisations on one platform; each row carries an `owner` dict for Request.meta.
`url_pattern` also picks up accounts stored as plain websites, so it must match only that platform's
own host (never a generic shape like "https://host/@user", which TikTok and others share).
"""
if ctx.conn is None:
return []
rows = ctx.conn.execute(
"""SELECT person_id, organization_id, url, handle, verified FROM social_account
WHERE (platform = %s OR (%s::text IS NOT NULL AND url ~* %s))
AND (person_id IS NOT NULL OR organization_id IS NOT NULL)
ORDER BY verified DESC, id""",
(platform, url_pattern, url_pattern),
).fetchall()
for row in rows:
row["owner"] = {"person_id": str(row["person_id"]) if row["person_id"] else None,
"org_id": None if row["person_id"] else str(row["organization_id"]),
"verified": row["verified"]}
return rows
def post_claim(*, owner: dict[str, Any], platform: str, url: str, text: str, created: Any,
title: str | None = None, data: dict | None = None) -> Claim:
"""A public post as a `statement` claim of the account owner (person, or organisation for party channels)."""
label = PLATFORM_LABELS[platform]
return Claim(
claim_type="statement", occurred_at=created,
person=PersonRef(ids={"politrack": owner["person_id"]}) if owner.get("person_id") else None,
org=OrgRef(ids={"politrack_org": owner["org_id"]}, create=False) if owner.get("org_id") else None,
summary_de=f"Auf {label}: „{excerpt(title or text, 200)}“",
summary_en=f"On {label}: \"{excerpt(title or text, 200)}\"",
data={"platform": platform, "account_verified": owner.get("verified", False), **(data or {})},
evidence=[Evidence(url=url, source=platform, doc_type="social_post", title=excerpt(title or text, 120),
published_at=created, full_text=text, excerpt=text, quote=excerpt(text, 500))],
facet=Statement(context="social_post", url=url, said_at=created, platform=platform, title=title,
excerpt=text),
)
@register
class BlueskyPosts(Adapter):
spec = BLUESKY
name = "social.bluesky"
description = "Public Bluesky posts of politicians"
schedule = "7 * * * *"
queue = "social"
def start(self, ctx: RunContext) -> Iterator[Request]:
for acc in accounts(ctx, "bluesky", r"bsky\.app/profile/"):
m = re.search(r"bsky\.app/profile/([^/?#]+)", acc["url"])
if m:
yield Request(url="https://public.api.bsky.app/xrpc/app.bsky.feed.getAuthorFeed",
params={"actor": m.group(1), "limit": 50, "filter": "posts_no_replies"}, kind="feed",
meta={**acc["owner"], "handle": m.group(1)})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
handle = response.meta["handle"]
for item in response.json().get("feed", []):
post = item.get("post") or {}
if item.get("reason"): # repost of someone else's post
continue
record = post.get("record") or {}
text = record.get("text") or ""
author = (post.get("author") or {}).get("handle") or handle
rkey = post.get("uri", "").rsplit("/", 1)[-1]
yield post_claim(owner=response.meta, platform="bluesky",
url=f"https://bsky.app/profile/{author}/post/{rkey}", text=text,
created=record.get("createdAt"),
data={"likes": post.get("likeCount"), "reposts": post.get("repostCount")})
@register
class MastodonPosts(Adapter):
spec = MASTODON
name = "social.mastodon"
description = "Public Mastodon/Fediverse posts of politicians"
schedule = "17 * * * *"
queue = "social"
def start(self, ctx: RunContext) -> Iterator[Request]:
for acc in accounts(ctx, "mastodon"): # instances have arbitrary hosts: platform only
m = re.match(r"^https?://([^/]+)/@([^/@]+)", acc["url"])
if m:
host, user = m.groups()
yield Request(url=f"https://{host}/api/v1/accounts/lookup", params={"acct": user}, kind="lookup",
meta={**acc["owner"], "host": host})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
meta = response.meta
if response.request.kind == "lookup":
account_id = response.json().get("id")
if account_id:
yield Request(url=f"https://{meta['host']}/api/v1/accounts/{account_id}/statuses",
params={"limit": 40, "exclude_replies": "true", "exclude_reblogs": "true"},
kind="statuses", meta=meta)
return
for status in response.json():
if status.get("visibility") not in (None, "public"):
continue
yield post_claim(owner=meta, platform="mastodon", url=status.get("url") or status.get("uri"),
text=html_to_text(status.get("content")), created=status.get("created_at"),
data={"favourites": status.get("favourites_count"), "reblogs": status.get("reblogs_count")})
@register
class TelegramChannels(Adapter):
spec = TELEGRAM
name = "social.telegram"
description = "Public Telegram broadcast channels of politicians (web preview, latest posts)"
schedule = "27 */2 * * *"
queue = "social"
def start(self, ctx: RunContext) -> Iterator[Request]:
for acc in accounts(ctx, "telegram", r"(t\.me|telegram\.me)/"):
m = re.search(r"(?:t\.me|telegram\.me)/(?:s/)?([A-Za-z0-9_]{4,})", acc["url"])
if m:
yield Request(url=f"https://t.me/s/{m.group(1)}", kind="channel",
meta={**acc["owner"], "channel": m.group(1)})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
soup = response.soup
subscribers = soup.select_one(".tgme_channel_info_counter .counter_value")
for msg in soup.select("div.tgme_widget_message[data-post]"):
text_el = msg.select_one(".tgme_widget_message_text")
time_el = msg.select_one(".tgme_widget_message_date time")
text = text_el.get_text("\n", strip=True) if text_el else ""
if not text:
continue # media-only posts
forwarded = msg.select_one(".tgme_widget_message_forwarded_from")
views = msg.select_one(".tgme_widget_message_views")
yield post_claim(
owner=response.meta, platform="telegram", url=f"https://t.me/{msg['data-post']}",
text=text, created=time_el.get("datetime") if time_el else None,
data={"views": views.get_text(strip=True) if views else None,
"forwarded_from": forwarded.get_text(" ", strip=True) if forwarded else None,
"channel_subscribers": subscribers.get_text(strip=True) if subscribers else None},
)
@register
class YouTubeVideos(Adapter):
spec = YOUTUBE
name = "social.youtube"
description = "Latest YouTube uploads of politicians' channels (requires POLITRACK_YOUTUBE_API_KEY)"
schedule = "37 */6 * * *"
queue = "social"
def start(self, ctx: RunContext) -> Iterator[Request]:
key = get_settings().youtube_api_key
if not key:
ctx.stats["skipped_no_api_key"] = 1
return
api = "https://www.googleapis.com/youtube/v3/channels"
for acc in accounts(ctx, "youtube", r"youtube\.com/"):
meta = acc["owner"]
if m := re.search(r"youtube\.com/channel/(UC[\w-]+)", acc["url"]):
params = {"id": m.group(1)}
elif m := re.search(r"youtube\.com/@([\w.-]+)", acc["url"]):
params = {"forHandle": f"@{m.group(1)}"}
elif m := re.search(r"youtube\.com/(?:user/|c/)?([\w.-]+)/?$", acc["url"]):
params = {"forUsername": m.group(1)}
else:
continue
yield Request(url=api, params={**params, "part": "contentDetails", "key": key}, kind="channel", meta=meta)
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
data = response.json()
meta = response.meta
if response.request.kind == "channel":
for item in data.get("items", []):
uploads = item["contentDetails"]["relatedPlaylists"]["uploads"]
yield Request(url="https://www.googleapis.com/youtube/v3/playlistItems", kind="uploads", meta=meta,
params={"playlistId": uploads, "part": "snippet", "maxResults": 25,
"key": get_settings().youtube_api_key})
return
for item in data.get("items", []):
snippet = item.get("snippet") or {}
video = (snippet.get("resourceId") or {}).get("videoId")
if not video:
continue
yield post_claim(owner=meta, platform="youtube",
url=f"https://www.youtube.com/watch?v={video}", title=snippet.get("title"),
text=f"{snippet.get('title', '')}\n{snippet.get('description', '')}".strip(),
created=snippet.get("publishedAt"))
@register
class XPosts(Adapter):
spec = X
name = "social.x"
description = "Posts of politicians on X via the official API v2 (requires POLITRACK_X_BEARER_TOKEN)"
schedule = "47 */6 * * *"
queue = "social"
def start(self, ctx: RunContext) -> Iterator[Request]:
token = get_settings().x_bearer_token
if not token:
ctx.stats["skipped_no_api_key"] = 1
return
for acc in accounts(ctx, "twitter", r"//(www\.)?(twitter|x)\.com/"):
m = re.search(r"(?:twitter|x)\.com/(?:#!/)?@?(\w{1,15})/?$", acc["url"])
if m:
yield Request(url=f"{X_API}users/by/username/{m.group(1)}", kind="user", headers=self.auth(),
meta=acc["owner"])
@staticmethod
def auth() -> dict[str, str]:
return {"Authorization": f"Bearer {get_settings().x_bearer_token}"}
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
data = response.json()
meta = response.meta
if response.request.kind == "user":
user = data.get("data") or {}
if user.get("id"):
yield Request(url=f"{X_API}users/{user['id']}/tweets", kind="tweets", headers=self.auth(),
params={"max_results": 50, "exclude": "replies,retweets",
"tweet.fields": "created_at,public_metrics"},
meta={**meta, "username": user.get("username")})
return
for tweet in data.get("data") or []:
metrics = tweet.get("public_metrics") or {}
yield post_claim(owner=meta, platform="x",
url=f"https://x.com/{meta['username']}/status/{tweet['id']}", text=tweet.get("text", ""),
created=tweet.get("created_at"),
data={"likes": metrics.get("like_count"), "reposts": metrics.get("retweet_count")})
+305
View File
@@ -0,0 +1,305 @@
"""Wikidata (CC0) via the public SPARQL endpoint.
Fills what the federal parliament API does not cover: state parliaments (Landtage), state
governments, mayors and other offices, party membership, careers (employers, positions abroad,
EU roles) and social-media handles. People are merged with parliament records through P2280
(Austrian Parliament ID = PAD_INTERN).
Seats in the Nationalrat/Bundesrat/European Parliament are mapped to the canonical role "mp" in
that chamber, so a Wikidata mandate and a Parliament mandate with the same start date are the
same claim (corroborated by construction).
Flow: list Austrian positions -> holders in batches of 40 positions -> for each batch of persons:
party memberships, careers, social handles. Batching keeps queries within the endpoint's 60 s limit.
"""
from __future__ import annotations
import json
import re
from collections.abc import Iterator
from datetime import date
from typing import Any
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Evidence, OrgRef, PersonRecord, PersonRef, Role, SocialAccount
SPARQL = "https://query.wikidata.org/sparql"
SPEC = SourceSpec(
slug="wikidata",
name="Wikidata",
kind="aggregator",
homepage="https://www.wikidata.org",
license="CC0",
access="api",
trust=65,
min_interval=2.0,
robots_exempt=(SPARQL,),
tos_notes="SPARQL endpoint. robots.txt disallows /sparql for crawlers, but the Wikimedia User-Agent policy "
"allows bots with a descriptive User-Agent to use the query service. Community-edited.",
)
POSITION_LIST_QUERY = """
SELECT DISTINCT ?position ?positionLabel WHERE {
?position wdt:P17 wd:Q40 .
[] ps:P39 ?position .
SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". }
}
"""
POSITIONS_QUERY = """
SELECT ?person ?personLabel ?givenLabel ?familyLabel ?birth ?death ?pad ?ep ?position ?positionLabel ?start ?end
?group ?groupLabel WHERE {
VALUES ?position { %(positions)s }
?person p:P39 ?st .
?st ps:P39 ?position .
OPTIONAL { ?st pq:P580 ?start }
OPTIONAL { ?st pq:P582 ?end }
FILTER(!BOUND(?end) || ?end >= "%(since)s-01-01"^^xsd:dateTime)
OPTIONAL { ?st pq:P4100 ?group }
OPTIONAL { ?person wdt:P2280 ?pad }
# Generic positions with country=Austria are also held by foreigners (e.g. "Bundessprecher" of the German
# Green Youth): require an Austrian connection - parliament id, citizenship or an Austrian party.
FILTER(BOUND(?pad) || EXISTS { ?person wdt:P27 wd:Q40 } || EXISTS { ?person wdt:P102 ?ap . ?ap wdt:P17 wd:Q40 })
OPTIONAL { ?person wdt:P1186 ?ep }
OPTIONAL { ?person wdt:P569 ?birth }
OPTIONAL { ?person wdt:P570 ?death }
OPTIONAL { ?person wdt:P735 ?given }
OPTIONAL { ?person wdt:P734 ?family }
SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". }
}
"""
PARTY_QUERY = """
SELECT ?person ?party ?partyLabel ?partyShort ?start ?end WHERE {
VALUES ?person { %(persons)s }
?person p:P102 ?st .
?st ps:P102 ?party .
?party wdt:P17 wd:Q40 .
OPTIONAL { ?st pq:P580 ?start }
OPTIONAL { ?st pq:P582 ?end }
OPTIONAL { ?party wdt:P1813 ?partyShort . FILTER(LANG(?partyShort) = "de") }
SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". }
}
"""
CAREER_QUERY = """
SELECT ?person ?org ?orgLabel ?start ?end ?kind WHERE {
VALUES ?person { %(persons)s }
{ ?person p:P108 ?st . ?st ps:P108 ?org . BIND("employer" AS ?kind) }
UNION
{ ?person p:P39 ?st . ?st ps:P39 ?org . FILTER NOT EXISTS { ?org wdt:P17 wd:Q40 } BIND("position" AS ?kind) }
OPTIONAL { ?st pq:P580 ?start }
OPTIONAL { ?st pq:P582 ?end }
SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". }
}
"""
# Social-media properties: P2002 X/Twitter, P12361 Bluesky, P4033 Mastodon, P3789 Telegram,
# P2397 YouTube channel id, P2003 Instagram, P2013 Facebook, P7085 TikTok
SOCIAL_QUERY = """
SELECT ?person ?prop ?value WHERE {
VALUES ?person { %(persons)s }
VALUES ?prop { wdt:P2002 wdt:P12361 wdt:P4033 wdt:P3789 wdt:P2397 wdt:P2003 wdt:P2013 wdt:P7085 }
?person ?prop ?value .
}
"""
SOCIAL_PROPS = {
"P2002": ("twitter", "https://x.com/{}"),
"P12361": ("bluesky", "https://bsky.app/profile/{}"),
"P4033": ("mastodon", None),
"P3789": ("telegram", "https://t.me/{}"),
"P2397": ("youtube", "https://www.youtube.com/channel/{}"),
"P2003": ("instagram", "https://www.instagram.com/{}"),
"P2013": ("facebook", "https://www.facebook.com/{}"),
"P7085": ("tiktok", "https://www.tiktok.com/@{}"),
}
EU_POLITICAL = re.compile(r"Europäischen Parlament|Europaparlament|Europäischen Kommission|EU-Kommissar|"
r"Member of the European Parliament", re.IGNORECASE)
EXCLUDE = re.compile(
r"professor|kommerzialrat|kommerzienrat|hofrat|ministerialrat|regierungsrat|gerichtshof|richter|"
r"botschafter|bischof|rektor|ökonomierat|medizinalrat|schulrat|studienrat|baurat|veterinärrat|"
r"bergrat|amtsrat|kammerrat|konsul",
re.IGNORECASE,
)
CHAMBER_LABELS = [
(re.compile(r"Abgeordnete[rn]? zum Nationalrat|Mitglied des Nationalrates", re.I), "nationalrat", "federal"),
(re.compile(r"Mitglied des (österreichischen )?Bundesrate?s", re.I), "bundesrat", "federal"),
(re.compile(r"Mitglied des Europäischen Parlaments|Member of the European Parliament", re.I),
"europaeisches-parlament", "eu"),
]
def classify_level(label: str) -> str:
if re.search(r"europäischen parlament|europaabgeordnet|eu-kommiss", label, re.I):
return "eu"
if re.search(r"bürgermeister|gemeinderat|stadtrat|vizebürgermeister|gemeinde", label, re.I):
return "municipal" if "Wiener Landtag" not in label else "state"
if re.search(r"landtag|landesrat|landeshauptmann|landesregierung|landesamtsdirektor", label, re.I):
return "state"
if re.search(r"bezirk", label, re.I):
return "district"
return "federal"
def wd_date(binding: dict, key: str) -> date | None:
value = binding.get(key, {}).get("value")
if not value or value.startswith("-"):
return None
try:
return date.fromisoformat(value[:10])
except ValueError:
return None
def qid(binding: dict, key: str) -> str | None:
value = binding.get(key, {}).get("value")
return value.rsplit("/", 1)[-1] if value else None
def label(binding: dict, key: str) -> str | None:
value = binding.get(key, {}).get("value")
return None if not value or re.fullmatch(r"Q\d+", value) else value
def sparql(query: str, kind: str, meta: dict | None = None) -> Request:
return Request(url=SPARQL, params={"query": query, "format": "json"}, kind=kind, meta=meta or {},
headers={"Accept": "application/sparql-results+json"})
def role_for(position_label: str) -> tuple[str, OrgRef | None, str]:
for pattern, slug, level in CHAMBER_LABELS:
if pattern.search(position_label):
return "mp", OrgRef(slug=slug), level
return position_label, None, classify_level(position_label)
@register
class WikidataPoliticians(Adapter):
spec = SPEC
name = "wikidata.politicians"
description = "Austrian office holders (all levels), party membership, careers, social handles"
schedule = "45 4 * * 0"
creates_persons = True
parser_version = 2
def start(self, ctx: RunContext) -> Iterator[Request]:
yield sparql(POSITION_LIST_QUERY, kind="positions")
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
bindings = json.loads(response.text, strict=False)["results"]["bindings"]
kind = response.request.kind
if kind == "positions":
positions = [qid(b, "position") for b in bindings
if label(b, "positionLabel") and not EXCLUDE.search(label(b, "positionLabel") or "")]
since = ctx.params.get("since_year", 2000)
for i in range(0, len(positions), 40):
values = " ".join(f"wd:{q}" for q in positions[i:i + 40])
yield sparql(POSITIONS_QUERY % {"positions": values, "since": since}, kind="holders")
elif kind == "holders":
yield from self.parse_holders(bindings)
elif kind == "parties":
yield from self.parse_parties(bindings)
elif kind == "careers":
yield from self.parse_careers(bindings)
elif kind == "social":
yield from self.parse_social(bindings)
def parse_holders(self, bindings: list[dict]) -> Iterator[Any]:
seen: set[str] = set()
for b in bindings:
person_qid, name, position = qid(b, "person"), label(b, "personLabel"), label(b, "positionLabel")
if not name or not position or EXCLUDE.search(position):
continue
pad = b.get("pad", {}).get("value")
ids = {"wikidata": person_qid}
if pad and pad.isdigit():
ids["parlament"] = str(int(pad))
if b.get("ep", {}).get("value"):
ids["ep"] = b["ep"]["value"]
ref = PersonRef(ids=ids)
if person_qid not in seen:
seen.add(person_qid)
yield PersonRecord(ref=ref, full_name=name, given_name=label(b, "givenLabel"),
family_name=label(b, "familyLabel"), birth_date=wd_date(b, "birth"),
death_date=wd_date(b, "death"))
start, end = wd_date(b, "start"), wd_date(b, "end") or wd_date(b, "death")
group_label = label(b, "groupLabel")
group = OrgRef(name=group_label, kind="parliamentary_group", ids={"wikidata": qid(b, "group")}) \
if group_label else None
role, body, level = role_for(position)
yield Claim(
claim_type="role", person=ref, org=body, counterpart_org=group, valid_from=start, valid_to=end,
summary_de=f"{name}: {position}" + (f" ({group_label})" if group_label else ""),
data={"position_qid": qid(b, "position"), "position_label": position},
evidence=[Evidence(url=f"https://www.wikidata.org/wiki/{person_qid}", doc_type="api_record",
title=f"Wikidata: {name}", locator=f"P39:{qid(b, 'position')}:{start}")],
facet=Role(role=role, level=level, body=body, start=start, end=end),
)
persons = sorted(seen)
for i in range(0, len(persons), 300):
values = " ".join(f"wd:{q}" for q in persons[i:i + 300])
yield sparql(PARTY_QUERY % {"persons": values}, kind="parties")
yield sparql(CAREER_QUERY % {"persons": values}, kind="careers")
yield sparql(SOCIAL_QUERY % {"persons": values}, kind="social")
def parse_parties(self, bindings: list[dict]) -> Iterator[Claim]:
for b in bindings:
party_label = label(b, "partyLabel")
if not party_label:
continue
person_qid = qid(b, "person")
party = OrgRef(name=party_label, short_name=b.get("partyShort", {}).get("value"), kind="party",
ids={"wikidata": qid(b, "party")})
start, end = wd_date(b, "start"), wd_date(b, "end")
yield Claim(
claim_type="role", person=PersonRef(ids={"wikidata": person_qid}), org=party,
valid_from=start, valid_to=end,
summary_de=f"Mitglied der Partei {party_label}", summary_en=f"Member of {party_label}",
evidence=[Evidence(url=f"https://www.wikidata.org/wiki/{person_qid}", doc_type="api_record",
title="Wikidata", locator=f"P102:{qid(b, 'party')}:{start}")],
facet=Role(role="party_member", level="party", body=party, party=party, start=start, end=end),
)
def parse_careers(self, bindings: list[dict]) -> Iterator[Claim]:
for b in bindings:
org_label = label(b, "orgLabel")
if not org_label or EXCLUDE.search(org_label):
continue
person_qid = qid(b, "person")
ref = PersonRef(ids={"wikidata": person_qid})
start, end = wd_date(b, "start"), wd_date(b, "end")
kind = b["kind"]["value"]
evidence = [Evidence(url=f"https://www.wikidata.org/wiki/{person_qid}", doc_type="api_record",
title="Wikidata", locator=f"{kind}:{qid(b, 'org')}:{start}")]
if kind == "position" and EU_POLITICAL.search(org_label):
role, body, level = role_for(org_label)
yield Claim(claim_type="role", person=ref, org=body, valid_from=start, valid_to=end,
summary_de=org_label, evidence=evidence,
facet=Role(role=role, level="eu", body=body, start=start, end=end))
continue
employer = OrgRef(name=org_label, ids={"wikidata": qid(b, "org")}) if kind == "employer" else None
yield Claim(
claim_type="career", person=ref, counterpart_org=employer, valid_from=start, valid_to=end,
occurred_at=start, key=("wikidata-career", person_qid, kind, qid(b, "org"), start),
summary_de=f"{'Arbeitet(e) bei' if kind == 'employer' else 'Funktion:'} {org_label}",
summary_en=f"{'Employed by' if kind == 'employer' else 'Position:'} {org_label}",
data={"kind": kind, "org_qid": qid(b, "org")}, evidence=evidence,
)
def parse_social(self, bindings: list[dict]) -> Iterator[SocialAccount]:
for b in bindings:
prop = b["prop"]["value"].rsplit("/", 1)[-1]
value = b["value"]["value"].strip()
platform, template = SOCIAL_PROPS[prop]
if platform == "mastodon":
user, _, host = value.lstrip("@").partition("@")
if not host:
continue
url = f"https://{host}/@{user}"
else:
url = template.format(value)
yield SocialAccount(platform=platform, url=url, handle=value,
person=PersonRef(ids={"wikidata": qid(b, "person")}), verified=False)
+83
View File
@@ -0,0 +1,83 @@
"""Wikipedia edits made from the networks of Parliament and federal government (classic transparency bot).
Anonymous Wikipedia edits are publicly attributed to the editor's IP address. We query the MediaWiki API
(list=usercontribs, uciprange) for the IP ranges registered to public institutions in the RIPE database
(config/ip_ranges.yaml) and record each edit as a `wiki_edit` claim of the institution, linked to the
person whose article was edited when the title matches a known politician. Individual IP addresses are
not displayed - only the institution owning the network.
The API is used per the MediaWiki API etiquette (descriptive User-Agent, sequential requests); robots.txt
disallows /w/ for crawlers, which does not apply to API clients (robots_exempt below).
"""
from __future__ import annotations
from collections.abc import Iterator
from datetime import date, timedelta
from typing import Any
import yaml
from politrack.config import get_settings
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Evidence, OrgRef, PersonRef
from politrack.text import excerpt
WIKIS = ("de.wikipedia.org", "en.wikipedia.org")
SPEC = SourceSpec(
slug="wikipedia", name="Wikipedia (Bearbeitungen aus Behördennetzen)", kind="aggregator",
homepage="https://de.wikipedia.org", license="CC BY-SA 4.0", access="api", trust=80, min_interval=1.0,
robots_exempt=tuple(f"https://{w}/w/api.php" for w in WIKIS),
tos_notes="MediaWiki API (API etiquette: descriptive User-Agent, no parallel requests). Edits from IP ranges "
"registered to Parliament / Bundeskanzleramt in the RIPE database.",
)
@register
class InstitutionalWikiEdits(Adapter):
spec = SPEC
name = "wikipedia.institution_edits"
description = "Wikipedia edits from the IP ranges of Parliament and federal government"
schedule = "15 7 * * *"
def start(self, ctx: RunContext) -> Iterator[Request]:
config = yaml.safe_load((get_settings().config_dir / "ip_ranges.yaml").read_text(encoding="utf-8"))
days = int(ctx.params.get("days", 3650 if not ctx.state.get("initialised") else 30))
end = (date.today() - timedelta(days=days)).isoformat() + "T00:00:00Z"
for network in config.get("networks") or []:
for cidr in network["ranges"]:
for wiki in WIKIS:
yield Request(
url=f"https://{wiki}/w/api.php", kind="contribs",
params={"action": "query", "list": "usercontribs", "uciprange": cidr, "uclimit": 500,
"ucend": end, "ucprop": "ids|title|timestamp|comment|sizediff", "format": "json",
"formatversion": 2},
meta={"wiki": wiki, "org": network["org"], "label": network["label"]}, conditional=False)
ctx.state.set("initialised", True)
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
data = response.json()
meta = response.meta
if "continue" in data:
yield Request(url=response.request.url, kind="contribs", meta=meta, conditional=False,
params={**response.request.params, **data["continue"]})
for edit in (data.get("query") or {}).get("usercontribs") or []:
title, revid = edit.get("title"), edit.get("revid")
if not title or not revid or ":" in title.split(" ")[0] and not title.startswith("Liste"):
continue # skip talk/user/meta pages ("Diskussion:", "Benutzer:")
size = edit.get("sizediff")
comment = edit.get("comment") or ""
diff = f"https://{meta['wiki']}/w/index.php?diff={revid}"
yield Claim(
claim_type="wiki_edit", org=OrgRef(slug=meta["org"], create=False),
counterpart_person=PersonRef(name=title, hints={}), key=("wiki", meta["wiki"], revid),
occurred_at=edit.get("timestamp"),
summary_de=f"Wikipedia-Bearbeitung aus dem Netz von {meta['label']}: „{title}“"
+ (f" ({size:+d} Zeichen)" if isinstance(size, int) else "")
+ (f" – {excerpt(comment, 120)}" if comment else ""),
summary_en=f"Wikipedia edit from the network of {meta['label']}: \"{title}\""
+ (f" ({size:+d} characters)" if isinstance(size, int) else ""),
data={"wiki": meta["wiki"], "title": title, "sizediff": size, "comment": comment},
evidence=[Evidence(url=diff, doc_type="web_page", title=f"Wikipedia: {title}",
published_at=edit.get("timestamp"), quote=comment or None)],
)