Politrack: track Austrian politicians from public sources
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
583ccba31c
187 files changed
+39890
No files matched your search
@@ -0,0 +1,5 @@
|
||||
"""Built-in source adapters. Every module in this package (and sub-packages) is imported by
|
||||
politrack.core.registry.load_builtin(); modules register their adapters with @register.
|
||||
|
||||
To add a source: create one module here (see docs/contributing/adapters.md).
|
||||
"""
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Bundeskanzleramt: government news archive and Council of Ministers minutes (Ministerratsprotokolle).
|
||||
|
||||
Official works (§7 UrhG); robots.txt only blocks /401.
|
||||
|
||||
* News ("Nachrichten der Bundesregierung", one index per year): stored with full text; the event
|
||||
extractor turns summits, trips abroad and bilateral meetings into travel/meeting claims.
|
||||
* Council of Ministers minutes (Beschlussprotokolle): every agenda item names the reporting office(s)
|
||||
("Bericht der Bundesministerin für Justiz, Zahl ..., betreffend ..."). Each becomes a
|
||||
government_decision claim for the office holder on that date (resolved via the office hint).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Document, Event, Evidence, OrgRef, Participation, PersonRef
|
||||
from politrack.text import excerpt, fingerprint
|
||||
|
||||
BASE = "https://www.bundeskanzleramt.gv.at"
|
||||
NEWS_INDEX = BASE + "/bundeskanzleramt/nachrichten-der-bundesregierung/{year}.html"
|
||||
MR_INDEX = BASE + "/bundeskanzleramt/die-bundesregierung/ministerratsprotokolle.html"
|
||||
SPEC = SourceSpec(
|
||||
slug="bundesregierung",
|
||||
name="Bundeskanzleramt – Bundesregierung",
|
||||
kind="official",
|
||||
homepage=BASE,
|
||||
license="Amtliches Werk (§7 UrhG)",
|
||||
access="scrape",
|
||||
trust=90,
|
||||
min_interval=2.0,
|
||||
tos_notes="Government news archive and Council of Ministers minutes. robots.txt only blocks /401.",
|
||||
)
|
||||
|
||||
MONTHS = {m: i for i, m in enumerate(["jänner", "februar", "märz", "april", "mai", "juni", "juli", "august",
|
||||
"september", "oktober", "november", "dezember"], start=1)}
|
||||
MONTHS["januar"] = 1
|
||||
GERMAN_DATE = re.compile(r"(\d{1,2})\.\s*([A-Za-zäÄ]+)\s+(\d{4})")
|
||||
# Office names may contain commas ("für Wohnen, Kunst, Kultur, Medien und Sport"); file numbers sometimes use
|
||||
# non-breaking hyphens or spaces ("2026 0.040.056"); subjects span lines (joined with " | ") and end at the
|
||||
# decision ("Der Ministerrat beschließt ...") or at the next item.
|
||||
ITEM = re.compile(r"(?:(?:Gemeinsamer\s+)?Bericht|Vortrag|Antrag)\s+(?P<offices>de[sr]\s+[^|]+?),?\s+"
|
||||
r"Zahl\s+(?P<zahl>\d[\d./\-\u2010-\u2015 ]*?)\s*,\s*betreffend\s+(?P<subject>.+?)"
|
||||
r"(?=\s*\|?\s*Der Ministerrat\b"
|
||||
r"|\s*\|\s*(?:\d+(?:\.\d+)*\.?\s+)?(?:Gemeinsamer\s+)?(?:Bericht|Vortrag|Antrag)\s+de[sr]\s|$)")
|
||||
OFFICE_SPLIT = re.compile(r",\s*(?=de[sr]\s)|\s+und\s+(?=de[sr]\s)")
|
||||
GENITIVE = [(r"\bBundeskanzlers\b", "Bundeskanzler"), (r"\bVizekanzlers\b", "Vizekanzler"),
|
||||
(r"\bBundesministers\b", "Bundesminister"), (r"\bStaatssekretärs\b", "Staatssekretär")]
|
||||
|
||||
|
||||
def german_date(text: str) -> date | None:
|
||||
m = GERMAN_DATE.search(text)
|
||||
if not m or m.group(2).lower() not in MONTHS:
|
||||
return None
|
||||
return date(int(m.group(3)), MONTHS[m.group(2).lower()], int(m.group(1)))
|
||||
|
||||
|
||||
def period_start(path: str) -> int:
|
||||
"""'.../ministerratsprotokolle-der-xxvi-regierungsperiode-2017-2018.html' -> 2018 (latest year named)."""
|
||||
return max((int(y) for y in re.findall(r"(20\d\d)", path)), default=0)
|
||||
|
||||
|
||||
def offices(text: str) -> list[str]:
|
||||
"""'des Bundeskanzlers, des Bundesministers für Wohnen, Kunst, Kultur, Medien und Sport und der
|
||||
Bundesministerin für Justiz' -> ['Bundeskanzler', 'Bundesminister für Wohnen, ...', 'Bundesministerin für Justiz']."""
|
||||
result = []
|
||||
for part in OFFICE_SPLIT.split(text):
|
||||
part = re.sub(r"^\s*de[sr]\s+", "", part).strip(" ,")
|
||||
for pattern, repl in GENITIVE:
|
||||
part = re.sub(pattern, repl, part)
|
||||
if part:
|
||||
result.append(part)
|
||||
return result
|
||||
|
||||
|
||||
def main_text(response: Response) -> tuple[str, str]:
|
||||
soup = response.soup
|
||||
main = soup.find("main") or soup
|
||||
title_el = main.find(class_="title") or main.find("h1")
|
||||
title = title_el.get_text(" ", strip=True) if title_el else (soup.title.string if soup.title else response.url)
|
||||
for noise in main.find_all(["nav", "aside", "script", "style", "form", "footer"]):
|
||||
noise.decompose()
|
||||
parts = [p.get_text(" ", strip=True) for p in main.find_all(["p", "li", "h2", "h3"])]
|
||||
return title, "\n".join(p for p in parts if p)
|
||||
|
||||
|
||||
@register
|
||||
class GovernmentNews(Adapter):
|
||||
spec = SPEC
|
||||
name = "bundesregierung.news"
|
||||
description = "Government news archive (summits, trips, meetings of government members)"
|
||||
schedule = "10 */3 * * *"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
first = int(ctx.params.get("since_year", date.today().year - 1))
|
||||
for year in range(first, date.today().year + 1):
|
||||
yield Request(url=NEWS_INDEX.format(year=year), kind="index", meta={"year": year})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
if response.request.kind == "index":
|
||||
year = response.meta["year"]
|
||||
for path in sorted(set(re.findall(
|
||||
rf'href="(/bundeskanzleramt/nachrichten-der-bundesregierung/{year}/[^"]+\.html)"', response.text))):
|
||||
yield Request(url=BASE + path, kind="article")
|
||||
return
|
||||
title, text = main_text(response)
|
||||
time_el = response.soup.find("time")
|
||||
yield Document(Evidence(url=response.request.url, doc_type="press_release", title=title,
|
||||
published_at=time_el.get("datetime") if time_el else None,
|
||||
excerpt=excerpt(text, 300), full_text=f"{title}\n{text}"))
|
||||
|
||||
|
||||
@register
|
||||
class CouncilOfMinisters(Adapter):
|
||||
spec = SPEC
|
||||
name = "bundesregierung.ministerrat"
|
||||
description = "Council of Ministers minutes: decisions per reporting minister"
|
||||
schedule = "20 6 * * 3,4"
|
||||
parser_version = 2
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
yield Request(url=MR_INDEX, kind="index")
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
kind = response.request.kind
|
||||
if kind == "index":
|
||||
periods = sorted(set(re.findall(r'href="(/bundeskanzleramt/die-bundesregierung/ministerratsprotokolle/'
|
||||
r'[^"?]+\.html)"', response.text)), key=period_start)
|
||||
# Period pages are named after the year a government took office ("...-xxviii-2025") and list all
|
||||
# its sessions, so the latest period that started before `since_year` may still contain newer ones.
|
||||
since = int(ctx.params.get("since_year", date.today().year - 1))
|
||||
started_before = [period_start(p) for p in periods if period_start(p) < since]
|
||||
keep_from = max(started_before, default=0)
|
||||
for path in reversed([p for p in periods if period_start(p) >= keep_from]): # newest first
|
||||
yield Request(url=BASE + path, kind="period")
|
||||
elif kind == "period":
|
||||
base = response.request.url.split("?")[0]
|
||||
for href in sorted(set(re.findall(r'href="([^"]+)"', response.text))):
|
||||
url = urljoin(response.url, href)
|
||||
if url.startswith(base.removesuffix(".html") + "/"):
|
||||
yield Request(url=url, kind="protocol")
|
||||
elif url.startswith(base + "?currentPage="):
|
||||
yield Request(url=url, kind="period")
|
||||
else:
|
||||
yield from self.parse_protocol(response)
|
||||
|
||||
def parse_protocol(self, response: Response) -> Iterator[Any]:
|
||||
title, text = main_text(response)
|
||||
held_on = german_date(title) or german_date(text[:300])
|
||||
url = response.request.url
|
||||
yield Document(Evidence(url=url, doc_type="protocol", title=title, published_at=held_on,
|
||||
excerpt=excerpt(text, 300), full_text=f"{title}\n{text}"))
|
||||
event_key = fingerprint("ministerrat", url)
|
||||
yield Event(key=event_key, kind="government_meeting", title=title, starts_at=held_on, location="Wien",
|
||||
country="AT", url=url, body=OrgRef(slug="bundesregierung"))
|
||||
flat = " | ".join(text.split("\n"))
|
||||
for n, m in enumerate(ITEM.finditer(flat)):
|
||||
subject = re.sub(r"\s*\|\s*", " ", m.group("subject")).strip(" .")
|
||||
zahl = re.sub(r"[\s\u2010-\u2015]+", "-", m.group("zahl").strip())
|
||||
for office in offices(m.group("offices")):
|
||||
yield Claim(
|
||||
claim_type="government_decision",
|
||||
person=PersonRef(name=None, hints={"office": office, "on": held_on}),
|
||||
key=("ministerrat", url, zahl, office), occurred_at=held_on,
|
||||
summary_de=f"Bericht im Ministerrat ({office}): {excerpt(subject, 220)}",
|
||||
summary_en=f"Report to the Council of Ministers ({office}): {excerpt(subject, 220)}",
|
||||
data={"office": office, "zahl": zahl, "item": n + 1},
|
||||
evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=held_on,
|
||||
quote=excerpt(re.sub(r"\s*\|\s*", " ", m.group(0)), 500),
|
||||
locator=f"item:{zahl}")],
|
||||
)
|
||||
# Participation of the reporting ministers in this session (calendar)
|
||||
for office in offices(m.group("offices")):
|
||||
yield Claim(
|
||||
claim_type="appearance", person=PersonRef(hints={"office": office, "on": held_on}),
|
||||
occurred_at=held_on, summary_de=f"Teilnahme am Ministerrat ({title})",
|
||||
evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=held_on)],
|
||||
facet=Participation(event_key=event_key, participation="reporting_minister"),
|
||||
)
|
||||
@@ -0,0 +1,193 @@
|
||||
"""EU institutions: Commission lobby meetings and the European Parliament (Austrian MEPs, declarations).
|
||||
|
||||
* European Commission - meetings with interest representatives (open data XLSX per Commission term).
|
||||
Every row names the cabinet ("Cabinet of Commissioner Magnus Brunner"). Only meetings of Austrian
|
||||
Commissioners' cabinets are kept (names passed from start(): Commission roles in the DB + a fallback list).
|
||||
* European Parliament Open Data API v2 (CC BY 4.0): current Austrian MEPs (authoritative mandate data),
|
||||
and their declarations (financial interests, conflicts of interest, gifts, ...).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import (
|
||||
Claim,
|
||||
Event,
|
||||
Evidence,
|
||||
OrgRef,
|
||||
Participation,
|
||||
PersonRecord,
|
||||
PersonRef,
|
||||
Role,
|
||||
)
|
||||
from politrack.text import excerpt, fingerprint, normalize
|
||||
|
||||
EC_SPEC = SourceSpec(
|
||||
slug="ec-meetings", name="Europäische Kommission – Treffen mit Interessenvertretern", kind="official",
|
||||
homepage="https://commission.europa.eu/about/service-standards-and-principles/transparency_en",
|
||||
license="EU open data (reuse permitted, attribution)", access="bulk", trust=90, min_interval=2.0,
|
||||
tos_notes="Open data export of meetings of Commissioners and their cabinets with interest representatives.",
|
||||
)
|
||||
EP_SPEC = SourceSpec(
|
||||
slug="europarl", name="Europäisches Parlament Open Data", kind="official",
|
||||
homepage="https://data.europarl.europa.eu", license="CC BY 4.0", access="api", trust=95, min_interval=1.0,
|
||||
tos_notes="EP Open Data API v2 (limit 500 requests / 5 min / endpoint).",
|
||||
)
|
||||
EC_EXPORT = "https://ec.europa.eu/transparency-initiative/meetings/data/meetings/dataxlsx"
|
||||
EC_TERMS = ("meetingscommissionrepresentatives2429", "meetingscommissionrepresentatives1924")
|
||||
EP_API = "https://data.europarl.europa.eu/api/v2"
|
||||
# Austrian Commissioners in the terms covered by EC_TERMS (fallback if the DB has no Commission roles yet).
|
||||
AUSTRIAN_COMMISSIONERS = ("Johannes Hahn", "Magnus Brunner")
|
||||
CABINET = re.compile(r"Cabinet of (?:Executive Vice-President|Vice-President|Commissioner|President)\s+(?P<name>.+)$")
|
||||
|
||||
|
||||
@register
|
||||
class CommissionMeetings(Adapter):
|
||||
spec = EC_SPEC
|
||||
name = "eu.commission_meetings"
|
||||
description = "Meetings of (Austrian) EU Commissioners and their cabinets with interest representatives"
|
||||
schedule = "0 5 * * 1"
|
||||
queue = "bulk"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
names = set(AUSTRIAN_COMMISSIONERS)
|
||||
if ctx.conn is not None:
|
||||
names |= {row["full_name"] for row in ctx.conn.execute(
|
||||
"""SELECT DISTINCT p.full_name FROM role r JOIN person p ON p.id = r.person_id
|
||||
JOIN organization o ON o.id = r.organization_id WHERE o.slug = 'europaeische-kommission'""")}
|
||||
for term in EC_TERMS:
|
||||
yield Request(url=EC_EXPORT, params={"name": term}, kind="xlsx", meta={"commissioners": sorted(names)})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
import openpyxl
|
||||
|
||||
wb = openpyxl.load_workbook(io.BytesIO(response.body), read_only=True, data_only=True)
|
||||
rows = list(wb.worksheets[0].iter_rows(values_only=True))
|
||||
header_index = next((i for i, r in enumerate(rows) if r and r[0] == "Name of cabinet"), None)
|
||||
if header_index is None:
|
||||
ctx.stats["unexpected_layout"] += 1
|
||||
return
|
||||
header = [str(h) for h in rows[header_index]]
|
||||
url = f"{EC_EXPORT}?name={response.request.params['name']}"
|
||||
wanted = {normalize(n) for n in response.meta.get("commissioners") or AUSTRIAN_COMMISSIONERS}
|
||||
for row in rows[header_index + 1:]:
|
||||
rec = dict(zip(header, row))
|
||||
cabinet = rec.get("Name of cabinet") or ""
|
||||
m = CABINET.search(cabinet.strip())
|
||||
if not m or not rec.get("Date of meeting"):
|
||||
continue
|
||||
commissioner = m.group("name").strip()
|
||||
if normalize(commissioner) not in wanted:
|
||||
ctx.stats["other_commissioners"] += 1
|
||||
continue
|
||||
org_name = (rec.get("Name of interest representative") or "").strip()
|
||||
held = str(rec["Date of meeting"])[:10]
|
||||
subject = (rec.get("Subject of the meeting") or "").strip()
|
||||
reps = rec.get("Name of EC representative") or ""
|
||||
titles = rec.get("Title of EC representative") or ""
|
||||
register_id = (rec.get("Transparency register ID") or "").strip()
|
||||
event_key = fingerprint("ec-meeting", cabinet, held, org_name, subject)
|
||||
yield Event(key=event_key, kind="meeting", title=f"{cabinet}: {org_name}", starts_at=held,
|
||||
location=rec.get("Location"), url=url, description=subject,
|
||||
body=OrgRef(slug="europaeische-kommission"))
|
||||
yield Claim(
|
||||
claim_type="meeting", person=PersonRef(name=commissioner), occurred_at=held,
|
||||
counterpart_org=OrgRef(name=org_name, kind="lobby",
|
||||
ids={"eu_transparency": register_id} if register_id else {}) if org_name else None,
|
||||
summary_de=f"Treffen ({'Kommissar persönlich' if 'Commissioner' in titles else 'Kabinett'}) mit "
|
||||
f"{org_name}: {excerpt(subject, 160)}",
|
||||
summary_en=f"Meeting ({'Commissioner' if 'Commissioner' in titles else 'cabinet'}) with {org_name}: "
|
||||
f"{excerpt(subject, 160)}",
|
||||
data={"cabinet": cabinet, "representatives": reps, "titles": titles, "location": rec.get("Location"),
|
||||
"register_id": register_id, "date_precision": "day"},
|
||||
evidence=[Evidence(url=url, doc_type="dataset_row", title=f"EU-Kommission: Treffen {cabinet}",
|
||||
published_at=held, quote=f"{held} | {org_name} | {subject}",
|
||||
locator=f"{held}:{org_name}:{subject[:60]}")],
|
||||
topics=("Europäische Union",),
|
||||
facet=Participation(event_key=event_key, participation="host"),
|
||||
)
|
||||
|
||||
|
||||
@register
|
||||
class EuropeanParliament(Adapter):
|
||||
spec = EP_SPEC
|
||||
name = "europarl.meps"
|
||||
description = "Current Austrian MEPs (mandates) and their declarations"
|
||||
schedule = "30 4 * * 2"
|
||||
creates_persons = True
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
yield Request(url=f"{EP_API}/meps/show-current", kind="meps",
|
||||
params={"country-of-representation": "AT", "format": "application/ld+json", "limit": 100})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
data = response.json().get("data") or []
|
||||
if response.request.kind == "meps":
|
||||
for mep in data:
|
||||
mep_id = str(mep.get("identifier"))
|
||||
given, family = mep.get("givenName"), mep.get("familyName")
|
||||
name = f"{given} {family}" if given and family else mep.get("label")
|
||||
ref = PersonRef(ids={"ep": mep_id}, name=name)
|
||||
yield PersonRecord(ref=ref, full_name=name, given_name=given, family_name=family,
|
||||
aliases=(mep.get("label") or "",), match_by_name=True)
|
||||
yield Request(url=f"{EP_API}/meps/{mep_id}", params={"format": "application/ld+json"}, kind="mep",
|
||||
meta={"mep_id": mep_id, "name": name, "group": mep.get("api:political-group")})
|
||||
yield Request(url=f"{EP_API}/meps-declarations", kind="declarations",
|
||||
params={"person-id": mep_id, "format": "application/ld+json", "limit": 100},
|
||||
meta={"mep_id": mep_id, "name": name})
|
||||
elif response.request.kind == "mep":
|
||||
yield from self.parse_mep(response, data[0] if data else {})
|
||||
else:
|
||||
yield from self.parse_declarations(response, data)
|
||||
|
||||
def parse_mep(self, response: Response, mep: dict) -> Iterator[Any]:
|
||||
meta = response.meta
|
||||
ref = PersonRef(ids={"ep": meta["mep_id"]})
|
||||
if mep.get("img"):
|
||||
yield PersonRecord(ref=ref, full_name=meta["name"], photo_url=mep["img"],
|
||||
birth_date=date.fromisoformat(mep["bday"][:10]) if mep.get("bday") else None)
|
||||
starts = [m.get("memberDuring", {}).get("startDate") for m in mep.get("hasMembership") or []
|
||||
if "EU_PARLIAMENT" in str(m.get("membershipClassification", ""))]
|
||||
starts = sorted(s for s in starts if s)
|
||||
body = OrgRef(slug="europaeisches-parlament")
|
||||
start = date.fromisoformat(starts[-1]) if starts else None
|
||||
url = f"https://www.europarl.europa.eu/meps/de/{meta['mep_id']}"
|
||||
yield Claim(
|
||||
claim_type="role", person=ref, org=body, valid_from=start,
|
||||
summary_de=f"Mitglied des Europäischen Parlaments (Fraktion {meta.get('group') or '?'})",
|
||||
summary_en=f"Member of the European Parliament ({meta.get('group') or '?'})",
|
||||
data={"ep_group": meta.get("group")},
|
||||
evidence=[Evidence(url=url, doc_type="api_record", title=f"Europäisches Parlament: {meta['name']}")],
|
||||
facet=Role(role="mp", level="eu", body=body, start=start),
|
||||
)
|
||||
|
||||
def parse_declarations(self, response: Response, docs: list[dict]) -> Iterator[Claim]:
|
||||
meta = response.meta
|
||||
for doc in docs:
|
||||
titles = []
|
||||
pdf = None
|
||||
for expression in doc.get("is_realized_by") or []:
|
||||
title = (expression.get("title") or {})
|
||||
titles.extend(v for v in title.values() if v)
|
||||
for manifestation in expression.get("is_embodied_by") or []:
|
||||
if "pdf" in str(manifestation.get("media_type", "")).lower() and not pdf:
|
||||
pdf = manifestation.get("is_exemplified_by")
|
||||
title = next((t for t in titles if t), doc.get("id"))
|
||||
url = f"https://www.europarl.europa.eu/{pdf}" if pdf and not pdf.startswith("http") else (
|
||||
pdf or f"https://data.europarl.europa.eu/{doc.get('id')}")
|
||||
kind = str(doc.get("work_type", "")).rsplit("/", 1)[-1]
|
||||
yield Claim(
|
||||
claim_type="declaration", person=PersonRef(ids={"ep": meta["mep_id"]}),
|
||||
key=("ep-declaration", doc.get("id")), occurred_at=doc.get("document_date"),
|
||||
summary_de=f"Erklärung im Europäischen Parlament: {excerpt(title, 200)}",
|
||||
summary_en=f"Declaration filed with the European Parliament: {excerpt(title, 200)}",
|
||||
data={"work_type": kind},
|
||||
evidence=[Evidence(url=url, doc_type="pdf", title=title, published_at=doc.get("document_date"))],
|
||||
topics=("Transparenz und Korruption",),
|
||||
)
|
||||
@@ -0,0 +1,70 @@
|
||||
"""GDELT DOC 2.0 API: news coverage of politicians across many more outlets (metadata only).
|
||||
|
||||
Free, open, max. one request every 5 s (we use 6 s and back off on the plain-text throttle notice).
|
||||
Coverage of German-language Austrian media is thin, so this is a supplement. Queries: active
|
||||
federal/EU politicians daily, former politicians in a weekly rotation ("what are they up to?").
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Document, Evidence
|
||||
|
||||
API = "https://api.gdeltproject.org/api/v2/doc/doc"
|
||||
SPEC = SourceSpec(
|
||||
slug="gdelt", name="Weitere Medien (via GDELT)", kind="aggregator", homepage="https://www.gdeltproject.org",
|
||||
license="GDELT open data (citation required)", access="api", trust=55, store_fulltext=False, keep_raw=False,
|
||||
min_interval=6.0, tos_notes="GDELT DOC 2.0 API: article metadata (URL, title, date). Max 1 request / 5 s.",
|
||||
)
|
||||
DOMAIN_SOURCES = {
|
||||
"orf.at": "orf", "derstandard.at": "derstandard", "diepresse.com": "diepresse", "kurier.at": "kurier",
|
||||
"profil.at": "profil", "falter.at": "falter", "krone.at": "krone", "sn.at": "sn", "kleinezeitung.at": "kleinezeitung",
|
||||
"ots.at": "apa-ots", "vol.at": "vol", "vienna.at": "vol",
|
||||
}
|
||||
|
||||
|
||||
@register
|
||||
class Gdelt(Adapter):
|
||||
spec = SPEC
|
||||
name = "gdelt.coverage"
|
||||
description = "Supplementary news coverage via GDELT (active and former politicians)"
|
||||
schedule = "0 1 * * *"
|
||||
queue = "bulk"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
if ctx.conn is None:
|
||||
return
|
||||
rows = ctx.conn.execute(
|
||||
"""SELECT p.id, p.given_name, p.family_name, s.status FROM person p
|
||||
JOIN person_status s ON s.person_id = p.id
|
||||
WHERE p.given_name IS NOT NULL AND p.family_name IS NOT NULL AND (
|
||||
(s.status = 'active' AND EXISTS (SELECT 1 FROM role r WHERE r.person_id = p.id
|
||||
AND r.level IN ('federal', 'eu') AND r.end_date IS NULL))
|
||||
OR (s.status IN ('former', 'party_member') AND s.last_office_end > current_date - interval '25 years'
|
||||
AND p.id %% 7 = %s))
|
||||
ORDER BY s.status, p.id LIMIT %s""",
|
||||
(date.today().weekday(), int(ctx.params.get("max_queries", 400))),
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
name = f"{row['given_name']} {row['family_name']}"
|
||||
yield Request(url=API, kind="artlist", meta={"person_id": row["id"]}, conditional=False,
|
||||
params={"query": f'"{name}" sourcecountry:AU', "mode": "artlist", "format": "json",
|
||||
"maxrecords": 50, "timespan": "7d" if row["status"] == "active" else "1m",
|
||||
"sort": "datedesc"})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
if not response.text.strip().startswith("{"):
|
||||
ctx.stats["throttled"] += 1 # plain-text rate-limit notice
|
||||
return
|
||||
for article in json.loads(response.text, strict=False).get("articles", []):
|
||||
domain = (article.get("domain") or "").removeprefix("www.")
|
||||
seen = article.get("seendate") or ""
|
||||
published = f"{seen[:4]}-{seen[4:6]}-{seen[6:8]}T{seen[9:11]}:{seen[11:13]}:00Z" if len(seen) >= 13 else None
|
||||
yield Document(Evidence(url=article["url"], source=DOMAIN_SOURCES.get(domain, "gdelt"),
|
||||
title=article.get("title"), doc_type="article", published_at=published,
|
||||
meta={"via": "gdelt", "domain": domain}))
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Press releases of the Wirtschafts- und Korruptionsstaatsanwaltschaft (WKStA, anti-corruption prosecutors).
|
||||
|
||||
Official statements on high-profile proceedings. They usually anonymise suspects ("ein ehemaliger
|
||||
Bundesminister"), so we store them as official documents (full text, searchable, RAG) and let the
|
||||
legal-matter detector propose claims - which, as always for legal matters, need human review.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Document, Evidence
|
||||
from politrack.sources.bundesregierung import german_date
|
||||
from politrack.text import excerpt
|
||||
|
||||
BASE = "https://www.justiz.gv.at"
|
||||
START = BASE + "/wksta/wirtschafts-und-korruptionsstaatsanwaltschaft.312.de.html"
|
||||
SPEC = SourceSpec(
|
||||
slug="wksta", name="WKStA – Pressemitteilungen", kind="official", homepage=START,
|
||||
license="Amtliches Werk (§7 UrhG)", access="scrape", trust=90, min_interval=3.0,
|
||||
tos_notes="Press releases of the anti-corruption prosecutors. robots.txt allows all.",
|
||||
)
|
||||
RELEASE = re.compile(r"/medienstelle/pressemitteilungen/pressemitteilungen-(\d{4})/")
|
||||
LISTING = re.compile(r"/medienstelle(?:/pressemitteilungen(?:/pressemitteilungen-\d{4})?)?\.[0-9a-f]+\.de\.html")
|
||||
|
||||
|
||||
def clean(url: str) -> str:
|
||||
return re.sub(r";jsessionid=[^?#]*", "", url)
|
||||
|
||||
|
||||
@register
|
||||
class WkstaPress(Adapter):
|
||||
spec = SPEC
|
||||
name = "justiz.wksta"
|
||||
description = "WKStA press releases (official documents for search and reviewed legal matters)"
|
||||
schedule = "40 8 * * *"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
yield Request(url=START, kind="listing", meta={"depth": 0})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
if response.request.kind == "listing":
|
||||
since = int(ctx.params.get("since_year", date.today().year - 2))
|
||||
for a in response.soup.find_all("a", href=True):
|
||||
url = clean(urljoin(response.url, a["href"]))
|
||||
year = RELEASE.search(url)
|
||||
if year and int(year.group(1)) >= since:
|
||||
yield Request(url=url, kind="release")
|
||||
elif LISTING.search(url) and response.meta["depth"] < 2:
|
||||
yield Request(url=url, kind="listing", meta={"depth": response.meta["depth"] + 1})
|
||||
return
|
||||
main = response.soup.find("main") or response.soup.find(id="content") or response.soup
|
||||
title_el = main.find("h1")
|
||||
title = title_el.get_text(" ", strip=True) if title_el else response.url
|
||||
text = "\n".join(p.get_text(" ", strip=True) for p in main.find_all(["p", "li"]) if p.get_text(strip=True))
|
||||
yield Document(Evidence(url=clean(response.request.url), doc_type="press_release", title=title,
|
||||
published_at=german_date(text[:400]) or german_date(title), excerpt=excerpt(text, 300),
|
||||
full_text=f"{title}\n{text}"))
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Headline monitoring via publishers' RSS feeds and Google-News sitemaps (listed in robots.txt).
|
||||
|
||||
Policy (docs/sources.md): only feeds/sitemaps publishers offer for automated discovery; store headline,
|
||||
teaser/keywords and link - never article text (store_fulltext=False, keep_raw=False). If a site declares
|
||||
a TDM reservation, the Loader drops even the teaser and linking uses the headline only.
|
||||
|
||||
The feed list lives in config/feeds.yaml (user-editable); every `source` there needs a SourceSpec below.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime, timezone
|
||||
from time import mktime
|
||||
from typing import Any
|
||||
|
||||
import feedparser
|
||||
import yaml
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register, register_spec
|
||||
from politrack.core.records import Document, Evidence
|
||||
from politrack.text import html_to_text
|
||||
|
||||
|
||||
def media(slug: str, name: str, homepage: str, trust: int, notes: str = "RSS/news sitemap only; headline + link.",
|
||||
license: str = "All rights reserved") -> SourceSpec:
|
||||
return register_spec(SourceSpec(slug=slug, name=name, kind="media", homepage=homepage, license=license,
|
||||
access="rss", trust=trust, store_fulltext=False, keep_raw=False,
|
||||
min_interval=2.0, tos_notes=notes))
|
||||
|
||||
|
||||
ORF = media("orf", "ORF.at", "https://orf.at", 75, "Public RSS feeds (rss.orf.at). Headline, teaser, link.")
|
||||
media("derstandard", "DER STANDARD", "https://www.derstandard.at", 75,
|
||||
"RSS + news sitemap. robots.txt blocks AI crawlers by name; we only use feed/sitemap and link out.")
|
||||
media("diepresse", "Die Presse", "https://www.diepresse.com", 75)
|
||||
media("kurier", "KURIER", "https://kurier.at", 70)
|
||||
media("profil", "profil", "https://www.profil.at", 75)
|
||||
media("falter", "FALTER", "https://www.falter.at", 75)
|
||||
media("krone", "Kronen Zeitung", "https://www.krone.at", 55)
|
||||
media("sn", "Salzburger Nachrichten", "https://www.sn.at", 75)
|
||||
media("kleinezeitung", "Kleine Zeitung", "https://www.kleinezeitung.at", 75)
|
||||
media("vol", "Vorarlberg Online", "https://www.vol.at", 65)
|
||||
media("apa-ots", "APA-OTS Presseaussendungen", "https://www.ots.at", 60,
|
||||
"tdmrep.json declares tdm-reservation=1 for the whole site: headline + link only.")
|
||||
|
||||
NS = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9", "news": "http://www.google.com/schemas/sitemap-news/0.9"}
|
||||
|
||||
|
||||
@register
|
||||
class NewsFeeds(Adapter):
|
||||
spec = ORF
|
||||
name = "news.feeds"
|
||||
description = "Headlines from outlets' RSS feeds and Google-News sitemaps (config/feeds.yaml)"
|
||||
schedule = "*/20 * * * *"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
config = yaml.safe_load((get_settings().config_dir / "feeds.yaml").read_text(encoding="utf-8"))
|
||||
for feed in config.get("feeds") or []:
|
||||
yield Request(url=feed["url"], kind=feed.get("type", "rss"), meta={"source": feed["source"]},
|
||||
skip_unchanged=True)
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
source = response.meta["source"]
|
||||
doc_type = "press_release" if source == "apa-ots" else "article"
|
||||
if response.request.kind == "sitemap":
|
||||
root = ET.fromstring(response.body)
|
||||
for node in root.findall("sm:url", NS):
|
||||
link, title = node.findtext("sm:loc", namespaces=NS), node.findtext("news:news/news:title", namespaces=NS)
|
||||
if not link or not title:
|
||||
continue
|
||||
keywords = node.findtext("news:news/news:keywords", namespaces=NS)
|
||||
yield Document(Evidence(url=link.strip(), source=source, title=html_to_text(title), doc_type=doc_type,
|
||||
published_at=node.findtext("news:news/news:publication_date", namespaces=NS),
|
||||
excerpt=f"Schlagworte: {keywords}" if keywords else None,
|
||||
meta={"sitemap": response.request.url}))
|
||||
return
|
||||
for entry in feedparser.parse(response.body).entries:
|
||||
if not entry.get("link"):
|
||||
continue
|
||||
published = (datetime.fromtimestamp(mktime(entry.published_parsed), tz=timezone.utc)
|
||||
if entry.get("published_parsed") else None)
|
||||
yield Document(Evidence(url=entry["link"], source=source, title=html_to_text(entry.get("title")),
|
||||
doc_type=doc_type, published_at=published, excerpt=html_to_text(entry.get("summary")),
|
||||
meta={"feed": response.request.url}))
|
||||
@@ -0,0 +1 @@
|
||||
"""Austrian Parliament adapters: persons, items (votes, immunity, questions, hearings), press, calendar."""
|
||||
@@ -0,0 +1,199 @@
|
||||
"""Shared spec and pure parsing helpers for the Austrian Parliament adapters (CC BY 4.0).
|
||||
|
||||
API (https://www.parlament.gv.at/recherchieren/open-data/):
|
||||
* list exports: POST /Filter/api/filter/data/<listeId>?js=eval&showAll=true (JSON body = filter)
|
||||
409 persons since 1918 · 101 parliamentary items · 251 speeches of a person
|
||||
110 Parlamentskorrespondenz (press releases) · 600 calendar (Termine)
|
||||
* detail pages: <page URL>?json=TRUE
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import date, datetime
|
||||
from typing import Any
|
||||
|
||||
from politrack.core import Request, SourceSpec, register_spec
|
||||
from politrack.text import html_to_text
|
||||
|
||||
BASE = "https://www.parlament.gv.at"
|
||||
CHAMBER_SLUG = {"NR": "nationalrat", "BR": "bundesrat"}
|
||||
CHAMBER_NAME_DE = {"NR": "Nationalrat", "BR": "Bundesrat"}
|
||||
GOVERNMENT_RE = re.compile(r"Bundeskanzler|Vizekanzler|Bundesminister|Staatssekretär", re.IGNORECASE)
|
||||
|
||||
SPEC = register_spec(SourceSpec(
|
||||
slug="parlament",
|
||||
name="Parlament Österreich (Open Data)",
|
||||
kind="official",
|
||||
homepage="https://www.parlament.gv.at/recherchieren/open-data/",
|
||||
license="CC BY 4.0",
|
||||
access="api",
|
||||
trust=95,
|
||||
store_fulltext=True,
|
||||
min_interval=1.0,
|
||||
tos_notes="Official JSON API (?json=TRUE, /Filter/api/filter/data/<list>). Attribution required. The "
|
||||
"Parlamentsdirektion may prohibit uses incompatible with the dignity of parliament.",
|
||||
))
|
||||
|
||||
|
||||
def list_request(list_id: int, body: dict, kind: str, meta: dict | None = None) -> Request:
|
||||
return Request(url=f"{BASE}/Filter/api/filter/data/{list_id}?js=eval&showAll=true", method="POST", json=body,
|
||||
kind=kind, meta=meta or {})
|
||||
|
||||
|
||||
def detail_request(path: str, kind: str, meta: dict | None = None) -> Request:
|
||||
return Request(url=f"{BASE}{path}", params={"json": "TRUE"}, kind=kind, meta=meta or {})
|
||||
|
||||
|
||||
def list_rows(data: dict) -> list[dict[str, Any]]:
|
||||
"""Turn a filter-API export into dicts keyed by column label (duplicates get a _<index> suffix)."""
|
||||
labels: list[str] = []
|
||||
for i, header in enumerate(data.get("header", [])):
|
||||
label = header.get("label") or header.get("feld_name") or f"col{i}"
|
||||
labels.append(label if label not in labels else f"{label}_{i}")
|
||||
return [dict(zip(labels, row)) for row in data.get("rows", [])]
|
||||
|
||||
|
||||
def parse_date(value: str | None) -> date | None:
|
||||
if not value:
|
||||
return None
|
||||
for fmt in ("%d.%m.%Y", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d"):
|
||||
try:
|
||||
return datetime.strptime(value[:19], fmt).date()
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def walk_dicts(obj: Any, required_key: str):
|
||||
"""Yield every nested dict that contains `required_key`."""
|
||||
if isinstance(obj, dict):
|
||||
if required_key in obj:
|
||||
yield obj
|
||||
for value in obj.values():
|
||||
yield from walk_dicts(value, required_key)
|
||||
elif isinstance(obj, list):
|
||||
for value in obj:
|
||||
yield from walk_dicts(value, required_key)
|
||||
|
||||
|
||||
# -- faction votes ------------------------------------------------------------------------
|
||||
|
||||
PARTY_TOKEN = re.compile(r"^[A-ZÄÖÜ][A-Za-zÄÖÜäöüß.\- ]{0,30}$")
|
||||
# Split votes ("teilweise", "bzw.", per-part results in parentheses) can't be attributed to factions.
|
||||
SPLIT_VOTE = re.compile(r"\(|\bbzw\.|\bteilweise\b|\btlw\.|\bmehrstimmig\b|\bgetrennt", re.IGNORECASE)
|
||||
|
||||
|
||||
def parse_party_list(value: str | None) -> list[str]:
|
||||
if not value:
|
||||
return []
|
||||
parts = [p.strip(" .;") for p in re.split(r",|\bund\b", value)]
|
||||
return [p for p in parts if PARTY_TOKEN.match(p)]
|
||||
|
||||
|
||||
def parse_vote_text(text: str) -> dict[str, Any] | None:
|
||||
"""'137. Sitzung …: Gesetzesvorschlag in dritter Lesung angenommen Dafür: ÖVP, SPÖ, dagegen: FPÖ'"""
|
||||
m = re.search(r"\b(angenommen|abgelehnt)\b", text, re.IGNORECASE)
|
||||
if not m:
|
||||
return None
|
||||
rest = text[m.end():]
|
||||
roll_call = "namentliche abstimmung" in text.lower()
|
||||
split = bool(SPLIT_VOTE.search(rest)) and not roll_call
|
||||
pro = re.search(r"Dafür:\s*(.*?)(?:,?\s*dagegen:|$)", rest, re.IGNORECASE)
|
||||
contra = re.search(r"dagegen:\s*(.*)$", rest, re.IGNORECASE)
|
||||
return {
|
||||
"stage": text[: m.start()].strip(" :,"),
|
||||
"result": "accepted" if m.group(1).lower() == "angenommen" else "rejected",
|
||||
"for": [] if split or roll_call else parse_party_list(pro.group(1) if pro else None),
|
||||
"against": [] if split or roll_call else parse_party_list(contra.group(1) if contra else None),
|
||||
"unanimous": "einstimmig" in text.lower(),
|
||||
"split": split,
|
||||
"roll_call": roll_call,
|
||||
}
|
||||
|
||||
|
||||
# -- roll-call votes ----------------------------------------------------------------------
|
||||
|
||||
ROLL_CALL_HEADER = re.compile(r"Mit\s*[„\"“]?\s*(Ja|Nein)\s*[“\"”]?\s*stimmten[^:]*:", re.IGNORECASE)
|
||||
NEW_VOTE = re.compile(r"\b(gelangen|kommen)\s+(?:\w+\s+){0,2}zur\s+Abstimmung\b", re.IGNORECASE)
|
||||
PERSON_LINK = re.compile(r"<a[^>]*href=\"[^\"]*/person/(\d+)\"[^>]*>.*?</a>", re.IGNORECASE | re.DOTALL)
|
||||
|
||||
|
||||
def parse_roll_call(html_text: str, anchor: str | None) -> dict[str, list[tuple[str, int | None]]]:
|
||||
"""Extract the "Mit „Ja“ stimmten …" / "Mit „Nein“ stimmten …" lists that follow `anchor`.
|
||||
|
||||
Returns {"for": [(label, pad|None)], "against": [...]}. Names are mostly surnames; where a surname is
|
||||
ambiguous the protocol links the person, which gives us the parliament id directly.
|
||||
"""
|
||||
start = 0
|
||||
if anchor:
|
||||
m = re.search(r'id="%s"' % re.escape(anchor), html_text)
|
||||
if not m:
|
||||
return {}
|
||||
start = m.start()
|
||||
first = ROLL_CALL_HEADER.search(html_text, start)
|
||||
if not first:
|
||||
return {}
|
||||
# The lists must belong to this anchor: they follow within a few numbered paragraphs and no new vote
|
||||
# may be called in between.
|
||||
between = html_text[start + 10 : first.start()]
|
||||
if anchor and (len(re.findall(r'class="[^"]*randnummer', between)) > 3 or NEW_VOTE.search(html_to_text(between))):
|
||||
return {}
|
||||
end_match = re.compile(r"<hr|class=\"[^\"]*randnummer").search(html_text, first.end())
|
||||
segment = html_text[first.start() : end_match.start() if end_match else len(html_text)]
|
||||
segment = PERSON_LINK.sub(lambda m: f" @@{m.group(1)}@@ ", segment)
|
||||
text = html_to_text(segment)
|
||||
|
||||
result: dict[str, list[tuple[str, int | None]]] = {"for": [], "against": []}
|
||||
headers = list(ROLL_CALL_HEADER.finditer(text))
|
||||
for i, header in enumerate(headers):
|
||||
block = text[header.end() : headers[i + 1].start() if i + 1 < len(headers) else len(text)]
|
||||
key = "for" if header.group(1).lower() == "ja" else "against"
|
||||
for token in re.split(r"[,;]|\.\s*$", block):
|
||||
token = token.strip(" .")
|
||||
if not token:
|
||||
continue
|
||||
link = re.fullmatch(r"@@(\d+)@@", token)
|
||||
result[key].append((token, int(link.group(1))) if link else (token, None))
|
||||
return result
|
||||
|
||||
|
||||
# -- immunity -----------------------------------------------------------------------------
|
||||
|
||||
STGB_OFFENSES = {
|
||||
"111": "Üble Nachrede", "115": "Beleidigung", "146": "Betrug", "153": "Untreue", "165": "Geldwäscherei",
|
||||
"201": "Vergewaltigung", "283": "Verhetzung", "288": "Falsche Beweisaussage", "293": "Fälschung eines Beweismittels",
|
||||
"295": "Unterdrückung eines Beweismittels", "297": "Verleumdung", "302": "Missbrauch der Amtsgewalt",
|
||||
"304": "Bestechlichkeit", "305": "Vorteilsannahme", "306": "Vorteilsannahme zur Beeinflussung",
|
||||
"307": "Bestechung", "308": "Verbotene Intervention", "310": "Verletzung des Amtsgeheimnisses",
|
||||
}
|
||||
|
||||
|
||||
def parse_immunity_report(text: str) -> dict[str, Any]:
|
||||
authority = re.search(r"Ersuchen (?:der|des) (.+?),\s*(?:do\.|GZ|Zl)", text)
|
||||
case_ref = re.search(r"\bGZ\.?\s*([^,]+?),", text)
|
||||
# "wegen des Verdachtes einer strafbaren Handlung nach § 283 Abs. 1 Z 1 und Z 2, Abs. 2 StGB"
|
||||
offense = re.search(r"wegen[^§]{0,90}?(§§? ?\d+.{0,120}?\b(?:StGB|VerbotsG|FinStrG|[A-Z][A-Za-z]*G)\b)", text)
|
||||
offense_text = offense.group(1).strip() if offense else None
|
||||
if offense_text:
|
||||
section = re.search(r"§ ?(\d+)", offense_text)
|
||||
if section and section.group(1) in STGB_OFFENSES and "StGB" in offense_text:
|
||||
offense_text = f"{offense_text} ({STGB_OFFENSES[section.group(1)]})"
|
||||
decision = re.search(r"(?:wird|werde)\s+(nicht\s+)?zugestimmt|(nicht\s+)?zuzustimmen", text)
|
||||
recommendation = None
|
||||
if decision:
|
||||
recommendation = "refuse" if (decision.group(1) or decision.group(2)) else "consent"
|
||||
connection = None
|
||||
if re.search(r"\bkein Zusammenhang\b", text):
|
||||
connection = False
|
||||
elif re.search(r"\bein Zusammenhang\b", text):
|
||||
connection = True
|
||||
quote = re.search(r"Als Ergebnis seiner Beratungen.*?(?:zugestimmt|nicht zugestimmt)\s*\.", text)
|
||||
return {
|
||||
"authority": authority.group(1).strip() if authority else None,
|
||||
"case_ref": case_ref.group(1).strip() if case_ref else None,
|
||||
"offense": offense_text,
|
||||
"recommendation": recommendation,
|
||||
"political_connection": connection,
|
||||
"quote": quote.group(0) if quote else None,
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
"""Parliament items (Gegenstände) of the configured legislative periods.
|
||||
|
||||
From one list export per period (list 101) we derive:
|
||||
* votes -> item detail -> VoteEvent + faction VotePositions
|
||||
roll calls -> stenographic protocol -> individual VotePositions
|
||||
* immunity requests ("Ersuchen um Zustimmung zur behördlichen Verfolgung") -> committee report -> Legal claim
|
||||
* written questions and answers (ITYP J / AB) -> parliamentary_question claims (directly from the list)
|
||||
* investigative-committee hearings ("öffentliche Befragung der Auskunftsperson ...") -> hearing Event +
|
||||
appearance claim for politicians who testified (other witnesses are not stored)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Adapter, Request, Response, RunContext, register
|
||||
from politrack.core.records import (
|
||||
Claim,
|
||||
Event,
|
||||
Evidence,
|
||||
Legal,
|
||||
OrgRef,
|
||||
Participation,
|
||||
PersonRef,
|
||||
VoteEvent,
|
||||
VotePosition,
|
||||
)
|
||||
from politrack.sources.parlament.common import (
|
||||
BASE,
|
||||
CHAMBER_SLUG,
|
||||
SPEC,
|
||||
detail_request,
|
||||
list_request,
|
||||
list_rows,
|
||||
parse_date,
|
||||
parse_immunity_report,
|
||||
parse_roll_call,
|
||||
parse_vote_text,
|
||||
walk_dicts,
|
||||
)
|
||||
from politrack.text import fingerprint, html_to_text, strip_titles
|
||||
|
||||
HEARING = re.compile(r"Befragung der Auskunftsperson\s+(?P<name>.+)$")
|
||||
WITNESS_TITLES = re.compile(r"^(?:(?:Oberstaatsanwalt|Staatsanwalt|Sektionschef(?:in)?|MinR|Generalsekretär(?:in)?|"
|
||||
r"Bundesminister(?:in)?|Abgeordnete[rn]?|Klubobmann|Klubobfrau|Bundeskanzler|"
|
||||
r"Vizekanzler|Präsident(?:in)?)\s+)+")
|
||||
|
||||
|
||||
@register
|
||||
class ParlamentItems(Adapter):
|
||||
spec = SPEC
|
||||
name = "parlament.items"
|
||||
description = "Votes (faction + roll call), immunity requests, written questions, U-committee hearings"
|
||||
schedule = "45 3 * * *"
|
||||
parser_version = 2
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
for gp in ctx.params.get("periods") or get_settings().parlament_periods:
|
||||
yield list_request(101, {"GP_CODE": [gp], "NRBR": ["NR"]}, kind="items", meta={"gp": gp})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
handler = {
|
||||
"items": self.parse_items,
|
||||
"votes": self.parse_votes,
|
||||
"rollcall": self.parse_rollcall,
|
||||
"immunity": self.parse_immunity_item,
|
||||
"immunity_report": self.parse_immunity_report,
|
||||
"hearing": self.parse_hearing_item,
|
||||
}[response.request.kind]
|
||||
yield from handler(ctx, response)
|
||||
|
||||
# -- list ---------------------------------------------------------------------------
|
||||
|
||||
def parse_items(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
for row in list_rows(response.json()):
|
||||
path, subject, kind = row.get("HIS_URL"), row.get("Betreff") or "", row.get("ITYP")
|
||||
if not path:
|
||||
continue
|
||||
if row.get("Dafür") or row.get("Dagegen") or row.get("Abstimmungstext"):
|
||||
# The list row changes when new votes happen, so the detail is re-fetched only then
|
||||
# (conditional request + unchanged-body detection do the rest).
|
||||
yield detail_request(path, kind="votes", meta={"title": subject, "ref": row.get("Nummer")})
|
||||
if subject.startswith("Ersuchen") and "behördlichen Verfolgung" in subject:
|
||||
yield detail_request(path, kind="immunity",
|
||||
meta={"title": subject, "date": row.get("Datum"), "status": row.get("Status")})
|
||||
if kind in ("J", "AB"):
|
||||
yield from self.question_claims(row, path)
|
||||
if kind == "KOMM" and HEARING.search(subject):
|
||||
yield detail_request(path, kind="hearing", meta={"title": subject, "date": row.get("Datum")})
|
||||
|
||||
def question_claims(self, row: dict, path: str) -> Iterator[Claim]:
|
||||
pads = row.get("Personen") or "[]"
|
||||
pads = json.loads(pads) if isinstance(pads, str) else pads
|
||||
subject = row.get("Betreff") or ""
|
||||
is_question = row.get("ITYP") == "J"
|
||||
asked = parse_date(row.get("Datum"))
|
||||
topics = tuple(json.loads(row.get("THEMEN") or "[]")) if isinstance(row.get("THEMEN"), str) else ()
|
||||
for pad in pads:
|
||||
if not str(pad).strip():
|
||||
continue
|
||||
verb_de = "stellte die schriftliche Anfrage" if is_question else "beantwortete die Anfrage"
|
||||
verb_en = "submitted the written question" if is_question else "answered the question"
|
||||
yield Claim(
|
||||
claim_type="parliamentary_question", person=PersonRef(ids={"parlament": str(pad)}),
|
||||
key=("question", path, str(pad)), occurred_at=asked,
|
||||
summary_de=f"{verb_de[0].upper()}{verb_de[1:]} „{subject}“ ({row.get('Nummer')})",
|
||||
summary_en=f"{verb_en[0].upper()}{verb_en[1:]} \"{subject}\" ({row.get('Nummer')})",
|
||||
data={"kind": "question" if is_question else "answer", "number": row.get("Nummer")},
|
||||
evidence=[Evidence(url=f"{BASE}{path}", doc_type="api_record", title=subject, published_at=asked)],
|
||||
topics=topics,
|
||||
)
|
||||
|
||||
# -- votes ----------------------------------------------------------------------------
|
||||
|
||||
def parse_votes(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
content = response.json().get("content") or {}
|
||||
path = response.request.url.removeprefix(BASE)
|
||||
title = html_to_text(content.get("title")) or response.meta.get("title") or path
|
||||
item_ref = html_to_text(content.get("zitation")) or response.meta.get("ref")
|
||||
item_url = f"{BASE}{path}"
|
||||
topics = tuple(b.get("label") for b in ((content.get("topics") or {}).get("data") or {}).get("bubbles") or [])
|
||||
eurovoc = [b.get("label") for b in ((content.get("eurovoc") or {}).get("data") or {}).get("bubbles") or []]
|
||||
|
||||
for stage in walk_dicts(content.get("phase") or content, "text"):
|
||||
text = html_to_text(stage.get("text"))
|
||||
if "angenommen" not in text and "abgelehnt" not in text:
|
||||
continue
|
||||
parsed = parse_vote_text(text)
|
||||
if not parsed or not (parsed["for"] or parsed["against"] or parsed["unanimous"] or parsed["split"]
|
||||
or parsed["roll_call"]):
|
||||
continue
|
||||
voted_on = parse_date(stage.get("date"))
|
||||
refs = stage.get("fsth") or []
|
||||
protocol_url = f"{BASE}{refs[0]['url']}" if refs and refs[0].get("url") else None
|
||||
chamber = "BR" if (refs and refs[0].get("gp_code") == "BR") or "Bundesrat" in text else "NR"
|
||||
chamber_ref = OrgRef(slug=CHAMBER_SLUG[chamber])
|
||||
event_key = fingerprint("vote", path, voted_on, parsed["stage"])
|
||||
yield VoteEvent(key=event_key, chamber=chamber_ref, title=title, voted_on=voted_on, result=parsed["result"],
|
||||
stage=parsed["stage"], item_ref=item_ref, roll_call=parsed["roll_call"], url=item_url,
|
||||
protocol_url=protocol_url, raw_text=text)
|
||||
evidence = [Evidence(url=item_url, doc_type="api_record", title=title, published_at=voted_on, quote=text,
|
||||
locator=f"stage:{voted_on}:{parsed['stage'][:80]}")]
|
||||
if protocol_url:
|
||||
evidence.append(Evidence(url=protocol_url, doc_type="protocol", published_at=voted_on,
|
||||
title=refs[0].get("title") or "Stenographisches Protokoll"))
|
||||
outcome = "angenommen" if parsed["result"] == "accepted" else "abgelehnt"
|
||||
for party, position in [(p, "for") for p in parsed["for"]] + [(p, "against") for p in parsed["against"]]:
|
||||
verb_de, verb_en = ("stimmte für", "voted for") if position == "for" else ("stimmte gegen", "voted against")
|
||||
yield Claim(
|
||||
claim_type="vote", org=OrgRef(name=party, short_name=party, kind="parliamentary_group"),
|
||||
counterpart_org=chamber_ref, occurred_at=voted_on,
|
||||
summary_de=f"{party} {verb_de}: {title} ({parsed['stage']}) – {outcome}",
|
||||
summary_en=f"{party} {verb_en}: {title} ({parsed['stage']})",
|
||||
data={"position": position, "result": parsed["result"], "item_ref": item_ref, "chamber": chamber,
|
||||
"eurovoc": eurovoc},
|
||||
evidence=evidence, topics=topics, facet=VotePosition(event_key=event_key, position=position),
|
||||
)
|
||||
if parsed["roll_call"]:
|
||||
# Individual votes are listed in the protocol after one of the referenced anchors; the
|
||||
# result is usually behind the last one, sometimes in another fragment (session break).
|
||||
refs_list = [r["url"].split("#", 1) for r in reversed(refs) if r.get("url") and "#" in r["url"]]
|
||||
if refs_list:
|
||||
yield Request(url=f"{BASE}{refs_list[0][0]}", kind="rollcall",
|
||||
meta={"event_key": event_key, "refs": refs_list, "tried": [refs_list[0][0]],
|
||||
"title": title, "chamber": chamber, "date": str(voted_on),
|
||||
"item_url": item_url, "topics": list(topics)})
|
||||
|
||||
def parse_rollcall(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
meta = response.meta
|
||||
doc = response.request.url.removeprefix(BASE)
|
||||
names: dict[str, list] = {}
|
||||
anchor_used = None
|
||||
for ref_doc, anchor in meta["refs"]:
|
||||
if ref_doc != doc:
|
||||
continue
|
||||
names = parse_roll_call(response.text, anchor)
|
||||
if names.get("for") or names.get("against"):
|
||||
anchor_used = anchor
|
||||
break
|
||||
if anchor_used is None:
|
||||
remaining = [d for d, _ in meta["refs"] if d not in meta["tried"]]
|
||||
if remaining:
|
||||
ctx.stats["rollcall_followup"] += 1
|
||||
yield Request(url=f"{BASE}{remaining[0]}", kind="rollcall",
|
||||
meta={**meta, "tried": [*meta["tried"], remaining[0]]})
|
||||
else:
|
||||
ctx.stats["rollcall_unparsed"] += 1
|
||||
return
|
||||
voted_on = parse_date(meta["date"])
|
||||
body = CHAMBER_SLUG[meta["chamber"]]
|
||||
source_url = f"{response.request.url}#{anchor_used}"
|
||||
evidence = [
|
||||
Evidence(url=source_url, doc_type="protocol", published_at=voted_on,
|
||||
title="Stenographisches Protokoll – Ergebnis der namentlichen Abstimmung", locator=anchor_used),
|
||||
Evidence(url=meta["item_url"], doc_type="api_record", title=meta["title"], published_at=voted_on,
|
||||
locator=f"rollcall:{voted_on}"),
|
||||
]
|
||||
for position, entries in names.items():
|
||||
answer = "Ja" if position == "for" else "Nein"
|
||||
for label, pad in entries:
|
||||
ref = (PersonRef(ids={"parlament": str(pad)}) if pad is not None else
|
||||
PersonRef(name=strip_titles(label), hints={"surname_in_body": body, "on": voted_on}))
|
||||
yield Claim(
|
||||
claim_type="vote", person=ref, counterpart_org=OrgRef(slug=body), occurred_at=voted_on,
|
||||
summary_de=f"Stimmte in namentlicher Abstimmung mit „{answer}“: {meta['title']}",
|
||||
summary_en=f"Voted \"{'yes' if position == 'for' else 'no'}\" in a roll-call vote: {meta['title']}",
|
||||
data={"position": position, "roll_call": True, "chamber": meta["chamber"], "label": label},
|
||||
evidence=evidence, topics=tuple(meta.get("topics") or ()),
|
||||
facet=VotePosition(event_key=meta["event_key"], position=position),
|
||||
)
|
||||
|
||||
# -- immunity -----------------------------------------------------------------------
|
||||
|
||||
def parse_immunity_item(self, ctx: RunContext, response: Response) -> Iterator[Request]:
|
||||
content = response.json().get("content") or {}
|
||||
report = next((d["link"] for group in content.get("documents") or [] for d in group.get("documents") or []
|
||||
if d.get("type") == "HTML"), None)
|
||||
if report:
|
||||
yield Request(url=f"{BASE}{report}", kind="immunity_report",
|
||||
meta={**response.meta, "item": response.request.url.removeprefix(BASE),
|
||||
"title": html_to_text(content.get("title")) or response.meta.get("title")})
|
||||
|
||||
def parse_immunity_report(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||||
meta = response.meta
|
||||
text = html_to_text(response.text)
|
||||
details = parse_immunity_report(text)
|
||||
title = meta["title"]
|
||||
name = re.sub(r"^.*?(Abgeordneten zum Nationalrat|Mitglied(es)? des Bundesrates)\s*(Klubobmann|Klubobfrau)?",
|
||||
"", title).strip()
|
||||
decided = str(meta.get("status")) == "5"
|
||||
stage = {"consent": "immunity_waived" if decided else "immunity_waiver_requested",
|
||||
"refuse": "immunity_refused" if decided else "immunity_waiver_requested"}.get(
|
||||
details["recommendation"] or "", "immunity_waiver_requested")
|
||||
stage_de = {"immunity_waived": "Nationalrat stimmte der behördlichen Verfolgung zu (Immunität aufgehoben)",
|
||||
"immunity_refused": "Nationalrat verweigerte die Zustimmung zur behördlichen Verfolgung",
|
||||
"immunity_waiver_requested": "Ersuchen um Aufhebung der Immunität"}[stage]
|
||||
offense = details["offense"]
|
||||
requested = parse_date(meta.get("date"))
|
||||
yield Claim(
|
||||
claim_type="legal_proceeding", person=PersonRef(name=strip_titles(name)),
|
||||
key=("immunity", meta["item"]), occurred_at=requested, sensitive=True,
|
||||
summary_de=stage_de + (f" – Verdacht nach {offense}" if offense else "")
|
||||
+ (f" (Ersuchen: {details['authority']})" if details["authority"] else ""),
|
||||
data={"political_connection": details["political_connection"],
|
||||
"recommendation": details["recommendation"], "item": meta["item"]},
|
||||
evidence=[
|
||||
Evidence(url=response.request.url, doc_type="register_entry", published_at=requested,
|
||||
title=f"Bericht des Immunitätsausschusses: {title}", quote=details["quote"], full_text=text),
|
||||
Evidence(url=f"{BASE}{meta['item']}", doc_type="api_record", title=title, published_at=requested),
|
||||
],
|
||||
topics=("Inneres und Recht",),
|
||||
facet=Legal(stage=stage, offense=offense, authority=details["authority"], case_ref=details["case_ref"],
|
||||
stage_date=requested),
|
||||
)
|
||||
|
||||
# -- investigative-committee hearings ----------------------------------------------
|
||||
|
||||
def parse_hearing_item(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
content = response.json().get("content") or {}
|
||||
title = html_to_text(content.get("title")) or response.meta["title"]
|
||||
m = HEARING.search(title)
|
||||
if not m:
|
||||
return
|
||||
witness = WITNESS_TITLES.sub("", strip_titles(m.group("name"))).strip()
|
||||
published = parse_date(response.meta.get("date"))
|
||||
protocol = next((d["link"] for group in content.get("documents") or [] for d in group.get("documents") or []
|
||||
if d.get("type") == "HTML"), None)
|
||||
url = f"{BASE}{protocol}" if protocol else response.request.url.split("?")[0]
|
||||
event_key = fingerprint("hearing", response.request.url)
|
||||
yield Event(key=event_key, kind="hearing", title=f"Befragung im Untersuchungsausschuss: {witness}",
|
||||
starts_at=published, location="Parlament, Wien", country="AT", url=url,
|
||||
description=title, body=OrgRef(slug="nationalrat"))
|
||||
yield Claim(
|
||||
claim_type="appearance", person=PersonRef(name=witness), occurred_at=published,
|
||||
summary_de=f"Befragung als Auskunftsperson im Untersuchungsausschuss ({title})",
|
||||
summary_en="Questioned as a witness by a parliamentary investigative committee",
|
||||
data={"context": "investigative_committee"},
|
||||
evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=published)],
|
||||
topics=("Transparenz und Korruption",),
|
||||
facet=Participation(event_key=event_key, participation="witness"),
|
||||
)
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Parliament press releases (Parlamentskorrespondenz, list 110) and calendar (Termine, list 600). CC BY 4.0.
|
||||
|
||||
Press releases are stored with full text; the people they link (/person/<id>) get a mention claim.
|
||||
The event extractor later turns visits, receptions and trips described in them into meeting/travel
|
||||
claims. The calendar fills the calendar view with sessions, committee meetings, hearings and events.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import date, timedelta
|
||||
from typing import Any
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, register
|
||||
from politrack.core.records import Claim, Document, Event, Evidence, OrgRef, PersonRef
|
||||
from politrack.sources.parlament.common import BASE, CHAMBER_SLUG, SPEC, list_request, list_rows, parse_date
|
||||
from politrack.text import excerpt, fingerprint, html_to_text
|
||||
|
||||
ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}")
|
||||
SKIP_TERMIN = re.compile(r"Guided Tour|Führung|Galeriebesuch|Besuch einer Plenarsitzung|Demokratiebildung", re.I)
|
||||
TERMIN_KIND = [
|
||||
(re.compile(r"Plenarsitzung|Bundesversammlung|Fest-|Gedenksitzung", re.I), "parliament_session"),
|
||||
(re.compile(r"Untersuchungsausschuss", re.I), "hearing"),
|
||||
(re.compile(r"Ausschuss|Enquete", re.I), "committee"),
|
||||
(re.compile(r"Pressekonferenz", re.I), "press_conference"),
|
||||
(re.compile(r"Internationales|Konferenz", re.I), "meeting"),
|
||||
]
|
||||
|
||||
|
||||
def release_body(content: list) -> str:
|
||||
"""The detail JSON is a list of render blocks; the article is the one wrapped in <div class=pk>."""
|
||||
for block in content:
|
||||
text = (block.get("data") or {}).get("text") if isinstance(block, dict) else None
|
||||
if isinstance(text, str) and re.search(r"class=\"?pk\b", text):
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
@register
|
||||
class ParlamentPress(Adapter):
|
||||
spec = SPEC
|
||||
name = "parlament.press"
|
||||
description = "Parlamentskorrespondenz: official press releases with linked people"
|
||||
schedule = "5 */2 * * *"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
last = ctx.state.get("last_date")
|
||||
backfill = int(ctx.params.get("backfill_days", 365))
|
||||
since = (date.fromisoformat(last) - timedelta(days=2)) if last else date.today() - timedelta(days=backfill)
|
||||
yield list_request(110, {"DATERANGE": [since.isoformat(), date.today().isoformat()]}, kind="list",
|
||||
meta={"since": since.isoformat()})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
if response.request.kind == "list":
|
||||
newest = response.meta["since"]
|
||||
for row in list_rows(response.json()):
|
||||
if not row.get("Link"):
|
||||
continue
|
||||
published = parse_date(row.get("Datum")) # "23.10.2026" (German) in list 110
|
||||
if published:
|
||||
newest = max(newest, published.isoformat())
|
||||
yield Request(url=f"{BASE}{row['Link']}", params={"json": "TRUE"}, kind="release",
|
||||
meta={k: row.get(k) for k in ("Titel", "Untertitel", "Datum", "Nr.", "Abothemen",
|
||||
"Stichworte")})
|
||||
if ctx.mode != "test":
|
||||
ctx.state.set("last_date", newest)
|
||||
return
|
||||
yield from self.parse_release(response)
|
||||
|
||||
def parse_release(self, response: Response) -> Iterator[Any]:
|
||||
meta = response.meta
|
||||
url = response.request.url
|
||||
body_html = release_body(response.json().get("content") or [])
|
||||
pads = sorted({p for p in re.findall(r"/person/(\d+)", body_html)})
|
||||
text = html_to_text(body_html)
|
||||
title, subtitle = meta.get("Titel") or "", meta.get("Untertitel") or ""
|
||||
topics_raw = meta.get("Abothemen")
|
||||
topics = tuple(json.loads(topics_raw)) if isinstance(topics_raw, str) and topics_raw.startswith("[") else ()
|
||||
published = parse_date(meta.get("Datum"))
|
||||
yield Document(Evidence(url=url, doc_type="press_release", title=title, published_at=published,
|
||||
excerpt=subtitle or excerpt(text, 300), full_text=f"{title}\n{subtitle}\n{text}",
|
||||
meta={"pk_number": meta.get("Nr."), "person_pads": pads, "topics": list(topics)}))
|
||||
for pad in pads:
|
||||
yield Claim(
|
||||
claim_type="media_mention", person=PersonRef(ids={"parlament": pad}), occurred_at=published,
|
||||
summary_de=f"Parlamentskorrespondenz: {title}",
|
||||
data={"context": "parlamentskorrespondenz"},
|
||||
evidence=[Evidence(url=url, doc_type="press_release", title=title, published_at=published,
|
||||
quote=subtitle or None, stance="mentions")],
|
||||
topics=topics,
|
||||
)
|
||||
|
||||
|
||||
@register
|
||||
class ParlamentCalendar(Adapter):
|
||||
spec = SPEC
|
||||
name = "parlament.calendar"
|
||||
description = "Parliament calendar: plenary sessions, committees, hearings, events (tours skipped)"
|
||||
schedule = "30 2 * * *"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
yield list_request(600, {}, kind="list")
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Event]:
|
||||
since = str(ctx.params.get("since_year", date.today().year - 2))
|
||||
for row in list_rows(response.json()):
|
||||
kind_label = (row.get("Terminart") or "").replace("<BR>", " ")
|
||||
starts = next((v for k, v in row.items() if k.startswith("Datum") and k != "Datum Bis"
|
||||
and isinstance(v, str) and ISO_DATE.match(v)), None)
|
||||
if not starts or starts[:4] < since or SKIP_TERMIN.search(f"{kind_label} {row.get('Bezeichnung') or ''}"):
|
||||
continue
|
||||
title = row.get("Bezeichnung") or kind_label
|
||||
kind = next((k for pattern, k in TERMIN_KIND if pattern.search(f"{kind_label} {title}")), "public_event")
|
||||
chamber = {"Nationalrat": "NR", "Bundesrat": "BR"}.get(row.get("Gremium") or "")
|
||||
ends = next((v for k, v in row.items() if k.startswith("Datum Bis") and isinstance(v, str)
|
||||
and ISO_DATE.match(v)), None)
|
||||
yield Event(
|
||||
key=fingerprint("termin", row.get("WENTRY_ID") or row.get("Link")), kind=kind, title=title,
|
||||
starts_at=starts, ends_at=ends, location=(row.get("Ort") or "").replace(" | ", ", ") or "Parlament, Wien",
|
||||
country="AT", url=f"{BASE}{row['Link']}" if row.get("Link") else None, description=kind_label,
|
||||
body=OrgRef(slug=CHAMBER_SLUG[chamber]) if chamber else None,
|
||||
)
|
||||
@@ -0,0 +1,239 @@
|
||||
"""Parliament: people, mandates, government functions, side-job disclosures, social links, speeches.
|
||||
|
||||
Flow: list 409 (all MPs since 1918) -> PersonRecord + mandate roles
|
||||
-> detail /person/<id> for members of the configured periods
|
||||
detail -> photo, social accounts, functions, side jobs (§6 Unv-Transparenz-G)
|
||||
-> list 251 (speeches of that person)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Adapter, Request, Response, RunContext, register
|
||||
from politrack.core.records import Claim, Evidence, OrgRef, PersonRecord, PersonRef, Role, SocialAccount, Statement
|
||||
from politrack.sources.parlament.common import (
|
||||
BASE,
|
||||
CHAMBER_NAME_DE,
|
||||
CHAMBER_SLUG,
|
||||
GOVERNMENT_RE,
|
||||
SPEC,
|
||||
detail_request,
|
||||
list_request,
|
||||
list_rows,
|
||||
parse_date,
|
||||
walk_dicts,
|
||||
)
|
||||
from politrack.text import firmenbuch_number, html_to_text, strip_titles
|
||||
|
||||
SOCIAL_PLATFORMS = {"TWITTER": "twitter", "FACEBOOK": "facebook", "INSTAGRAM": "instagram", "TIKTOK": "tiktok",
|
||||
"YOUTUBE": "youtube", "LINKEDIN": "linkedin", "WEBSITE": "website", "BLUESKY": "bluesky",
|
||||
"MASTODON": "mastodon", "TELEGRAM": "telegram", "THREADS": "threads"}
|
||||
|
||||
|
||||
FN_IN_TEXT = re.compile(r"\bFN\s*\d{1,6}\s?[a-zA-Z]\b")
|
||||
POSTCODE_CITY = re.compile(r"(?:(?<=\d)|\b)\d{4}\s+[A-ZÄÖÜ][\wäöüß .\-]*$")
|
||||
LEGAL_FORM_END = re.compile(r"(?i:gesmbh|gmbh|ges\.\s?m\.\s?b\.\s?h\.?|m\.\s?b\.\s?h\.?|aktiengesellschaft|\bag|\bkg|\bog|"
|
||||
r"e\.\s?u\.|\bse|privatstiftung|genossenschaft|sparkasse)(?=[A-ZÄÖÜ\s,]|$)")
|
||||
|
||||
|
||||
STREET_WORD = r"\S*(?:straße|strasse|str\.|gasse|platz|weg|kai|ring|allee|zeile|lände|markt|steig)"
|
||||
|
||||
|
||||
def strip_street(text: str) -> str:
|
||||
"""'Verein Musterhilfe Hauptstraße 12' -> 'Verein Musterhilfe' (greedy name, shortest street suffix)."""
|
||||
m = (re.match(rf"^(.*\S)\s+{STREET_WORD}\s*\d+[a-z]?(?:/\d+)*\s*$", text, re.IGNORECASE)
|
||||
or re.match(r"^(.*\S)\s+\S+\s*\d+[a-z]?(?:/\d+)*\s*$", text))
|
||||
return m.group(1) if m else text
|
||||
|
||||
|
||||
def employer(cell: str) -> tuple[str | None, str | None]:
|
||||
"""Employer name without business address, plus its company register number.
|
||||
|
||||
'Hypo Tirol Bank AG, Meraner Straße 8, 6020 Innsbruck' -> ('Hypo Tirol Bank AG', None)
|
||||
'25 PersonaldienstleistungsgesmbHFranz Josef Kai 95020 Salzburg FN306527a' (parliament's JSON glues
|
||||
the lines together) -> ('25 PersonaldienstleistungsgesmbH', '306527a')
|
||||
"""
|
||||
cell = re.sub(r"^lit\.\s*\w\s*:\s*", "", cell).strip()
|
||||
fn_match = FN_IN_TEXT.search(cell)
|
||||
fn = firmenbuch_number(fn_match.group(0)) if fn_match else None
|
||||
name = FN_IN_TEXT.sub("", cell).split(",")[0].strip(" *")
|
||||
address = POSTCODE_CITY.search(name)
|
||||
if address:
|
||||
head = name[:address.start()]
|
||||
ends = [m.end() for m in LEGAL_FORM_END.finditer(head)]
|
||||
# cut after the last legal form; otherwise drop "<street> <number>" before the postcode
|
||||
name = head[:ends[-1]] if ends else strip_street(head)
|
||||
name = name.strip(" ,-*")
|
||||
return (name if len(name) >= 3 and not re.fullmatch(r"[\d\s.€-]+", name) else None), fn
|
||||
|
||||
|
||||
@register
|
||||
class ParlamentPersons(Adapter):
|
||||
spec = SPEC
|
||||
name = "parlament.persons"
|
||||
description = "Members of Nationalrat and Bundesrat since 1918, mandates, functions, side jobs, speeches"
|
||||
schedule = "15 3 * * *"
|
||||
creates_persons = True
|
||||
parser_version = 3
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
yield list_request(409, {"NRBR": ["NR", "BR"]}, kind="persons")
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
kind = response.request.kind
|
||||
if kind == "persons":
|
||||
yield from self.parse_persons(ctx, response)
|
||||
elif kind == "person":
|
||||
yield from self.parse_person(ctx, response)
|
||||
elif kind == "speeches":
|
||||
yield from self.parse_speeches(ctx, response)
|
||||
|
||||
# -- persons & mandates -------------------------------------------------------------
|
||||
|
||||
def parse_persons(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
periods = set(ctx.params.get("periods") or get_settings().parlament_periods)
|
||||
for row in list_rows(response.json()):
|
||||
attr = row.get("Attribute") or {}
|
||||
if isinstance(attr, str):
|
||||
attr = json.loads(attr)
|
||||
m = re.search(r"/person/(\d+)", attr.get("uri") or "")
|
||||
if not m:
|
||||
continue
|
||||
pad = m.group(1)
|
||||
ref = PersonRef(ids={"parlament": pad})
|
||||
display = attr.get("zit") or strip_titles(row.get("Name") or "")
|
||||
yield PersonRecord(ref=ref, full_name=display, given_name=row.get("vorname"), family_name=row.get("nachname"),
|
||||
gender={"M": "male", "W": "female"}.get(row.get("GESCHL_CODE") or attr.get("geschlecht")))
|
||||
mandates = attr.get("mandate_detail") or []
|
||||
for mandate in mandates:
|
||||
claim = self.mandate_claim(pad, display, mandate)
|
||||
if claim:
|
||||
yield claim
|
||||
if any(md.get("gp_code") in periods for md in mandates):
|
||||
yield detail_request(f"/person/{pad}", kind="person", meta={"pad": pad, "name": display})
|
||||
|
||||
def mandate_claim(self, pad: str, name: str, m: dict) -> Claim | None:
|
||||
code = m.get("mand_code") or "NR"
|
||||
if code not in CHAMBER_SLUG:
|
||||
return None
|
||||
party_code = m.get("wahlpartei_code")
|
||||
party_name = m.get("wahlpartei_full_txt") or m.get("wahlpartei_txt") or party_code
|
||||
party = OrgRef(name=party_name, short_name=party_code, kind="party") if party_name else None
|
||||
start, end = parse_date(m.get("mandat_von")), parse_date(m.get("mandat_bis"))
|
||||
gp = m.get("gp_code") or ""
|
||||
region = m.get("wahlkreis_bundesland")
|
||||
label = party_code or party_name or "?"
|
||||
body = OrgRef(slug=CHAMBER_SLUG[code])
|
||||
return Claim(
|
||||
claim_type="role",
|
||||
person=PersonRef(ids={"parlament": pad}),
|
||||
org=body,
|
||||
counterpart_org=party,
|
||||
valid_from=start,
|
||||
valid_to=end,
|
||||
summary_de=f"{name}: Mandat im {CHAMBER_NAME_DE[code]} ({gp}. GP) für {label}"
|
||||
+ (f", Wahlkreis {region}" if region else ""),
|
||||
summary_en=f"{name}: member of the {CHAMBER_NAME_DE[code]} ({gp} legislative period) for {label}",
|
||||
data={"gp": gp, "party_code": party_code, "faction_code": m.get("fraktionscode"), "region": region},
|
||||
evidence=[Evidence(url=f"{BASE}/person/{pad}", doc_type="api_record", title=f"Parlament: {name}",
|
||||
locator=f"mandat:{code}:{gp}:{start}")],
|
||||
facet=Role(role="mp", level="federal", body=body, party=party, region=region, start=start, end=end),
|
||||
)
|
||||
|
||||
# -- person detail ----------------------------------------------------------------
|
||||
|
||||
def parse_person(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
data = response.json()
|
||||
content = data.get("content") or {}
|
||||
pad, name = response.meta["pad"], response.meta["name"]
|
||||
ref = PersonRef(ids={"parlament": pad})
|
||||
url = f"{BASE}/person/{pad}"
|
||||
photo = ((data.get("meta") or {}).get("openGraph") or {}).get("image")
|
||||
if photo:
|
||||
yield PersonRecord(ref=ref, full_name=name, photo_url=photo)
|
||||
|
||||
for account in (content.get("kontakt") or {}).get("socialMedia") or []:
|
||||
platform = SOCIAL_PLATFORMS.get((account.get("type") or "").upper(), (account.get("type") or "website").lower())
|
||||
yield SocialAccount(platform=platform, url=account["url"], person=ref, verified=True)
|
||||
|
||||
for fn in walk_dicts(content.get("biografie") or {}, "funktion_text"):
|
||||
if fn.get("gremium") in CHAMBER_SLUG:
|
||||
continue
|
||||
text = fn.get("funktion_text") or fn.get("bez")
|
||||
if not text:
|
||||
continue
|
||||
start, end = parse_date(fn.get("funktion_von")), parse_date(fn.get("funktion_bis"))
|
||||
is_gov = bool(GOVERNMENT_RE.search(text))
|
||||
body = OrgRef(slug="bundesregierung") if is_gov else None
|
||||
yield Claim(
|
||||
claim_type="role", person=ref, org=body, valid_from=start, valid_to=end,
|
||||
summary_de=f"{name}: {text}",
|
||||
evidence=[Evidence(url=url, doc_type="api_record", title=f"Parlament: {name}",
|
||||
locator=f"funktion:{text}:{start}")],
|
||||
facet=Role(role=text, level="federal" if is_gov else "other", body=body, start=start, end=end),
|
||||
)
|
||||
|
||||
yield from self.side_jobs(ref, pad, name, content.get("unvtrans") or {})
|
||||
yield list_request(251, {"PAD_INTERN": [int(pad)]}, kind="speeches", meta={"pad": pad, "name": name})
|
||||
|
||||
def side_jobs(self, ref: PersonRef, pad: str, name: str, unv: dict) -> Iterator[Claim]:
|
||||
"""Disclosures under §6 Unv-Transparenz-G: leading positions, paid side jobs, honorary positions."""
|
||||
tables = unv.get("unvtransData") or {}
|
||||
pdf = next((d["link"] for group in unv.get("linkliste") or [] for d in group.get("documents") or []
|
||||
if d.get("link", "").endswith(f"/person/{pad}/BezBegrBVGPar9.pdf")), None)
|
||||
for key, table in tables.items():
|
||||
m = re.fullmatch(r"(z\d+)Table", key)
|
||||
if not m or not isinstance(table, dict):
|
||||
continue
|
||||
header = html_to_text(tables.get(f"{m.group(1)}Header")) or key
|
||||
for row in (table.get("data") or {}).get("rows") or []:
|
||||
cells = [c for c in (html_to_text(c) for c in row if isinstance(c, str)) if c]
|
||||
if not cells:
|
||||
continue
|
||||
name_, fn = employer(cells[0]) if m.group(1) in ("z1", "z2", "z3") else (None, None)
|
||||
if name_: # never keep the (business) address that parliament publishes with the employer
|
||||
cells = [name_ + (f" (FN {fn})" if fn else ""), *cells[1:]]
|
||||
text = " – ".join(cells)
|
||||
yield Claim(
|
||||
claim_type="side_job", person=ref, key=("unvtrans", pad, m.group(1), text),
|
||||
counterpart_org=OrgRef(name=name_, kind="company" if LEGAL_FORM_END.search(name_) else "other",
|
||||
ids={"firmenbuch": fn} if fn else {}) if name_ else None,
|
||||
summary_de=f"{name} – {header.split(' - ')[0]}: {text}",
|
||||
data={"category": header, "cells": cells, "unpaid": "*" in "".join(str(c) for c in row),
|
||||
"employer": name_, "firmenbuch": fn},
|
||||
evidence=[Evidence(url=f"{BASE}{pdf}" if pdf else f"{BASE}/person/{pad}", doc_type="register_entry",
|
||||
title=f"Unvereinbarkeits-/Transparenzliste {name}", quote=text,
|
||||
locator=f"unvtrans:{m.group(1)}")],
|
||||
topics=("Transparenz und Korruption",),
|
||||
)
|
||||
|
||||
# -- speeches -----------------------------------------------------------------------
|
||||
|
||||
def parse_speeches(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||||
periods = set(ctx.params.get("periods") or get_settings().parlament_periods)
|
||||
pad, name = response.meta["pad"], response.meta["name"]
|
||||
for row in list_rows(response.json()):
|
||||
if row.get("GP") not in periods and row.get("GREMIUM") != "B":
|
||||
continue
|
||||
html_cells = " ".join(str(v) for v in row.values() if isinstance(v, str))
|
||||
protocol = re.search(r'href="(/dokument/[^"]+)"', html_cells)
|
||||
session = html_to_text(row.get("Sitzung"))
|
||||
topic = html_to_text(row.get("Rede"))
|
||||
said_at = row.get("fromDate")
|
||||
url = f"{BASE}{protocol.group(1)}" if protocol else f"{BASE}/person/{pad}"
|
||||
chamber = "Bundesrat" if row.get("GREMIUM") == "B" else "Nationalrat"
|
||||
yield Claim(
|
||||
claim_type="statement", person=PersonRef(ids={"parlament": pad}),
|
||||
key=("parl-speech", pad, row.get("Sortier"), topic), occurred_at=said_at,
|
||||
summary_de=f"{name} sprach im {chamber} zu: {topic}",
|
||||
summary_en=f"{name} spoke in the {chamber} on: {topic}",
|
||||
data={"session": session, "gp": row.get("GP"), "item_title": topic},
|
||||
evidence=[Evidence(url=url, doc_type="protocol", title=session, published_at=parse_date(said_at),
|
||||
locator=url.split("#", 1)[-1] if "#" in url else "")],
|
||||
facet=Statement(context="parliament_speech", url=url, said_at=said_at, medium=chamber, title=topic),
|
||||
)
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Political advertising on online platforms.
|
||||
|
||||
* Google Political Ads Transparency Report - public download bundle (zip with CSVs, ~300 MB). We keep
|
||||
ads served in Austria (Regions contains "AT"): advertiser, period, impressions bracket, spend range.
|
||||
Note: Google stopped serving political ads in the EU in October 2025 (EU TTPA regulation), so this
|
||||
is historical data (2019-2025).
|
||||
* Meta Ad Library API (political and issue ads reaching Austria) - requires an identity-verified developer
|
||||
token (POLITRACK_META_AD_LIBRARY_TOKEN) and page ids in config/meta_pages.yaml; skipped otherwise.
|
||||
|
||||
Advertisers are resolved to parties/organisations by name where possible (e.g. "FPÖ", "SPÖ Wien").
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import zipfile
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Evidence, Money, OrgRef
|
||||
from politrack.text import format_eur_de
|
||||
|
||||
GOOGLE_BUNDLE = "https://storage.googleapis.com/political-csv/google-political-ads-transparency-bundle.zip"
|
||||
GOOGLE_SPEC = SourceSpec(
|
||||
slug="google-political-ads", name="Google Political Ads Transparency Report", kind="aggregator",
|
||||
homepage="https://adstransparency.google.com/political", license="Google transparency data (public download)",
|
||||
access="bulk", trust=85, keep_raw=False, min_interval=5.0,
|
||||
tos_notes="Public CSV bundle of political ads. ~300 MB; fetched monthly with conditional requests.",
|
||||
)
|
||||
META_SPEC = SourceSpec(
|
||||
slug="meta-adlibrary", name="Meta Ad Library", kind="social", homepage="https://www.facebook.com/ads/library/api",
|
||||
license="Meta Platform Terms", access="api", trust=80, min_interval=2.0,
|
||||
tos_notes="Official Ad Library API for political and issue ads (identity-verified developer token required).",
|
||||
)
|
||||
META_API = "https://graph.facebook.com/v21.0/ads_archive"
|
||||
|
||||
|
||||
def col(row: dict[str, str], *names: str) -> str | None:
|
||||
lowered = {k.lower(): v for k, v in row.items()}
|
||||
for name in names:
|
||||
if lowered.get(name.lower()) not in (None, ""):
|
||||
return lowered[name.lower()]
|
||||
return None
|
||||
|
||||
|
||||
@register
|
||||
class GooglePoliticalAds(Adapter):
|
||||
spec = GOOGLE_SPEC
|
||||
name = "ads.google"
|
||||
description = "Political ads served in Austria on Google/YouTube (historical, until 10/2025)"
|
||||
schedule = "0 3 1 * *"
|
||||
queue = "bulk"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
yield Request(url=GOOGLE_BUNDLE, kind="bundle")
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||||
since = ctx.params.get("since") or get_settings().ad_archive_since
|
||||
with zipfile.ZipFile(io.BytesIO(response.body)) as bundle:
|
||||
name = next((n for n in bundle.namelist() if n.endswith("creative-stats.csv")), None)
|
||||
if name is None:
|
||||
ctx.stats["creative_stats_missing"] += 1
|
||||
return
|
||||
with bundle.open(name) as fh:
|
||||
for row in csv.DictReader(io.TextIOWrapper(fh, encoding="utf-8", errors="replace")):
|
||||
regions = (col(row, "Regions") or "").replace(" ", "").split(",")
|
||||
if "AT" not in regions:
|
||||
continue
|
||||
start = col(row, "Date_Range_Start") or ""
|
||||
if start < since:
|
||||
continue
|
||||
advertiser = col(row, "Advertiser_Name") or "?"
|
||||
low, high = col(row, "Spend_Range_Min_EUR"), col(row, "Spend_Range_Max_EUR")
|
||||
ad_url = col(row, "Ad_URL") or GOOGLE_BUNDLE
|
||||
payer = OrgRef(name=advertiser, kind="other", ids={"google_advertiser": col(row, "Advertiser_ID") or ""})
|
||||
spend = f"{format_eur_de(low)}–{format_eur_de(high)} €" if low and high else None
|
||||
yield Claim(
|
||||
claim_type="money_flow", org=payer, occurred_at=start,
|
||||
summary_de=f"Politische Google-Werbung von {advertiser} ({start} – {col(row, 'Date_Range_End')})"
|
||||
+ (f", Ausgaben {spend}" if spend else ""),
|
||||
summary_en=f"Political Google ad by {advertiser} ({start} – {col(row, 'Date_Range_End')})",
|
||||
data={"impressions": col(row, "Impressions"), "ad_type": col(row, "Ad_Type"),
|
||||
"age_targeting": col(row, "Age_Targeting"), "gender_targeting": col(row, "Gender_Targeting"),
|
||||
"geo_included": col(row, "Geo_Targeting_Included")},
|
||||
evidence=[Evidence(url=ad_url, doc_type="dataset_row", title=f"Google Ads Transparency: {advertiser}",
|
||||
published_at=start)],
|
||||
facet=Money(kind="political_ad", payer=payer, payer_name=advertiser,
|
||||
payee=OrgRef(name="Google", kind="company"), payee_name="Google",
|
||||
paid_on=date.fromisoformat(start[:10]) if start else None,
|
||||
amount_category=spend, reference=col(row, "Ad_ID"), category=col(row, "Ad_Type")),
|
||||
)
|
||||
|
||||
|
||||
@register
|
||||
class MetaPoliticalAds(Adapter):
|
||||
spec = META_SPEC
|
||||
name = "ads.meta"
|
||||
description = "Political/issue ads reaching Austria from Meta's Ad Library (token + page list required)"
|
||||
schedule = "0 4 * * 4"
|
||||
queue = "bulk"
|
||||
|
||||
FIELDS = ("id,page_id,page_name,bylines,ad_delivery_start_time,ad_delivery_stop_time,spend,impressions,"
|
||||
"currency,ad_snapshot_url,ad_creative_bodies")
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
token = get_settings().meta_ad_library_token
|
||||
path = get_settings().config_dir / "meta_pages.yaml"
|
||||
if not token or not path.exists():
|
||||
ctx.stats["skipped_not_configured"] = 1
|
||||
return
|
||||
pages = [str(p["page_id"]) for p in (yaml.safe_load(path.read_text(encoding="utf-8")) or {}).get("pages", [])]
|
||||
for i in range(0, len(pages), 10):
|
||||
yield Request(url=META_API, kind="ads", conditional=False, params={
|
||||
"access_token": token, "ad_reached_countries": '["AT"]', "ad_type": "POLITICAL_AND_ISSUE_ADS",
|
||||
"search_page_ids": ",".join(pages[i:i + 10]), "fields": self.FIELDS, "limit": 250,
|
||||
"ad_delivery_date_min": ctx.params.get("since") or get_settings().ad_archive_since})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
data = response.json()
|
||||
next_url = (data.get("paging") or {}).get("next")
|
||||
if next_url:
|
||||
yield Request(url=next_url, kind="ads", conditional=False)
|
||||
for ad in data.get("data") or []:
|
||||
spend = ad.get("spend") or {}
|
||||
payer_name = ad.get("bylines") or ad.get("page_name") or "?"
|
||||
payer = OrgRef(name=ad.get("page_name") or payer_name, kind="other", ids={"facebook_page": ad.get("page_id")})
|
||||
rng = f"{spend.get('lower_bound')}–{spend.get('upper_bound')} {ad.get('currency') or 'EUR'}" if spend else None
|
||||
start = (ad.get("ad_delivery_start_time") or "")[:10]
|
||||
yield Claim(
|
||||
claim_type="money_flow", org=payer, occurred_at=start or None,
|
||||
summary_de=f"Politische Meta-Werbung von {payer_name} ab {start}" + (f", Ausgaben {rng}" if rng else ""),
|
||||
data={"impressions": ad.get("impressions"), "bodies": (ad.get("ad_creative_bodies") or [])[:2]},
|
||||
evidence=[Evidence(url=ad.get("ad_snapshot_url") or f"https://www.facebook.com/ads/library/?id={ad['id']}",
|
||||
doc_type="dataset_row", title=f"Meta Ad Library: {payer_name}", published_at=start or None)],
|
||||
facet=Money(kind="political_ad", payer=payer, payer_name=payer_name,
|
||||
payee=OrgRef(name="Meta Platforms", kind="company"), payee_name="Meta",
|
||||
paid_on=date.fromisoformat(start) if start else None, amount_category=rng,
|
||||
reference=ad.get("id")),
|
||||
)
|
||||
@@ -0,0 +1,254 @@
|
||||
"""Public procurement: contract award notices published as open "core data" (Kerndaten, §62/64 BVergG 2018).
|
||||
|
||||
Every contracting authority (or the platform it uses) publishes a "Kerndatenquelle" (KDQ): an XML index
|
||||
<kdq><item id lastmod><url>…</url></item>…</kdq> listed as a dataset on data.gv.at. Each item is one
|
||||
notice; since 2023/24 notices use the EU eForms UBL schema. We keep ContractAwardNotice documents
|
||||
(who bought what from whom, for how much) and skip calls for tenders.
|
||||
|
||||
Correlations this enables: suppliers that are also party donors, sponsors or side-job employers of
|
||||
politicians; contracts by ministries led by a given minister (organisation pages).
|
||||
|
||||
Volume: tens of thousands of notices; first run backfills from POLITRACK_PROCUREMENT_SINCE, later runs only
|
||||
fetch items whose lastmod changed. Runs on the `bulk` queue.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Evidence, Money, OrgRef
|
||||
from politrack.text import firmenbuch_number, format_eur_de
|
||||
|
||||
HUB = "https://www.data.gv.at/api/hub/search/search"
|
||||
SPEC = SourceSpec(
|
||||
slug="vergabe", name="Öffentliche Aufträge (Kerndaten, data.gv.at)", kind="official",
|
||||
homepage="https://www.data.gv.at", license="Open data (free licence required by §64 BVergG 2018)", access="bulk",
|
||||
trust=90, min_interval=1.0,
|
||||
tos_notes="Contracting authorities must publish award core data as open, machine-readable data under a free "
|
||||
"licence (BVergG 2018). We read each publisher's Kerndatenquelle (KDQ) XML and its eForms notices.",
|
||||
)
|
||||
NS = {
|
||||
"cbc": "urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2",
|
||||
"cac": "urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2",
|
||||
"efac": "http://data.europa.eu/p27/eforms-ubl-extension-aggregate-components/1",
|
||||
"efbc": "http://data.europa.eu/p27/eforms-ubl-extension-basic-components/1",
|
||||
"efext": "http://data.europa.eu/p27/eforms-ubl-extensions/1",
|
||||
"ext": "urn:oasis:names:specification:ubl:schema:xsd:CommonExtensionComponents-2",
|
||||
}
|
||||
CATALOG_QUERIES = ("Kerndaten", "Kerndatenquelle", "KDQ BVergG")
|
||||
|
||||
|
||||
def local(tag: str) -> str:
|
||||
return tag.rsplit("}", 1)[-1]
|
||||
|
||||
|
||||
def text(node: ET.Element | None, path: str) -> str | None:
|
||||
if node is None:
|
||||
return None
|
||||
found = node.find(path, NS)
|
||||
return found.text.strip() if found is not None and found.text else None
|
||||
|
||||
|
||||
def parse_kerndaten(root: ET.Element) -> dict[str, Any] | None:
|
||||
"""Austrian core-data format (BVergG 2018, schema brz.gv.at/eproc/kd): KD_8_2_* = awarded contracts.
|
||||
|
||||
Still used by many publishers (e-shops, below-threshold awards). Contractor NATIONALID is usually the
|
||||
company register number ("183286p"), occasionally the VAT id.
|
||||
"""
|
||||
if not local(root.tag).startswith("KD_8_2"):
|
||||
return None
|
||||
|
||||
def t(node: ET.Element | None, path: str) -> str | None:
|
||||
found = node.find(path) if node is not None else None
|
||||
return found.text.strip() if found is not None and found.text else None
|
||||
|
||||
buyer = root.find("{*}CONTRACTING_BODY/{*}ADDRESS_CONTRACTING_BODY")
|
||||
title = " ".join(p.text.strip() for p in root.iterfind("{*}OBJECT_CONTRACT/{*}TITLE/{*}P") if p.text)
|
||||
cpv = root.find("{*}OBJECT_CONTRACT/{*}CPV_MAIN/{*}CPV_CODE")
|
||||
contracts = []
|
||||
for awarded in root.iterfind(".//{*}AWARD_CONTRACT/{*}AWARDED_CONTRACT"):
|
||||
contracts.append({
|
||||
"id": None,
|
||||
"title": None,
|
||||
"awarded": t(awarded, "{*}DATE_CONCLUSION_CONTRACT"),
|
||||
"amount": t(awarded, "{*}VAL_TOTAL"),
|
||||
"winners": [{"name": t(c, "{*}ADDRESS_CONTRACTOR/{*}OFFICIALNAME"),
|
||||
"company_id": t(c, "{*}ADDRESS_CONTRACTOR/{*}NATIONALID")}
|
||||
for c in awarded.iterfind("{*}CONTRACTOR")],
|
||||
})
|
||||
return {
|
||||
"notice_id": t(root, "{*}EXTENSION[@KEY='Referenz']") or t(root, "{*}OBJECT_CONTRACT/{*}REFERENCE_NUMBER"),
|
||||
"issued": None,
|
||||
"title": title or None,
|
||||
"cpv": cpv.get("CODE") if cpv is not None else None,
|
||||
"total": None,
|
||||
"buyers": [{"name": t(buyer, "{*}OFFICIALNAME"), "company_id": None}] if buyer is not None else [],
|
||||
"contracts": contracts,
|
||||
}
|
||||
|
||||
|
||||
def parse_award_notice(xml: bytes) -> dict[str, Any] | None:
|
||||
"""Extract buyer, winners, amounts, title, CPV and dates from an award notice: eForms
|
||||
ContractAwardNotice, or the Austrian core-data format (parse_kerndaten)."""
|
||||
root = ET.fromstring(xml)
|
||||
if local(root.tag).startswith("KD_"):
|
||||
return parse_kerndaten(root)
|
||||
if local(root.tag) != "ContractAwardNotice":
|
||||
return None
|
||||
orgs: dict[str, dict[str, Any]] = {}
|
||||
for org in root.iterfind(".//efac:Organizations/efac:Organization/efac:Company", NS):
|
||||
org_id = text(org, "cac:PartyIdentification/cbc:ID")
|
||||
if org_id:
|
||||
orgs[org_id] = {"name": text(org, "cac:PartyName/cbc:Name"),
|
||||
"company_id": text(org, "cac:PartyLegalEntity/cbc:CompanyID"),
|
||||
"city": text(org, "cac:PostalAddress/cbc:CityName")}
|
||||
buyer_ids = [e.text for e in root.iterfind("cac:ContractingParty/cac:Party/cac:PartyIdentification/cbc:ID", NS)
|
||||
if e.text]
|
||||
result = root.find(".//efac:NoticeResult", NS)
|
||||
tender_amounts = {text(t, "cbc:ID"): text(t, "cac:LegalMonetaryTotal/cbc:PayableAmount")
|
||||
for t in (result.iterfind("efac:LotTender", NS) if result is not None else [])}
|
||||
tendering_parties = {text(tp, "cbc:ID"): [e.text for e in tp.iterfind("efac:Tenderer/cbc:ID", NS) if e.text]
|
||||
for tp in (result.iterfind("efac:TenderingParty", NS) if result is not None else [])}
|
||||
lot_tender_party = {text(t, "cbc:ID"): text(t, "efac:TenderingParty/cbc:ID")
|
||||
for t in (result.iterfind("efac:LotTender", NS) if result is not None else [])}
|
||||
contracts = []
|
||||
for contract in (result.iterfind("efac:SettledContract", NS) if result is not None else []):
|
||||
tender_id = text(contract, "efac:LotTender/cbc:ID")
|
||||
winners = tendering_parties.get(lot_tender_party.get(tender_id), [])
|
||||
contracts.append({
|
||||
"id": text(contract, "cbc:ID"),
|
||||
"title": text(contract, "cbc:Title"),
|
||||
"awarded": text(contract, "cbc:AwardDate") or text(contract, "cbc:IssueDate"),
|
||||
"amount": tender_amounts.get(tender_id),
|
||||
"winners": [orgs.get(w, {"name": None}) | {"ref": w} for w in winners],
|
||||
})
|
||||
project_title = text(root, "cac:ProcurementProject/cbc:Name")
|
||||
return {
|
||||
"notice_id": text(root, "cbc:ID"),
|
||||
"issued": text(root, "cbc:IssueDate"),
|
||||
"title": project_title,
|
||||
"cpv": text(root, "cac:ProcurementProject/cac:MainCommodityClassification/cbc:ItemClassificationCode"),
|
||||
"total": text(root, ".//efac:NoticeResult/cbc:TotalAmount"),
|
||||
"buyers": [orgs.get(b, {"name": None}) | {"ref": b} for b in buyer_ids],
|
||||
"contracts": contracts,
|
||||
}
|
||||
|
||||
|
||||
def org_ref(org: dict[str, Any], kind: str) -> OrgRef | None:
|
||||
if not org.get("name"):
|
||||
return None
|
||||
company_id = (org.get("company_id") or "").replace(" ", "")
|
||||
ids = {}
|
||||
if company_id.upper().startswith("ATU"):
|
||||
ids["uid"] = company_id.upper()
|
||||
elif fn := firmenbuch_number(company_id):
|
||||
ids["firmenbuch"] = fn
|
||||
return OrgRef(name=org["name"], kind=kind, ids=ids)
|
||||
|
||||
|
||||
def as_date(value: str | None) -> date | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return date.fromisoformat(value[:10])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
@register
|
||||
class PublicContracts(Adapter):
|
||||
spec = SPEC
|
||||
name = "vergabe.awards"
|
||||
description = "Contract awards of Austrian public bodies (eForms and BVergG core data via data.gv.at)"
|
||||
schedule = "0 2 * * *"
|
||||
queue = "bulk"
|
||||
parser_version = 2
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
for query in CATALOG_QUERIES:
|
||||
yield Request(url=HUB, params={"q": query, "limit": 100, "page": 0}, kind="catalog", conditional=False)
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
kind = response.request.kind
|
||||
if kind == "catalog":
|
||||
result = response.json().get("result") or {}
|
||||
for dataset in result.get("results") or []:
|
||||
for dist in dataset.get("distributions") or []:
|
||||
url = (dist.get("access_url") or dist.get("download_url") or [None])
|
||||
url = url[0] if isinstance(url, list) else url
|
||||
if url and url.startswith("http") and "data.gv.at/katalog" not in url:
|
||||
if ctx.params.get("only_url") and ctx.params["only_url"] not in url:
|
||||
continue # targeted runs: -p only_url=bbg.gv.at
|
||||
title = (dataset.get("title") or {}).get("de") or ""
|
||||
yield Request(url=url, kind="kdq", meta={"publisher": title.split("von", 1)[-1].strip(" '")})
|
||||
if result.get("count", 0) > (response.request.params["page"] + 1) * 100:
|
||||
yield Request(url=HUB, params={**response.request.params, "page": response.request.params["page"] + 1},
|
||||
kind="catalog", conditional=False)
|
||||
elif kind == "kdq":
|
||||
yield from self.parse_kdq(ctx, response)
|
||||
elif kind == "notice":
|
||||
yield from self.parse_notice(ctx, response)
|
||||
|
||||
def parse_kdq(self, ctx: RunContext, response: Response) -> Iterator[Request]:
|
||||
try:
|
||||
root = ET.fromstring(response.body)
|
||||
except ET.ParseError:
|
||||
ctx.stats["kdq_not_xml"] += 1
|
||||
return
|
||||
state_key = f"kdq:{response.request.url}"
|
||||
since = ctx.params.get("since") or ctx.state.get(state_key) or get_settings().procurement_since
|
||||
limit = int(ctx.params.get("max_notices_per_source", 3000))
|
||||
items = [(item.get("lastmod") or "", item.findtext("{*}url") or item.findtext("url"))
|
||||
for item in root.iter() if local(item.tag) == "item"]
|
||||
# Oldest first so an interrupted backfill resumes where it stopped (cursor = newest lastmod handled).
|
||||
items = sorted(i for i in items if i[1] and i[0] > since)[:limit]
|
||||
for lastmod, url in items:
|
||||
# lastmod in meta (not in the URL) + conditional requests: unchanged notices cost a 304 at most.
|
||||
# The cursor advances only when a notice has been parsed (see advance_cursor), so an interrupted
|
||||
# run resumes with the first notice it did not get to.
|
||||
yield Request(url=url.strip(), kind="notice", meta={"publisher": response.meta.get("publisher"),
|
||||
"lastmod": lastmod, "state_key": state_key})
|
||||
|
||||
@staticmethod
|
||||
def advance_cursor(ctx: RunContext, response: Response) -> None:
|
||||
key, lastmod = response.meta.get("state_key"), response.meta.get("lastmod")
|
||||
if key and lastmod and lastmod > (ctx.state.get(key) or ""):
|
||||
ctx.state.set(key, lastmod)
|
||||
|
||||
def parse_notice(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||||
self.advance_cursor(ctx, response) # notices arrive oldest-first per publisher
|
||||
try:
|
||||
notice = parse_award_notice(response.body)
|
||||
except ET.ParseError:
|
||||
ctx.stats["notice_not_xml"] += 1
|
||||
return
|
||||
if notice is None:
|
||||
ctx.stats["not_an_award"] += 1
|
||||
return
|
||||
buyer = next((org_ref(b, "public_body") for b in notice["buyers"] if b.get("name")), None)
|
||||
if buyer is None:
|
||||
return
|
||||
for contract in notice["contracts"]:
|
||||
for winner in contract["winners"]:
|
||||
supplier = org_ref(winner, "company")
|
||||
if supplier is None:
|
||||
continue
|
||||
amount = contract["amount"] or (notice["total"] if len(notice["contracts"]) == 1 else None)
|
||||
awarded = as_date(contract["awarded"]) or as_date(notice["issued"])
|
||||
title = contract["title"] or notice["title"] or "Auftrag"
|
||||
yield Claim(
|
||||
claim_type="money_flow", org=buyer, counterpart_org=supplier, occurred_at=awarded,
|
||||
summary_de=f"Auftrag „{title}“ an {supplier.name}" + (f" – {format_eur_de(amount)} €" if amount else ""),
|
||||
summary_en=f"Contract \"{title}\" awarded to {supplier.name}" + (f" – EUR {amount}" if amount else ""),
|
||||
data={"notice_id": notice["notice_id"], "cpv": notice["cpv"], "publisher": response.meta.get("publisher")},
|
||||
evidence=[Evidence(url=response.request.url, doc_type="register_entry", title=f"Vergabe: {title}",
|
||||
published_at=awarded, quote=f"{buyer.name} → {supplier.name}: {title}")],
|
||||
facet=Money(kind="contract", amount=amount, payer=buyer, payer_name=buyer.name, payee=supplier,
|
||||
payee_name=supplier.name, paid_on=awarded,
|
||||
reference=f"{notice['notice_id']}/{contract['id']}", category=notice["cpv"]),
|
||||
)
|
||||
@@ -0,0 +1,551 @@
|
||||
"""Rechnungshof (Court of Audit): published party donations and parties' annual financial reports.
|
||||
|
||||
Official works (§7 UrhG); no robots restrictions.
|
||||
|
||||
Donations: HTML pages + one CSV per year/election. The CSVs come in UTF-8, Windows-1252 and Mac Roman,
|
||||
with or without title rows, and one older file without header (date; amount; donor; address; party).
|
||||
|
||||
Annual reports (Rechenschaftsberichte, 2023+ as XLSX): income statement per organisation level
|
||||
(public funding "Fördermittel", membership fees, donations, sponsoring, ad income...) and named lists:
|
||||
donations > EUR 500, sponsors, advertisers in party media, party-owned companies, loans, affiliated
|
||||
organisations.
|
||||
|
||||
Data minimisation: donor addresses and postcodes are never mapped; private donors are stored as names
|
||||
only (no person entity is created); company donors become organisation entities (contract/donor correlation).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Document, Evidence, Money, OrgRef
|
||||
from politrack.text import firmenbuch_number, format_eur_de, normalize, normalize_org_name
|
||||
|
||||
BASE = "https://www.rechnungshof.gv.at"
|
||||
SPEC = SourceSpec(
|
||||
slug="rechnungshof",
|
||||
name="Rechnungshof Österreich",
|
||||
kind="official",
|
||||
homepage=BASE,
|
||||
license="Amtliches Werk (§7 UrhG)",
|
||||
access="scrape",
|
||||
trust=95,
|
||||
min_interval=2.0,
|
||||
tos_notes="Party donation lists and audited annual party reports. Official works; no robots restrictions.",
|
||||
)
|
||||
|
||||
DONATION_INDEX = [
|
||||
"/rh/home/was-wir-tun/was-wir-tun_5/Kontrolle_der_Parteien.html",
|
||||
"/rh/home/was-wir-tun/was-wir-tun_5/was-wir-tun_5/Parteispenden/Parteispenden_2024.html",
|
||||
]
|
||||
REPORTS_INDEX = "/rh/home/was-wir-tun/was-wir-tun_5/was-wir-tun_6/Kontrolle_der_Parteien.html"
|
||||
SECTION_PREFIX = "/rh/home/was-wir-tun/"
|
||||
DONATION_LINK = re.compile(r"spende", re.IGNORECASE)
|
||||
PARTY_PREFIXES = {"ovp": "oevp", "osterreichische volkspartei": "oevp", "die neue volkspartei": "oevp",
|
||||
"spo": "spoe", "sozialdemokratische": "spoe", "fpo": "fpoe", "freiheitliche": "fpoe",
|
||||
"die freiheitliche": "fpoe", "neos": "neos", "die grunen": "gruene", "grune": "gruene",
|
||||
"kpo": "kpoe", "kommunistische": "kpoe", "bier": "bier", "die bierpartei": "bier", "mfg": "mfg"}
|
||||
COMPANY = re.compile(r"\b(GmbH|AG|KG|OG|e\.U\.|GesmbH|Ges\.m\.b\.H|Gesellschaft|Verein|Stiftung|Bank|Holding|"
|
||||
r"Genossenschaft|Verband|Kammer|Sparkasse|Versicherung|Privatstiftung)\b", re.IGNORECASE)
|
||||
|
||||
|
||||
# -- shared helpers -----------------------------------------------------------------------
|
||||
|
||||
def decode(raw: bytes) -> str:
|
||||
"""UTF-8, Windows-1252 or Mac Roman: pick the decoding that yields real umlauts."""
|
||||
try:
|
||||
return raw.decode("utf-8-sig")
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
candidates = [raw.decode(enc, errors="replace") for enc in ("cp1252", "mac_roman")]
|
||||
return max(candidates, key=lambda t: sum(t.count(c) for c in "äöüÄÖÜß"))
|
||||
|
||||
|
||||
def parse_amount(value: Any) -> Decimal | None:
|
||||
if isinstance(value, (int, float, Decimal)):
|
||||
return Decimal(str(value)).quantize(Decimal("0.01"))
|
||||
value = (value or "").strip().replace(".", "").replace(",", ".").replace("€", "").strip()
|
||||
try:
|
||||
return Decimal(value)
|
||||
except InvalidOperation:
|
||||
return None
|
||||
|
||||
|
||||
def parse_de_date(value: str) -> date | None:
|
||||
for fmt in ("%d.%m.%Y", "%d.%m.%y"):
|
||||
try:
|
||||
return datetime.strptime((value or "").strip(), fmt).date()
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def format_de(amount: Decimal) -> str:
|
||||
return format_eur_de(amount)
|
||||
|
||||
|
||||
POSTAL_ADDRESS = re.compile(r"^\d{4}\s")
|
||||
|
||||
|
||||
def clean_name(value: str) -> str:
|
||||
"""Guard against malformed cells ('"2522 Ort Straße 4;Partei"') - an address never ends up in a name."""
|
||||
value = (value or "").split(";")[-1].strip()
|
||||
return "" if POSTAL_ADDRESS.match(value) else value
|
||||
|
||||
|
||||
def party_ref(label: str) -> OrgRef | None:
|
||||
if not label:
|
||||
return None
|
||||
norm = normalize(label)
|
||||
for prefix, slug in PARTY_PREFIXES.items():
|
||||
if norm.startswith(prefix):
|
||||
return OrgRef(slug=slug)
|
||||
return OrgRef(name=label.strip(), kind="party")
|
||||
|
||||
|
||||
def donor_ref(name: str) -> OrgRef | None:
|
||||
"""Organisations become entities (for correlations); private persons stay a plain name."""
|
||||
return OrgRef(name=name, kind="company") if COMPANY.search(name) else None
|
||||
|
||||
|
||||
def column(header: list[str], *needles: str) -> int | None:
|
||||
for i, h in enumerate(header):
|
||||
h = normalize(h)
|
||||
if any(n in h for n in needles):
|
||||
return i
|
||||
return None
|
||||
|
||||
|
||||
DATE_CELL = re.compile(r"^\d{1,2}\.\d{1,2}\.\d{2,4}$")
|
||||
AMOUNT_CELL = re.compile(r"^[\d.]+(,\d{1,2})?$")
|
||||
|
||||
|
||||
def find_columns(rows: list[list[str]]) -> tuple[int, dict[str, int | None]]:
|
||||
"""Locate the header row (older files have a title row above it) and map the columns we use.
|
||||
Address/postcode columns are never mapped."""
|
||||
for i, row in enumerate(rows[:5]):
|
||||
norm = [normalize(c) for c in row]
|
||||
if any("betrag" in c for c in norm) and any("name" in c or "spender" in c for c in norm):
|
||||
return i + 1, {
|
||||
"party": column(row, "partei"),
|
||||
"date": column(row, "eingang") if column(row, "eingang") is not None else column(row, "datum"),
|
||||
"donor": column(row, "name", "spender"),
|
||||
"amount": column(row, "betrag"),
|
||||
"currency": column(row, "wahrung", "waehrung"),
|
||||
"recipient": column(row, "empfang", "empfaeng"),
|
||||
"note": column(row, "anmerk"),
|
||||
}
|
||||
first = next((r for r in rows if any(c.strip() for c in r)), [])
|
||||
date_i = next((i for i, c in enumerate(first) if DATE_CELL.match(c.strip())), None)
|
||||
amount_i = next((i for i, c in enumerate(first) if AMOUNT_CELL.match(c.strip()) and i != date_i), None)
|
||||
if date_i is None or amount_i is None:
|
||||
return 0, {}
|
||||
return 0, {"party": len(first) - 1, "date": date_i, "donor": amount_i + 1, "amount": amount_i,
|
||||
"currency": None, "recipient": len(first) - 1, "note": None}
|
||||
|
||||
|
||||
def read_rows(text: str) -> tuple[dict[str, int | None], list[list[str]]]:
|
||||
"""Parse the semicolon CSV; repairs rows broken by stray line breaks."""
|
||||
reader = list(csv.reader(io.StringIO(text), delimiter=";"))
|
||||
if not reader:
|
||||
return {}, []
|
||||
start, idx = find_columns(reader)
|
||||
if not idx:
|
||||
return {}, []
|
||||
width = max(i for i in idx.values() if i is not None) + 1
|
||||
rows, pending = [], []
|
||||
for row in reader[start:]:
|
||||
if pending:
|
||||
pending[-1] += " " + (row[0] if row else "")
|
||||
pending.extend(row[1:])
|
||||
else:
|
||||
pending = list(row)
|
||||
if len(pending) >= width - 1:
|
||||
rows.append(pending)
|
||||
pending = []
|
||||
return idx, [r for r in rows if any(c.strip() for c in r)]
|
||||
|
||||
|
||||
# -- donations ----------------------------------------------------------------------------
|
||||
|
||||
@register
|
||||
class RechnungshofDonations(Adapter):
|
||||
spec = SPEC
|
||||
name = "rechnungshof.donations"
|
||||
description = "Published party donations (CSV per year / election)"
|
||||
schedule = "30 5 * * *"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
for path in DONATION_INDEX:
|
||||
yield Request(url=urljoin(BASE, path), kind="page", meta={"depth": 0})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
if response.request.kind == "page":
|
||||
depth = response.meta.get("depth", 0)
|
||||
for a in response.soup.find_all("a", href=True):
|
||||
href = urljoin(response.url, a["href"])
|
||||
path = href.removeprefix(BASE)
|
||||
if href.lower().endswith(".csv") and DONATION_LINK.search(href):
|
||||
yield Request(url=href, kind="csv", meta={"page": response.url})
|
||||
elif (depth < 2 and path.startswith(SECTION_PREFIX) and path.endswith(".html")
|
||||
and DONATION_LINK.search(path)):
|
||||
yield Request(url=href, kind="page", meta={"depth": depth + 1})
|
||||
return
|
||||
yield from self.parse_csv(ctx, response)
|
||||
|
||||
def parse_csv(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||||
idx, rows = read_rows(decode(response.body))
|
||||
if idx.get("donor") is None or idx.get("amount") is None:
|
||||
ctx.stats["unparsed_files"] += 1
|
||||
return
|
||||
|
||||
def cell(row: list[str], key: str) -> str:
|
||||
i = idx[key]
|
||||
value = row[i].strip() if i is not None and i < len(row) else ""
|
||||
return clean_name(value) if key in ("party", "donor", "recipient") else value
|
||||
|
||||
for n, row in enumerate(rows):
|
||||
party_label, donor, recipient = cell(row, "party"), cell(row, "donor"), cell(row, "recipient")
|
||||
amount, received = parse_amount(cell(row, "amount")), parse_de_date(cell(row, "date"))
|
||||
if not donor or amount is None:
|
||||
continue
|
||||
currency = cell(row, "currency") or "EUR"
|
||||
currency = "EUR" if currency.lower().startswith("eur") else currency
|
||||
party = party_ref(party_label) or party_ref(recipient)
|
||||
line = " | ".join(c for c in (cell(row, "date"), party_label, donor, cell(row, "amount"), currency,
|
||||
recipient) if c)
|
||||
yield Claim(
|
||||
claim_type="donation", org=party, occurred_at=received,
|
||||
summary_de=f"{recipient or party_label} erhielt {format_de(amount)} {currency} von {donor}",
|
||||
summary_en=f"{recipient or party_label} received {currency} {amount:,.2f} from {donor}",
|
||||
data={"party_label": party_label, "recipient": recipient, "note": cell(row, "note")},
|
||||
evidence=[
|
||||
Evidence(url=response.meta["page"], doc_type="register_entry",
|
||||
title="Rechnungshof: veröffentlichte Parteispenden", quote=line, locator=f"row:{line}"),
|
||||
Evidence(url=response.request.url, doc_type="register_entry",
|
||||
title="Rechnungshof: Parteispenden (CSV)", locator=f"row:{n + 1}"),
|
||||
],
|
||||
topics=("Transparenz und Korruption",),
|
||||
facet=Money(kind="donation", amount=amount, currency=currency, payer=donor_ref(donor), payer_name=donor,
|
||||
payee=party, payee_name=recipient or party_label, paid_on=received),
|
||||
)
|
||||
|
||||
|
||||
# -- annual party reports -------------------------------------------------------------------
|
||||
|
||||
# Lines of the income statement (§ 5 Abs. 4/5 PartG), matched on the normalised label. Order matters:
|
||||
# the first matching pattern wins ("Inserate und Werbeeinschaltungen" is an expense, "Inserate" income).
|
||||
FINANCE_LINES = [
|
||||
(re.compile(r"^(gesamt)?summe (der )?ertrage"), "total_income"),
|
||||
(re.compile(r"^(gesamt)?summe (der )?aufwendungen"), "total_expenses"),
|
||||
(re.compile(r"^fordermittel"), "public_funding"),
|
||||
(re.compile(r"^mitgliedsbeitrage"), "membership_fees"),
|
||||
(re.compile(r"^beitrage der .*angehorenden mandatare"), "mandate_holder_levies"),
|
||||
(re.compile(r"^geldspenden"), "donations"),
|
||||
(re.compile(r"^sponsoring"), "sponsoring"),
|
||||
(re.compile(r"^inserate und werbeeinschaltungen"), "ad_spending"),
|
||||
(re.compile(r"^inserate\b"), "ad_income"),
|
||||
(re.compile(r"^ertrage aus nahestehenden organisationen"), "from_affiliates"),
|
||||
(re.compile(r"^personalaufwand"), "staff_costs"),
|
||||
(re.compile(r"^au(ss|ß)enwerbung"), "outdoor_advertising"),
|
||||
(re.compile(r"^direktwerbung"), "direct_marketing"),
|
||||
]
|
||||
|
||||
|
||||
def finance_lines(rows: list[list[Any]], year: str) -> dict[str, Decimal]:
|
||||
"""Income/expense lines of one sheet, summed over all blocks in it.
|
||||
|
||||
Some parties stack one block per organisation in a sheet (e.g. every state below the other); each block
|
||||
has its own header row with the years. The value is taken from the reporting-year column of the latest
|
||||
header (columns are "2024, 2023" in some files and "2023, 2024" in others); without a header, the first
|
||||
number after the label.
|
||||
"""
|
||||
total: dict[str, Decimal] = {}
|
||||
block: dict[str, Decimal] = {}
|
||||
year_col: int | None = None
|
||||
for row in rows:
|
||||
years = [i for i, c in enumerate(row) if str(c).strip() in (year, f"{year}.0")]
|
||||
if years and sum(1 for c in row if re.fullmatch(r"(19|20)\d\d(\.0)?", str(c).strip() if c is not None else "")) >= 1:
|
||||
year_col = years[0]
|
||||
for i, cell in enumerate(row):
|
||||
if not isinstance(cell, str) or len(cell.strip()) < 4:
|
||||
continue
|
||||
label = re.sub(r"\s+", " ", normalize(cell))
|
||||
key = next((k for pattern, k in FINANCE_LINES if pattern.search(label)), None)
|
||||
if key:
|
||||
value = number(row[year_col]) if year_col is not None and year_col > i and year_col < len(row) else None
|
||||
if value is None:
|
||||
value = next((number(c) for c in row[i + 1:] if number(c) is not None), None)
|
||||
if value is not None:
|
||||
if key in block: # the same line again: a new block (next organisation) starts
|
||||
for k, v in block.items():
|
||||
total[k] = total.get(k, Decimal(0)) + v
|
||||
block = {}
|
||||
block[key] = value
|
||||
break # only the first text cell of a row is its label
|
||||
for k, v in block.items():
|
||||
total[k] = total.get(k, Decimal(0)) + v
|
||||
return total
|
||||
|
||||
|
||||
def sum_levels(levels: dict[str, dict[str, Decimal]]) -> tuple[dict[str, Decimal], dict[str, dict[str, Decimal]]]:
|
||||
"""Sum all organisation levels (federal, states, capitals, districts ...) -> (totals, levels used).
|
||||
|
||||
Overview sheets that add up other levels are left out: titles listing several levels ("BO&LOs&LH-St.&NTG")
|
||||
and sheets whose public funding equals the sum of all the others. Totals (income, expenses) are only
|
||||
reported when every level states them - a partial sum would be misleading.
|
||||
"""
|
||||
levels = {t: v for t, v in levels.items() if "&" not in t}
|
||||
funding = {title: v.get("public_funding", Decimal(0)) for title, v in levels.items()}
|
||||
for title, value in funding.items():
|
||||
others = sum(v for t, v in funding.items() if t != title)
|
||||
if value > 0 and len(funding) > 2 and abs(value - others) <= Decimal(1):
|
||||
levels = {t: v for t, v in levels.items() if t != title}
|
||||
break
|
||||
totals: dict[str, Decimal] = {}
|
||||
for values in levels.values():
|
||||
for key, value in values.items():
|
||||
totals[key] = totals.get(key, Decimal(0)) + value
|
||||
for key in ("total_income", "total_expenses"):
|
||||
if key in totals and not all(key in v for v in levels.values()):
|
||||
del totals[key]
|
||||
return totals, levels
|
||||
|
||||
|
||||
AFFILIATE_SKIP = re.compile(
|
||||
r"^(anlage|liste|ertrag|gem\.|gemass|\(?§|name$|zvr|reg\.? ?nr|organisationseinheit|nahestehende|"
|
||||
r"bundesorganisation|landesorganisation|zweigvereine:?$|[a-z]\)|[a-z]\. anlage|keine|leermeldung|rechtsform|zweck)")
|
||||
ZVR = re.compile(r"\(?ZVR(?:-Zahl)?:?\s*(\d{6,10})\)?", re.IGNORECASE)
|
||||
|
||||
|
||||
def affiliate_from_row(row: list[Any]) -> tuple[str, str | None] | None:
|
||||
"""Name (and association-register number) of an affiliated organisation from one row of the list."""
|
||||
cells = [str(c).strip() for c in row if c is not None and str(c).strip()]
|
||||
number_cell = next((c for c in cells if re.fullmatch(r"\d{6,10}", c)), None)
|
||||
for cell in cells:
|
||||
if re.fullmatch(r"[\d\s./-]+", cell) or cell.lower() in ("partei", "verein", "gmbh", "stiftung"):
|
||||
continue
|
||||
if AFFILIATE_SKIP.search(normalize(cell)) or len(cell) <= 3:
|
||||
return None
|
||||
zvr = ZVR.search(cell)
|
||||
name = ZVR.sub("", cell)
|
||||
name = re.sub(r'"?,?\s*Kurzbezeichnung:?\s*"?([^"]+)"?\s*$', r" (\1)", name).strip(' "„“,;-')
|
||||
return (name, (zvr.group(1) if zvr else number_cell)) if len(name) > 3 else None
|
||||
return None
|
||||
|
||||
|
||||
def transliterate(text: str) -> str:
|
||||
"""'Kaernten Koefer Gruenen' -> 'Kärnten Köfer Grünen' (file names; not after vowels: 'Neue', 'Steuer')."""
|
||||
for plain, umlaut in (("ae", "ä"), ("oe", "ö"), ("ue", "ü")):
|
||||
text = re.sub(rf"(?<![aeiouqAEIOUQ]){plain}", umlaut, text)
|
||||
text = re.sub(rf"(?<![aeiouqAEIOUQ]){plain.capitalize()}", umlaut.upper(), text)
|
||||
return text
|
||||
|
||||
|
||||
def report_party_name(sheets: dict[str, list[list[Any]]], url: str) -> str:
|
||||
"""Cover sheet, else the 'Territoriale Gliederung ... : <party>' line, else the file name."""
|
||||
cover = [str(c).strip(" -–") for row in sheets.get("Deckblatt", []) for c in row if c and str(c).strip(" -–")]
|
||||
names = [c for c in cover if len(c) > 2 and not re.search(
|
||||
r"(?i)rechenschaftsbericht|bericht|kalenderjahr|jahr\b|partg|gemäß|name der|politische partei$|^\d", c)]
|
||||
if names:
|
||||
return names[0]
|
||||
for rows in sheets.values():
|
||||
for row in rows:
|
||||
for cell in row:
|
||||
if isinstance(cell, str) and (m := re.search(r"Territoriale Gliederung[^:]*:\s*(.+)$", cell.strip())):
|
||||
return m.group(1).strip()
|
||||
stem = url.rsplit("/", 1)[-1].rsplit(".", 1)[0]
|
||||
stem = re.sub(r"(?i)^Rechenschaftsbericht[_-]?\d{4}[_-]?|[_-](gepr(ue|ü)ft|Anlagen|Verm(oe|ö)gensaufstellung)$", "", stem)
|
||||
return transliterate(re.sub(r"[_-]+", " ", stem)).strip()
|
||||
|
||||
|
||||
def sheets_titled(sheets: dict[str, list[list[Any]]], *keywords: str) -> list[list[Any]]:
|
||||
"""All rows of every sheet whose (normalised) title contains one of the keywords."""
|
||||
return [row for title, rows in sheets.items() if any(k in normalize(title) for k in keywords) for row in rows]
|
||||
|
||||
|
||||
def number(value: Any) -> Decimal | None:
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, (int, float, Decimal)):
|
||||
return Decimal(str(value)).quantize(Decimal("0.01"))
|
||||
text = str(value).strip()
|
||||
if re.fullmatch(r"-?\d+\.\d{1,2}", text): # '301561.19' - plain decimal point (some exports)
|
||||
return Decimal(text).quantize(Decimal("0.01"))
|
||||
return parse_amount(text) if re.fullmatch(r"[\d.,\s€-]+", text or "x") else None
|
||||
|
||||
|
||||
def sheet_lists(rows: list[list[Any]], header_needles: tuple[str, ...]) -> Iterator[dict[str, Any]]:
|
||||
"""Yield dict rows of every table in a sheet whose header contains all needles (case-insensitive)."""
|
||||
header: list[str] | None = None
|
||||
for row in rows:
|
||||
cells = ["" if c is None else str(c).strip() for c in row]
|
||||
lowered = [normalize(c) for c in cells]
|
||||
if all(any(n in c for c in lowered) for n in header_needles):
|
||||
header = lowered
|
||||
continue
|
||||
if header is None or not any(cells):
|
||||
continue
|
||||
if any("leermeldung" in c or "keine" in c for c in lowered[:2]):
|
||||
header = None
|
||||
continue
|
||||
record = {header[i]: row[i] for i in range(min(len(header), len(row))) if header[i]}
|
||||
amount = next((number(row[i]) for i in range(len(row) - 1, -1, -1) if number(row[i]) is not None), None)
|
||||
if amount is not None and any(isinstance(v, str) and v.strip() for v in record.values()):
|
||||
record["__amount"] = amount
|
||||
yield record
|
||||
|
||||
|
||||
def pick(record: dict[str, Any], *needles: str) -> str:
|
||||
for key, value in record.items():
|
||||
if any(n in key for n in needles) and isinstance(value, str) and value.strip():
|
||||
return value.strip()
|
||||
return ""
|
||||
|
||||
|
||||
@register
|
||||
class RechnungshofPartyReports(Adapter):
|
||||
spec = SPEC
|
||||
name = "rechnungshof.party_reports"
|
||||
description = "Parties' annual financial reports: funding, income, sponsors, advertisers, companies, loans"
|
||||
schedule = "0 6 * * 1"
|
||||
parser_version = 3
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
yield Request(url=f"{BASE}{REPORTS_INDEX}", kind="index")
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
if response.request.kind == "index":
|
||||
for a in response.soup.find_all("a", href=True):
|
||||
href = urljoin(response.url, a["href"])
|
||||
if re.search(r"Rechenschaftsbericht[_-]?\d{4}.*\.xlsx$", href, re.I):
|
||||
yield Request(url=href, kind="xlsx")
|
||||
elif re.search(r"(Ergebnis[_-]der[_-]Pruefung|Rechenschaftsbericht).*\.pdf$", href, re.I):
|
||||
yield Document(Evidence(url=href, doc_type="pdf", title=a.get_text(" ", strip=True) or href.rsplit("/", 1)[-1],
|
||||
meta={"kind": "party_report_pdf"}))
|
||||
return
|
||||
yield from self.parse_report(ctx, response)
|
||||
|
||||
def parse_report(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
import openpyxl
|
||||
|
||||
wb = openpyxl.load_workbook(io.BytesIO(response.body), data_only=True, read_only=True)
|
||||
sheets = {ws.title.strip(): [list(r) for r in ws.iter_rows(values_only=True)] for ws in wb.worksheets}
|
||||
url = response.request.url
|
||||
year_match = re.search(r"Rechenschaftsbericht[_-]?(\d{4})", url) or re.search(r"(\d{4})", url)
|
||||
year = year_match.group(1) if year_match else "?"
|
||||
audited = bool(re.search(r"gepr(ue|ü)ft", url, re.IGNORECASE))
|
||||
party_name = report_party_name(sheets, url)
|
||||
party = party_ref(party_name)
|
||||
version_de = "vom Rechnungshof geprüft" if audited else "eingereicht"
|
||||
evidence = [Evidence(url=url, doc_type="register_entry",
|
||||
title=f"Rechenschaftsbericht {year} ({version_de}): {party_name}")]
|
||||
|
||||
# Income statement per organisation level (federal, states, capitals, districts...), then summed.
|
||||
levels = {title: lines for title, rows in sheets.items()
|
||||
if (lines := finance_lines(rows, year)) and ("public_funding" in lines or "total_income" in lines)}
|
||||
totals, levels = sum_levels(levels)
|
||||
if totals:
|
||||
income = totals.get("total_income")
|
||||
yield Claim(
|
||||
claim_type="party_finance", org=party,
|
||||
key=("rh-report", normalize_org_name(party_name), year, "audited" if audited else "submitted"),
|
||||
valid_from=date(int(year), 1, 1) if year.isdigit() else None,
|
||||
valid_to=date(int(year), 12, 31) if year.isdigit() else None,
|
||||
summary_de=f"Rechenschaftsbericht {year} ({version_de}): "
|
||||
+ (f"Erträge {format_de(income)} €, " if income is not None else "")
|
||||
+ f"davon Fördermittel {format_de(totals.get('public_funding', Decimal(0)))} €"
|
||||
+ " (Summe aller ausgewiesenen Gliederungen)",
|
||||
summary_en=f"Annual report {year} ({'audited' if audited else 'submitted'}): "
|
||||
+ (f"income EUR {income:,.2f}, " if income is not None else "")
|
||||
+ f"public funding EUR {totals.get('public_funding', 0):,.2f}",
|
||||
data={"year": year, "audited": audited, **{k: str(v) for k, v in totals.items()},
|
||||
"levels": {t: {k: str(v) for k, v in lines.items()} for t, lines in levels.items()}},
|
||||
evidence=evidence, topics=("Transparenz und Korruption",),
|
||||
)
|
||||
if totals.get("public_funding"):
|
||||
yield Claim(
|
||||
claim_type="money_flow", org=party, valid_from=date(int(year), 1, 1) if year.isdigit() else None,
|
||||
summary_de=f"{party_name} erhielt {year} öffentliche Fördermittel von {format_de(totals['public_funding'])} €",
|
||||
summary_en=f"{party_name} received EUR {totals['public_funding']:,.2f} in public funding in {year}",
|
||||
evidence=evidence, topics=("Transparenz und Korruption",),
|
||||
facet=Money(kind="party_funding", amount=totals["public_funding"], payee=party,
|
||||
payee_name=party_name, payer_name="Öffentliche Parteienförderung", period=year),
|
||||
)
|
||||
else:
|
||||
ctx.stats["reports_without_income_statement"] += 1
|
||||
|
||||
def money_claims(rows: list[list[Any]], needles: tuple[str, ...], kind: str, who_keys: tuple[str, ...],
|
||||
label_de: str) -> Iterator[Claim]:
|
||||
for rec in sheet_lists(rows, needles):
|
||||
who = clean_name(pick(rec, *who_keys))
|
||||
if not who:
|
||||
continue
|
||||
recipient = pick(rec, "empfanger", "empfager")
|
||||
medium = pick(rec, "medium")
|
||||
amount = rec["__amount"]
|
||||
yield Claim(
|
||||
claim_type="donation" if kind == "donation" else "money_flow", org=party,
|
||||
valid_from=date(int(year), 1, 1) if year.isdigit() else None,
|
||||
summary_de=f"{label_de}: {who} → {recipient or party_name}"
|
||||
+ (f" ({medium})" if medium else "") + f", {format_de(amount)} € ({year})",
|
||||
data={"year": year, "recipient": recipient, "medium": medium},
|
||||
evidence=[Evidence(url=url, doc_type="register_entry", title=evidence[0].title,
|
||||
quote=f"{who} | {recipient} | {medium} | {amount}", locator=f"{kind}:{who}:{amount}")],
|
||||
topics=("Transparenz und Korruption",),
|
||||
facet=Money(kind=kind, amount=amount, payer=donor_ref(who), payer_name=who, payee=party,
|
||||
payee_name=recipient or party_name, period=year, category=medium or None),
|
||||
)
|
||||
|
||||
# List sheets are named differently by every party ("Spenden", "Spendenliste (§ 6 PartG)", "Spenden,
|
||||
# Sponsoring, Inserate" ...): select by title keyword, find the tables by their header.
|
||||
yield from money_claims(sheets_titled(sheets, "spende"), ("empfanger", "name", "plz"), "donation", ("name",),
|
||||
"Spende laut Rechenschaftsbericht")
|
||||
yield from money_claims(sheets_titled(sheets, "sponsor"), ("sponsor",), "sponsorship", ("sponsor",), "Sponsoring")
|
||||
yield from money_claims(sheets_titled(sheets, "inserat"), ("inserent",), "party_ad", ("inserent",),
|
||||
"Inserat in Parteimedium")
|
||||
for rec in sheet_lists(sheets_titled(sheets, "kredit", "darlehen", "verbindlichkeit"), ("kreditgeber",)):
|
||||
lender = clean_name(pick(rec, "kreditgeber"))
|
||||
if lender:
|
||||
yield Claim(
|
||||
claim_type="money_flow", org=party,
|
||||
summary_de=f"Kredit/Darlehen von {lender} an {party_name}: {format_de(rec['__amount'])} € ({year})",
|
||||
evidence=evidence, facet=Money(kind="loan", amount=rec["__amount"], payer=OrgRef(name=lender, kind="company"),
|
||||
payer_name=lender, payee=party, payee_name=party_name, period=year),
|
||||
)
|
||||
for row in sheets_titled(sheets, "beteiligung"):
|
||||
cells = [str(c).strip() for c in row if c is not None and str(c).strip()]
|
||||
fn = next((f for f in (firmenbuch_number(c) for c in cells[1:3]) if f), None)
|
||||
if fn:
|
||||
company = re.sub(r"\s*\d\)$", "", cells[0]) # footnote markers: "Werbeagentur 1)"
|
||||
yield Claim(
|
||||
claim_type="ownership", org=party,
|
||||
counterpart_org=OrgRef(name=company, kind="company", ids={"firmenbuch": fn}),
|
||||
key=("rh-ownership", normalize_org_name(party_name), year, fn),
|
||||
summary_de=f"{party_name} hält eine Beteiligung an {company} (FN {fn}) – laut Rechenschaftsbericht {year}",
|
||||
evidence=evidence, topics=("Transparenz und Korruption",),
|
||||
)
|
||||
for title, rows in sheets.items():
|
||||
# list of affiliated organisations (not the sheet of income received from them)
|
||||
if "nahestehend" not in normalize(title) or "ertr" in normalize(title):
|
||||
continue
|
||||
for row in rows:
|
||||
affiliate = affiliate_from_row(row)
|
||||
if affiliate:
|
||||
name, zvr = affiliate
|
||||
yield Claim(
|
||||
claim_type="affiliation", org=party,
|
||||
counterpart_org=OrgRef(name=name, kind="ngo", ids={"zvr": zvr} if zvr else {}),
|
||||
key=("rh-affiliate", normalize_org_name(party_name), year, normalize_org_name(name)),
|
||||
summary_de=f"Nahestehende Organisation von {party_name}: {name} ({year})",
|
||||
evidence=evidence,
|
||||
)
|
||||
ctx.stats["reports"] += 1
|
||||
@@ -0,0 +1,106 @@
|
||||
"""RTR / KommAustria media transparency (MedKF-TG): advertising and media subsidies paid by public bodies.
|
||||
|
||||
Open data API (no usage restrictions, attribution): https://data.rtr.at/api/v1/tables/<table>.json
|
||||
* mt-bekanntgabe2024 (half-yearly since 2024): payer, medium, media owner, campaign, amount, ad sujet link
|
||||
* mt-bekanntgabe (quarterly until 2023): payer, medium/owner, amount
|
||||
Each row -> money_flow (kind government_ad for advertising, subsidy for media funding) from the public body
|
||||
(ministry, state government, municipality, public company...) to the media owner.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Evidence, Money, OrgRef
|
||||
from politrack.text import format_eur_de
|
||||
|
||||
API = "https://data.rtr.at/api/v1/tables"
|
||||
PAGE_SIZE = 5000
|
||||
SPEC = SourceSpec(
|
||||
slug="rtr-medientransparenz", name="RTR/KommAustria Medientransparenz", kind="official",
|
||||
homepage="https://www.rtr.at/medien/was_wir_tun/medientransparenz/",
|
||||
license="Amtliches Werk (§7 UrhG); attribution \"KommAustria – data.rtr.at\"", access="api",
|
||||
trust=95, min_interval=2.0,
|
||||
# data.rtr.at/robots.txt disallows everything for crawlers, but RTR documents this API as "der Endpunkt zur
|
||||
# automatischen Abfrage der Daten" (rtr.at/rtr/service/opendata/OD_Schnittstellenbeschreibung.de.html).
|
||||
robots_exempt=(API + "/",),
|
||||
tos_notes="Government advertising and media funding reported under MedKF-TG. The REST API is documented by "
|
||||
"RTR for automated queries (OD_Schnittstellenbeschreibung); robots.txt on data.rtr.at blocks "
|
||||
"crawlers only. Terms (OD_Nutzungsbedingungen): free works under §7 UrhG, attribution requested, "
|
||||
"no use for defamatory/discriminatory purposes.",
|
||||
)
|
||||
KIND = {2: "government_ad", 4: "subsidy"}
|
||||
|
||||
|
||||
def half_years(since_year: int) -> list[str]:
|
||||
today = date.today()
|
||||
return [f"{y}{h}" for y in range(since_year, today.year + 1) for h in (1, 2)
|
||||
if (y, h) <= (today.year, 1 if today.month <= 6 else 2)]
|
||||
|
||||
|
||||
def quarters(since_year: int, until_year: int = 2023) -> list[str]:
|
||||
return [f"{y}{q}" for y in range(since_year, until_year + 1) for q in (1, 2, 3, 4)]
|
||||
|
||||
|
||||
@register
|
||||
class MediaTransparency(Adapter):
|
||||
spec = SPEC
|
||||
name = "rtr.medientransparenz"
|
||||
description = "Government advertising and media subsidies per public body and media owner"
|
||||
schedule = "0 4 * * 2"
|
||||
queue = "bulk"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
for period in half_years(int(ctx.params.get("since_year", 2024))):
|
||||
yield Request(url=f"{API}/mt-bekanntgabe2024.json", params={"halbjahr": period, "size": PAGE_SIZE, "page": 0},
|
||||
kind="halbjahr", meta={"period": period})
|
||||
for period in quarters(int(ctx.params.get("legacy_since_year", 2021))):
|
||||
yield Request(url=f"{API}/mt-bekanntgabe.json", params={"quartal": period, "size": PAGE_SIZE, "page": 0},
|
||||
kind="quartal", meta={"period": period})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
rows = response.json().get("data") or []
|
||||
params = response.request.params or {}
|
||||
if len(rows) >= PAGE_SIZE:
|
||||
yield Request(url=response.request.url, params={**params, "page": params.get("page", 0) + 1},
|
||||
kind=response.request.kind, meta=response.meta)
|
||||
period = response.meta["period"]
|
||||
label = f"{period[:4]}-H{period[4:]}" if response.request.kind == "halbjahr" else f"{period[:4]}-Q{period[4:]}"
|
||||
min_euro = float(ctx.params.get("min_euro", 0))
|
||||
for row in rows:
|
||||
amount = row.get("euro")
|
||||
if row.get("leermeldung") or amount is None or float(amount) < min_euro:
|
||||
continue
|
||||
kind = KIND.get(row.get("bekanntgabe"), "other")
|
||||
payer_name = (row.get("rechtstraeger") or "").strip().strip('"')
|
||||
owner = (row.get("medieninhaber") or row.get("mediumMedieninhaber") or "").strip()
|
||||
medium = (row.get("medium") or row.get("mediumMedieninhaber") or "").strip()
|
||||
if not payer_name or not owner:
|
||||
continue
|
||||
payer = OrgRef(name=payer_name, kind="public_body",
|
||||
ids={"rtr": str(row["rechtstraeger_id"])} if row.get("rechtstraeger_id") else {})
|
||||
payee = OrgRef(name=owner, kind="media")
|
||||
campaign = row.get("kampagne")
|
||||
what = "Werbung" if kind == "government_ad" else "Förderung"
|
||||
evidence_url = (f"{API}/{'mt-bekanntgabe2024' if response.request.kind == 'halbjahr' else 'mt-bekanntgabe'}"
|
||||
f".json?{'halbjahr' if response.request.kind == 'halbjahr' else 'quartal'}={period}"
|
||||
+ (f"&rechtstraeger_id={row['rechtstraeger_id']}" if row.get("rechtstraeger_id") else ""))
|
||||
yield Claim(
|
||||
claim_type="money_flow", org=payer, counterpart_org=payee,
|
||||
summary_de=f"{payer_name}: {what} in {medium} ({owner}), {format_eur_de(amount)} € ({label})"
|
||||
+ (f" – Kampagne „{campaign}“" if campaign else ""),
|
||||
summary_en=f"{payer_name}: {'advertising' if kind == 'government_ad' else 'funding'} in {medium} "
|
||||
f"({owner}), EUR {amount:,.2f} ({label})",
|
||||
data={"medium": medium, "category": row.get("kategorie"), "subcategory": row.get("subkategorie"),
|
||||
"campaign": campaign, "sujet": row.get("sujet"), "period": label},
|
||||
evidence=[Evidence(url=evidence_url, doc_type="dataset_row",
|
||||
title=f"Medientransparenz {label}: {payer_name}",
|
||||
quote=f"{payer_name} → {medium} ({owner}): {amount} €")],
|
||||
topics=("Information und Medien",),
|
||||
facet=Money(kind=kind, amount=amount, payer=payer, payer_name=payer_name, payee=payee, payee_name=owner,
|
||||
period=label, reference=f"{campaign or ''}|{medium}|{row.get('sujet_dateiname') or ''}",
|
||||
category=row.get("kategorie")),
|
||||
)
|
||||
@@ -0,0 +1,279 @@
|
||||
"""Public posts of politicians on open platforms: Bluesky, Mastodon, Telegram (public channels), YouTube,
|
||||
and X via the official (paid) API when a token is configured.
|
||||
|
||||
Accounts come from the `social_account` table: parliament profiles (verified), Wikidata
|
||||
P2002/P12361/P4033/P3789/P2397 (unverified until confirmed) and the curated config/social_accounts.yaml
|
||||
(enricher `config.social_accounts`). Accounts belong to a person or to an organisation (party channels).
|
||||
|
||||
Not collected (see docs/sources.md): X without the official API (ToS forbid scraping and proxies such as
|
||||
Nitter), Facebook/Instagram (ToS), WhatsApp channels (no public read access, ToS forbid automated
|
||||
collection), Signal (private by design, no public channels), private Telegram groups.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Evidence, OrgRef, PersonRef, Statement
|
||||
from politrack.text import excerpt, html_to_text
|
||||
|
||||
|
||||
def social_spec(slug: str, name: str, homepage: str, notes: str, robots_exempt: tuple[str, ...] = ()) -> SourceSpec:
|
||||
return SourceSpec(slug=slug, name=name, kind="social", homepage=homepage, license="Per-post copyright of the author",
|
||||
access="api", trust=40, store_fulltext=True, keep_raw=True, min_interval=2.0,
|
||||
robots_exempt=robots_exempt, tos_notes=notes)
|
||||
|
||||
|
||||
BLUESKY = social_spec("bluesky", "Bluesky", "https://bsky.app",
|
||||
"Public AT Protocol AppView API, no auth for public reads. Posts by the politicians themselves.")
|
||||
MASTODON = social_spec("mastodon", "Mastodon / Fediverse", "https://joinmastodon.org",
|
||||
"Public REST API of the account's instance (no auth for public statuses). Respect instance rules.")
|
||||
TELEGRAM = social_spec("telegram", "Telegram (öffentliche Kanäle)", "https://telegram.org",
|
||||
"Public web preview of *public broadcast channels* (t.me/s/<channel>); no robots.txt restrictions. "
|
||||
"Private groups/chats are never accessed. Low frequency, latest posts only.")
|
||||
YOUTUBE = social_spec("youtube", "YouTube", "https://www.youtube.com",
|
||||
"YouTube Data API v3 (free quota, API key required via POLITRACK_YOUTUBE_API_KEY).")
|
||||
X_API = "https://api.x.com/2/"
|
||||
X = SourceSpec(slug="x", name="X (Twitter)", kind="social", homepage="https://x.com",
|
||||
license="Per-post copyright of the author; X Developer Agreement", access="api", trust=40,
|
||||
store_fulltext=False, keep_raw=False, min_interval=2.0, robots_exempt=(X_API,),
|
||||
tos_notes="Official X API v2 only (paid tier, POLITRACK_X_BEARER_TOKEN). Scraping x.com or using "
|
||||
"mirrors/proxies (Nitter etc.) violates the ToS and is not done. The Developer Agreement requires "
|
||||
"removing posts that were deleted on X; no raw archive or full text is kept, but the "
|
||||
"deletion sync is still open (docs/roadmap.md).")
|
||||
|
||||
PLATFORM_LABELS = {"bluesky": "Bluesky", "mastodon": "Mastodon", "telegram": "Telegram", "youtube": "YouTube",
|
||||
"x": "X"}
|
||||
|
||||
|
||||
def accounts(ctx: RunContext, platform: str, url_pattern: str | None = None) -> list[dict[str, Any]]:
|
||||
"""Accounts of persons or organisations on one platform; each row carries an `owner` dict for Request.meta.
|
||||
|
||||
`url_pattern` also picks up accounts stored as plain websites, so it must match only that platform's
|
||||
own host (never a generic shape like "https://host/@user", which TikTok and others share).
|
||||
"""
|
||||
if ctx.conn is None:
|
||||
return []
|
||||
rows = ctx.conn.execute(
|
||||
"""SELECT person_id, organization_id, url, handle, verified FROM social_account
|
||||
WHERE (platform = %s OR (%s::text IS NOT NULL AND url ~* %s))
|
||||
AND (person_id IS NOT NULL OR organization_id IS NOT NULL)
|
||||
ORDER BY verified DESC, id""",
|
||||
(platform, url_pattern, url_pattern),
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
row["owner"] = {"person_id": str(row["person_id"]) if row["person_id"] else None,
|
||||
"org_id": None if row["person_id"] else str(row["organization_id"]),
|
||||
"verified": row["verified"]}
|
||||
return rows
|
||||
|
||||
|
||||
def post_claim(*, owner: dict[str, Any], platform: str, url: str, text: str, created: Any,
|
||||
title: str | None = None, data: dict | None = None) -> Claim:
|
||||
"""A public post as a `statement` claim of the account owner (person, or organisation for party channels)."""
|
||||
label = PLATFORM_LABELS[platform]
|
||||
return Claim(
|
||||
claim_type="statement", occurred_at=created,
|
||||
person=PersonRef(ids={"politrack": owner["person_id"]}) if owner.get("person_id") else None,
|
||||
org=OrgRef(ids={"politrack_org": owner["org_id"]}, create=False) if owner.get("org_id") else None,
|
||||
summary_de=f"Auf {label}: „{excerpt(title or text, 200)}“",
|
||||
summary_en=f"On {label}: \"{excerpt(title or text, 200)}\"",
|
||||
data={"platform": platform, "account_verified": owner.get("verified", False), **(data or {})},
|
||||
evidence=[Evidence(url=url, source=platform, doc_type="social_post", title=excerpt(title or text, 120),
|
||||
published_at=created, full_text=text, excerpt=text, quote=excerpt(text, 500))],
|
||||
facet=Statement(context="social_post", url=url, said_at=created, platform=platform, title=title,
|
||||
excerpt=text),
|
||||
)
|
||||
|
||||
|
||||
@register
|
||||
class BlueskyPosts(Adapter):
|
||||
spec = BLUESKY
|
||||
name = "social.bluesky"
|
||||
description = "Public Bluesky posts of politicians"
|
||||
schedule = "7 * * * *"
|
||||
queue = "social"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
for acc in accounts(ctx, "bluesky", r"bsky\.app/profile/"):
|
||||
m = re.search(r"bsky\.app/profile/([^/?#]+)", acc["url"])
|
||||
if m:
|
||||
yield Request(url="https://public.api.bsky.app/xrpc/app.bsky.feed.getAuthorFeed",
|
||||
params={"actor": m.group(1), "limit": 50, "filter": "posts_no_replies"}, kind="feed",
|
||||
meta={**acc["owner"], "handle": m.group(1)})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||||
handle = response.meta["handle"]
|
||||
for item in response.json().get("feed", []):
|
||||
post = item.get("post") or {}
|
||||
if item.get("reason"): # repost of someone else's post
|
||||
continue
|
||||
record = post.get("record") or {}
|
||||
text = record.get("text") or ""
|
||||
author = (post.get("author") or {}).get("handle") or handle
|
||||
rkey = post.get("uri", "").rsplit("/", 1)[-1]
|
||||
yield post_claim(owner=response.meta, platform="bluesky",
|
||||
url=f"https://bsky.app/profile/{author}/post/{rkey}", text=text,
|
||||
created=record.get("createdAt"),
|
||||
data={"likes": post.get("likeCount"), "reposts": post.get("repostCount")})
|
||||
|
||||
|
||||
@register
|
||||
class MastodonPosts(Adapter):
|
||||
spec = MASTODON
|
||||
name = "social.mastodon"
|
||||
description = "Public Mastodon/Fediverse posts of politicians"
|
||||
schedule = "17 * * * *"
|
||||
queue = "social"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
for acc in accounts(ctx, "mastodon"): # instances have arbitrary hosts: platform only
|
||||
m = re.match(r"^https?://([^/]+)/@([^/@]+)", acc["url"])
|
||||
if m:
|
||||
host, user = m.groups()
|
||||
yield Request(url=f"https://{host}/api/v1/accounts/lookup", params={"acct": user}, kind="lookup",
|
||||
meta={**acc["owner"], "host": host})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
meta = response.meta
|
||||
if response.request.kind == "lookup":
|
||||
account_id = response.json().get("id")
|
||||
if account_id:
|
||||
yield Request(url=f"https://{meta['host']}/api/v1/accounts/{account_id}/statuses",
|
||||
params={"limit": 40, "exclude_replies": "true", "exclude_reblogs": "true"},
|
||||
kind="statuses", meta=meta)
|
||||
return
|
||||
for status in response.json():
|
||||
if status.get("visibility") not in (None, "public"):
|
||||
continue
|
||||
yield post_claim(owner=meta, platform="mastodon", url=status.get("url") or status.get("uri"),
|
||||
text=html_to_text(status.get("content")), created=status.get("created_at"),
|
||||
data={"favourites": status.get("favourites_count"), "reblogs": status.get("reblogs_count")})
|
||||
|
||||
|
||||
@register
|
||||
class TelegramChannels(Adapter):
|
||||
spec = TELEGRAM
|
||||
name = "social.telegram"
|
||||
description = "Public Telegram broadcast channels of politicians (web preview, latest posts)"
|
||||
schedule = "27 */2 * * *"
|
||||
queue = "social"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
for acc in accounts(ctx, "telegram", r"(t\.me|telegram\.me)/"):
|
||||
m = re.search(r"(?:t\.me|telegram\.me)/(?:s/)?([A-Za-z0-9_]{4,})", acc["url"])
|
||||
if m:
|
||||
yield Request(url=f"https://t.me/s/{m.group(1)}", kind="channel",
|
||||
meta={**acc["owner"], "channel": m.group(1)})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||||
soup = response.soup
|
||||
subscribers = soup.select_one(".tgme_channel_info_counter .counter_value")
|
||||
for msg in soup.select("div.tgme_widget_message[data-post]"):
|
||||
text_el = msg.select_one(".tgme_widget_message_text")
|
||||
time_el = msg.select_one(".tgme_widget_message_date time")
|
||||
text = text_el.get_text("\n", strip=True) if text_el else ""
|
||||
if not text:
|
||||
continue # media-only posts
|
||||
forwarded = msg.select_one(".tgme_widget_message_forwarded_from")
|
||||
views = msg.select_one(".tgme_widget_message_views")
|
||||
yield post_claim(
|
||||
owner=response.meta, platform="telegram", url=f"https://t.me/{msg['data-post']}",
|
||||
text=text, created=time_el.get("datetime") if time_el else None,
|
||||
data={"views": views.get_text(strip=True) if views else None,
|
||||
"forwarded_from": forwarded.get_text(" ", strip=True) if forwarded else None,
|
||||
"channel_subscribers": subscribers.get_text(strip=True) if subscribers else None},
|
||||
)
|
||||
|
||||
|
||||
@register
|
||||
class YouTubeVideos(Adapter):
|
||||
spec = YOUTUBE
|
||||
name = "social.youtube"
|
||||
description = "Latest YouTube uploads of politicians' channels (requires POLITRACK_YOUTUBE_API_KEY)"
|
||||
schedule = "37 */6 * * *"
|
||||
queue = "social"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
key = get_settings().youtube_api_key
|
||||
if not key:
|
||||
ctx.stats["skipped_no_api_key"] = 1
|
||||
return
|
||||
api = "https://www.googleapis.com/youtube/v3/channels"
|
||||
for acc in accounts(ctx, "youtube", r"youtube\.com/"):
|
||||
meta = acc["owner"]
|
||||
if m := re.search(r"youtube\.com/channel/(UC[\w-]+)", acc["url"]):
|
||||
params = {"id": m.group(1)}
|
||||
elif m := re.search(r"youtube\.com/@([\w.-]+)", acc["url"]):
|
||||
params = {"forHandle": f"@{m.group(1)}"}
|
||||
elif m := re.search(r"youtube\.com/(?:user/|c/)?([\w.-]+)/?$", acc["url"]):
|
||||
params = {"forUsername": m.group(1)}
|
||||
else:
|
||||
continue
|
||||
yield Request(url=api, params={**params, "part": "contentDetails", "key": key}, kind="channel", meta=meta)
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
data = response.json()
|
||||
meta = response.meta
|
||||
if response.request.kind == "channel":
|
||||
for item in data.get("items", []):
|
||||
uploads = item["contentDetails"]["relatedPlaylists"]["uploads"]
|
||||
yield Request(url="https://www.googleapis.com/youtube/v3/playlistItems", kind="uploads", meta=meta,
|
||||
params={"playlistId": uploads, "part": "snippet", "maxResults": 25,
|
||||
"key": get_settings().youtube_api_key})
|
||||
return
|
||||
for item in data.get("items", []):
|
||||
snippet = item.get("snippet") or {}
|
||||
video = (snippet.get("resourceId") or {}).get("videoId")
|
||||
if not video:
|
||||
continue
|
||||
yield post_claim(owner=meta, platform="youtube",
|
||||
url=f"https://www.youtube.com/watch?v={video}", title=snippet.get("title"),
|
||||
text=f"{snippet.get('title', '')}\n{snippet.get('description', '')}".strip(),
|
||||
created=snippet.get("publishedAt"))
|
||||
|
||||
|
||||
@register
|
||||
class XPosts(Adapter):
|
||||
spec = X
|
||||
name = "social.x"
|
||||
description = "Posts of politicians on X via the official API v2 (requires POLITRACK_X_BEARER_TOKEN)"
|
||||
schedule = "47 */6 * * *"
|
||||
queue = "social"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
token = get_settings().x_bearer_token
|
||||
if not token:
|
||||
ctx.stats["skipped_no_api_key"] = 1
|
||||
return
|
||||
for acc in accounts(ctx, "twitter", r"//(www\.)?(twitter|x)\.com/"):
|
||||
m = re.search(r"(?:twitter|x)\.com/(?:#!/)?@?(\w{1,15})/?$", acc["url"])
|
||||
if m:
|
||||
yield Request(url=f"{X_API}users/by/username/{m.group(1)}", kind="user", headers=self.auth(),
|
||||
meta=acc["owner"])
|
||||
|
||||
@staticmethod
|
||||
def auth() -> dict[str, str]:
|
||||
return {"Authorization": f"Bearer {get_settings().x_bearer_token}"}
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
data = response.json()
|
||||
meta = response.meta
|
||||
if response.request.kind == "user":
|
||||
user = data.get("data") or {}
|
||||
if user.get("id"):
|
||||
yield Request(url=f"{X_API}users/{user['id']}/tweets", kind="tweets", headers=self.auth(),
|
||||
params={"max_results": 50, "exclude": "replies,retweets",
|
||||
"tweet.fields": "created_at,public_metrics"},
|
||||
meta={**meta, "username": user.get("username")})
|
||||
return
|
||||
for tweet in data.get("data") or []:
|
||||
metrics = tweet.get("public_metrics") or {}
|
||||
yield post_claim(owner=meta, platform="x",
|
||||
url=f"https://x.com/{meta['username']}/status/{tweet['id']}", text=tweet.get("text", ""),
|
||||
created=tweet.get("created_at"),
|
||||
data={"likes": metrics.get("like_count"), "reposts": metrics.get("retweet_count")})
|
||||
@@ -0,0 +1,305 @@
|
||||
"""Wikidata (CC0) via the public SPARQL endpoint.
|
||||
|
||||
Fills what the federal parliament API does not cover: state parliaments (Landtage), state
|
||||
governments, mayors and other offices, party membership, careers (employers, positions abroad,
|
||||
EU roles) and social-media handles. People are merged with parliament records through P2280
|
||||
(Austrian Parliament ID = PAD_INTERN).
|
||||
|
||||
Seats in the Nationalrat/Bundesrat/European Parliament are mapped to the canonical role "mp" in
|
||||
that chamber, so a Wikidata mandate and a Parliament mandate with the same start date are the
|
||||
same claim (corroborated by construction).
|
||||
|
||||
Flow: list Austrian positions -> holders in batches of 40 positions -> for each batch of persons:
|
||||
party memberships, careers, social handles. Batching keeps queries within the endpoint's 60 s limit.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Evidence, OrgRef, PersonRecord, PersonRef, Role, SocialAccount
|
||||
|
||||
SPARQL = "https://query.wikidata.org/sparql"
|
||||
|
||||
SPEC = SourceSpec(
|
||||
slug="wikidata",
|
||||
name="Wikidata",
|
||||
kind="aggregator",
|
||||
homepage="https://www.wikidata.org",
|
||||
license="CC0",
|
||||
access="api",
|
||||
trust=65,
|
||||
min_interval=2.0,
|
||||
robots_exempt=(SPARQL,),
|
||||
tos_notes="SPARQL endpoint. robots.txt disallows /sparql for crawlers, but the Wikimedia User-Agent policy "
|
||||
"allows bots with a descriptive User-Agent to use the query service. Community-edited.",
|
||||
)
|
||||
|
||||
POSITION_LIST_QUERY = """
|
||||
SELECT DISTINCT ?position ?positionLabel WHERE {
|
||||
?position wdt:P17 wd:Q40 .
|
||||
[] ps:P39 ?position .
|
||||
SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". }
|
||||
}
|
||||
"""
|
||||
|
||||
POSITIONS_QUERY = """
|
||||
SELECT ?person ?personLabel ?givenLabel ?familyLabel ?birth ?death ?pad ?ep ?position ?positionLabel ?start ?end
|
||||
?group ?groupLabel WHERE {
|
||||
VALUES ?position { %(positions)s }
|
||||
?person p:P39 ?st .
|
||||
?st ps:P39 ?position .
|
||||
OPTIONAL { ?st pq:P580 ?start }
|
||||
OPTIONAL { ?st pq:P582 ?end }
|
||||
FILTER(!BOUND(?end) || ?end >= "%(since)s-01-01"^^xsd:dateTime)
|
||||
OPTIONAL { ?st pq:P4100 ?group }
|
||||
OPTIONAL { ?person wdt:P2280 ?pad }
|
||||
# Generic positions with country=Austria are also held by foreigners (e.g. "Bundessprecher" of the German
|
||||
# Green Youth): require an Austrian connection - parliament id, citizenship or an Austrian party.
|
||||
FILTER(BOUND(?pad) || EXISTS { ?person wdt:P27 wd:Q40 } || EXISTS { ?person wdt:P102 ?ap . ?ap wdt:P17 wd:Q40 })
|
||||
OPTIONAL { ?person wdt:P1186 ?ep }
|
||||
OPTIONAL { ?person wdt:P569 ?birth }
|
||||
OPTIONAL { ?person wdt:P570 ?death }
|
||||
OPTIONAL { ?person wdt:P735 ?given }
|
||||
OPTIONAL { ?person wdt:P734 ?family }
|
||||
SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". }
|
||||
}
|
||||
"""
|
||||
|
||||
PARTY_QUERY = """
|
||||
SELECT ?person ?party ?partyLabel ?partyShort ?start ?end WHERE {
|
||||
VALUES ?person { %(persons)s }
|
||||
?person p:P102 ?st .
|
||||
?st ps:P102 ?party .
|
||||
?party wdt:P17 wd:Q40 .
|
||||
OPTIONAL { ?st pq:P580 ?start }
|
||||
OPTIONAL { ?st pq:P582 ?end }
|
||||
OPTIONAL { ?party wdt:P1813 ?partyShort . FILTER(LANG(?partyShort) = "de") }
|
||||
SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". }
|
||||
}
|
||||
"""
|
||||
|
||||
CAREER_QUERY = """
|
||||
SELECT ?person ?org ?orgLabel ?start ?end ?kind WHERE {
|
||||
VALUES ?person { %(persons)s }
|
||||
{ ?person p:P108 ?st . ?st ps:P108 ?org . BIND("employer" AS ?kind) }
|
||||
UNION
|
||||
{ ?person p:P39 ?st . ?st ps:P39 ?org . FILTER NOT EXISTS { ?org wdt:P17 wd:Q40 } BIND("position" AS ?kind) }
|
||||
OPTIONAL { ?st pq:P580 ?start }
|
||||
OPTIONAL { ?st pq:P582 ?end }
|
||||
SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". }
|
||||
}
|
||||
"""
|
||||
|
||||
# Social-media properties: P2002 X/Twitter, P12361 Bluesky, P4033 Mastodon, P3789 Telegram,
|
||||
# P2397 YouTube channel id, P2003 Instagram, P2013 Facebook, P7085 TikTok
|
||||
SOCIAL_QUERY = """
|
||||
SELECT ?person ?prop ?value WHERE {
|
||||
VALUES ?person { %(persons)s }
|
||||
VALUES ?prop { wdt:P2002 wdt:P12361 wdt:P4033 wdt:P3789 wdt:P2397 wdt:P2003 wdt:P2013 wdt:P7085 }
|
||||
?person ?prop ?value .
|
||||
}
|
||||
"""
|
||||
SOCIAL_PROPS = {
|
||||
"P2002": ("twitter", "https://x.com/{}"),
|
||||
"P12361": ("bluesky", "https://bsky.app/profile/{}"),
|
||||
"P4033": ("mastodon", None),
|
||||
"P3789": ("telegram", "https://t.me/{}"),
|
||||
"P2397": ("youtube", "https://www.youtube.com/channel/{}"),
|
||||
"P2003": ("instagram", "https://www.instagram.com/{}"),
|
||||
"P2013": ("facebook", "https://www.facebook.com/{}"),
|
||||
"P7085": ("tiktok", "https://www.tiktok.com/@{}"),
|
||||
}
|
||||
|
||||
EU_POLITICAL = re.compile(r"Europäischen Parlament|Europaparlament|Europäischen Kommission|EU-Kommissar|"
|
||||
r"Member of the European Parliament", re.IGNORECASE)
|
||||
EXCLUDE = re.compile(
|
||||
r"professor|kommerzialrat|kommerzienrat|hofrat|ministerialrat|regierungsrat|gerichtshof|richter|"
|
||||
r"botschafter|bischof|rektor|ökonomierat|medizinalrat|schulrat|studienrat|baurat|veterinärrat|"
|
||||
r"bergrat|amtsrat|kammerrat|konsul",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
CHAMBER_LABELS = [
|
||||
(re.compile(r"Abgeordnete[rn]? zum Nationalrat|Mitglied des Nationalrates", re.I), "nationalrat", "federal"),
|
||||
(re.compile(r"Mitglied des (österreichischen )?Bundesrate?s", re.I), "bundesrat", "federal"),
|
||||
(re.compile(r"Mitglied des Europäischen Parlaments|Member of the European Parliament", re.I),
|
||||
"europaeisches-parlament", "eu"),
|
||||
]
|
||||
|
||||
|
||||
def classify_level(label: str) -> str:
|
||||
if re.search(r"europäischen parlament|europaabgeordnet|eu-kommiss", label, re.I):
|
||||
return "eu"
|
||||
if re.search(r"bürgermeister|gemeinderat|stadtrat|vizebürgermeister|gemeinde", label, re.I):
|
||||
return "municipal" if "Wiener Landtag" not in label else "state"
|
||||
if re.search(r"landtag|landesrat|landeshauptmann|landesregierung|landesamtsdirektor", label, re.I):
|
||||
return "state"
|
||||
if re.search(r"bezirk", label, re.I):
|
||||
return "district"
|
||||
return "federal"
|
||||
|
||||
|
||||
def wd_date(binding: dict, key: str) -> date | None:
|
||||
value = binding.get(key, {}).get("value")
|
||||
if not value or value.startswith("-"):
|
||||
return None
|
||||
try:
|
||||
return date.fromisoformat(value[:10])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def qid(binding: dict, key: str) -> str | None:
|
||||
value = binding.get(key, {}).get("value")
|
||||
return value.rsplit("/", 1)[-1] if value else None
|
||||
|
||||
|
||||
def label(binding: dict, key: str) -> str | None:
|
||||
value = binding.get(key, {}).get("value")
|
||||
return None if not value or re.fullmatch(r"Q\d+", value) else value
|
||||
|
||||
|
||||
def sparql(query: str, kind: str, meta: dict | None = None) -> Request:
|
||||
return Request(url=SPARQL, params={"query": query, "format": "json"}, kind=kind, meta=meta or {},
|
||||
headers={"Accept": "application/sparql-results+json"})
|
||||
|
||||
|
||||
def role_for(position_label: str) -> tuple[str, OrgRef | None, str]:
|
||||
for pattern, slug, level in CHAMBER_LABELS:
|
||||
if pattern.search(position_label):
|
||||
return "mp", OrgRef(slug=slug), level
|
||||
return position_label, None, classify_level(position_label)
|
||||
|
||||
|
||||
@register
|
||||
class WikidataPoliticians(Adapter):
|
||||
spec = SPEC
|
||||
name = "wikidata.politicians"
|
||||
description = "Austrian office holders (all levels), party membership, careers, social handles"
|
||||
schedule = "45 4 * * 0"
|
||||
creates_persons = True
|
||||
parser_version = 2
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
yield sparql(POSITION_LIST_QUERY, kind="positions")
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
bindings = json.loads(response.text, strict=False)["results"]["bindings"]
|
||||
kind = response.request.kind
|
||||
if kind == "positions":
|
||||
positions = [qid(b, "position") for b in bindings
|
||||
if label(b, "positionLabel") and not EXCLUDE.search(label(b, "positionLabel") or "")]
|
||||
since = ctx.params.get("since_year", 2000)
|
||||
for i in range(0, len(positions), 40):
|
||||
values = " ".join(f"wd:{q}" for q in positions[i:i + 40])
|
||||
yield sparql(POSITIONS_QUERY % {"positions": values, "since": since}, kind="holders")
|
||||
elif kind == "holders":
|
||||
yield from self.parse_holders(bindings)
|
||||
elif kind == "parties":
|
||||
yield from self.parse_parties(bindings)
|
||||
elif kind == "careers":
|
||||
yield from self.parse_careers(bindings)
|
||||
elif kind == "social":
|
||||
yield from self.parse_social(bindings)
|
||||
|
||||
def parse_holders(self, bindings: list[dict]) -> Iterator[Any]:
|
||||
seen: set[str] = set()
|
||||
for b in bindings:
|
||||
person_qid, name, position = qid(b, "person"), label(b, "personLabel"), label(b, "positionLabel")
|
||||
if not name or not position or EXCLUDE.search(position):
|
||||
continue
|
||||
pad = b.get("pad", {}).get("value")
|
||||
ids = {"wikidata": person_qid}
|
||||
if pad and pad.isdigit():
|
||||
ids["parlament"] = str(int(pad))
|
||||
if b.get("ep", {}).get("value"):
|
||||
ids["ep"] = b["ep"]["value"]
|
||||
ref = PersonRef(ids=ids)
|
||||
if person_qid not in seen:
|
||||
seen.add(person_qid)
|
||||
yield PersonRecord(ref=ref, full_name=name, given_name=label(b, "givenLabel"),
|
||||
family_name=label(b, "familyLabel"), birth_date=wd_date(b, "birth"),
|
||||
death_date=wd_date(b, "death"))
|
||||
start, end = wd_date(b, "start"), wd_date(b, "end") or wd_date(b, "death")
|
||||
group_label = label(b, "groupLabel")
|
||||
group = OrgRef(name=group_label, kind="parliamentary_group", ids={"wikidata": qid(b, "group")}) \
|
||||
if group_label else None
|
||||
role, body, level = role_for(position)
|
||||
yield Claim(
|
||||
claim_type="role", person=ref, org=body, counterpart_org=group, valid_from=start, valid_to=end,
|
||||
summary_de=f"{name}: {position}" + (f" ({group_label})" if group_label else ""),
|
||||
data={"position_qid": qid(b, "position"), "position_label": position},
|
||||
evidence=[Evidence(url=f"https://www.wikidata.org/wiki/{person_qid}", doc_type="api_record",
|
||||
title=f"Wikidata: {name}", locator=f"P39:{qid(b, 'position')}:{start}")],
|
||||
facet=Role(role=role, level=level, body=body, start=start, end=end),
|
||||
)
|
||||
persons = sorted(seen)
|
||||
for i in range(0, len(persons), 300):
|
||||
values = " ".join(f"wd:{q}" for q in persons[i:i + 300])
|
||||
yield sparql(PARTY_QUERY % {"persons": values}, kind="parties")
|
||||
yield sparql(CAREER_QUERY % {"persons": values}, kind="careers")
|
||||
yield sparql(SOCIAL_QUERY % {"persons": values}, kind="social")
|
||||
|
||||
def parse_parties(self, bindings: list[dict]) -> Iterator[Claim]:
|
||||
for b in bindings:
|
||||
party_label = label(b, "partyLabel")
|
||||
if not party_label:
|
||||
continue
|
||||
person_qid = qid(b, "person")
|
||||
party = OrgRef(name=party_label, short_name=b.get("partyShort", {}).get("value"), kind="party",
|
||||
ids={"wikidata": qid(b, "party")})
|
||||
start, end = wd_date(b, "start"), wd_date(b, "end")
|
||||
yield Claim(
|
||||
claim_type="role", person=PersonRef(ids={"wikidata": person_qid}), org=party,
|
||||
valid_from=start, valid_to=end,
|
||||
summary_de=f"Mitglied der Partei {party_label}", summary_en=f"Member of {party_label}",
|
||||
evidence=[Evidence(url=f"https://www.wikidata.org/wiki/{person_qid}", doc_type="api_record",
|
||||
title="Wikidata", locator=f"P102:{qid(b, 'party')}:{start}")],
|
||||
facet=Role(role="party_member", level="party", body=party, party=party, start=start, end=end),
|
||||
)
|
||||
|
||||
def parse_careers(self, bindings: list[dict]) -> Iterator[Claim]:
|
||||
for b in bindings:
|
||||
org_label = label(b, "orgLabel")
|
||||
if not org_label or EXCLUDE.search(org_label):
|
||||
continue
|
||||
person_qid = qid(b, "person")
|
||||
ref = PersonRef(ids={"wikidata": person_qid})
|
||||
start, end = wd_date(b, "start"), wd_date(b, "end")
|
||||
kind = b["kind"]["value"]
|
||||
evidence = [Evidence(url=f"https://www.wikidata.org/wiki/{person_qid}", doc_type="api_record",
|
||||
title="Wikidata", locator=f"{kind}:{qid(b, 'org')}:{start}")]
|
||||
if kind == "position" and EU_POLITICAL.search(org_label):
|
||||
role, body, level = role_for(org_label)
|
||||
yield Claim(claim_type="role", person=ref, org=body, valid_from=start, valid_to=end,
|
||||
summary_de=org_label, evidence=evidence,
|
||||
facet=Role(role=role, level="eu", body=body, start=start, end=end))
|
||||
continue
|
||||
employer = OrgRef(name=org_label, ids={"wikidata": qid(b, "org")}) if kind == "employer" else None
|
||||
yield Claim(
|
||||
claim_type="career", person=ref, counterpart_org=employer, valid_from=start, valid_to=end,
|
||||
occurred_at=start, key=("wikidata-career", person_qid, kind, qid(b, "org"), start),
|
||||
summary_de=f"{'Arbeitet(e) bei' if kind == 'employer' else 'Funktion:'} {org_label}",
|
||||
summary_en=f"{'Employed by' if kind == 'employer' else 'Position:'} {org_label}",
|
||||
data={"kind": kind, "org_qid": qid(b, "org")}, evidence=evidence,
|
||||
)
|
||||
|
||||
def parse_social(self, bindings: list[dict]) -> Iterator[SocialAccount]:
|
||||
for b in bindings:
|
||||
prop = b["prop"]["value"].rsplit("/", 1)[-1]
|
||||
value = b["value"]["value"].strip()
|
||||
platform, template = SOCIAL_PROPS[prop]
|
||||
if platform == "mastodon":
|
||||
user, _, host = value.lstrip("@").partition("@")
|
||||
if not host:
|
||||
continue
|
||||
url = f"https://{host}/@{user}"
|
||||
else:
|
||||
url = template.format(value)
|
||||
yield SocialAccount(platform=platform, url=url, handle=value,
|
||||
person=PersonRef(ids={"wikidata": qid(b, "person")}), verified=False)
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Wikipedia edits made from the networks of Parliament and federal government (classic transparency bot).
|
||||
|
||||
Anonymous Wikipedia edits are publicly attributed to the editor's IP address. We query the MediaWiki API
|
||||
(list=usercontribs, uciprange) for the IP ranges registered to public institutions in the RIPE database
|
||||
(config/ip_ranges.yaml) and record each edit as a `wiki_edit` claim of the institution, linked to the
|
||||
person whose article was edited when the title matches a known politician. Individual IP addresses are
|
||||
not displayed - only the institution owning the network.
|
||||
|
||||
The API is used per the MediaWiki API etiquette (descriptive User-Agent, sequential requests); robots.txt
|
||||
disallows /w/ for crawlers, which does not apply to API clients (robots_exempt below).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
from datetime import date, timedelta
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Evidence, OrgRef, PersonRef
|
||||
from politrack.text import excerpt
|
||||
|
||||
WIKIS = ("de.wikipedia.org", "en.wikipedia.org")
|
||||
SPEC = SourceSpec(
|
||||
slug="wikipedia", name="Wikipedia (Bearbeitungen aus Behördennetzen)", kind="aggregator",
|
||||
homepage="https://de.wikipedia.org", license="CC BY-SA 4.0", access="api", trust=80, min_interval=1.0,
|
||||
robots_exempt=tuple(f"https://{w}/w/api.php" for w in WIKIS),
|
||||
tos_notes="MediaWiki API (API etiquette: descriptive User-Agent, no parallel requests). Edits from IP ranges "
|
||||
"registered to Parliament / Bundeskanzleramt in the RIPE database.",
|
||||
)
|
||||
|
||||
|
||||
@register
|
||||
class InstitutionalWikiEdits(Adapter):
|
||||
spec = SPEC
|
||||
name = "wikipedia.institution_edits"
|
||||
description = "Wikipedia edits from the IP ranges of Parliament and federal government"
|
||||
schedule = "15 7 * * *"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
config = yaml.safe_load((get_settings().config_dir / "ip_ranges.yaml").read_text(encoding="utf-8"))
|
||||
days = int(ctx.params.get("days", 3650 if not ctx.state.get("initialised") else 30))
|
||||
end = (date.today() - timedelta(days=days)).isoformat() + "T00:00:00Z"
|
||||
for network in config.get("networks") or []:
|
||||
for cidr in network["ranges"]:
|
||||
for wiki in WIKIS:
|
||||
yield Request(
|
||||
url=f"https://{wiki}/w/api.php", kind="contribs",
|
||||
params={"action": "query", "list": "usercontribs", "uciprange": cidr, "uclimit": 500,
|
||||
"ucend": end, "ucprop": "ids|title|timestamp|comment|sizediff", "format": "json",
|
||||
"formatversion": 2},
|
||||
meta={"wiki": wiki, "org": network["org"], "label": network["label"]}, conditional=False)
|
||||
ctx.state.set("initialised", True)
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
data = response.json()
|
||||
meta = response.meta
|
||||
if "continue" in data:
|
||||
yield Request(url=response.request.url, kind="contribs", meta=meta, conditional=False,
|
||||
params={**response.request.params, **data["continue"]})
|
||||
for edit in (data.get("query") or {}).get("usercontribs") or []:
|
||||
title, revid = edit.get("title"), edit.get("revid")
|
||||
if not title or not revid or ":" in title.split(" ")[0] and not title.startswith("Liste"):
|
||||
continue # skip talk/user/meta pages ("Diskussion:", "Benutzer:")
|
||||
size = edit.get("sizediff")
|
||||
comment = edit.get("comment") or ""
|
||||
diff = f"https://{meta['wiki']}/w/index.php?diff={revid}"
|
||||
yield Claim(
|
||||
claim_type="wiki_edit", org=OrgRef(slug=meta["org"], create=False),
|
||||
counterpart_person=PersonRef(name=title, hints={}), key=("wiki", meta["wiki"], revid),
|
||||
occurred_at=edit.get("timestamp"),
|
||||
summary_de=f"Wikipedia-Bearbeitung aus dem Netz von {meta['label']}: „{title}“"
|
||||
+ (f" ({size:+d} Zeichen)" if isinstance(size, int) else "")
|
||||
+ (f" – {excerpt(comment, 120)}" if comment else ""),
|
||||
summary_en=f"Wikipedia edit from the network of {meta['label']}: \"{title}\""
|
||||
+ (f" ({size:+d} characters)" if isinstance(size, int) else ""),
|
||||
data={"wiki": meta["wiki"], "title": title, "sizediff": size, "comment": comment},
|
||||
evidence=[Evidence(url=diff, doc_type="web_page", title=f"Wikipedia: {title}",
|
||||
published_at=edit.get("timestamp"), quote=comment or None)],
|
||||
)
|
||||
Reference in new issue
Block a user