simonandClaude Opus 5.5
583ccba31c
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
306 lines
14 KiB
Python
306 lines
14 KiB
Python
"""Wikidata (CC0) via the public SPARQL endpoint.
|
|
|
|
Fills what the federal parliament API does not cover: state parliaments (Landtage), state
|
|
governments, mayors and other offices, party membership, careers (employers, positions abroad,
|
|
EU roles) and social-media handles. People are merged with parliament records through P2280
|
|
(Austrian Parliament ID = PAD_INTERN).
|
|
|
|
Seats in the Nationalrat/Bundesrat/European Parliament are mapped to the canonical role "mp" in
|
|
that chamber, so a Wikidata mandate and a Parliament mandate with the same start date are the
|
|
same claim (corroborated by construction).
|
|
|
|
Flow: list Austrian positions -> holders in batches of 40 positions -> for each batch of persons:
|
|
party memberships, careers, social handles. Batching keeps queries within the endpoint's 60 s limit.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
from collections.abc import Iterator
|
|
from datetime import date
|
|
from typing import Any
|
|
|
|
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
|
from politrack.core.records import Claim, Evidence, OrgRef, PersonRecord, PersonRef, Role, SocialAccount
|
|
|
|
SPARQL = "https://query.wikidata.org/sparql"
|
|
|
|
SPEC = SourceSpec(
|
|
slug="wikidata",
|
|
name="Wikidata",
|
|
kind="aggregator",
|
|
homepage="https://www.wikidata.org",
|
|
license="CC0",
|
|
access="api",
|
|
trust=65,
|
|
min_interval=2.0,
|
|
robots_exempt=(SPARQL,),
|
|
tos_notes="SPARQL endpoint. robots.txt disallows /sparql for crawlers, but the Wikimedia User-Agent policy "
|
|
"allows bots with a descriptive User-Agent to use the query service. Community-edited.",
|
|
)
|
|
|
|
POSITION_LIST_QUERY = """
|
|
SELECT DISTINCT ?position ?positionLabel WHERE {
|
|
?position wdt:P17 wd:Q40 .
|
|
[] ps:P39 ?position .
|
|
SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". }
|
|
}
|
|
"""
|
|
|
|
POSITIONS_QUERY = """
|
|
SELECT ?person ?personLabel ?givenLabel ?familyLabel ?birth ?death ?pad ?ep ?position ?positionLabel ?start ?end
|
|
?group ?groupLabel WHERE {
|
|
VALUES ?position { %(positions)s }
|
|
?person p:P39 ?st .
|
|
?st ps:P39 ?position .
|
|
OPTIONAL { ?st pq:P580 ?start }
|
|
OPTIONAL { ?st pq:P582 ?end }
|
|
FILTER(!BOUND(?end) || ?end >= "%(since)s-01-01"^^xsd:dateTime)
|
|
OPTIONAL { ?st pq:P4100 ?group }
|
|
OPTIONAL { ?person wdt:P2280 ?pad }
|
|
# Generic positions with country=Austria are also held by foreigners (e.g. "Bundessprecher" of the German
|
|
# Green Youth): require an Austrian connection - parliament id, citizenship or an Austrian party.
|
|
FILTER(BOUND(?pad) || EXISTS { ?person wdt:P27 wd:Q40 } || EXISTS { ?person wdt:P102 ?ap . ?ap wdt:P17 wd:Q40 })
|
|
OPTIONAL { ?person wdt:P1186 ?ep }
|
|
OPTIONAL { ?person wdt:P569 ?birth }
|
|
OPTIONAL { ?person wdt:P570 ?death }
|
|
OPTIONAL { ?person wdt:P735 ?given }
|
|
OPTIONAL { ?person wdt:P734 ?family }
|
|
SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". }
|
|
}
|
|
"""
|
|
|
|
PARTY_QUERY = """
|
|
SELECT ?person ?party ?partyLabel ?partyShort ?start ?end WHERE {
|
|
VALUES ?person { %(persons)s }
|
|
?person p:P102 ?st .
|
|
?st ps:P102 ?party .
|
|
?party wdt:P17 wd:Q40 .
|
|
OPTIONAL { ?st pq:P580 ?start }
|
|
OPTIONAL { ?st pq:P582 ?end }
|
|
OPTIONAL { ?party wdt:P1813 ?partyShort . FILTER(LANG(?partyShort) = "de") }
|
|
SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". }
|
|
}
|
|
"""
|
|
|
|
CAREER_QUERY = """
|
|
SELECT ?person ?org ?orgLabel ?start ?end ?kind WHERE {
|
|
VALUES ?person { %(persons)s }
|
|
{ ?person p:P108 ?st . ?st ps:P108 ?org . BIND("employer" AS ?kind) }
|
|
UNION
|
|
{ ?person p:P39 ?st . ?st ps:P39 ?org . FILTER NOT EXISTS { ?org wdt:P17 wd:Q40 } BIND("position" AS ?kind) }
|
|
OPTIONAL { ?st pq:P580 ?start }
|
|
OPTIONAL { ?st pq:P582 ?end }
|
|
SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". }
|
|
}
|
|
"""
|
|
|
|
# Social-media properties: P2002 X/Twitter, P12361 Bluesky, P4033 Mastodon, P3789 Telegram,
|
|
# P2397 YouTube channel id, P2003 Instagram, P2013 Facebook, P7085 TikTok
|
|
SOCIAL_QUERY = """
|
|
SELECT ?person ?prop ?value WHERE {
|
|
VALUES ?person { %(persons)s }
|
|
VALUES ?prop { wdt:P2002 wdt:P12361 wdt:P4033 wdt:P3789 wdt:P2397 wdt:P2003 wdt:P2013 wdt:P7085 }
|
|
?person ?prop ?value .
|
|
}
|
|
"""
|
|
SOCIAL_PROPS = {
|
|
"P2002": ("twitter", "https://x.com/{}"),
|
|
"P12361": ("bluesky", "https://bsky.app/profile/{}"),
|
|
"P4033": ("mastodon", None),
|
|
"P3789": ("telegram", "https://t.me/{}"),
|
|
"P2397": ("youtube", "https://www.youtube.com/channel/{}"),
|
|
"P2003": ("instagram", "https://www.instagram.com/{}"),
|
|
"P2013": ("facebook", "https://www.facebook.com/{}"),
|
|
"P7085": ("tiktok", "https://www.tiktok.com/@{}"),
|
|
}
|
|
|
|
EU_POLITICAL = re.compile(r"Europäischen Parlament|Europaparlament|Europäischen Kommission|EU-Kommissar|"
|
|
r"Member of the European Parliament", re.IGNORECASE)
|
|
EXCLUDE = re.compile(
|
|
r"professor|kommerzialrat|kommerzienrat|hofrat|ministerialrat|regierungsrat|gerichtshof|richter|"
|
|
r"botschafter|bischof|rektor|ökonomierat|medizinalrat|schulrat|studienrat|baurat|veterinärrat|"
|
|
r"bergrat|amtsrat|kammerrat|konsul",
|
|
re.IGNORECASE,
|
|
)
|
|
CHAMBER_LABELS = [
|
|
(re.compile(r"Abgeordnete[rn]? zum Nationalrat|Mitglied des Nationalrates", re.I), "nationalrat", "federal"),
|
|
(re.compile(r"Mitglied des (österreichischen )?Bundesrate?s", re.I), "bundesrat", "federal"),
|
|
(re.compile(r"Mitglied des Europäischen Parlaments|Member of the European Parliament", re.I),
|
|
"europaeisches-parlament", "eu"),
|
|
]
|
|
|
|
|
|
def classify_level(label: str) -> str:
|
|
if re.search(r"europäischen parlament|europaabgeordnet|eu-kommiss", label, re.I):
|
|
return "eu"
|
|
if re.search(r"bürgermeister|gemeinderat|stadtrat|vizebürgermeister|gemeinde", label, re.I):
|
|
return "municipal" if "Wiener Landtag" not in label else "state"
|
|
if re.search(r"landtag|landesrat|landeshauptmann|landesregierung|landesamtsdirektor", label, re.I):
|
|
return "state"
|
|
if re.search(r"bezirk", label, re.I):
|
|
return "district"
|
|
return "federal"
|
|
|
|
|
|
def wd_date(binding: dict, key: str) -> date | None:
|
|
value = binding.get(key, {}).get("value")
|
|
if not value or value.startswith("-"):
|
|
return None
|
|
try:
|
|
return date.fromisoformat(value[:10])
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def qid(binding: dict, key: str) -> str | None:
|
|
value = binding.get(key, {}).get("value")
|
|
return value.rsplit("/", 1)[-1] if value else None
|
|
|
|
|
|
def label(binding: dict, key: str) -> str | None:
|
|
value = binding.get(key, {}).get("value")
|
|
return None if not value or re.fullmatch(r"Q\d+", value) else value
|
|
|
|
|
|
def sparql(query: str, kind: str, meta: dict | None = None) -> Request:
|
|
return Request(url=SPARQL, params={"query": query, "format": "json"}, kind=kind, meta=meta or {},
|
|
headers={"Accept": "application/sparql-results+json"})
|
|
|
|
|
|
def role_for(position_label: str) -> tuple[str, OrgRef | None, str]:
|
|
for pattern, slug, level in CHAMBER_LABELS:
|
|
if pattern.search(position_label):
|
|
return "mp", OrgRef(slug=slug), level
|
|
return position_label, None, classify_level(position_label)
|
|
|
|
|
|
@register
|
|
class WikidataPoliticians(Adapter):
|
|
spec = SPEC
|
|
name = "wikidata.politicians"
|
|
description = "Austrian office holders (all levels), party membership, careers, social handles"
|
|
schedule = "45 4 * * 0"
|
|
creates_persons = True
|
|
parser_version = 2
|
|
|
|
def start(self, ctx: RunContext) -> Iterator[Request]:
|
|
yield sparql(POSITION_LIST_QUERY, kind="positions")
|
|
|
|
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
|
bindings = json.loads(response.text, strict=False)["results"]["bindings"]
|
|
kind = response.request.kind
|
|
if kind == "positions":
|
|
positions = [qid(b, "position") for b in bindings
|
|
if label(b, "positionLabel") and not EXCLUDE.search(label(b, "positionLabel") or "")]
|
|
since = ctx.params.get("since_year", 2000)
|
|
for i in range(0, len(positions), 40):
|
|
values = " ".join(f"wd:{q}" for q in positions[i:i + 40])
|
|
yield sparql(POSITIONS_QUERY % {"positions": values, "since": since}, kind="holders")
|
|
elif kind == "holders":
|
|
yield from self.parse_holders(bindings)
|
|
elif kind == "parties":
|
|
yield from self.parse_parties(bindings)
|
|
elif kind == "careers":
|
|
yield from self.parse_careers(bindings)
|
|
elif kind == "social":
|
|
yield from self.parse_social(bindings)
|
|
|
|
def parse_holders(self, bindings: list[dict]) -> Iterator[Any]:
|
|
seen: set[str] = set()
|
|
for b in bindings:
|
|
person_qid, name, position = qid(b, "person"), label(b, "personLabel"), label(b, "positionLabel")
|
|
if not name or not position or EXCLUDE.search(position):
|
|
continue
|
|
pad = b.get("pad", {}).get("value")
|
|
ids = {"wikidata": person_qid}
|
|
if pad and pad.isdigit():
|
|
ids["parlament"] = str(int(pad))
|
|
if b.get("ep", {}).get("value"):
|
|
ids["ep"] = b["ep"]["value"]
|
|
ref = PersonRef(ids=ids)
|
|
if person_qid not in seen:
|
|
seen.add(person_qid)
|
|
yield PersonRecord(ref=ref, full_name=name, given_name=label(b, "givenLabel"),
|
|
family_name=label(b, "familyLabel"), birth_date=wd_date(b, "birth"),
|
|
death_date=wd_date(b, "death"))
|
|
start, end = wd_date(b, "start"), wd_date(b, "end") or wd_date(b, "death")
|
|
group_label = label(b, "groupLabel")
|
|
group = OrgRef(name=group_label, kind="parliamentary_group", ids={"wikidata": qid(b, "group")}) \
|
|
if group_label else None
|
|
role, body, level = role_for(position)
|
|
yield Claim(
|
|
claim_type="role", person=ref, org=body, counterpart_org=group, valid_from=start, valid_to=end,
|
|
summary_de=f"{name}: {position}" + (f" ({group_label})" if group_label else ""),
|
|
data={"position_qid": qid(b, "position"), "position_label": position},
|
|
evidence=[Evidence(url=f"https://www.wikidata.org/wiki/{person_qid}", doc_type="api_record",
|
|
title=f"Wikidata: {name}", locator=f"P39:{qid(b, 'position')}:{start}")],
|
|
facet=Role(role=role, level=level, body=body, start=start, end=end),
|
|
)
|
|
persons = sorted(seen)
|
|
for i in range(0, len(persons), 300):
|
|
values = " ".join(f"wd:{q}" for q in persons[i:i + 300])
|
|
yield sparql(PARTY_QUERY % {"persons": values}, kind="parties")
|
|
yield sparql(CAREER_QUERY % {"persons": values}, kind="careers")
|
|
yield sparql(SOCIAL_QUERY % {"persons": values}, kind="social")
|
|
|
|
def parse_parties(self, bindings: list[dict]) -> Iterator[Claim]:
|
|
for b in bindings:
|
|
party_label = label(b, "partyLabel")
|
|
if not party_label:
|
|
continue
|
|
person_qid = qid(b, "person")
|
|
party = OrgRef(name=party_label, short_name=b.get("partyShort", {}).get("value"), kind="party",
|
|
ids={"wikidata": qid(b, "party")})
|
|
start, end = wd_date(b, "start"), wd_date(b, "end")
|
|
yield Claim(
|
|
claim_type="role", person=PersonRef(ids={"wikidata": person_qid}), org=party,
|
|
valid_from=start, valid_to=end,
|
|
summary_de=f"Mitglied der Partei {party_label}", summary_en=f"Member of {party_label}",
|
|
evidence=[Evidence(url=f"https://www.wikidata.org/wiki/{person_qid}", doc_type="api_record",
|
|
title="Wikidata", locator=f"P102:{qid(b, 'party')}:{start}")],
|
|
facet=Role(role="party_member", level="party", body=party, party=party, start=start, end=end),
|
|
)
|
|
|
|
def parse_careers(self, bindings: list[dict]) -> Iterator[Claim]:
|
|
for b in bindings:
|
|
org_label = label(b, "orgLabel")
|
|
if not org_label or EXCLUDE.search(org_label):
|
|
continue
|
|
person_qid = qid(b, "person")
|
|
ref = PersonRef(ids={"wikidata": person_qid})
|
|
start, end = wd_date(b, "start"), wd_date(b, "end")
|
|
kind = b["kind"]["value"]
|
|
evidence = [Evidence(url=f"https://www.wikidata.org/wiki/{person_qid}", doc_type="api_record",
|
|
title="Wikidata", locator=f"{kind}:{qid(b, 'org')}:{start}")]
|
|
if kind == "position" and EU_POLITICAL.search(org_label):
|
|
role, body, level = role_for(org_label)
|
|
yield Claim(claim_type="role", person=ref, org=body, valid_from=start, valid_to=end,
|
|
summary_de=org_label, evidence=evidence,
|
|
facet=Role(role=role, level="eu", body=body, start=start, end=end))
|
|
continue
|
|
employer = OrgRef(name=org_label, ids={"wikidata": qid(b, "org")}) if kind == "employer" else None
|
|
yield Claim(
|
|
claim_type="career", person=ref, counterpart_org=employer, valid_from=start, valid_to=end,
|
|
occurred_at=start, key=("wikidata-career", person_qid, kind, qid(b, "org"), start),
|
|
summary_de=f"{'Arbeitet(e) bei' if kind == 'employer' else 'Funktion:'} {org_label}",
|
|
summary_en=f"{'Employed by' if kind == 'employer' else 'Position:'} {org_label}",
|
|
data={"kind": kind, "org_qid": qid(b, "org")}, evidence=evidence,
|
|
)
|
|
|
|
def parse_social(self, bindings: list[dict]) -> Iterator[SocialAccount]:
|
|
for b in bindings:
|
|
prop = b["prop"]["value"].rsplit("/", 1)[-1]
|
|
value = b["value"]["value"].strip()
|
|
platform, template = SOCIAL_PROPS[prop]
|
|
if platform == "mastodon":
|
|
user, _, host = value.lstrip("@").partition("@")
|
|
if not host:
|
|
continue
|
|
url = f"https://{host}/@{user}"
|
|
else:
|
|
url = template.format(value)
|
|
yield SocialAccount(platform=platform, url=url, handle=value,
|
|
person=PersonRef(ids={"wikidata": qid(b, "person")}), verified=False)
|