"""Wikidata (CC0) via the public SPARQL endpoint. Fills what the federal parliament API does not cover: state parliaments (Landtage), state governments, mayors and other offices, party membership, careers (employers, positions abroad, EU roles) and social-media handles. People are merged with parliament records through P2280 (Austrian Parliament ID = PAD_INTERN). Seats in the Nationalrat/Bundesrat/European Parliament are mapped to the canonical role "mp" in that chamber, so a Wikidata mandate and a Parliament mandate with the same start date are the same claim (corroborated by construction). Flow: list Austrian positions -> holders in batches of 40 positions -> for each batch of persons: party memberships, careers, social handles. Batching keeps queries within the endpoint's 60 s limit. """ from __future__ import annotations import json import re from collections.abc import Iterator from datetime import date from typing import Any from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register from politrack.core.records import Claim, Evidence, OrgRef, PersonRecord, PersonRef, Role, SocialAccount SPARQL = "https://query.wikidata.org/sparql" SPEC = SourceSpec( slug="wikidata", name="Wikidata", kind="aggregator", homepage="https://www.wikidata.org", license="CC0", access="api", trust=65, min_interval=2.0, robots_exempt=(SPARQL,), tos_notes="SPARQL endpoint. robots.txt disallows /sparql for crawlers, but the Wikimedia User-Agent policy " "allows bots with a descriptive User-Agent to use the query service. Community-edited.", ) POSITION_LIST_QUERY = """ SELECT DISTINCT ?position ?positionLabel WHERE { ?position wdt:P17 wd:Q40 . [] ps:P39 ?position . SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". } } """ POSITIONS_QUERY = """ SELECT ?person ?personLabel ?givenLabel ?familyLabel ?birth ?death ?pad ?ep ?position ?positionLabel ?start ?end ?group ?groupLabel WHERE { VALUES ?position { %(positions)s } ?person p:P39 ?st . ?st ps:P39 ?position . OPTIONAL { ?st pq:P580 ?start } OPTIONAL { ?st pq:P582 ?end } FILTER(!BOUND(?end) || ?end >= "%(since)s-01-01"^^xsd:dateTime) OPTIONAL { ?st pq:P4100 ?group } OPTIONAL { ?person wdt:P2280 ?pad } # Generic positions with country=Austria are also held by foreigners (e.g. "Bundessprecher" of the German # Green Youth): require an Austrian connection - parliament id, citizenship or an Austrian party. FILTER(BOUND(?pad) || EXISTS { ?person wdt:P27 wd:Q40 } || EXISTS { ?person wdt:P102 ?ap . ?ap wdt:P17 wd:Q40 }) OPTIONAL { ?person wdt:P1186 ?ep } OPTIONAL { ?person wdt:P569 ?birth } OPTIONAL { ?person wdt:P570 ?death } OPTIONAL { ?person wdt:P735 ?given } OPTIONAL { ?person wdt:P734 ?family } SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". } } """ PARTY_QUERY = """ SELECT ?person ?party ?partyLabel ?partyShort ?start ?end WHERE { VALUES ?person { %(persons)s } ?person p:P102 ?st . ?st ps:P102 ?party . ?party wdt:P17 wd:Q40 . OPTIONAL { ?st pq:P580 ?start } OPTIONAL { ?st pq:P582 ?end } OPTIONAL { ?party wdt:P1813 ?partyShort . FILTER(LANG(?partyShort) = "de") } SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". } } """ CAREER_QUERY = """ SELECT ?person ?org ?orgLabel ?start ?end ?kind WHERE { VALUES ?person { %(persons)s } { ?person p:P108 ?st . ?st ps:P108 ?org . BIND("employer" AS ?kind) } UNION { ?person p:P39 ?st . ?st ps:P39 ?org . FILTER NOT EXISTS { ?org wdt:P17 wd:Q40 } BIND("position" AS ?kind) } OPTIONAL { ?st pq:P580 ?start } OPTIONAL { ?st pq:P582 ?end } SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". } } """ # Social-media properties: P2002 X/Twitter, P12361 Bluesky, P4033 Mastodon, P3789 Telegram, # P2397 YouTube channel id, P2003 Instagram, P2013 Facebook, P7085 TikTok SOCIAL_QUERY = """ SELECT ?person ?prop ?value WHERE { VALUES ?person { %(persons)s } VALUES ?prop { wdt:P2002 wdt:P12361 wdt:P4033 wdt:P3789 wdt:P2397 wdt:P2003 wdt:P2013 wdt:P7085 } ?person ?prop ?value . } """ SOCIAL_PROPS = { "P2002": ("twitter", "https://x.com/{}"), "P12361": ("bluesky", "https://bsky.app/profile/{}"), "P4033": ("mastodon", None), "P3789": ("telegram", "https://t.me/{}"), "P2397": ("youtube", "https://www.youtube.com/channel/{}"), "P2003": ("instagram", "https://www.instagram.com/{}"), "P2013": ("facebook", "https://www.facebook.com/{}"), "P7085": ("tiktok", "https://www.tiktok.com/@{}"), } EU_POLITICAL = re.compile(r"Europäischen Parlament|Europaparlament|Europäischen Kommission|EU-Kommissar|" r"Member of the European Parliament", re.IGNORECASE) EXCLUDE = re.compile( r"professor|kommerzialrat|kommerzienrat|hofrat|ministerialrat|regierungsrat|gerichtshof|richter|" r"botschafter|bischof|rektor|ökonomierat|medizinalrat|schulrat|studienrat|baurat|veterinärrat|" r"bergrat|amtsrat|kammerrat|konsul", re.IGNORECASE, ) CHAMBER_LABELS = [ (re.compile(r"Abgeordnete[rn]? zum Nationalrat|Mitglied des Nationalrates", re.I), "nationalrat", "federal"), (re.compile(r"Mitglied des (österreichischen )?Bundesrate?s", re.I), "bundesrat", "federal"), (re.compile(r"Mitglied des Europäischen Parlaments|Member of the European Parliament", re.I), "europaeisches-parlament", "eu"), ] def classify_level(label: str) -> str: if re.search(r"europäischen parlament|europaabgeordnet|eu-kommiss", label, re.I): return "eu" if re.search(r"bürgermeister|gemeinderat|stadtrat|vizebürgermeister|gemeinde", label, re.I): return "municipal" if "Wiener Landtag" not in label else "state" if re.search(r"landtag|landesrat|landeshauptmann|landesregierung|landesamtsdirektor", label, re.I): return "state" if re.search(r"bezirk", label, re.I): return "district" return "federal" def wd_date(binding: dict, key: str) -> date | None: value = binding.get(key, {}).get("value") if not value or value.startswith("-"): return None try: return date.fromisoformat(value[:10]) except ValueError: return None def qid(binding: dict, key: str) -> str | None: value = binding.get(key, {}).get("value") return value.rsplit("/", 1)[-1] if value else None def label(binding: dict, key: str) -> str | None: value = binding.get(key, {}).get("value") return None if not value or re.fullmatch(r"Q\d+", value) else value def sparql(query: str, kind: str, meta: dict | None = None) -> Request: return Request(url=SPARQL, params={"query": query, "format": "json"}, kind=kind, meta=meta or {}, headers={"Accept": "application/sparql-results+json"}) def role_for(position_label: str) -> tuple[str, OrgRef | None, str]: for pattern, slug, level in CHAMBER_LABELS: if pattern.search(position_label): return "mp", OrgRef(slug=slug), level return position_label, None, classify_level(position_label) @register class WikidataPoliticians(Adapter): spec = SPEC name = "wikidata.politicians" description = "Austrian office holders (all levels), party membership, careers, social handles" schedule = "45 4 * * 0" creates_persons = True parser_version = 2 def start(self, ctx: RunContext) -> Iterator[Request]: yield sparql(POSITION_LIST_QUERY, kind="positions") def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: bindings = json.loads(response.text, strict=False)["results"]["bindings"] kind = response.request.kind if kind == "positions": positions = [qid(b, "position") for b in bindings if label(b, "positionLabel") and not EXCLUDE.search(label(b, "positionLabel") or "")] since = ctx.params.get("since_year", 2000) for i in range(0, len(positions), 40): values = " ".join(f"wd:{q}" for q in positions[i:i + 40]) yield sparql(POSITIONS_QUERY % {"positions": values, "since": since}, kind="holders") elif kind == "holders": yield from self.parse_holders(bindings) elif kind == "parties": yield from self.parse_parties(bindings) elif kind == "careers": yield from self.parse_careers(bindings) elif kind == "social": yield from self.parse_social(bindings) def parse_holders(self, bindings: list[dict]) -> Iterator[Any]: seen: set[str] = set() for b in bindings: person_qid, name, position = qid(b, "person"), label(b, "personLabel"), label(b, "positionLabel") if not name or not position or EXCLUDE.search(position): continue pad = b.get("pad", {}).get("value") ids = {"wikidata": person_qid} if pad and pad.isdigit(): ids["parlament"] = str(int(pad)) if b.get("ep", {}).get("value"): ids["ep"] = b["ep"]["value"] ref = PersonRef(ids=ids) if person_qid not in seen: seen.add(person_qid) yield PersonRecord(ref=ref, full_name=name, given_name=label(b, "givenLabel"), family_name=label(b, "familyLabel"), birth_date=wd_date(b, "birth"), death_date=wd_date(b, "death")) start, end = wd_date(b, "start"), wd_date(b, "end") or wd_date(b, "death") group_label = label(b, "groupLabel") group = OrgRef(name=group_label, kind="parliamentary_group", ids={"wikidata": qid(b, "group")}) \ if group_label else None role, body, level = role_for(position) yield Claim( claim_type="role", person=ref, org=body, counterpart_org=group, valid_from=start, valid_to=end, summary_de=f"{name}: {position}" + (f" ({group_label})" if group_label else ""), data={"position_qid": qid(b, "position"), "position_label": position}, evidence=[Evidence(url=f"https://www.wikidata.org/wiki/{person_qid}", doc_type="api_record", title=f"Wikidata: {name}", locator=f"P39:{qid(b, 'position')}:{start}")], facet=Role(role=role, level=level, body=body, start=start, end=end), ) persons = sorted(seen) for i in range(0, len(persons), 300): values = " ".join(f"wd:{q}" for q in persons[i:i + 300]) yield sparql(PARTY_QUERY % {"persons": values}, kind="parties") yield sparql(CAREER_QUERY % {"persons": values}, kind="careers") yield sparql(SOCIAL_QUERY % {"persons": values}, kind="social") def parse_parties(self, bindings: list[dict]) -> Iterator[Claim]: for b in bindings: party_label = label(b, "partyLabel") if not party_label: continue person_qid = qid(b, "person") party = OrgRef(name=party_label, short_name=b.get("partyShort", {}).get("value"), kind="party", ids={"wikidata": qid(b, "party")}) start, end = wd_date(b, "start"), wd_date(b, "end") yield Claim( claim_type="role", person=PersonRef(ids={"wikidata": person_qid}), org=party, valid_from=start, valid_to=end, summary_de=f"Mitglied der Partei {party_label}", summary_en=f"Member of {party_label}", evidence=[Evidence(url=f"https://www.wikidata.org/wiki/{person_qid}", doc_type="api_record", title="Wikidata", locator=f"P102:{qid(b, 'party')}:{start}")], facet=Role(role="party_member", level="party", body=party, party=party, start=start, end=end), ) def parse_careers(self, bindings: list[dict]) -> Iterator[Claim]: for b in bindings: org_label = label(b, "orgLabel") if not org_label or EXCLUDE.search(org_label): continue person_qid = qid(b, "person") ref = PersonRef(ids={"wikidata": person_qid}) start, end = wd_date(b, "start"), wd_date(b, "end") kind = b["kind"]["value"] evidence = [Evidence(url=f"https://www.wikidata.org/wiki/{person_qid}", doc_type="api_record", title="Wikidata", locator=f"{kind}:{qid(b, 'org')}:{start}")] if kind == "position" and EU_POLITICAL.search(org_label): role, body, level = role_for(org_label) yield Claim(claim_type="role", person=ref, org=body, valid_from=start, valid_to=end, summary_de=org_label, evidence=evidence, facet=Role(role=role, level="eu", body=body, start=start, end=end)) continue employer = OrgRef(name=org_label, ids={"wikidata": qid(b, "org")}) if kind == "employer" else None yield Claim( claim_type="career", person=ref, counterpart_org=employer, valid_from=start, valid_to=end, occurred_at=start, key=("wikidata-career", person_qid, kind, qid(b, "org"), start), summary_de=f"{'Arbeitet(e) bei' if kind == 'employer' else 'Funktion:'} {org_label}", summary_en=f"{'Employed by' if kind == 'employer' else 'Position:'} {org_label}", data={"kind": kind, "org_qid": qid(b, "org")}, evidence=evidence, ) def parse_social(self, bindings: list[dict]) -> Iterator[SocialAccount]: for b in bindings: prop = b["prop"]["value"].rsplit("/", 1)[-1] value = b["value"]["value"].strip() platform, template = SOCIAL_PROPS[prop] if platform == "mastodon": user, _, host = value.lstrip("@").partition("@") if not host: continue url = f"https://{host}/@{user}" else: url = template.format(value) yield SocialAccount(platform=platform, url=url, handle=value, person=PersonRef(ids={"wikidata": qid(b, "person")}), verified=False)