"""Wikipedia edits made from the networks of Parliament and federal government (classic transparency bot). Anonymous Wikipedia edits are publicly attributed to the editor's IP address. We query the MediaWiki API (list=usercontribs, uciprange) for the IP ranges registered to public institutions in the RIPE database (config/ip_ranges.yaml) and record each edit as a `wiki_edit` claim of the institution, linked to the person whose article was edited when the title matches a known politician. Individual IP addresses are not displayed - only the institution owning the network. The API is used per the MediaWiki API etiquette (descriptive User-Agent, sequential requests); robots.txt disallows /w/ for crawlers, which does not apply to API clients (robots_exempt below). """ from __future__ import annotations from collections.abc import Iterator from datetime import date, timedelta from typing import Any import yaml from politrack.config import get_settings from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register from politrack.core.records import Claim, Evidence, OrgRef, PersonRef from politrack.text import excerpt WIKIS = ("de.wikipedia.org", "en.wikipedia.org") SPEC = SourceSpec( slug="wikipedia", name="Wikipedia (Bearbeitungen aus Behördennetzen)", kind="aggregator", homepage="https://de.wikipedia.org", license="CC BY-SA 4.0", access="api", trust=80, min_interval=1.0, robots_exempt=tuple(f"https://{w}/w/api.php" for w in WIKIS), tos_notes="MediaWiki API (API etiquette: descriptive User-Agent, no parallel requests). Edits from IP ranges " "registered to Parliament / Bundeskanzleramt in the RIPE database.", ) @register class InstitutionalWikiEdits(Adapter): spec = SPEC name = "wikipedia.institution_edits" description = "Wikipedia edits from the IP ranges of Parliament and federal government" schedule = "15 7 * * *" def start(self, ctx: RunContext) -> Iterator[Request]: config = yaml.safe_load((get_settings().config_dir / "ip_ranges.yaml").read_text(encoding="utf-8")) days = int(ctx.params.get("days", 3650 if not ctx.state.get("initialised") else 30)) end = (date.today() - timedelta(days=days)).isoformat() + "T00:00:00Z" for network in config.get("networks") or []: for cidr in network["ranges"]: for wiki in WIKIS: yield Request( url=f"https://{wiki}/w/api.php", kind="contribs", params={"action": "query", "list": "usercontribs", "uciprange": cidr, "uclimit": 500, "ucend": end, "ucprop": "ids|title|timestamp|comment|sizediff", "format": "json", "formatversion": 2}, meta={"wiki": wiki, "org": network["org"], "label": network["label"]}, conditional=False) ctx.state.set("initialised", True) def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: data = response.json() meta = response.meta if "continue" in data: yield Request(url=response.request.url, kind="contribs", meta=meta, conditional=False, params={**response.request.params, **data["continue"]}) for edit in (data.get("query") or {}).get("usercontribs") or []: title, revid = edit.get("title"), edit.get("revid") if not title or not revid or ":" in title.split(" ")[0] and not title.startswith("Liste"): continue # skip talk/user/meta pages ("Diskussion:", "Benutzer:") size = edit.get("sizediff") comment = edit.get("comment") or "" diff = f"https://{meta['wiki']}/w/index.php?diff={revid}" yield Claim( claim_type="wiki_edit", org=OrgRef(slug=meta["org"], create=False), counterpart_person=PersonRef(name=title, hints={}), key=("wiki", meta["wiki"], revid), occurred_at=edit.get("timestamp"), summary_de=f"Wikipedia-Bearbeitung aus dem Netz von {meta['label']}: „{title}“" + (f" ({size:+d} Zeichen)" if isinstance(size, int) else "") + (f" – {excerpt(comment, 120)}" if comment else ""), summary_en=f"Wikipedia edit from the network of {meta['label']}: \"{title}\"" + (f" ({size:+d} characters)" if isinstance(size, int) else ""), data={"wiki": meta["wiki"], "title": title, "sizediff": size, "comment": comment}, evidence=[Evidence(url=diff, doc_type="web_page", title=f"Wikipedia: {title}", published_at=edit.get("timestamp"), quote=comment or None)], )