simonandClaude Opus 5.5
583ccba31c
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
84 lines
4.7 KiB
Python
84 lines
4.7 KiB
Python
"""Wikipedia edits made from the networks of Parliament and federal government (classic transparency bot).
|
||
|
||
Anonymous Wikipedia edits are publicly attributed to the editor's IP address. We query the MediaWiki API
|
||
(list=usercontribs, uciprange) for the IP ranges registered to public institutions in the RIPE database
|
||
(config/ip_ranges.yaml) and record each edit as a `wiki_edit` claim of the institution, linked to the
|
||
person whose article was edited when the title matches a known politician. Individual IP addresses are
|
||
not displayed - only the institution owning the network.
|
||
|
||
The API is used per the MediaWiki API etiquette (descriptive User-Agent, sequential requests); robots.txt
|
||
disallows /w/ for crawlers, which does not apply to API clients (robots_exempt below).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from collections.abc import Iterator
|
||
from datetime import date, timedelta
|
||
from typing import Any
|
||
|
||
import yaml
|
||
|
||
from politrack.config import get_settings
|
||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||
from politrack.core.records import Claim, Evidence, OrgRef, PersonRef
|
||
from politrack.text import excerpt
|
||
|
||
WIKIS = ("de.wikipedia.org", "en.wikipedia.org")
|
||
SPEC = SourceSpec(
|
||
slug="wikipedia", name="Wikipedia (Bearbeitungen aus Behördennetzen)", kind="aggregator",
|
||
homepage="https://de.wikipedia.org", license="CC BY-SA 4.0", access="api", trust=80, min_interval=1.0,
|
||
robots_exempt=tuple(f"https://{w}/w/api.php" for w in WIKIS),
|
||
tos_notes="MediaWiki API (API etiquette: descriptive User-Agent, no parallel requests). Edits from IP ranges "
|
||
"registered to Parliament / Bundeskanzleramt in the RIPE database.",
|
||
)
|
||
|
||
|
||
@register
|
||
class InstitutionalWikiEdits(Adapter):
|
||
spec = SPEC
|
||
name = "wikipedia.institution_edits"
|
||
description = "Wikipedia edits from the IP ranges of Parliament and federal government"
|
||
schedule = "15 7 * * *"
|
||
|
||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||
config = yaml.safe_load((get_settings().config_dir / "ip_ranges.yaml").read_text(encoding="utf-8"))
|
||
days = int(ctx.params.get("days", 3650 if not ctx.state.get("initialised") else 30))
|
||
end = (date.today() - timedelta(days=days)).isoformat() + "T00:00:00Z"
|
||
for network in config.get("networks") or []:
|
||
for cidr in network["ranges"]:
|
||
for wiki in WIKIS:
|
||
yield Request(
|
||
url=f"https://{wiki}/w/api.php", kind="contribs",
|
||
params={"action": "query", "list": "usercontribs", "uciprange": cidr, "uclimit": 500,
|
||
"ucend": end, "ucprop": "ids|title|timestamp|comment|sizediff", "format": "json",
|
||
"formatversion": 2},
|
||
meta={"wiki": wiki, "org": network["org"], "label": network["label"]}, conditional=False)
|
||
ctx.state.set("initialised", True)
|
||
|
||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||
data = response.json()
|
||
meta = response.meta
|
||
if "continue" in data:
|
||
yield Request(url=response.request.url, kind="contribs", meta=meta, conditional=False,
|
||
params={**response.request.params, **data["continue"]})
|
||
for edit in (data.get("query") or {}).get("usercontribs") or []:
|
||
title, revid = edit.get("title"), edit.get("revid")
|
||
if not title or not revid or ":" in title.split(" ")[0] and not title.startswith("Liste"):
|
||
continue # skip talk/user/meta pages ("Diskussion:", "Benutzer:")
|
||
size = edit.get("sizediff")
|
||
comment = edit.get("comment") or ""
|
||
diff = f"https://{meta['wiki']}/w/index.php?diff={revid}"
|
||
yield Claim(
|
||
claim_type="wiki_edit", org=OrgRef(slug=meta["org"], create=False),
|
||
counterpart_person=PersonRef(name=title, hints={}), key=("wiki", meta["wiki"], revid),
|
||
occurred_at=edit.get("timestamp"),
|
||
summary_de=f"Wikipedia-Bearbeitung aus dem Netz von {meta['label']}: „{title}“"
|
||
+ (f" ({size:+d} Zeichen)" if isinstance(size, int) else "")
|
||
+ (f" – {excerpt(comment, 120)}" if comment else ""),
|
||
summary_en=f"Wikipedia edit from the network of {meta['label']}: \"{title}\""
|
||
+ (f" ({size:+d} characters)" if isinstance(size, int) else ""),
|
||
data={"wiki": meta["wiki"], "title": title, "sizediff": size, "comment": comment},
|
||
evidence=[Evidence(url=diff, doc_type="web_page", title=f"Wikipedia: {title}",
|
||
published_at=edit.get("timestamp"), quote=comment or None)],
|
||
)
|