Politrack: track Austrian politicians from public sources
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
583ccba31c
187 files changed
+39890
No files matched your search
@@ -0,0 +1,83 @@
|
||||
"""Wikipedia edits made from the networks of Parliament and federal government (classic transparency bot).
|
||||
|
||||
Anonymous Wikipedia edits are publicly attributed to the editor's IP address. We query the MediaWiki API
|
||||
(list=usercontribs, uciprange) for the IP ranges registered to public institutions in the RIPE database
|
||||
(config/ip_ranges.yaml) and record each edit as a `wiki_edit` claim of the institution, linked to the
|
||||
person whose article was edited when the title matches a known politician. Individual IP addresses are
|
||||
not displayed - only the institution owning the network.
|
||||
|
||||
The API is used per the MediaWiki API etiquette (descriptive User-Agent, sequential requests); robots.txt
|
||||
disallows /w/ for crawlers, which does not apply to API clients (robots_exempt below).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
from datetime import date, timedelta
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Evidence, OrgRef, PersonRef
|
||||
from politrack.text import excerpt
|
||||
|
||||
WIKIS = ("de.wikipedia.org", "en.wikipedia.org")
|
||||
SPEC = SourceSpec(
|
||||
slug="wikipedia", name="Wikipedia (Bearbeitungen aus Behördennetzen)", kind="aggregator",
|
||||
homepage="https://de.wikipedia.org", license="CC BY-SA 4.0", access="api", trust=80, min_interval=1.0,
|
||||
robots_exempt=tuple(f"https://{w}/w/api.php" for w in WIKIS),
|
||||
tos_notes="MediaWiki API (API etiquette: descriptive User-Agent, no parallel requests). Edits from IP ranges "
|
||||
"registered to Parliament / Bundeskanzleramt in the RIPE database.",
|
||||
)
|
||||
|
||||
|
||||
@register
|
||||
class InstitutionalWikiEdits(Adapter):
|
||||
spec = SPEC
|
||||
name = "wikipedia.institution_edits"
|
||||
description = "Wikipedia edits from the IP ranges of Parliament and federal government"
|
||||
schedule = "15 7 * * *"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
config = yaml.safe_load((get_settings().config_dir / "ip_ranges.yaml").read_text(encoding="utf-8"))
|
||||
days = int(ctx.params.get("days", 3650 if not ctx.state.get("initialised") else 30))
|
||||
end = (date.today() - timedelta(days=days)).isoformat() + "T00:00:00Z"
|
||||
for network in config.get("networks") or []:
|
||||
for cidr in network["ranges"]:
|
||||
for wiki in WIKIS:
|
||||
yield Request(
|
||||
url=f"https://{wiki}/w/api.php", kind="contribs",
|
||||
params={"action": "query", "list": "usercontribs", "uciprange": cidr, "uclimit": 500,
|
||||
"ucend": end, "ucprop": "ids|title|timestamp|comment|sizediff", "format": "json",
|
||||
"formatversion": 2},
|
||||
meta={"wiki": wiki, "org": network["org"], "label": network["label"]}, conditional=False)
|
||||
ctx.state.set("initialised", True)
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
data = response.json()
|
||||
meta = response.meta
|
||||
if "continue" in data:
|
||||
yield Request(url=response.request.url, kind="contribs", meta=meta, conditional=False,
|
||||
params={**response.request.params, **data["continue"]})
|
||||
for edit in (data.get("query") or {}).get("usercontribs") or []:
|
||||
title, revid = edit.get("title"), edit.get("revid")
|
||||
if not title or not revid or ":" in title.split(" ")[0] and not title.startswith("Liste"):
|
||||
continue # skip talk/user/meta pages ("Diskussion:", "Benutzer:")
|
||||
size = edit.get("sizediff")
|
||||
comment = edit.get("comment") or ""
|
||||
diff = f"https://{meta['wiki']}/w/index.php?diff={revid}"
|
||||
yield Claim(
|
||||
claim_type="wiki_edit", org=OrgRef(slug=meta["org"], create=False),
|
||||
counterpart_person=PersonRef(name=title, hints={}), key=("wiki", meta["wiki"], revid),
|
||||
occurred_at=edit.get("timestamp"),
|
||||
summary_de=f"Wikipedia-Bearbeitung aus dem Netz von {meta['label']}: „{title}“"
|
||||
+ (f" ({size:+d} Zeichen)" if isinstance(size, int) else "")
|
||||
+ (f" – {excerpt(comment, 120)}" if comment else ""),
|
||||
summary_en=f"Wikipedia edit from the network of {meta['label']}: \"{title}\""
|
||||
+ (f" ({size:+d} characters)" if isinstance(size, int) else ""),
|
||||
data={"wiki": meta["wiki"], "title": title, "sizediff": size, "comment": comment},
|
||||
evidence=[Evidence(url=diff, doc_type="web_page", title=f"Wikipedia: {title}",
|
||||
published_at=edit.get("timestamp"), quote=comment or None)],
|
||||
)
|
||||
Reference in new issue
Block a user