Simon DiesenreiterandClaude Opus 5.5
9da5a1dc48
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
82 lines
4.1 KiB
Python
82 lines
4.1 KiB
Python
"""Housekeeping enrichers that work directly on canonical rows (no new facts, only derived structure).
|
|
|
|
* enrich.roles - give open-ended offices the party of the holder's party membership at that time
|
|
(Wikidata offices carry no party), close open roles at the holder's date of death
|
|
* enrich.rollcall - derive each faction's majority position from individual roll-call votes, so party
|
|
pages show roll calls too ("ÖVP mostly for: ... 70 yes, 2 no")
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections import Counter, defaultdict
|
|
from collections.abc import Iterator
|
|
|
|
from politrack.core import Enricher, RunContext, register
|
|
from politrack.core.records import Claim, Evidence, OrgRef, VotePosition
|
|
from politrack.core.spec import DERIVED
|
|
|
|
|
|
@register
|
|
class RoleMaintenance(Enricher):
|
|
spec = DERIVED
|
|
name = "enrich.roles"
|
|
description = "Assign parties to offices from party memberships; close roles at death"
|
|
schedule = "50 5 * * *"
|
|
|
|
def run(self, ctx: RunContext) -> None:
|
|
conn = ctx.conn
|
|
ctx.stats["roles_with_party"] = conn.execute(
|
|
"""UPDATE role r SET party_id = pm.party_id
|
|
FROM role pm JOIN claim c ON c.id = pm.claim_id
|
|
WHERE r.party_id IS NULL AND r.level <> 'party' AND pm.role = 'party_member' AND c.merged_into IS NULL
|
|
AND pm.person_id = r.person_id
|
|
AND (pm.start_date IS NULL OR r.end_date IS NULL OR pm.start_date <= r.end_date)
|
|
AND (pm.end_date IS NULL OR r.start_date IS NULL OR pm.end_date >= r.start_date)"""
|
|
).rowcount
|
|
ctx.stats["roles_closed_at_death"] = conn.execute(
|
|
"""UPDATE role r SET end_date = p.death_date FROM person p
|
|
WHERE p.id = r.person_id AND r.end_date IS NULL AND p.death_date IS NOT NULL"""
|
|
).rowcount
|
|
return None
|
|
|
|
|
|
@register
|
|
class RollCallFactions(Enricher):
|
|
spec = DERIVED
|
|
name = "enrich.rollcall"
|
|
description = "Faction majority positions derived from individual roll-call votes"
|
|
schedule = "55 4 * * *"
|
|
|
|
def run(self, ctx: RunContext) -> Iterator[Claim]:
|
|
rows = ctx.conn.execute(
|
|
"""SELECT ve.event_key, ve.item_title, ve.voted_on, ve.url, ch.slug AS chamber, vp.party_id, vp.position,
|
|
o.short_name, o.name
|
|
FROM vote_event ve JOIN vote_position vp ON vp.vote_event_id = ve.id
|
|
JOIN organization ch ON ch.id = ve.chamber_id
|
|
JOIN organization o ON o.id = vp.party_id
|
|
JOIN claim c ON c.id = vp.claim_id AND c.merged_into IS NULL
|
|
WHERE ve.roll_call AND vp.person_id IS NOT NULL"""
|
|
).fetchall()
|
|
tally: dict[tuple, Counter] = defaultdict(Counter)
|
|
info: dict[tuple, dict] = {}
|
|
for r in rows:
|
|
key = (r["event_key"], r["party_id"])
|
|
tally[key][r["position"]] += 1
|
|
info[key] = r
|
|
for key, counts in tally.items():
|
|
r = info[key]
|
|
position, _ = counts.most_common(1)[0]
|
|
party = r["short_name"] or r["name"]
|
|
yield Claim(
|
|
claim_type="vote", org=OrgRef(ids={"politrack_org": str(r["party_id"])}, create=False),
|
|
counterpart_org=OrgRef(slug=r["chamber"]), occurred_at=r["voted_on"],
|
|
summary_de=f"{party} stimmte mehrheitlich {'für' if position == 'for' else 'gegen'}: {r['item_title']} "
|
|
f"(namentliche Abstimmung, {counts['for']} Ja, {counts['against']} Nein)",
|
|
summary_en=f"{party} mostly voted {'for' if position == 'for' else 'against'}: {r['item_title']} "
|
|
f"(roll call, {counts['for']} yes / {counts['against']} no)",
|
|
data={"position": position, "roll_call": True, "yes": counts["for"], "no": counts["against"]},
|
|
evidence=[Evidence(url=r["url"], source="parlament", doc_type="api_record", title=r["item_title"],
|
|
published_at=r["voted_on"], locator=f"rollcall:{r['voted_on']}")],
|
|
facet=VotePosition(event_key=r["event_key"], position=position),
|
|
)
|