Politrack: track Austrian politicians from public sources

Initial import of Politrack, a self-hosted service that tracks Austrian
politicians from federal to municipal level - roles, votes, money flows,
statements, meetings, social media and legal proceedings - with every claim
linked to its sources.

- backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse ->
  load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive
  for reparsing, and a claim/evidence model with corroboration
- sources: Parliament, Wikidata, EU Parliament/Commission, Council of
  Ministers, Rechnungshof party reports and donations, procurement, RTR media
  transparency, political ad libraries, WKStA, news feeds, Wikipedia edits,
  Bluesky, Mastodon, public Telegram channels, YouTube and X (official API)
- web: Next.js frontend in German and English with typed API client
- mcp: MCP server for AI agents and RAG
- docs: architecture, data model, sources, legal notes, Unraid deployment and
  contributor guides; Docker Compose setup for Unraid

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
simonandClaude Opus 5.5 committed 2026-10-28 20:02:01 +01:00
commit 583ccba31c
187 files changed
+39890

No files matched your search

+17
View File
@@ -0,0 +1,17 @@
"""Read queries shared by the REST API and the MCP server (one module per domain)."""
from politrack.queries.admin import list_sources, list_topics, pending_claims, recent_runs, set_review, stats
from politrack.queries.claims import calendar, list_claims, timeline_histogram
from politrack.queries.common import SourceFilter
from politrack.queries.money import list_money, money_summary, social_feed, wiki_edits
from politrack.queries.organizations import get_organization, get_party, list_organizations, list_parties
from politrack.queries.people import former_overview, get_person, list_persons, person_network
from politrack.queries.search import search
from politrack.queries.votes import list_votes, person_votes, vote_detail
__all__ = [
"SourceFilter", "calendar", "former_overview", "get_organization", "get_party", "get_person", "list_claims",
"list_money", "list_organizations", "list_parties", "list_persons", "list_sources", "list_topics", "list_votes",
"money_summary", "pending_claims", "recent_runs", "person_network", "person_votes", "search", "set_review", "social_feed",
"stats", "timeline_histogram", "vote_detail", "wiki_edits",
]
+69
View File
@@ -0,0 +1,69 @@
"""Sources, topics, statistics, moderation."""
from __future__ import annotations
from psycopg import Connection
def list_sources(conn: Connection) -> list[dict]:
return conn.execute(
"""
SELECT s.slug, s.name, s.kind, s.homepage, s.license, s.access_method, s.default_trust, s.tos_notes,
(SELECT count(*) FROM document d WHERE d.source_id = s.id) AS documents
FROM source s ORDER BY s.kind, s.default_trust DESC, s.name
"""
).fetchall()
def list_topics(conn: Connection) -> list[dict]:
return conn.execute("SELECT slug, name_de, name_en FROM topic ORDER BY name_de").fetchall()
def pending_claims(conn: Connection, limit: int = 50) -> list[dict]:
return conn.execute(
"""
SELECT c.id, c.claim_type, c.summary_de, c.occurred_at, p.full_name,
(SELECT jsonb_agg(jsonb_build_object('url', d.url, 'title', d.title))
FROM claim_evidence e JOIN document d ON d.id = e.document_id WHERE e.claim_id = c.id) AS evidence
FROM claim c LEFT JOIN person p ON p.id = c.person_id
WHERE c.review_status = 'pending' AND c.merged_into IS NULL
ORDER BY c.occurred_at DESC NULLS LAST LIMIT %s
""",
(limit,),
).fetchall()
def set_review(conn: Connection, claim_id: int, decision: str, reviewer: str) -> None:
if decision not in ("published", "rejected"):
raise ValueError("decision must be 'published' or 'rejected'")
conn.execute(
"UPDATE claim SET review_status = %s, reviewed_by = %s, reviewed_at = now() WHERE id = %s",
(decision, reviewer, claim_id),
)
def stats(conn: Connection) -> dict:
return conn.execute(
"""
SELECT (SELECT count(*) FROM person) AS persons,
(SELECT count(*) FROM person_status WHERE status = 'active') AS active_persons,
(SELECT count(*) FROM organization WHERE kind = 'party') AS parties,
(SELECT count(*) FROM claim WHERE merged_into IS NULL AND review_status = 'published') AS claims,
(SELECT count(*) FROM claim WHERE merged_into IS NULL AND status = 'corroborated') AS corroborated,
(SELECT count(*) FROM claim WHERE review_status = 'pending') AS pending_review,
(SELECT count(*) FROM document) AS documents,
(SELECT count(*) FROM vote_event) AS vote_events,
(SELECT count(*) FROM vote_event WHERE roll_call) AS roll_call_votes,
(SELECT count(*) FROM event) AS events,
(SELECT count(*) FROM money_flow) AS money_flows,
(SELECT max(finished_at) FROM run WHERE status IN ('ok', 'partial')) AS last_update
"""
).fetchone()
def recent_runs(conn: Connection, limit: int = 50) -> list[dict]:
return conn.execute(
"SELECT adapter, mode, status, started_at, finished_at, stats, left(error, 2000) AS error FROM run "
"ORDER BY id DESC LIMIT %s",
(limit,),
).fetchall()
+157
View File
@@ -0,0 +1,157 @@
"""Claims (the generic fact list), timeline histogram and calendar."""
from __future__ import annotations
from datetime import date
from psycopg import Connection
from politrack.queries.common import VISIBLE, SourceFilter
def list_claims(
conn: Connection,
*,
sources: SourceFilter,
person: str | None = None,
party: str | None = None,
organization: str | None = None,
claim_types: list[str] | None = None,
topics: list[str] | None = None,
since: date | None = None,
until: date | None = None,
q: str | None = None,
include_pending: bool = False,
limit: int = 50,
offset: int = 0,
) -> list[dict]:
source_sql, params = sources.sql("s")
params.update({
"person": person, "party": party, "organization": organization, "claim_types": claim_types, "topics": topics, "since": since,
"until": until, "q": q, "limit": limit, "offset": offset, "include_pending": include_pending,
})
return conn.execute(
f"""
SELECT c.id, c.claim_type, c.summary_de, c.summary_en, c.occurred_at, c.valid_from, c.valid_to, c.status,
c.sensitive, c.review_status, c.data,
p.slug AS person_slug, p.full_name AS person_name,
o.slug AS organization_slug, o.short_name AS organization_short, o.name AS organization_name,
lp.stage AS legal_stage, lp.offense AS legal_offense, lp.authority AS legal_authority,
cp.slug AS counterpart_slug, cp.name AS counterpart_name, cp.kind AS counterpart_kind,
cpp.slug AS counterpart_person_slug, cpp.full_name AS counterpart_person_name,
to_jsonb(mf) - 'claim_id' AS money, st.platform AS platform,
(SELECT jsonb_agg(jsonb_build_object('slug', t.slug, 'name_de', t.name_de, 'name_en', t.name_en))
FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id WHERE ta.claim_id = c.id) AS topics,
ev.evidence, ev.source_count
FROM claim c
LEFT JOIN person p ON p.id = c.person_id
LEFT JOIN organization o ON o.id = c.organization_id
LEFT JOIN legal_proceeding lp ON lp.claim_id = c.id
LEFT JOIN organization cp ON cp.id = c.counterpart_org_id
LEFT JOIN person cpp ON cpp.id = c.counterpart_person_id
LEFT JOIN money_flow mf ON mf.claim_id = c.id
LEFT JOIN statement st ON st.claim_id = c.id
CROSS JOIN LATERAL (
SELECT jsonb_agg(jsonb_build_object(
'url', d.url, 'title', d.title, 'source', s.slug, 'source_name', s.name, 'source_kind', s.kind,
'published_at', d.published_at, 'quote', e.quote, 'stance', e.stance, 'archive_url', d.archive_url)
ORDER BY s.default_trust DESC, d.published_at) AS evidence,
count(DISTINCT s.id) AS source_count
FROM claim_evidence e JOIN document d ON d.id = e.document_id JOIN source s ON s.id = d.source_id
WHERE e.claim_id = c.id AND {source_sql}
) ev
WHERE ev.evidence IS NOT NULL AND {VISIBLE}
AND (%(person)s::text IS NULL OR p.slug = %(person)s OR cpp.slug = %(person)s)
AND (%(party)s::text IS NULL OR o.slug = %(party)s
OR c.counterpart_org_id = (SELECT id FROM organization WHERE slug = %(party)s))
AND (%(organization)s::text IS NULL OR o.slug = %(organization)s
OR c.counterpart_org_id = (SELECT id FROM organization WHERE slug = %(organization)s))
AND (%(claim_types)s::text[] IS NULL OR c.claim_type = ANY(%(claim_types)s))
AND (%(topics)s::text[] IS NULL OR c.claim_type IN ('role', 'career') OR EXISTS (
SELECT 1 FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id
WHERE ta.claim_id = c.id AND t.slug = ANY(%(topics)s)))
AND (%(since)s::date IS NULL OR COALESCE(c.occurred_at::date, c.valid_to, c.valid_from, current_date) >= %(since)s)
AND (%(until)s::date IS NULL OR COALESCE(c.occurred_at::date, c.valid_from) <= %(until)s)
AND (%(q)s::text IS NULL OR c.summary_de ILIKE '%%' || %(q)s || '%%')
ORDER BY COALESCE(c.occurred_at, c.valid_from::timestamptz, c.created_at) DESC, c.id DESC
LIMIT %(limit)s OFFSET %(offset)s
""",
params,
).fetchall()
def timeline_histogram(conn: Connection, *, sources: SourceFilter, person: str | None = None,
party: str | None = None) -> list[dict]:
"""Claim counts per month and type - drives the timeline overview."""
source_sql, params = sources.sql("s")
params.update({"person": person, "party": party, "include_pending": False})
return conn.execute(
f"""
SELECT to_char(date_trunc('month', COALESCE(c.occurred_at, c.valid_from::timestamptz)), 'YYYY-MM') AS month,
c.claim_type, count(DISTINCT c.id) AS count
FROM claim c
LEFT JOIN person p ON p.id = c.person_id
LEFT JOIN person cpp ON cpp.id = c.counterpart_person_id
LEFT JOIN organization o ON o.id = c.organization_id
JOIN claim_evidence e ON e.claim_id = c.id JOIN document d ON d.id = e.document_id
JOIN source s ON s.id = d.source_id
WHERE {VISIBLE} AND {source_sql} AND COALESCE(c.occurred_at, c.valid_from::timestamptz) IS NOT NULL
AND (%(person)s::text IS NULL OR p.slug = %(person)s OR cpp.slug = %(person)s)
AND (%(party)s::text IS NULL OR o.slug = %(party)s)
AND c.claim_type <> 'role'
GROUP BY 1, 2 ORDER BY 1
""",
params,
).fetchall()
def calendar(conn: Connection, *, sources: SourceFilter, start: date, end: date, person: str | None = None,
party: str | None = None, kinds: list[str] | None = None) -> dict:
"""Events (sessions, trips, meetings, appearances) plus per-day claim counts in [start, end]."""
source_sql, params = sources.sql("s")
params.update({"start": start, "end": end, "person": person, "party": party, "kinds": kinds,
"include_pending": False})
events = conn.execute(
"""
SELECT ev.id, ev.kind, ev.title, ev.starts_at, ev.ends_at, ev.location_name, ev.country, ev.url,
ev.description, o.name AS body,
COALESCE(jsonb_agg(DISTINCT jsonb_build_object('slug', p.slug, 'name', p.full_name))
FILTER (WHERE p.id IS NOT NULL), '[]'::jsonb) AS participants
FROM event ev
LEFT JOIN organization o ON o.id = ev.organization_id
LEFT JOIN event_participant ep ON ep.event_id = ev.id
LEFT JOIN claim c ON c.id = ep.claim_id
LEFT JOIN person p ON p.id = ep.person_id AND c.merged_into IS NULL
WHERE ev.starts_at::date BETWEEN %(start)s AND %(end)s
AND (%(kinds)s::text[] IS NULL OR ev.kind = ANY(%(kinds)s))
AND (%(person)s::text IS NULL OR EXISTS (
SELECT 1 FROM event_participant ep2 JOIN person p2 ON p2.id = ep2.person_id
JOIN claim c2 ON c2.id = ep2.claim_id AND c2.merged_into IS NULL
WHERE ep2.event_id = ev.id AND p2.slug = %(person)s))
AND (%(party)s::text IS NULL OR EXISTS (
SELECT 1 FROM event_participant ep3 JOIN person_status ps3 ON ps3.person_id = ep3.person_id
JOIN organization o3 ON o3.id = ANY(ps3.current_party_ids)
WHERE ep3.event_id = ev.id AND o3.slug = %(party)s))
GROUP BY ev.id, o.name
ORDER BY ev.starts_at
LIMIT 2000
""",
params,
).fetchall()
days = conn.execute(
f"""
SELECT c.occurred_at::date AS day, c.claim_type, count(DISTINCT c.id) AS count
FROM claim c
LEFT JOIN person p ON p.id = c.person_id
LEFT JOIN person cpp ON cpp.id = c.counterpart_person_id
LEFT JOIN organization o ON o.id = c.organization_id
JOIN claim_evidence e ON e.claim_id = c.id JOIN document d ON d.id = e.document_id
JOIN source s ON s.id = d.source_id
WHERE {VISIBLE} AND {source_sql} AND c.occurred_at::date BETWEEN %(start)s AND %(end)s
AND (%(person)s::text IS NULL OR p.slug = %(person)s OR cpp.slug = %(person)s)
AND (%(party)s::text IS NULL OR o.slug = %(party)s)
GROUP BY 1, 2 ORDER BY 1
""",
params,
).fetchall()
return {"events": events, "days": days}
+41
View File
@@ -0,0 +1,41 @@
"""Rules shared by all read queries.
* Source filtering happens at the evidence level: a claim is returned only if at least one document comes
from an allowed source, and only allowed evidence is shown.
* Claims merged into another claim (corroboration) are hidden; the survivor carries all evidence.
* Claims pending human review are hidden unless include_pending is set (moderation).
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
VISIBLE = ("c.merged_into IS NULL AND (c.review_status = 'published' OR "
"(%(include_pending)s AND c.review_status = 'pending'))")
@dataclass
class SourceFilter:
exclude_sources: list[str] = field(default_factory=list)
min_trust: int = 0
def sql(self, alias: str = "s") -> tuple[str, dict[str, Any]]:
return (
f"(NOT ({alias}.slug = ANY(%(exclude_sources)s)) AND {alias}.default_trust >= %(min_trust)s)",
{"exclude_sources": self.exclude_sources, "min_trust": self.min_trust},
)
def evidence_json(source_sql: str) -> str:
"""Lateral subquery aggregating the allowed evidence of claim `c`."""
return f"""
CROSS JOIN LATERAL (
SELECT jsonb_agg(jsonb_build_object(
'url', d.url, 'title', d.title, 'source', s.slug, 'source_name', s.name, 'source_kind', s.kind,
'published_at', d.published_at, 'quote', e.quote, 'stance', e.stance, 'archive_url', d.archive_url)
ORDER BY s.default_trust DESC, d.published_at) AS evidence,
count(DISTINCT s.id) AS source_count
FROM claim_evidence e JOIN document d ON d.id = e.document_id JOIN source s ON s.id = d.source_id
WHERE e.claim_id = c.id AND {source_sql}
) ev"""
+114
View File
@@ -0,0 +1,114 @@
"""Money flows (donations, contracts, government ads, party funding, political ads), social posts, wiki edits."""
from __future__ import annotations
from datetime import date
from psycopg import Connection
from politrack.queries.common import VISIBLE, SourceFilter, evidence_json
def list_money(conn: Connection, *, sources: SourceFilter, kinds: list[str] | None = None, org: str | None = None,
person: str | None = None, q: str | None = None, since: date | None = None, until: date | None = None,
min_amount: float | None = None, limit: int = 50, offset: int = 0) -> list[dict]:
"""Money flows, filterable by kind, organisation (either side), connected person and period.
`person` = flows of organisations connected to that person (side jobs, careers, party) - the
"follow the money" view.
"""
source_sql, params = sources.sql("s")
params.update({"kinds": kinds, "org": org, "person": person, "q": q, "since": since, "until": until,
"min_amount": min_amount, "limit": limit, "offset": offset, "include_pending": False})
return conn.execute(
f"""
WITH person_orgs AS (
SELECT c.counterpart_org_id AS org_id FROM claim c JOIN person p ON p.id = c.person_id
WHERE p.slug = %(person)s AND c.counterpart_org_id IS NOT NULL AND c.merged_into IS NULL
UNION SELECT r.party_id FROM role r JOIN person p ON p.id = r.person_id WHERE p.slug = %(person)s
)
SELECT c.id, c.summary_de, c.summary_en, c.occurred_at, c.status, c.data, mf.kind, mf.amount, mf.amount_category,
mf.currency, mf.paid_on, mf.period, mf.reference, mf.category, mf.payer_name, mf.payee_name,
pa.slug AS payer_slug, pa.kind AS payer_kind, pe.slug AS payee_slug, pe.kind AS payee_kind,
ev.evidence, ev.source_count
FROM money_flow mf
JOIN claim c ON c.id = mf.claim_id
LEFT JOIN organization pa ON pa.id = mf.payer_org_id
LEFT JOIN organization pe ON pe.id = mf.payee_org_id
{evidence_json(source_sql)}
WHERE ev.evidence IS NOT NULL AND {VISIBLE}
AND (%(kinds)s::text[] IS NULL OR mf.kind = ANY(%(kinds)s))
AND (%(org)s::text IS NULL OR pa.slug = %(org)s OR pe.slug = %(org)s)
AND (%(person)s::text IS NULL OR mf.payer_org_id IN (SELECT org_id FROM person_orgs)
OR mf.payee_org_id IN (SELECT org_id FROM person_orgs))
AND (%(q)s::text IS NULL OR mf.payer_name ILIKE '%%' || %(q)s || '%%' OR mf.payee_name ILIKE '%%' || %(q)s || '%%'
OR c.summary_de ILIKE '%%' || %(q)s || '%%')
AND (%(since)s::date IS NULL OR COALESCE(mf.paid_on, c.occurred_at::date, c.valid_from) >= %(since)s)
AND (%(until)s::date IS NULL OR COALESCE(mf.paid_on, c.occurred_at::date, c.valid_from) <= %(until)s)
AND (%(min_amount)s::numeric IS NULL OR mf.amount >= %(min_amount)s)
ORDER BY COALESCE(mf.paid_on, c.occurred_at::date, c.valid_from) DESC NULLS LAST, mf.amount DESC NULLS LAST
LIMIT %(limit)s OFFSET %(offset)s
""",
params,
).fetchall()
def money_summary(conn: Connection, *, kinds: list[str] | None = None, since: date | None = None) -> list[dict]:
"""Top payers/payees per kind (for the money overview page)."""
return conn.execute(
"""
SELECT mf.kind, o.slug, o.name, o.kind AS org_kind,
CASE WHEN o.id = mf.payer_org_id THEN 'payer' ELSE 'payee' END AS side,
sum(mf.amount) AS total, count(*) AS count
FROM money_flow mf JOIN claim c ON c.id = mf.claim_id AND c.merged_into IS NULL
JOIN organization o ON o.id IN (mf.payer_org_id, mf.payee_org_id)
WHERE (%(kinds)s::text[] IS NULL OR mf.kind = ANY(%(kinds)s))
AND (%(since)s::date IS NULL OR COALESCE(mf.paid_on, c.valid_from) >= %(since)s)
GROUP BY 1, 2, 3, 4, 5
ORDER BY total DESC NULLS LAST LIMIT 100
""",
{"kinds": kinds, "since": since},
).fetchall()
def social_feed(conn: Connection, *, sources: SourceFilter, person: str | None = None, party: str | None = None,
platforms: list[str] | None = None, limit: int = 50, offset: int = 0) -> list[dict]:
"""Public posts of politicians and of party/institution channels (`party` = the party's own channels
plus posts of its current members)."""
source_sql, params = sources.sql("s")
params.update({"person": person, "party": party, "platforms": platforms, "limit": limit, "offset": offset,
"include_pending": False})
return conn.execute(
f"""
SELECT c.id, st.platform, st.said_at, st.excerpt, st.url, st.title, c.data, p.slug AS person_slug,
p.full_name AS person_name, o.slug AS org_slug, o.name AS org_name, ev.evidence
FROM statement st JOIN claim c ON c.id = st.claim_id
LEFT JOIN person p ON p.id = st.person_id
LEFT JOIN organization o ON o.id = c.organization_id AND st.person_id IS NULL
{evidence_json(source_sql)}
WHERE st.context = 'social_post' AND ev.evidence IS NOT NULL AND {VISIBLE}
AND (%(person)s::text IS NULL OR p.slug = %(person)s)
AND (%(platforms)s::text[] IS NULL OR st.platform = ANY(%(platforms)s))
AND (%(party)s::text IS NULL OR o.slug = %(party)s OR EXISTS (
SELECT 1 FROM person_status ps JOIN organization po ON po.id = ANY(ps.current_party_ids)
WHERE ps.person_id = p.id AND po.slug = %(party)s))
ORDER BY st.said_at DESC NULLS LAST
LIMIT %(limit)s OFFSET %(offset)s
""",
params,
).fetchall()
def wiki_edits(conn: Connection, *, limit: int = 100, offset: int = 0) -> list[dict]:
return conn.execute(
"""
SELECT c.id, c.occurred_at, c.summary_de, c.summary_en, c.data, o.slug AS org_slug, o.name AS org_name,
p.slug AS person_slug, p.full_name AS person_name,
(SELECT d.url FROM claim_evidence e JOIN document d ON d.id = e.document_id WHERE e.claim_id = c.id LIMIT 1) AS url
FROM claim c JOIN organization o ON o.id = c.organization_id
LEFT JOIN person p ON p.id = c.counterpart_person_id
WHERE c.claim_type = 'wiki_edit' AND c.merged_into IS NULL
ORDER BY c.occurred_at DESC NULLS LAST LIMIT %s OFFSET %s
""",
(limit, offset),
).fetchall()
+160
View File
@@ -0,0 +1,160 @@
"""Parties and organisations (companies, public bodies, media owners, lobby groups)."""
from __future__ import annotations
from psycopg import Connection
def list_parties(conn: Connection) -> list[dict]:
return conn.execute(
"""
SELECT o.slug, o.name, o.short_name, o.color, o.logo_url,
(SELECT count(*) FROM person_status ps WHERE ps.status = 'active' AND o.id = ANY(ps.current_party_ids))
AS active_officials,
count(DISTINCT r.person_id) AS total_people
FROM organization o LEFT JOIN role r ON r.party_id = o.id
WHERE o.kind = 'party'
GROUP BY o.id
ORDER BY active_officials DESC, total_people DESC, o.name
"""
).fetchall()
def get_party(conn: Connection, slug: str) -> dict | None:
party = conn.execute(
"SELECT id, slug, name, short_name, color, logo_url, kind FROM organization WHERE slug = %s", (slug,)
).fetchone()
if not party:
return None
pid = party["id"]
party["members"] = conn.execute(
"""
SELECT DISTINCT ON (p.id) p.slug, p.full_name, p.photo_url, r.role, r.level, r.region, r.start_date,
ch.name AS body
FROM role r
JOIN claim c ON c.id = r.claim_id AND c.merged_into IS NULL
JOIN person p ON p.id = r.person_id
JOIN person_status ps ON ps.person_id = p.id AND ps.status = 'active'
LEFT JOIN organization ch ON ch.id = r.organization_id
WHERE r.party_id = %s AND (r.end_date IS NULL OR r.end_date >= current_date) AND r.level <> 'party'
AND r.role = ANY(ps.current_roles)
ORDER BY p.id, CASE r.level WHEN 'eu' THEN 0 WHEN 'federal' THEN 1 WHEN 'state' THEN 2 ELSE 3 END
""",
(pid,),
).fetchall()
party["former_members"] = conn.execute(
"""
SELECT p.slug, p.full_name, ps.last_office_end,
(SELECT max(c.occurred_at) FROM claim c WHERE c.person_id = p.id AND c.claim_type = 'media_mention')
AS last_mention
FROM person p JOIN person_status ps ON ps.person_id = p.id
WHERE ps.status IN ('former', 'party_member') AND EXISTS (SELECT 1 FROM role r WHERE r.person_id = p.id AND r.party_id = %s)
ORDER BY last_mention DESC NULLS LAST, ps.last_office_end DESC NULLS LAST LIMIT 50
""",
(pid,),
).fetchall()
party["money_by_year"] = conn.execute(
"""
SELECT COALESCE(extract(year FROM mf.paid_on)::int, left(mf.period, 4)::int) AS year, mf.kind,
sum(mf.amount) AS total, count(*) AS count
FROM money_flow mf JOIN claim c ON c.id = mf.claim_id AND c.merged_into IS NULL
WHERE (mf.payee_org_id = %(id)s OR c.organization_id = %(id)s)
AND mf.kind IN ('donation', 'sponsorship', 'party_ad', 'party_funding', 'loan')
AND COALESCE(mf.paid_on::text, mf.period) ~ '^\\d{4}'
GROUP BY 1, 2 ORDER BY 1 DESC, 2
""",
{"id": pid},
).fetchall()
party["finance"] = conn.execute(
"""SELECT c.data, c.valid_from FROM claim c WHERE c.organization_id = %s AND c.claim_type = 'party_finance'
AND c.merged_into IS NULL ORDER BY c.valid_from DESC NULLS LAST LIMIT 5""",
(pid,),
).fetchall()
party["vote_stats"] = conn.execute(
"SELECT vp.position, count(*) AS count FROM vote_position vp WHERE vp.organization_id = %s GROUP BY 1", (pid,)
).fetchall()
party["dissent"] = conn.execute(
"""
SELECT p.slug, p.full_name, count(*) AS dissenting_votes
FROM vote_position vp
JOIN person p ON p.id = vp.person_id
JOIN vote_position fp ON fp.vote_event_id = vp.vote_event_id AND fp.organization_id = vp.party_id
WHERE vp.party_id = %s AND vp.position <> fp.position
GROUP BY p.id ORDER BY dissenting_votes DESC LIMIT 20
""",
(pid,),
).fetchall()
party["related"] = conn.execute(
"""SELECT c.claim_type, o.slug, o.name, o.kind FROM claim c JOIN organization o ON o.id = c.counterpart_org_id
WHERE c.organization_id = %s AND c.claim_type IN ('ownership', 'affiliation') AND c.merged_into IS NULL
ORDER BY c.claim_type, o.name""",
(pid,),
).fetchall()
return party
def list_organizations(conn: Connection, *, q: str | None = None, kind: str | None = None, limit: int = 50,
offset: int = 0) -> list[dict]:
return conn.execute(
"""
SELECT o.slug, o.name, o.short_name, o.kind,
(SELECT sum(amount) FROM money_flow WHERE payer_org_id = o.id) AS paid,
(SELECT sum(amount) FROM money_flow WHERE payee_org_id = o.id) AS received
FROM organization o
WHERE (%(q)s::text IS NULL OR o.name ILIKE '%%' || %(q)s || '%%'
OR EXISTS (SELECT 1 FROM alias a WHERE a.entity_type = 'organization' AND a.entity_id = o.id
AND a.alias_norm LIKE '%%' || lower(unaccent(%(q)s)) || '%%'))
AND (%(kind)s::text IS NULL OR o.kind = %(kind)s)
ORDER BY COALESCE((SELECT sum(amount) FROM money_flow WHERE payer_org_id = o.id OR payee_org_id = o.id), 0) DESC,
o.name
LIMIT %(limit)s OFFSET %(offset)s
""",
{"q": q, "kind": kind, "limit": limit, "offset": offset},
).fetchall()
def get_organization(conn: Connection, slug: str) -> dict | None:
org = conn.execute(
"""SELECT o.id, o.slug, o.name, o.short_name, o.kind, parent.slug AS parent_slug, parent.name AS parent_name,
(SELECT jsonb_object_agg(i.scheme, i.value) FROM identifier i
WHERE i.entity_type = 'organization' AND i.entity_id = o.id) AS identifiers
FROM organization o LEFT JOIN organization parent ON parent.id = o.parent_id WHERE o.slug = %s""",
(slug,),
).fetchone()
if not org:
return None
oid = org["id"]
org["money"] = conn.execute(
"""
SELECT mf.kind, CASE WHEN mf.payer_org_id = %(id)s THEN 'paid' ELSE 'received' END AS direction,
sum(mf.amount) AS total, count(*) AS count, min(COALESCE(mf.paid_on::text, mf.period)) AS first,
max(COALESCE(mf.paid_on::text, mf.period)) AS last
FROM money_flow mf JOIN claim c ON c.id = mf.claim_id AND c.merged_into IS NULL
WHERE mf.payer_org_id = %(id)s OR mf.payee_org_id = %(id)s
GROUP BY 1, 2 ORDER BY 3 DESC NULLS LAST
""",
{"id": oid},
).fetchall()
org["counterparties"] = conn.execute(
"""
SELECT o.slug, o.name, o.kind, mf.kind AS money_kind,
CASE WHEN mf.payer_org_id = %(id)s THEN 'paid_to' ELSE 'received_from' END AS direction,
sum(mf.amount) AS total, count(*) AS count
FROM money_flow mf JOIN claim c ON c.id = mf.claim_id AND c.merged_into IS NULL
JOIN organization o ON o.id = CASE WHEN mf.payer_org_id = %(id)s THEN mf.payee_org_id ELSE mf.payer_org_id END
WHERE mf.payer_org_id = %(id)s OR mf.payee_org_id = %(id)s
GROUP BY 1, 2, 3, 4, 5 ORDER BY total DESC NULLS LAST LIMIT 50
""",
{"id": oid},
).fetchall()
org["people"] = conn.execute(
"""
SELECT DISTINCT ON (p.id, c.claim_type) p.slug, p.full_name, c.claim_type AS via, c.summary_de
FROM claim c JOIN person p ON p.id = c.person_id
WHERE c.counterpart_org_id = %(id)s AND c.merged_into IS NULL AND c.review_status = 'published'
AND c.claim_type IN ('side_job', 'career', 'meeting', 'role', 'lobbying')
ORDER BY p.id, c.claim_type LIMIT 100
""",
{"id": oid},
).fetchall()
return org
+146
View File
@@ -0,0 +1,146 @@
"""Persons: lists, profiles, former politicians, connections (network)."""
from __future__ import annotations
from psycopg import Connection
def list_persons(conn: Connection, *, q: str | None = None, party: str | None = None, level: str | None = None,
status: str | None = "active", limit: int = 50, offset: int = 0) -> list[dict]:
"""status: active | former | party_member | deceased | None (all)."""
return conn.execute(
"""
SELECT p.slug, p.full_name, p.photo_url, ps.status, ps.verified AS status_verified, ps.last_office_end,
ps.current_roles,
(SELECT jsonb_agg(DISTINCT jsonb_build_object('slug', o.slug, 'short_name', o.short_name, 'color', o.color))
FROM organization o WHERE o.id = ANY(ps.current_party_ids)) AS parties
FROM person p
JOIN person_status ps ON ps.person_id = p.id
WHERE (%(status)s::text IS NULL OR ps.status = %(status)s)
AND (%(q)s::text IS NULL OR p.full_name ILIKE '%%' || %(q)s || '%%'
OR EXISTS (SELECT 1 FROM alias a WHERE a.entity_type = 'person' AND a.entity_id = p.id
AND a.alias_norm LIKE '%%' || lower(unaccent(%(q)s)) || '%%'))
AND (%(party)s::text IS NULL OR EXISTS (
SELECT 1 FROM role r JOIN organization o ON o.id = r.party_id
WHERE r.person_id = p.id AND o.slug = %(party)s
AND (ps.status <> 'active' OR o.id = ANY(ps.current_party_ids))))
AND (%(level)s::text IS NULL OR EXISTS (
SELECT 1 FROM role r WHERE r.person_id = p.id AND r.level = %(level)s
AND (ps.status <> 'active' OR r.role = ANY(ps.current_roles))))
AND EXISTS (SELECT 1 FROM role r WHERE r.person_id = p.id)
ORDER BY p.family_name NULLS LAST, p.full_name
LIMIT %(limit)s OFFSET %(offset)s
""",
{"q": q, "party": party, "level": level, "status": status, "limit": limit, "offset": offset},
).fetchall()
def get_person(conn: Connection, slug: str) -> dict | None:
person = conn.execute(
"""
SELECT p.id, p.slug, p.full_name, p.given_name, p.family_name, p.honorific_prefix, p.birth_date, p.death_date,
p.photo_url, pi.wikidata_qid, pi.parlament_pad, pi.ep_id, p.summary_de, p.summary_en, ps.status,
ps.verified AS status_verified, ps.last_office_end
FROM person p JOIN person_status ps ON ps.person_id = p.id JOIN person_identifier pi ON pi.person_id = p.id
WHERE p.slug = %s
""",
(slug,),
).fetchone()
if not person:
return None
person["roles"] = conn.execute(
"""
SELECT r.role, r.level, r.region, r.start_date, r.end_date, o.name AS body, o.slug AS body_slug,
pa.slug AS party_slug, pa.short_name AS party_short, pa.color AS party_color, c.status AS claim_status,
(SELECT jsonb_agg(DISTINCT jsonb_build_object('url', d.url, 'source', s.slug))
FROM claim_evidence e JOIN document d ON d.id = e.document_id JOIN source s ON s.id = d.source_id
WHERE e.claim_id = r.claim_id) AS evidence
FROM role r
JOIN claim c ON c.id = r.claim_id AND c.merged_into IS NULL
LEFT JOIN organization o ON o.id = r.organization_id
LEFT JOIN organization pa ON pa.id = r.party_id
WHERE r.person_id = %s
ORDER BY r.end_date DESC NULLS FIRST, r.start_date DESC NULLS LAST
""",
(person["id"],),
).fetchall()
person["social_accounts"] = conn.execute(
"SELECT platform, handle, url, verified FROM social_account WHERE person_id = %s ORDER BY verified DESC, platform",
(person["id"],),
).fetchall()
person["claim_counts"] = {
r["claim_type"]: r["count"]
for r in conn.execute(
"SELECT claim_type, count(*) AS count FROM claim WHERE person_id = %s AND merged_into IS NULL "
"AND review_status = 'published' GROUP BY claim_type",
(person["id"],),
)
}
return person
def former_overview(conn: Connection, *, party: str | None = None, limit: int = 50, offset: int = 0) -> list[dict]:
"""Former politicians, most recently in the news first ("Was macht eigentlich ...?")."""
return conn.execute(
"""
SELECT p.slug, p.full_name, p.photo_url, ps.status, ps.last_office_end,
(SELECT r.role FROM role r JOIN claim c ON c.id = r.claim_id AND c.merged_into IS NULL
WHERE r.person_id = p.id AND r.level <> 'party' ORDER BY r.end_date DESC NULLS LAST LIMIT 1) AS last_role,
latest.summary_de AS latest_news, latest.occurred_at AS latest_news_at, latest.url AS latest_news_url,
career.summary_de AS latest_career
FROM person p
JOIN person_status ps ON ps.person_id = p.id
LEFT JOIN LATERAL (
SELECT c.summary_de, c.occurred_at, d.url FROM claim c
JOIN claim_evidence e ON e.claim_id = c.id JOIN document d ON d.id = e.document_id
WHERE c.person_id = p.id AND c.claim_type IN ('media_mention', 'statement') AND c.merged_into IS NULL
ORDER BY c.occurred_at DESC NULLS LAST LIMIT 1) latest ON true
LEFT JOIN LATERAL (
SELECT c.summary_de FROM claim c WHERE c.person_id = p.id AND c.claim_type = 'career'
AND c.merged_into IS NULL ORDER BY c.valid_from DESC NULLS LAST LIMIT 1) career ON true
WHERE ps.status IN ('former', 'party_member')
AND (%(party)s::text IS NULL OR EXISTS (
SELECT 1 FROM role r JOIN organization o ON o.id = r.party_id
WHERE r.person_id = p.id AND o.slug = %(party)s))
AND (latest.occurred_at IS NOT NULL OR career.summary_de IS NOT NULL)
ORDER BY latest.occurred_at DESC NULLS LAST, ps.last_office_end DESC NULLS LAST
LIMIT %(limit)s OFFSET %(offset)s
""",
{"party": party, "limit": limit, "offset": offset},
).fetchall()
def person_network(conn: Connection, slug: str) -> dict:
"""Organisations connected to a person and what flows through them.
Connections: employers/boards from side-job disclosures and careers, parties, orgs met (meetings).
For each organisation: donations it made, contracts it won, ads/funding it received or paid.
"""
person = conn.execute("SELECT id FROM person WHERE slug = %s", (slug,)).fetchone()
if not person:
return {"organizations": []}
orgs = conn.execute(
"""
WITH links AS (
SELECT c.counterpart_org_id AS org_id, c.claim_type AS via, max(c.summary_de) AS label
FROM claim c
WHERE c.person_id = %(pid)s AND c.counterpart_org_id IS NOT NULL AND c.merged_into IS NULL
AND c.review_status = 'published'
AND c.claim_type IN ('side_job', 'career', 'meeting', 'role', 'lobbying')
GROUP BY 1, 2
UNION
SELECT r.party_id, 'party', NULL FROM role r WHERE r.person_id = %(pid)s AND r.party_id IS NOT NULL
)
SELECT o.id, o.slug, o.name, o.kind, array_agg(DISTINCT l.via) AS via, max(l.label) AS label,
(SELECT jsonb_object_agg(k, v) FROM (
SELECT mf.kind || CASE WHEN mf.payer_org_id = o.id THEN '_paid' ELSE '_received' END AS k,
jsonb_build_object('total', sum(mf.amount), 'count', count(*)) AS v
FROM money_flow mf JOIN claim mc ON mc.id = mf.claim_id AND mc.merged_into IS NULL
WHERE mf.payer_org_id = o.id OR mf.payee_org_id = o.id GROUP BY 1) x) AS money
FROM links l JOIN organization o ON o.id = l.org_id
GROUP BY o.id
ORDER BY (o.kind = 'party') DESC, o.name
""",
{"pid": person["id"]},
).fetchall()
return {"organizations": orgs}
+68
View File
@@ -0,0 +1,68 @@
"""Hybrid search (German full text + optional vector similarity)."""
from __future__ import annotations
from datetime import date
import httpx
from psycopg import Connection
from politrack.config import get_settings
from politrack.queries.common import VISIBLE, SourceFilter
def search(conn: Connection, q: str, *, sources: SourceFilter, since: date | None = None, until: date | None = None,
limit: int = 20) -> list[dict]:
"""Hybrid search: German full-text + (if configured) vector similarity, fused by reciprocal rank."""
settings = get_settings()
source_sql, params = sources.sql("s")
params.update({"q": q, "limit": limit * 3, "since": since, "until": until, "include_pending": False})
base = f"""
SELECT ch.id, ch.text, ch.claim_id, ch.document_id, c.summary_de, c.claim_type, c.occurred_at,
p.slug AS person_slug, p.full_name AS person_name, o.slug AS organization_slug,
dd.url AS document_url, dd.title AS document_title, dd.published_at AS document_date,
(SELECT jsonb_agg(DISTINCT jsonb_build_object('url', d.url, 'title', d.title, 'source', s.slug))
FROM claim_evidence e JOIN document d ON d.id = e.document_id JOIN source s ON s.id = d.source_id
WHERE e.claim_id = ch.claim_id AND {source_sql}) AS evidence
FROM chunk ch
LEFT JOIN claim c ON c.id = ch.claim_id
LEFT JOIN person p ON p.id = c.person_id
LEFT JOIN organization o ON o.id = c.organization_id
LEFT JOIN document dd ON dd.id = ch.document_id
LEFT JOIN source ds ON ds.id = dd.source_id
WHERE (ch.claim_id IS NULL OR ({VISIBLE}))
AND (ch.document_id IS NULL OR (NOT (ds.slug = ANY(%(exclude_sources)s)) AND ds.default_trust >= %(min_trust)s))
AND (%(since)s::date IS NULL OR COALESCE(c.occurred_at::date, c.valid_from, dd.published_at::date) >= %(since)s)
AND (%(until)s::date IS NULL OR COALESCE(c.occurred_at::date, c.valid_from, dd.published_at::date) <= %(until)s)
"""
fts = conn.execute(
base + " AND ch.tsv @@ websearch_to_tsquery('german', %(q)s)"
" ORDER BY ts_rank_cd(ch.tsv, websearch_to_tsquery('german', %(q)s)) DESC LIMIT %(limit)s",
params,
).fetchall()
ranked: dict[int, tuple[float, dict]] = {}
for rank, row in enumerate(fts):
ranked[row["id"]] = (1 / (60 + rank), row)
if settings.ollama_url:
try:
resp = httpx.post(f"{settings.ollama_url}/api/embed",
json={"model": settings.embedding_model, "input": [q]}, timeout=30)
resp.raise_for_status()
params["vec"] = str(resp.json()["embeddings"][0])
vec_rows = conn.execute(
base + " AND ch.embedding IS NOT NULL ORDER BY ch.embedding <=> %(vec)s::vector LIMIT %(limit)s",
params,
).fetchall()
for rank, row in enumerate(vec_rows):
score = ranked.get(row["id"], (0.0, row))[0] + 1 / (60 + rank)
ranked[row["id"]] = (score, row)
except httpx.HTTPError:
pass
results = [row | {"score": score} for score, row in sorted(ranked.values(), key=lambda x: -x[0])]
# Hide results whose only evidence comes from sources the user excluded.
return [r for r in results if r["evidence"] or r["claim_id"] is None][:limit]
# -- moderation -----------------------------------------------------------------
+127
View File
@@ -0,0 +1,127 @@
"""Votes: per person (individual roll calls + faction votes during mandates, with "say vs. vote"), lists, details."""
from __future__ import annotations
from datetime import date
from psycopg import Connection
# Did the person speak on this item on the day of the vote? Speech titles are the item titles of the
# parliament's speech list; vote_event titles come from the item page - similar, not identical.
SPOKE = """
(SELECT jsonb_build_object('url', st.url, 'title', st.title) FROM statement st
JOIN claim sc ON sc.id = st.claim_id AND sc.merged_into IS NULL
WHERE st.person_id = me.id AND st.context = 'parliament_speech' AND st.said_at::date = ve.voted_on
AND similarity(st.title, ve.item_title) > 0.35
ORDER BY similarity(st.title, ve.item_title) DESC LIMIT 1)
"""
def person_votes(conn: Connection, slug: str, *, topics: list[str] | None = None, limit: int = 100,
offset: int = 0) -> list[dict]:
"""Votes concerning a person.
* individual=true: roll-call votes with the person's own vote; dissent=true when it differs from the
majority of their faction (vote_position.party_id = their faction that day).
* individual=false: faction votes cast while the person held a mandate for that faction.
* spoke: the person's speech on that item on the same day ("say vs. vote"), if any.
"""
return conn.execute(
f"""
WITH me AS (SELECT id FROM person WHERE slug = %(slug)s)
SELECT ve.id, ve.item_title, ve.item_ref, ve.stage, ve.voted_on, ve.result, ve.url, ve.protocol_url,
ve.roll_call, vp.position, fo.short_name AS faction, ch.name AS chamber, true AS individual,
(fp.position IS NOT NULL AND fp.position <> vp.position) AS dissent, {SPOKE} AS spoke,
(SELECT jsonb_agg(DISTINCT t.slug) FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id
WHERE ta.claim_id = vp.claim_id) AS topics
FROM vote_position vp
JOIN me ON me.id = vp.person_id
JOIN claim c ON c.id = vp.claim_id AND c.merged_into IS NULL
JOIN vote_event ve ON ve.id = vp.vote_event_id
JOIN organization ch ON ch.id = ve.chamber_id
LEFT JOIN organization fo ON fo.id = vp.party_id
LEFT JOIN vote_position fp ON fp.vote_event_id = ve.id AND fp.organization_id = vp.party_id
WHERE (%(topics)s::text[] IS NULL OR EXISTS (
SELECT 1 FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id
WHERE ta.claim_id = vp.claim_id AND t.slug = ANY(%(topics)s)))
UNION ALL
SELECT ve.id, ve.item_title, ve.item_ref, ve.stage, ve.voted_on, ve.result, ve.url, ve.protocol_url,
ve.roll_call, vp.position, o.short_name, ch.name, false, false, {SPOKE},
(SELECT jsonb_agg(DISTINCT t.slug) FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id
WHERE ta.claim_id = vp.claim_id)
FROM me
JOIN role r ON r.person_id = me.id AND r.role = 'mp'
JOIN claim rc ON rc.id = r.claim_id AND rc.merged_into IS NULL
JOIN vote_event ve ON ve.chamber_id = r.organization_id AND NOT ve.roll_call
AND ve.voted_on BETWEEN r.start_date AND COALESCE(r.end_date, 'infinity'::date)
JOIN vote_position vp ON vp.vote_event_id = ve.id AND vp.organization_id = r.party_id
JOIN organization o ON o.id = vp.organization_id
JOIN organization ch ON ch.id = ve.chamber_id
WHERE (%(topics)s::text[] IS NULL OR EXISTS (
SELECT 1 FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id
WHERE ta.claim_id = vp.claim_id AND t.slug = ANY(%(topics)s)))
ORDER BY voted_on DESC NULLS LAST
LIMIT %(limit)s OFFSET %(offset)s
""",
{"slug": slug, "topics": topics, "limit": limit, "offset": offset},
).fetchall()
def list_votes(conn: Connection, *, party: str | None = None, topics: list[str] | None = None,
roll_call: bool | None = None, since: date | None = None, until: date | None = None,
limit: int = 50, offset: int = 0) -> list[dict]:
return conn.execute(
"""
SELECT ve.id, ve.item_title, ve.item_ref, ve.stage, ve.voted_on, ve.result, ve.url, ve.protocol_url,
ve.raw_text, ve.roll_call, ch.name AS chamber,
COALESCE(jsonb_agg(jsonb_build_object('party', o.short_name, 'slug', o.slug, 'color', o.color,
'position', vp.position) ORDER BY o.short_name)
FILTER (WHERE o.id IS NOT NULL), '[]'::jsonb) AS positions,
(SELECT count(*) FROM vote_position x WHERE x.vote_event_id = ve.id AND x.person_id IS NOT NULL)
AS individual_votes
FROM vote_event ve
JOIN organization ch ON ch.id = ve.chamber_id
LEFT JOIN vote_position vp ON vp.vote_event_id = ve.id AND vp.organization_id IS NOT NULL
LEFT JOIN organization o ON o.id = vp.organization_id
WHERE (%(party)s::text IS NULL OR EXISTS (
SELECT 1 FROM vote_position v2 JOIN organization o2 ON o2.id = v2.organization_id
WHERE v2.vote_event_id = ve.id AND o2.slug = %(party)s))
AND (%(topics)s::text[] IS NULL OR EXISTS (
SELECT 1 FROM vote_position v3 JOIN topic_assignment ta ON ta.claim_id = v3.claim_id
JOIN topic t ON t.id = ta.topic_id WHERE v3.vote_event_id = ve.id AND t.slug = ANY(%(topics)s)))
AND (%(roll_call)s::boolean IS NULL OR ve.roll_call = %(roll_call)s)
AND (%(since)s::date IS NULL OR ve.voted_on >= %(since)s)
AND (%(until)s::date IS NULL OR ve.voted_on <= %(until)s)
GROUP BY ve.id, ch.name
ORDER BY ve.voted_on DESC NULLS LAST, ve.id DESC
LIMIT %(limit)s OFFSET %(offset)s
""",
{"party": party, "topics": topics, "roll_call": roll_call, "since": since, "until": until,
"limit": limit, "offset": offset},
).fetchall()
def vote_detail(conn: Connection, vote_id: int) -> dict | None:
vote = conn.execute(
"SELECT ve.*, ch.name AS chamber FROM vote_event ve JOIN organization ch ON ch.id = ve.chamber_id WHERE ve.id = %s",
(vote_id,),
).fetchone()
if not vote:
return None
vote["positions"] = conn.execute(
"""SELECT o.short_name AS party, o.slug, o.color, vp.position FROM vote_position vp
JOIN organization o ON o.id = vp.organization_id WHERE vp.vote_event_id = %s ORDER BY o.short_name""",
(vote_id,),
).fetchall()
vote["individual"] = conn.execute(
"""
SELECT p.slug, p.full_name, vp.position, o.short_name AS faction,
(fp.position IS NOT NULL AND fp.position <> vp.position) AS dissent
FROM vote_position vp JOIN person p ON p.id = vp.person_id
LEFT JOIN organization o ON o.id = vp.party_id
LEFT JOIN vote_position fp ON fp.vote_event_id = vp.vote_event_id AND fp.organization_id = vp.party_id
WHERE vp.vote_event_id = %s ORDER BY o.short_name, p.family_name
""",
(vote_id,),
).fetchall()
return vote