Politrack: track Austrian politicians from public sources
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
583ccba31c
187 files changed
+39890
No files matched your search
@@ -0,0 +1,17 @@
|
||||
"""Read queries shared by the REST API and the MCP server (one module per domain)."""
|
||||
|
||||
from politrack.queries.admin import list_sources, list_topics, pending_claims, recent_runs, set_review, stats
|
||||
from politrack.queries.claims import calendar, list_claims, timeline_histogram
|
||||
from politrack.queries.common import SourceFilter
|
||||
from politrack.queries.money import list_money, money_summary, social_feed, wiki_edits
|
||||
from politrack.queries.organizations import get_organization, get_party, list_organizations, list_parties
|
||||
from politrack.queries.people import former_overview, get_person, list_persons, person_network
|
||||
from politrack.queries.search import search
|
||||
from politrack.queries.votes import list_votes, person_votes, vote_detail
|
||||
|
||||
__all__ = [
|
||||
"SourceFilter", "calendar", "former_overview", "get_organization", "get_party", "get_person", "list_claims",
|
||||
"list_money", "list_organizations", "list_parties", "list_persons", "list_sources", "list_topics", "list_votes",
|
||||
"money_summary", "pending_claims", "recent_runs", "person_network", "person_votes", "search", "set_review", "social_feed",
|
||||
"stats", "timeline_histogram", "vote_detail", "wiki_edits",
|
||||
]
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Sources, topics, statistics, moderation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from psycopg import Connection
|
||||
|
||||
|
||||
def list_sources(conn: Connection) -> list[dict]:
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT s.slug, s.name, s.kind, s.homepage, s.license, s.access_method, s.default_trust, s.tos_notes,
|
||||
(SELECT count(*) FROM document d WHERE d.source_id = s.id) AS documents
|
||||
FROM source s ORDER BY s.kind, s.default_trust DESC, s.name
|
||||
"""
|
||||
).fetchall()
|
||||
|
||||
|
||||
def list_topics(conn: Connection) -> list[dict]:
|
||||
return conn.execute("SELECT slug, name_de, name_en FROM topic ORDER BY name_de").fetchall()
|
||||
|
||||
|
||||
def pending_claims(conn: Connection, limit: int = 50) -> list[dict]:
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT c.id, c.claim_type, c.summary_de, c.occurred_at, p.full_name,
|
||||
(SELECT jsonb_agg(jsonb_build_object('url', d.url, 'title', d.title))
|
||||
FROM claim_evidence e JOIN document d ON d.id = e.document_id WHERE e.claim_id = c.id) AS evidence
|
||||
FROM claim c LEFT JOIN person p ON p.id = c.person_id
|
||||
WHERE c.review_status = 'pending' AND c.merged_into IS NULL
|
||||
ORDER BY c.occurred_at DESC NULLS LAST LIMIT %s
|
||||
""",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
|
||||
|
||||
def set_review(conn: Connection, claim_id: int, decision: str, reviewer: str) -> None:
|
||||
if decision not in ("published", "rejected"):
|
||||
raise ValueError("decision must be 'published' or 'rejected'")
|
||||
conn.execute(
|
||||
"UPDATE claim SET review_status = %s, reviewed_by = %s, reviewed_at = now() WHERE id = %s",
|
||||
(decision, reviewer, claim_id),
|
||||
)
|
||||
|
||||
|
||||
def stats(conn: Connection) -> dict:
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT (SELECT count(*) FROM person) AS persons,
|
||||
(SELECT count(*) FROM person_status WHERE status = 'active') AS active_persons,
|
||||
(SELECT count(*) FROM organization WHERE kind = 'party') AS parties,
|
||||
(SELECT count(*) FROM claim WHERE merged_into IS NULL AND review_status = 'published') AS claims,
|
||||
(SELECT count(*) FROM claim WHERE merged_into IS NULL AND status = 'corroborated') AS corroborated,
|
||||
(SELECT count(*) FROM claim WHERE review_status = 'pending') AS pending_review,
|
||||
(SELECT count(*) FROM document) AS documents,
|
||||
(SELECT count(*) FROM vote_event) AS vote_events,
|
||||
(SELECT count(*) FROM vote_event WHERE roll_call) AS roll_call_votes,
|
||||
(SELECT count(*) FROM event) AS events,
|
||||
(SELECT count(*) FROM money_flow) AS money_flows,
|
||||
(SELECT max(finished_at) FROM run WHERE status IN ('ok', 'partial')) AS last_update
|
||||
"""
|
||||
).fetchone()
|
||||
|
||||
|
||||
def recent_runs(conn: Connection, limit: int = 50) -> list[dict]:
|
||||
return conn.execute(
|
||||
"SELECT adapter, mode, status, started_at, finished_at, stats, left(error, 2000) AS error FROM run "
|
||||
"ORDER BY id DESC LIMIT %s",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
@@ -0,0 +1,157 @@
|
||||
"""Claims (the generic fact list), timeline histogram and calendar."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
from psycopg import Connection
|
||||
|
||||
from politrack.queries.common import VISIBLE, SourceFilter
|
||||
|
||||
|
||||
def list_claims(
|
||||
conn: Connection,
|
||||
*,
|
||||
sources: SourceFilter,
|
||||
person: str | None = None,
|
||||
party: str | None = None,
|
||||
organization: str | None = None,
|
||||
claim_types: list[str] | None = None,
|
||||
topics: list[str] | None = None,
|
||||
since: date | None = None,
|
||||
until: date | None = None,
|
||||
q: str | None = None,
|
||||
include_pending: bool = False,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> list[dict]:
|
||||
source_sql, params = sources.sql("s")
|
||||
params.update({
|
||||
"person": person, "party": party, "organization": organization, "claim_types": claim_types, "topics": topics, "since": since,
|
||||
"until": until, "q": q, "limit": limit, "offset": offset, "include_pending": include_pending,
|
||||
})
|
||||
return conn.execute(
|
||||
f"""
|
||||
SELECT c.id, c.claim_type, c.summary_de, c.summary_en, c.occurred_at, c.valid_from, c.valid_to, c.status,
|
||||
c.sensitive, c.review_status, c.data,
|
||||
p.slug AS person_slug, p.full_name AS person_name,
|
||||
o.slug AS organization_slug, o.short_name AS organization_short, o.name AS organization_name,
|
||||
lp.stage AS legal_stage, lp.offense AS legal_offense, lp.authority AS legal_authority,
|
||||
cp.slug AS counterpart_slug, cp.name AS counterpart_name, cp.kind AS counterpart_kind,
|
||||
cpp.slug AS counterpart_person_slug, cpp.full_name AS counterpart_person_name,
|
||||
to_jsonb(mf) - 'claim_id' AS money, st.platform AS platform,
|
||||
(SELECT jsonb_agg(jsonb_build_object('slug', t.slug, 'name_de', t.name_de, 'name_en', t.name_en))
|
||||
FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id WHERE ta.claim_id = c.id) AS topics,
|
||||
ev.evidence, ev.source_count
|
||||
FROM claim c
|
||||
LEFT JOIN person p ON p.id = c.person_id
|
||||
LEFT JOIN organization o ON o.id = c.organization_id
|
||||
LEFT JOIN legal_proceeding lp ON lp.claim_id = c.id
|
||||
LEFT JOIN organization cp ON cp.id = c.counterpart_org_id
|
||||
LEFT JOIN person cpp ON cpp.id = c.counterpart_person_id
|
||||
LEFT JOIN money_flow mf ON mf.claim_id = c.id
|
||||
LEFT JOIN statement st ON st.claim_id = c.id
|
||||
CROSS JOIN LATERAL (
|
||||
SELECT jsonb_agg(jsonb_build_object(
|
||||
'url', d.url, 'title', d.title, 'source', s.slug, 'source_name', s.name, 'source_kind', s.kind,
|
||||
'published_at', d.published_at, 'quote', e.quote, 'stance', e.stance, 'archive_url', d.archive_url)
|
||||
ORDER BY s.default_trust DESC, d.published_at) AS evidence,
|
||||
count(DISTINCT s.id) AS source_count
|
||||
FROM claim_evidence e JOIN document d ON d.id = e.document_id JOIN source s ON s.id = d.source_id
|
||||
WHERE e.claim_id = c.id AND {source_sql}
|
||||
) ev
|
||||
WHERE ev.evidence IS NOT NULL AND {VISIBLE}
|
||||
AND (%(person)s::text IS NULL OR p.slug = %(person)s OR cpp.slug = %(person)s)
|
||||
AND (%(party)s::text IS NULL OR o.slug = %(party)s
|
||||
OR c.counterpart_org_id = (SELECT id FROM organization WHERE slug = %(party)s))
|
||||
AND (%(organization)s::text IS NULL OR o.slug = %(organization)s
|
||||
OR c.counterpart_org_id = (SELECT id FROM organization WHERE slug = %(organization)s))
|
||||
AND (%(claim_types)s::text[] IS NULL OR c.claim_type = ANY(%(claim_types)s))
|
||||
AND (%(topics)s::text[] IS NULL OR c.claim_type IN ('role', 'career') OR EXISTS (
|
||||
SELECT 1 FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id
|
||||
WHERE ta.claim_id = c.id AND t.slug = ANY(%(topics)s)))
|
||||
AND (%(since)s::date IS NULL OR COALESCE(c.occurred_at::date, c.valid_to, c.valid_from, current_date) >= %(since)s)
|
||||
AND (%(until)s::date IS NULL OR COALESCE(c.occurred_at::date, c.valid_from) <= %(until)s)
|
||||
AND (%(q)s::text IS NULL OR c.summary_de ILIKE '%%' || %(q)s || '%%')
|
||||
ORDER BY COALESCE(c.occurred_at, c.valid_from::timestamptz, c.created_at) DESC, c.id DESC
|
||||
LIMIT %(limit)s OFFSET %(offset)s
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
|
||||
|
||||
def timeline_histogram(conn: Connection, *, sources: SourceFilter, person: str | None = None,
|
||||
party: str | None = None) -> list[dict]:
|
||||
"""Claim counts per month and type - drives the timeline overview."""
|
||||
source_sql, params = sources.sql("s")
|
||||
params.update({"person": person, "party": party, "include_pending": False})
|
||||
return conn.execute(
|
||||
f"""
|
||||
SELECT to_char(date_trunc('month', COALESCE(c.occurred_at, c.valid_from::timestamptz)), 'YYYY-MM') AS month,
|
||||
c.claim_type, count(DISTINCT c.id) AS count
|
||||
FROM claim c
|
||||
LEFT JOIN person p ON p.id = c.person_id
|
||||
LEFT JOIN person cpp ON cpp.id = c.counterpart_person_id
|
||||
LEFT JOIN organization o ON o.id = c.organization_id
|
||||
JOIN claim_evidence e ON e.claim_id = c.id JOIN document d ON d.id = e.document_id
|
||||
JOIN source s ON s.id = d.source_id
|
||||
WHERE {VISIBLE} AND {source_sql} AND COALESCE(c.occurred_at, c.valid_from::timestamptz) IS NOT NULL
|
||||
AND (%(person)s::text IS NULL OR p.slug = %(person)s OR cpp.slug = %(person)s)
|
||||
AND (%(party)s::text IS NULL OR o.slug = %(party)s)
|
||||
AND c.claim_type <> 'role'
|
||||
GROUP BY 1, 2 ORDER BY 1
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
|
||||
|
||||
def calendar(conn: Connection, *, sources: SourceFilter, start: date, end: date, person: str | None = None,
|
||||
party: str | None = None, kinds: list[str] | None = None) -> dict:
|
||||
"""Events (sessions, trips, meetings, appearances) plus per-day claim counts in [start, end]."""
|
||||
source_sql, params = sources.sql("s")
|
||||
params.update({"start": start, "end": end, "person": person, "party": party, "kinds": kinds,
|
||||
"include_pending": False})
|
||||
events = conn.execute(
|
||||
"""
|
||||
SELECT ev.id, ev.kind, ev.title, ev.starts_at, ev.ends_at, ev.location_name, ev.country, ev.url,
|
||||
ev.description, o.name AS body,
|
||||
COALESCE(jsonb_agg(DISTINCT jsonb_build_object('slug', p.slug, 'name', p.full_name))
|
||||
FILTER (WHERE p.id IS NOT NULL), '[]'::jsonb) AS participants
|
||||
FROM event ev
|
||||
LEFT JOIN organization o ON o.id = ev.organization_id
|
||||
LEFT JOIN event_participant ep ON ep.event_id = ev.id
|
||||
LEFT JOIN claim c ON c.id = ep.claim_id
|
||||
LEFT JOIN person p ON p.id = ep.person_id AND c.merged_into IS NULL
|
||||
WHERE ev.starts_at::date BETWEEN %(start)s AND %(end)s
|
||||
AND (%(kinds)s::text[] IS NULL OR ev.kind = ANY(%(kinds)s))
|
||||
AND (%(person)s::text IS NULL OR EXISTS (
|
||||
SELECT 1 FROM event_participant ep2 JOIN person p2 ON p2.id = ep2.person_id
|
||||
JOIN claim c2 ON c2.id = ep2.claim_id AND c2.merged_into IS NULL
|
||||
WHERE ep2.event_id = ev.id AND p2.slug = %(person)s))
|
||||
AND (%(party)s::text IS NULL OR EXISTS (
|
||||
SELECT 1 FROM event_participant ep3 JOIN person_status ps3 ON ps3.person_id = ep3.person_id
|
||||
JOIN organization o3 ON o3.id = ANY(ps3.current_party_ids)
|
||||
WHERE ep3.event_id = ev.id AND o3.slug = %(party)s))
|
||||
GROUP BY ev.id, o.name
|
||||
ORDER BY ev.starts_at
|
||||
LIMIT 2000
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
days = conn.execute(
|
||||
f"""
|
||||
SELECT c.occurred_at::date AS day, c.claim_type, count(DISTINCT c.id) AS count
|
||||
FROM claim c
|
||||
LEFT JOIN person p ON p.id = c.person_id
|
||||
LEFT JOIN person cpp ON cpp.id = c.counterpart_person_id
|
||||
LEFT JOIN organization o ON o.id = c.organization_id
|
||||
JOIN claim_evidence e ON e.claim_id = c.id JOIN document d ON d.id = e.document_id
|
||||
JOIN source s ON s.id = d.source_id
|
||||
WHERE {VISIBLE} AND {source_sql} AND c.occurred_at::date BETWEEN %(start)s AND %(end)s
|
||||
AND (%(person)s::text IS NULL OR p.slug = %(person)s OR cpp.slug = %(person)s)
|
||||
AND (%(party)s::text IS NULL OR o.slug = %(party)s)
|
||||
GROUP BY 1, 2 ORDER BY 1
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
return {"events": events, "days": days}
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Rules shared by all read queries.
|
||||
|
||||
* Source filtering happens at the evidence level: a claim is returned only if at least one document comes
|
||||
from an allowed source, and only allowed evidence is shown.
|
||||
* Claims merged into another claim (corroboration) are hidden; the survivor carries all evidence.
|
||||
* Claims pending human review are hidden unless include_pending is set (moderation).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
VISIBLE = ("c.merged_into IS NULL AND (c.review_status = 'published' OR "
|
||||
"(%(include_pending)s AND c.review_status = 'pending'))")
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceFilter:
|
||||
exclude_sources: list[str] = field(default_factory=list)
|
||||
min_trust: int = 0
|
||||
|
||||
def sql(self, alias: str = "s") -> tuple[str, dict[str, Any]]:
|
||||
return (
|
||||
f"(NOT ({alias}.slug = ANY(%(exclude_sources)s)) AND {alias}.default_trust >= %(min_trust)s)",
|
||||
{"exclude_sources": self.exclude_sources, "min_trust": self.min_trust},
|
||||
)
|
||||
|
||||
|
||||
def evidence_json(source_sql: str) -> str:
|
||||
"""Lateral subquery aggregating the allowed evidence of claim `c`."""
|
||||
return f"""
|
||||
CROSS JOIN LATERAL (
|
||||
SELECT jsonb_agg(jsonb_build_object(
|
||||
'url', d.url, 'title', d.title, 'source', s.slug, 'source_name', s.name, 'source_kind', s.kind,
|
||||
'published_at', d.published_at, 'quote', e.quote, 'stance', e.stance, 'archive_url', d.archive_url)
|
||||
ORDER BY s.default_trust DESC, d.published_at) AS evidence,
|
||||
count(DISTINCT s.id) AS source_count
|
||||
FROM claim_evidence e JOIN document d ON d.id = e.document_id JOIN source s ON s.id = d.source_id
|
||||
WHERE e.claim_id = c.id AND {source_sql}
|
||||
) ev"""
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Money flows (donations, contracts, government ads, party funding, political ads), social posts, wiki edits."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
from psycopg import Connection
|
||||
|
||||
from politrack.queries.common import VISIBLE, SourceFilter, evidence_json
|
||||
|
||||
|
||||
def list_money(conn: Connection, *, sources: SourceFilter, kinds: list[str] | None = None, org: str | None = None,
|
||||
person: str | None = None, q: str | None = None, since: date | None = None, until: date | None = None,
|
||||
min_amount: float | None = None, limit: int = 50, offset: int = 0) -> list[dict]:
|
||||
"""Money flows, filterable by kind, organisation (either side), connected person and period.
|
||||
|
||||
`person` = flows of organisations connected to that person (side jobs, careers, party) - the
|
||||
"follow the money" view.
|
||||
"""
|
||||
source_sql, params = sources.sql("s")
|
||||
params.update({"kinds": kinds, "org": org, "person": person, "q": q, "since": since, "until": until,
|
||||
"min_amount": min_amount, "limit": limit, "offset": offset, "include_pending": False})
|
||||
return conn.execute(
|
||||
f"""
|
||||
WITH person_orgs AS (
|
||||
SELECT c.counterpart_org_id AS org_id FROM claim c JOIN person p ON p.id = c.person_id
|
||||
WHERE p.slug = %(person)s AND c.counterpart_org_id IS NOT NULL AND c.merged_into IS NULL
|
||||
UNION SELECT r.party_id FROM role r JOIN person p ON p.id = r.person_id WHERE p.slug = %(person)s
|
||||
)
|
||||
SELECT c.id, c.summary_de, c.summary_en, c.occurred_at, c.status, c.data, mf.kind, mf.amount, mf.amount_category,
|
||||
mf.currency, mf.paid_on, mf.period, mf.reference, mf.category, mf.payer_name, mf.payee_name,
|
||||
pa.slug AS payer_slug, pa.kind AS payer_kind, pe.slug AS payee_slug, pe.kind AS payee_kind,
|
||||
ev.evidence, ev.source_count
|
||||
FROM money_flow mf
|
||||
JOIN claim c ON c.id = mf.claim_id
|
||||
LEFT JOIN organization pa ON pa.id = mf.payer_org_id
|
||||
LEFT JOIN organization pe ON pe.id = mf.payee_org_id
|
||||
{evidence_json(source_sql)}
|
||||
WHERE ev.evidence IS NOT NULL AND {VISIBLE}
|
||||
AND (%(kinds)s::text[] IS NULL OR mf.kind = ANY(%(kinds)s))
|
||||
AND (%(org)s::text IS NULL OR pa.slug = %(org)s OR pe.slug = %(org)s)
|
||||
AND (%(person)s::text IS NULL OR mf.payer_org_id IN (SELECT org_id FROM person_orgs)
|
||||
OR mf.payee_org_id IN (SELECT org_id FROM person_orgs))
|
||||
AND (%(q)s::text IS NULL OR mf.payer_name ILIKE '%%' || %(q)s || '%%' OR mf.payee_name ILIKE '%%' || %(q)s || '%%'
|
||||
OR c.summary_de ILIKE '%%' || %(q)s || '%%')
|
||||
AND (%(since)s::date IS NULL OR COALESCE(mf.paid_on, c.occurred_at::date, c.valid_from) >= %(since)s)
|
||||
AND (%(until)s::date IS NULL OR COALESCE(mf.paid_on, c.occurred_at::date, c.valid_from) <= %(until)s)
|
||||
AND (%(min_amount)s::numeric IS NULL OR mf.amount >= %(min_amount)s)
|
||||
ORDER BY COALESCE(mf.paid_on, c.occurred_at::date, c.valid_from) DESC NULLS LAST, mf.amount DESC NULLS LAST
|
||||
LIMIT %(limit)s OFFSET %(offset)s
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
|
||||
|
||||
def money_summary(conn: Connection, *, kinds: list[str] | None = None, since: date | None = None) -> list[dict]:
|
||||
"""Top payers/payees per kind (for the money overview page)."""
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT mf.kind, o.slug, o.name, o.kind AS org_kind,
|
||||
CASE WHEN o.id = mf.payer_org_id THEN 'payer' ELSE 'payee' END AS side,
|
||||
sum(mf.amount) AS total, count(*) AS count
|
||||
FROM money_flow mf JOIN claim c ON c.id = mf.claim_id AND c.merged_into IS NULL
|
||||
JOIN organization o ON o.id IN (mf.payer_org_id, mf.payee_org_id)
|
||||
WHERE (%(kinds)s::text[] IS NULL OR mf.kind = ANY(%(kinds)s))
|
||||
AND (%(since)s::date IS NULL OR COALESCE(mf.paid_on, c.valid_from) >= %(since)s)
|
||||
GROUP BY 1, 2, 3, 4, 5
|
||||
ORDER BY total DESC NULLS LAST LIMIT 100
|
||||
""",
|
||||
{"kinds": kinds, "since": since},
|
||||
).fetchall()
|
||||
|
||||
|
||||
def social_feed(conn: Connection, *, sources: SourceFilter, person: str | None = None, party: str | None = None,
|
||||
platforms: list[str] | None = None, limit: int = 50, offset: int = 0) -> list[dict]:
|
||||
"""Public posts of politicians and of party/institution channels (`party` = the party's own channels
|
||||
plus posts of its current members)."""
|
||||
source_sql, params = sources.sql("s")
|
||||
params.update({"person": person, "party": party, "platforms": platforms, "limit": limit, "offset": offset,
|
||||
"include_pending": False})
|
||||
return conn.execute(
|
||||
f"""
|
||||
SELECT c.id, st.platform, st.said_at, st.excerpt, st.url, st.title, c.data, p.slug AS person_slug,
|
||||
p.full_name AS person_name, o.slug AS org_slug, o.name AS org_name, ev.evidence
|
||||
FROM statement st JOIN claim c ON c.id = st.claim_id
|
||||
LEFT JOIN person p ON p.id = st.person_id
|
||||
LEFT JOIN organization o ON o.id = c.organization_id AND st.person_id IS NULL
|
||||
{evidence_json(source_sql)}
|
||||
WHERE st.context = 'social_post' AND ev.evidence IS NOT NULL AND {VISIBLE}
|
||||
AND (%(person)s::text IS NULL OR p.slug = %(person)s)
|
||||
AND (%(platforms)s::text[] IS NULL OR st.platform = ANY(%(platforms)s))
|
||||
AND (%(party)s::text IS NULL OR o.slug = %(party)s OR EXISTS (
|
||||
SELECT 1 FROM person_status ps JOIN organization po ON po.id = ANY(ps.current_party_ids)
|
||||
WHERE ps.person_id = p.id AND po.slug = %(party)s))
|
||||
ORDER BY st.said_at DESC NULLS LAST
|
||||
LIMIT %(limit)s OFFSET %(offset)s
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
|
||||
|
||||
def wiki_edits(conn: Connection, *, limit: int = 100, offset: int = 0) -> list[dict]:
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT c.id, c.occurred_at, c.summary_de, c.summary_en, c.data, o.slug AS org_slug, o.name AS org_name,
|
||||
p.slug AS person_slug, p.full_name AS person_name,
|
||||
(SELECT d.url FROM claim_evidence e JOIN document d ON d.id = e.document_id WHERE e.claim_id = c.id LIMIT 1) AS url
|
||||
FROM claim c JOIN organization o ON o.id = c.organization_id
|
||||
LEFT JOIN person p ON p.id = c.counterpart_person_id
|
||||
WHERE c.claim_type = 'wiki_edit' AND c.merged_into IS NULL
|
||||
ORDER BY c.occurred_at DESC NULLS LAST LIMIT %s OFFSET %s
|
||||
""",
|
||||
(limit, offset),
|
||||
).fetchall()
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Parties and organisations (companies, public bodies, media owners, lobby groups)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from psycopg import Connection
|
||||
|
||||
|
||||
def list_parties(conn: Connection) -> list[dict]:
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT o.slug, o.name, o.short_name, o.color, o.logo_url,
|
||||
(SELECT count(*) FROM person_status ps WHERE ps.status = 'active' AND o.id = ANY(ps.current_party_ids))
|
||||
AS active_officials,
|
||||
count(DISTINCT r.person_id) AS total_people
|
||||
FROM organization o LEFT JOIN role r ON r.party_id = o.id
|
||||
WHERE o.kind = 'party'
|
||||
GROUP BY o.id
|
||||
ORDER BY active_officials DESC, total_people DESC, o.name
|
||||
"""
|
||||
).fetchall()
|
||||
|
||||
|
||||
def get_party(conn: Connection, slug: str) -> dict | None:
|
||||
party = conn.execute(
|
||||
"SELECT id, slug, name, short_name, color, logo_url, kind FROM organization WHERE slug = %s", (slug,)
|
||||
).fetchone()
|
||||
if not party:
|
||||
return None
|
||||
pid = party["id"]
|
||||
party["members"] = conn.execute(
|
||||
"""
|
||||
SELECT DISTINCT ON (p.id) p.slug, p.full_name, p.photo_url, r.role, r.level, r.region, r.start_date,
|
||||
ch.name AS body
|
||||
FROM role r
|
||||
JOIN claim c ON c.id = r.claim_id AND c.merged_into IS NULL
|
||||
JOIN person p ON p.id = r.person_id
|
||||
JOIN person_status ps ON ps.person_id = p.id AND ps.status = 'active'
|
||||
LEFT JOIN organization ch ON ch.id = r.organization_id
|
||||
WHERE r.party_id = %s AND (r.end_date IS NULL OR r.end_date >= current_date) AND r.level <> 'party'
|
||||
AND r.role = ANY(ps.current_roles)
|
||||
ORDER BY p.id, CASE r.level WHEN 'eu' THEN 0 WHEN 'federal' THEN 1 WHEN 'state' THEN 2 ELSE 3 END
|
||||
""",
|
||||
(pid,),
|
||||
).fetchall()
|
||||
party["former_members"] = conn.execute(
|
||||
"""
|
||||
SELECT p.slug, p.full_name, ps.last_office_end,
|
||||
(SELECT max(c.occurred_at) FROM claim c WHERE c.person_id = p.id AND c.claim_type = 'media_mention')
|
||||
AS last_mention
|
||||
FROM person p JOIN person_status ps ON ps.person_id = p.id
|
||||
WHERE ps.status IN ('former', 'party_member') AND EXISTS (SELECT 1 FROM role r WHERE r.person_id = p.id AND r.party_id = %s)
|
||||
ORDER BY last_mention DESC NULLS LAST, ps.last_office_end DESC NULLS LAST LIMIT 50
|
||||
""",
|
||||
(pid,),
|
||||
).fetchall()
|
||||
party["money_by_year"] = conn.execute(
|
||||
"""
|
||||
SELECT COALESCE(extract(year FROM mf.paid_on)::int, left(mf.period, 4)::int) AS year, mf.kind,
|
||||
sum(mf.amount) AS total, count(*) AS count
|
||||
FROM money_flow mf JOIN claim c ON c.id = mf.claim_id AND c.merged_into IS NULL
|
||||
WHERE (mf.payee_org_id = %(id)s OR c.organization_id = %(id)s)
|
||||
AND mf.kind IN ('donation', 'sponsorship', 'party_ad', 'party_funding', 'loan')
|
||||
AND COALESCE(mf.paid_on::text, mf.period) ~ '^\\d{4}'
|
||||
GROUP BY 1, 2 ORDER BY 1 DESC, 2
|
||||
""",
|
||||
{"id": pid},
|
||||
).fetchall()
|
||||
party["finance"] = conn.execute(
|
||||
"""SELECT c.data, c.valid_from FROM claim c WHERE c.organization_id = %s AND c.claim_type = 'party_finance'
|
||||
AND c.merged_into IS NULL ORDER BY c.valid_from DESC NULLS LAST LIMIT 5""",
|
||||
(pid,),
|
||||
).fetchall()
|
||||
party["vote_stats"] = conn.execute(
|
||||
"SELECT vp.position, count(*) AS count FROM vote_position vp WHERE vp.organization_id = %s GROUP BY 1", (pid,)
|
||||
).fetchall()
|
||||
party["dissent"] = conn.execute(
|
||||
"""
|
||||
SELECT p.slug, p.full_name, count(*) AS dissenting_votes
|
||||
FROM vote_position vp
|
||||
JOIN person p ON p.id = vp.person_id
|
||||
JOIN vote_position fp ON fp.vote_event_id = vp.vote_event_id AND fp.organization_id = vp.party_id
|
||||
WHERE vp.party_id = %s AND vp.position <> fp.position
|
||||
GROUP BY p.id ORDER BY dissenting_votes DESC LIMIT 20
|
||||
""",
|
||||
(pid,),
|
||||
).fetchall()
|
||||
party["related"] = conn.execute(
|
||||
"""SELECT c.claim_type, o.slug, o.name, o.kind FROM claim c JOIN organization o ON o.id = c.counterpart_org_id
|
||||
WHERE c.organization_id = %s AND c.claim_type IN ('ownership', 'affiliation') AND c.merged_into IS NULL
|
||||
ORDER BY c.claim_type, o.name""",
|
||||
(pid,),
|
||||
).fetchall()
|
||||
return party
|
||||
|
||||
|
||||
def list_organizations(conn: Connection, *, q: str | None = None, kind: str | None = None, limit: int = 50,
|
||||
offset: int = 0) -> list[dict]:
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT o.slug, o.name, o.short_name, o.kind,
|
||||
(SELECT sum(amount) FROM money_flow WHERE payer_org_id = o.id) AS paid,
|
||||
(SELECT sum(amount) FROM money_flow WHERE payee_org_id = o.id) AS received
|
||||
FROM organization o
|
||||
WHERE (%(q)s::text IS NULL OR o.name ILIKE '%%' || %(q)s || '%%'
|
||||
OR EXISTS (SELECT 1 FROM alias a WHERE a.entity_type = 'organization' AND a.entity_id = o.id
|
||||
AND a.alias_norm LIKE '%%' || lower(unaccent(%(q)s)) || '%%'))
|
||||
AND (%(kind)s::text IS NULL OR o.kind = %(kind)s)
|
||||
ORDER BY COALESCE((SELECT sum(amount) FROM money_flow WHERE payer_org_id = o.id OR payee_org_id = o.id), 0) DESC,
|
||||
o.name
|
||||
LIMIT %(limit)s OFFSET %(offset)s
|
||||
""",
|
||||
{"q": q, "kind": kind, "limit": limit, "offset": offset},
|
||||
).fetchall()
|
||||
|
||||
|
||||
def get_organization(conn: Connection, slug: str) -> dict | None:
|
||||
org = conn.execute(
|
||||
"""SELECT o.id, o.slug, o.name, o.short_name, o.kind, parent.slug AS parent_slug, parent.name AS parent_name,
|
||||
(SELECT jsonb_object_agg(i.scheme, i.value) FROM identifier i
|
||||
WHERE i.entity_type = 'organization' AND i.entity_id = o.id) AS identifiers
|
||||
FROM organization o LEFT JOIN organization parent ON parent.id = o.parent_id WHERE o.slug = %s""",
|
||||
(slug,),
|
||||
).fetchone()
|
||||
if not org:
|
||||
return None
|
||||
oid = org["id"]
|
||||
org["money"] = conn.execute(
|
||||
"""
|
||||
SELECT mf.kind, CASE WHEN mf.payer_org_id = %(id)s THEN 'paid' ELSE 'received' END AS direction,
|
||||
sum(mf.amount) AS total, count(*) AS count, min(COALESCE(mf.paid_on::text, mf.period)) AS first,
|
||||
max(COALESCE(mf.paid_on::text, mf.period)) AS last
|
||||
FROM money_flow mf JOIN claim c ON c.id = mf.claim_id AND c.merged_into IS NULL
|
||||
WHERE mf.payer_org_id = %(id)s OR mf.payee_org_id = %(id)s
|
||||
GROUP BY 1, 2 ORDER BY 3 DESC NULLS LAST
|
||||
""",
|
||||
{"id": oid},
|
||||
).fetchall()
|
||||
org["counterparties"] = conn.execute(
|
||||
"""
|
||||
SELECT o.slug, o.name, o.kind, mf.kind AS money_kind,
|
||||
CASE WHEN mf.payer_org_id = %(id)s THEN 'paid_to' ELSE 'received_from' END AS direction,
|
||||
sum(mf.amount) AS total, count(*) AS count
|
||||
FROM money_flow mf JOIN claim c ON c.id = mf.claim_id AND c.merged_into IS NULL
|
||||
JOIN organization o ON o.id = CASE WHEN mf.payer_org_id = %(id)s THEN mf.payee_org_id ELSE mf.payer_org_id END
|
||||
WHERE mf.payer_org_id = %(id)s OR mf.payee_org_id = %(id)s
|
||||
GROUP BY 1, 2, 3, 4, 5 ORDER BY total DESC NULLS LAST LIMIT 50
|
||||
""",
|
||||
{"id": oid},
|
||||
).fetchall()
|
||||
org["people"] = conn.execute(
|
||||
"""
|
||||
SELECT DISTINCT ON (p.id, c.claim_type) p.slug, p.full_name, c.claim_type AS via, c.summary_de
|
||||
FROM claim c JOIN person p ON p.id = c.person_id
|
||||
WHERE c.counterpart_org_id = %(id)s AND c.merged_into IS NULL AND c.review_status = 'published'
|
||||
AND c.claim_type IN ('side_job', 'career', 'meeting', 'role', 'lobbying')
|
||||
ORDER BY p.id, c.claim_type LIMIT 100
|
||||
""",
|
||||
{"id": oid},
|
||||
).fetchall()
|
||||
return org
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Persons: lists, profiles, former politicians, connections (network)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from psycopg import Connection
|
||||
|
||||
|
||||
def list_persons(conn: Connection, *, q: str | None = None, party: str | None = None, level: str | None = None,
|
||||
status: str | None = "active", limit: int = 50, offset: int = 0) -> list[dict]:
|
||||
"""status: active | former | party_member | deceased | None (all)."""
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT p.slug, p.full_name, p.photo_url, ps.status, ps.verified AS status_verified, ps.last_office_end,
|
||||
ps.current_roles,
|
||||
(SELECT jsonb_agg(DISTINCT jsonb_build_object('slug', o.slug, 'short_name', o.short_name, 'color', o.color))
|
||||
FROM organization o WHERE o.id = ANY(ps.current_party_ids)) AS parties
|
||||
FROM person p
|
||||
JOIN person_status ps ON ps.person_id = p.id
|
||||
WHERE (%(status)s::text IS NULL OR ps.status = %(status)s)
|
||||
AND (%(q)s::text IS NULL OR p.full_name ILIKE '%%' || %(q)s || '%%'
|
||||
OR EXISTS (SELECT 1 FROM alias a WHERE a.entity_type = 'person' AND a.entity_id = p.id
|
||||
AND a.alias_norm LIKE '%%' || lower(unaccent(%(q)s)) || '%%'))
|
||||
AND (%(party)s::text IS NULL OR EXISTS (
|
||||
SELECT 1 FROM role r JOIN organization o ON o.id = r.party_id
|
||||
WHERE r.person_id = p.id AND o.slug = %(party)s
|
||||
AND (ps.status <> 'active' OR o.id = ANY(ps.current_party_ids))))
|
||||
AND (%(level)s::text IS NULL OR EXISTS (
|
||||
SELECT 1 FROM role r WHERE r.person_id = p.id AND r.level = %(level)s
|
||||
AND (ps.status <> 'active' OR r.role = ANY(ps.current_roles))))
|
||||
AND EXISTS (SELECT 1 FROM role r WHERE r.person_id = p.id)
|
||||
ORDER BY p.family_name NULLS LAST, p.full_name
|
||||
LIMIT %(limit)s OFFSET %(offset)s
|
||||
""",
|
||||
{"q": q, "party": party, "level": level, "status": status, "limit": limit, "offset": offset},
|
||||
).fetchall()
|
||||
|
||||
|
||||
def get_person(conn: Connection, slug: str) -> dict | None:
|
||||
person = conn.execute(
|
||||
"""
|
||||
SELECT p.id, p.slug, p.full_name, p.given_name, p.family_name, p.honorific_prefix, p.birth_date, p.death_date,
|
||||
p.photo_url, pi.wikidata_qid, pi.parlament_pad, pi.ep_id, p.summary_de, p.summary_en, ps.status,
|
||||
ps.verified AS status_verified, ps.last_office_end
|
||||
FROM person p JOIN person_status ps ON ps.person_id = p.id JOIN person_identifier pi ON pi.person_id = p.id
|
||||
WHERE p.slug = %s
|
||||
""",
|
||||
(slug,),
|
||||
).fetchone()
|
||||
if not person:
|
||||
return None
|
||||
person["roles"] = conn.execute(
|
||||
"""
|
||||
SELECT r.role, r.level, r.region, r.start_date, r.end_date, o.name AS body, o.slug AS body_slug,
|
||||
pa.slug AS party_slug, pa.short_name AS party_short, pa.color AS party_color, c.status AS claim_status,
|
||||
(SELECT jsonb_agg(DISTINCT jsonb_build_object('url', d.url, 'source', s.slug))
|
||||
FROM claim_evidence e JOIN document d ON d.id = e.document_id JOIN source s ON s.id = d.source_id
|
||||
WHERE e.claim_id = r.claim_id) AS evidence
|
||||
FROM role r
|
||||
JOIN claim c ON c.id = r.claim_id AND c.merged_into IS NULL
|
||||
LEFT JOIN organization o ON o.id = r.organization_id
|
||||
LEFT JOIN organization pa ON pa.id = r.party_id
|
||||
WHERE r.person_id = %s
|
||||
ORDER BY r.end_date DESC NULLS FIRST, r.start_date DESC NULLS LAST
|
||||
""",
|
||||
(person["id"],),
|
||||
).fetchall()
|
||||
person["social_accounts"] = conn.execute(
|
||||
"SELECT platform, handle, url, verified FROM social_account WHERE person_id = %s ORDER BY verified DESC, platform",
|
||||
(person["id"],),
|
||||
).fetchall()
|
||||
person["claim_counts"] = {
|
||||
r["claim_type"]: r["count"]
|
||||
for r in conn.execute(
|
||||
"SELECT claim_type, count(*) AS count FROM claim WHERE person_id = %s AND merged_into IS NULL "
|
||||
"AND review_status = 'published' GROUP BY claim_type",
|
||||
(person["id"],),
|
||||
)
|
||||
}
|
||||
return person
|
||||
|
||||
|
||||
def former_overview(conn: Connection, *, party: str | None = None, limit: int = 50, offset: int = 0) -> list[dict]:
|
||||
"""Former politicians, most recently in the news first ("Was macht eigentlich ...?")."""
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT p.slug, p.full_name, p.photo_url, ps.status, ps.last_office_end,
|
||||
(SELECT r.role FROM role r JOIN claim c ON c.id = r.claim_id AND c.merged_into IS NULL
|
||||
WHERE r.person_id = p.id AND r.level <> 'party' ORDER BY r.end_date DESC NULLS LAST LIMIT 1) AS last_role,
|
||||
latest.summary_de AS latest_news, latest.occurred_at AS latest_news_at, latest.url AS latest_news_url,
|
||||
career.summary_de AS latest_career
|
||||
FROM person p
|
||||
JOIN person_status ps ON ps.person_id = p.id
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT c.summary_de, c.occurred_at, d.url FROM claim c
|
||||
JOIN claim_evidence e ON e.claim_id = c.id JOIN document d ON d.id = e.document_id
|
||||
WHERE c.person_id = p.id AND c.claim_type IN ('media_mention', 'statement') AND c.merged_into IS NULL
|
||||
ORDER BY c.occurred_at DESC NULLS LAST LIMIT 1) latest ON true
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT c.summary_de FROM claim c WHERE c.person_id = p.id AND c.claim_type = 'career'
|
||||
AND c.merged_into IS NULL ORDER BY c.valid_from DESC NULLS LAST LIMIT 1) career ON true
|
||||
WHERE ps.status IN ('former', 'party_member')
|
||||
AND (%(party)s::text IS NULL OR EXISTS (
|
||||
SELECT 1 FROM role r JOIN organization o ON o.id = r.party_id
|
||||
WHERE r.person_id = p.id AND o.slug = %(party)s))
|
||||
AND (latest.occurred_at IS NOT NULL OR career.summary_de IS NOT NULL)
|
||||
ORDER BY latest.occurred_at DESC NULLS LAST, ps.last_office_end DESC NULLS LAST
|
||||
LIMIT %(limit)s OFFSET %(offset)s
|
||||
""",
|
||||
{"party": party, "limit": limit, "offset": offset},
|
||||
).fetchall()
|
||||
|
||||
|
||||
def person_network(conn: Connection, slug: str) -> dict:
|
||||
"""Organisations connected to a person and what flows through them.
|
||||
|
||||
Connections: employers/boards from side-job disclosures and careers, parties, orgs met (meetings).
|
||||
For each organisation: donations it made, contracts it won, ads/funding it received or paid.
|
||||
"""
|
||||
person = conn.execute("SELECT id FROM person WHERE slug = %s", (slug,)).fetchone()
|
||||
if not person:
|
||||
return {"organizations": []}
|
||||
orgs = conn.execute(
|
||||
"""
|
||||
WITH links AS (
|
||||
SELECT c.counterpart_org_id AS org_id, c.claim_type AS via, max(c.summary_de) AS label
|
||||
FROM claim c
|
||||
WHERE c.person_id = %(pid)s AND c.counterpart_org_id IS NOT NULL AND c.merged_into IS NULL
|
||||
AND c.review_status = 'published'
|
||||
AND c.claim_type IN ('side_job', 'career', 'meeting', 'role', 'lobbying')
|
||||
GROUP BY 1, 2
|
||||
UNION
|
||||
SELECT r.party_id, 'party', NULL FROM role r WHERE r.person_id = %(pid)s AND r.party_id IS NOT NULL
|
||||
)
|
||||
SELECT o.id, o.slug, o.name, o.kind, array_agg(DISTINCT l.via) AS via, max(l.label) AS label,
|
||||
(SELECT jsonb_object_agg(k, v) FROM (
|
||||
SELECT mf.kind || CASE WHEN mf.payer_org_id = o.id THEN '_paid' ELSE '_received' END AS k,
|
||||
jsonb_build_object('total', sum(mf.amount), 'count', count(*)) AS v
|
||||
FROM money_flow mf JOIN claim mc ON mc.id = mf.claim_id AND mc.merged_into IS NULL
|
||||
WHERE mf.payer_org_id = o.id OR mf.payee_org_id = o.id GROUP BY 1) x) AS money
|
||||
FROM links l JOIN organization o ON o.id = l.org_id
|
||||
GROUP BY o.id
|
||||
ORDER BY (o.kind = 'party') DESC, o.name
|
||||
""",
|
||||
{"pid": person["id"]},
|
||||
).fetchall()
|
||||
return {"organizations": orgs}
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Hybrid search (German full text + optional vector similarity)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
import httpx
|
||||
from psycopg import Connection
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.queries.common import VISIBLE, SourceFilter
|
||||
|
||||
|
||||
def search(conn: Connection, q: str, *, sources: SourceFilter, since: date | None = None, until: date | None = None,
|
||||
limit: int = 20) -> list[dict]:
|
||||
"""Hybrid search: German full-text + (if configured) vector similarity, fused by reciprocal rank."""
|
||||
settings = get_settings()
|
||||
source_sql, params = sources.sql("s")
|
||||
params.update({"q": q, "limit": limit * 3, "since": since, "until": until, "include_pending": False})
|
||||
base = f"""
|
||||
SELECT ch.id, ch.text, ch.claim_id, ch.document_id, c.summary_de, c.claim_type, c.occurred_at,
|
||||
p.slug AS person_slug, p.full_name AS person_name, o.slug AS organization_slug,
|
||||
dd.url AS document_url, dd.title AS document_title, dd.published_at AS document_date,
|
||||
(SELECT jsonb_agg(DISTINCT jsonb_build_object('url', d.url, 'title', d.title, 'source', s.slug))
|
||||
FROM claim_evidence e JOIN document d ON d.id = e.document_id JOIN source s ON s.id = d.source_id
|
||||
WHERE e.claim_id = ch.claim_id AND {source_sql}) AS evidence
|
||||
FROM chunk ch
|
||||
LEFT JOIN claim c ON c.id = ch.claim_id
|
||||
LEFT JOIN person p ON p.id = c.person_id
|
||||
LEFT JOIN organization o ON o.id = c.organization_id
|
||||
LEFT JOIN document dd ON dd.id = ch.document_id
|
||||
LEFT JOIN source ds ON ds.id = dd.source_id
|
||||
WHERE (ch.claim_id IS NULL OR ({VISIBLE}))
|
||||
AND (ch.document_id IS NULL OR (NOT (ds.slug = ANY(%(exclude_sources)s)) AND ds.default_trust >= %(min_trust)s))
|
||||
AND (%(since)s::date IS NULL OR COALESCE(c.occurred_at::date, c.valid_from, dd.published_at::date) >= %(since)s)
|
||||
AND (%(until)s::date IS NULL OR COALESCE(c.occurred_at::date, c.valid_from, dd.published_at::date) <= %(until)s)
|
||||
"""
|
||||
fts = conn.execute(
|
||||
base + " AND ch.tsv @@ websearch_to_tsquery('german', %(q)s)"
|
||||
" ORDER BY ts_rank_cd(ch.tsv, websearch_to_tsquery('german', %(q)s)) DESC LIMIT %(limit)s",
|
||||
params,
|
||||
).fetchall()
|
||||
ranked: dict[int, tuple[float, dict]] = {}
|
||||
for rank, row in enumerate(fts):
|
||||
ranked[row["id"]] = (1 / (60 + rank), row)
|
||||
|
||||
if settings.ollama_url:
|
||||
try:
|
||||
resp = httpx.post(f"{settings.ollama_url}/api/embed",
|
||||
json={"model": settings.embedding_model, "input": [q]}, timeout=30)
|
||||
resp.raise_for_status()
|
||||
params["vec"] = str(resp.json()["embeddings"][0])
|
||||
vec_rows = conn.execute(
|
||||
base + " AND ch.embedding IS NOT NULL ORDER BY ch.embedding <=> %(vec)s::vector LIMIT %(limit)s",
|
||||
params,
|
||||
).fetchall()
|
||||
for rank, row in enumerate(vec_rows):
|
||||
score = ranked.get(row["id"], (0.0, row))[0] + 1 / (60 + rank)
|
||||
ranked[row["id"]] = (score, row)
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
|
||||
results = [row | {"score": score} for score, row in sorted(ranked.values(), key=lambda x: -x[0])]
|
||||
# Hide results whose only evidence comes from sources the user excluded.
|
||||
return [r for r in results if r["evidence"] or r["claim_id"] is None][:limit]
|
||||
|
||||
|
||||
# -- moderation -----------------------------------------------------------------
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Votes: per person (individual roll calls + faction votes during mandates, with "say vs. vote"), lists, details."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
from psycopg import Connection
|
||||
|
||||
# Did the person speak on this item on the day of the vote? Speech titles are the item titles of the
|
||||
# parliament's speech list; vote_event titles come from the item page - similar, not identical.
|
||||
SPOKE = """
|
||||
(SELECT jsonb_build_object('url', st.url, 'title', st.title) FROM statement st
|
||||
JOIN claim sc ON sc.id = st.claim_id AND sc.merged_into IS NULL
|
||||
WHERE st.person_id = me.id AND st.context = 'parliament_speech' AND st.said_at::date = ve.voted_on
|
||||
AND similarity(st.title, ve.item_title) > 0.35
|
||||
ORDER BY similarity(st.title, ve.item_title) DESC LIMIT 1)
|
||||
"""
|
||||
|
||||
|
||||
def person_votes(conn: Connection, slug: str, *, topics: list[str] | None = None, limit: int = 100,
|
||||
offset: int = 0) -> list[dict]:
|
||||
"""Votes concerning a person.
|
||||
|
||||
* individual=true: roll-call votes with the person's own vote; dissent=true when it differs from the
|
||||
majority of their faction (vote_position.party_id = their faction that day).
|
||||
* individual=false: faction votes cast while the person held a mandate for that faction.
|
||||
* spoke: the person's speech on that item on the same day ("say vs. vote"), if any.
|
||||
"""
|
||||
return conn.execute(
|
||||
f"""
|
||||
WITH me AS (SELECT id FROM person WHERE slug = %(slug)s)
|
||||
SELECT ve.id, ve.item_title, ve.item_ref, ve.stage, ve.voted_on, ve.result, ve.url, ve.protocol_url,
|
||||
ve.roll_call, vp.position, fo.short_name AS faction, ch.name AS chamber, true AS individual,
|
||||
(fp.position IS NOT NULL AND fp.position <> vp.position) AS dissent, {SPOKE} AS spoke,
|
||||
(SELECT jsonb_agg(DISTINCT t.slug) FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id
|
||||
WHERE ta.claim_id = vp.claim_id) AS topics
|
||||
FROM vote_position vp
|
||||
JOIN me ON me.id = vp.person_id
|
||||
JOIN claim c ON c.id = vp.claim_id AND c.merged_into IS NULL
|
||||
JOIN vote_event ve ON ve.id = vp.vote_event_id
|
||||
JOIN organization ch ON ch.id = ve.chamber_id
|
||||
LEFT JOIN organization fo ON fo.id = vp.party_id
|
||||
LEFT JOIN vote_position fp ON fp.vote_event_id = ve.id AND fp.organization_id = vp.party_id
|
||||
WHERE (%(topics)s::text[] IS NULL OR EXISTS (
|
||||
SELECT 1 FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id
|
||||
WHERE ta.claim_id = vp.claim_id AND t.slug = ANY(%(topics)s)))
|
||||
UNION ALL
|
||||
SELECT ve.id, ve.item_title, ve.item_ref, ve.stage, ve.voted_on, ve.result, ve.url, ve.protocol_url,
|
||||
ve.roll_call, vp.position, o.short_name, ch.name, false, false, {SPOKE},
|
||||
(SELECT jsonb_agg(DISTINCT t.slug) FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id
|
||||
WHERE ta.claim_id = vp.claim_id)
|
||||
FROM me
|
||||
JOIN role r ON r.person_id = me.id AND r.role = 'mp'
|
||||
JOIN claim rc ON rc.id = r.claim_id AND rc.merged_into IS NULL
|
||||
JOIN vote_event ve ON ve.chamber_id = r.organization_id AND NOT ve.roll_call
|
||||
AND ve.voted_on BETWEEN r.start_date AND COALESCE(r.end_date, 'infinity'::date)
|
||||
JOIN vote_position vp ON vp.vote_event_id = ve.id AND vp.organization_id = r.party_id
|
||||
JOIN organization o ON o.id = vp.organization_id
|
||||
JOIN organization ch ON ch.id = ve.chamber_id
|
||||
WHERE (%(topics)s::text[] IS NULL OR EXISTS (
|
||||
SELECT 1 FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id
|
||||
WHERE ta.claim_id = vp.claim_id AND t.slug = ANY(%(topics)s)))
|
||||
ORDER BY voted_on DESC NULLS LAST
|
||||
LIMIT %(limit)s OFFSET %(offset)s
|
||||
""",
|
||||
{"slug": slug, "topics": topics, "limit": limit, "offset": offset},
|
||||
).fetchall()
|
||||
|
||||
|
||||
def list_votes(conn: Connection, *, party: str | None = None, topics: list[str] | None = None,
|
||||
roll_call: bool | None = None, since: date | None = None, until: date | None = None,
|
||||
limit: int = 50, offset: int = 0) -> list[dict]:
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT ve.id, ve.item_title, ve.item_ref, ve.stage, ve.voted_on, ve.result, ve.url, ve.protocol_url,
|
||||
ve.raw_text, ve.roll_call, ch.name AS chamber,
|
||||
COALESCE(jsonb_agg(jsonb_build_object('party', o.short_name, 'slug', o.slug, 'color', o.color,
|
||||
'position', vp.position) ORDER BY o.short_name)
|
||||
FILTER (WHERE o.id IS NOT NULL), '[]'::jsonb) AS positions,
|
||||
(SELECT count(*) FROM vote_position x WHERE x.vote_event_id = ve.id AND x.person_id IS NOT NULL)
|
||||
AS individual_votes
|
||||
FROM vote_event ve
|
||||
JOIN organization ch ON ch.id = ve.chamber_id
|
||||
LEFT JOIN vote_position vp ON vp.vote_event_id = ve.id AND vp.organization_id IS NOT NULL
|
||||
LEFT JOIN organization o ON o.id = vp.organization_id
|
||||
WHERE (%(party)s::text IS NULL OR EXISTS (
|
||||
SELECT 1 FROM vote_position v2 JOIN organization o2 ON o2.id = v2.organization_id
|
||||
WHERE v2.vote_event_id = ve.id AND o2.slug = %(party)s))
|
||||
AND (%(topics)s::text[] IS NULL OR EXISTS (
|
||||
SELECT 1 FROM vote_position v3 JOIN topic_assignment ta ON ta.claim_id = v3.claim_id
|
||||
JOIN topic t ON t.id = ta.topic_id WHERE v3.vote_event_id = ve.id AND t.slug = ANY(%(topics)s)))
|
||||
AND (%(roll_call)s::boolean IS NULL OR ve.roll_call = %(roll_call)s)
|
||||
AND (%(since)s::date IS NULL OR ve.voted_on >= %(since)s)
|
||||
AND (%(until)s::date IS NULL OR ve.voted_on <= %(until)s)
|
||||
GROUP BY ve.id, ch.name
|
||||
ORDER BY ve.voted_on DESC NULLS LAST, ve.id DESC
|
||||
LIMIT %(limit)s OFFSET %(offset)s
|
||||
""",
|
||||
{"party": party, "topics": topics, "roll_call": roll_call, "since": since, "until": until,
|
||||
"limit": limit, "offset": offset},
|
||||
).fetchall()
|
||||
|
||||
|
||||
def vote_detail(conn: Connection, vote_id: int) -> dict | None:
|
||||
vote = conn.execute(
|
||||
"SELECT ve.*, ch.name AS chamber FROM vote_event ve JOIN organization ch ON ch.id = ve.chamber_id WHERE ve.id = %s",
|
||||
(vote_id,),
|
||||
).fetchone()
|
||||
if not vote:
|
||||
return None
|
||||
vote["positions"] = conn.execute(
|
||||
"""SELECT o.short_name AS party, o.slug, o.color, vp.position FROM vote_position vp
|
||||
JOIN organization o ON o.id = vp.organization_id WHERE vp.vote_event_id = %s ORDER BY o.short_name""",
|
||||
(vote_id,),
|
||||
).fetchall()
|
||||
vote["individual"] = conn.execute(
|
||||
"""
|
||||
SELECT p.slug, p.full_name, vp.position, o.short_name AS faction,
|
||||
(fp.position IS NOT NULL AND fp.position <> vp.position) AS dissent
|
||||
FROM vote_position vp JOIN person p ON p.id = vp.person_id
|
||||
LEFT JOIN organization o ON o.id = vp.party_id
|
||||
LEFT JOIN vote_position fp ON fp.vote_event_id = vp.vote_event_id AND fp.organization_id = vp.party_id
|
||||
WHERE vp.vote_event_id = %s ORDER BY o.short_name, p.family_name
|
||||
""",
|
||||
(vote_id,),
|
||||
).fetchall()
|
||||
return vote
|
||||
Reference in new issue
Block a user