Politrack: track Austrian politicians from public sources
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
583ccba31c
187 files changed
+39890
No files matched your search
@@ -0,0 +1,68 @@
|
||||
"""Hybrid search (German full text + optional vector similarity)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
import httpx
|
||||
from psycopg import Connection
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.queries.common import VISIBLE, SourceFilter
|
||||
|
||||
|
||||
def search(conn: Connection, q: str, *, sources: SourceFilter, since: date | None = None, until: date | None = None,
|
||||
limit: int = 20) -> list[dict]:
|
||||
"""Hybrid search: German full-text + (if configured) vector similarity, fused by reciprocal rank."""
|
||||
settings = get_settings()
|
||||
source_sql, params = sources.sql("s")
|
||||
params.update({"q": q, "limit": limit * 3, "since": since, "until": until, "include_pending": False})
|
||||
base = f"""
|
||||
SELECT ch.id, ch.text, ch.claim_id, ch.document_id, c.summary_de, c.claim_type, c.occurred_at,
|
||||
p.slug AS person_slug, p.full_name AS person_name, o.slug AS organization_slug,
|
||||
dd.url AS document_url, dd.title AS document_title, dd.published_at AS document_date,
|
||||
(SELECT jsonb_agg(DISTINCT jsonb_build_object('url', d.url, 'title', d.title, 'source', s.slug))
|
||||
FROM claim_evidence e JOIN document d ON d.id = e.document_id JOIN source s ON s.id = d.source_id
|
||||
WHERE e.claim_id = ch.claim_id AND {source_sql}) AS evidence
|
||||
FROM chunk ch
|
||||
LEFT JOIN claim c ON c.id = ch.claim_id
|
||||
LEFT JOIN person p ON p.id = c.person_id
|
||||
LEFT JOIN organization o ON o.id = c.organization_id
|
||||
LEFT JOIN document dd ON dd.id = ch.document_id
|
||||
LEFT JOIN source ds ON ds.id = dd.source_id
|
||||
WHERE (ch.claim_id IS NULL OR ({VISIBLE}))
|
||||
AND (ch.document_id IS NULL OR (NOT (ds.slug = ANY(%(exclude_sources)s)) AND ds.default_trust >= %(min_trust)s))
|
||||
AND (%(since)s::date IS NULL OR COALESCE(c.occurred_at::date, c.valid_from, dd.published_at::date) >= %(since)s)
|
||||
AND (%(until)s::date IS NULL OR COALESCE(c.occurred_at::date, c.valid_from, dd.published_at::date) <= %(until)s)
|
||||
"""
|
||||
fts = conn.execute(
|
||||
base + " AND ch.tsv @@ websearch_to_tsquery('german', %(q)s)"
|
||||
" ORDER BY ts_rank_cd(ch.tsv, websearch_to_tsquery('german', %(q)s)) DESC LIMIT %(limit)s",
|
||||
params,
|
||||
).fetchall()
|
||||
ranked: dict[int, tuple[float, dict]] = {}
|
||||
for rank, row in enumerate(fts):
|
||||
ranked[row["id"]] = (1 / (60 + rank), row)
|
||||
|
||||
if settings.ollama_url:
|
||||
try:
|
||||
resp = httpx.post(f"{settings.ollama_url}/api/embed",
|
||||
json={"model": settings.embedding_model, "input": [q]}, timeout=30)
|
||||
resp.raise_for_status()
|
||||
params["vec"] = str(resp.json()["embeddings"][0])
|
||||
vec_rows = conn.execute(
|
||||
base + " AND ch.embedding IS NOT NULL ORDER BY ch.embedding <=> %(vec)s::vector LIMIT %(limit)s",
|
||||
params,
|
||||
).fetchall()
|
||||
for rank, row in enumerate(vec_rows):
|
||||
score = ranked.get(row["id"], (0.0, row))[0] + 1 / (60 + rank)
|
||||
ranked[row["id"]] = (score, row)
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
|
||||
results = [row | {"score": score} for score, row in sorted(ranked.values(), key=lambda x: -x[0])]
|
||||
# Hide results whose only evidence comes from sources the user excluded.
|
||||
return [r for r in results if r["evidence"] or r["claim_id"] is None][:limit]
|
||||
|
||||
|
||||
# -- moderation -----------------------------------------------------------------
|
||||
Reference in new issue
Block a user