simonandClaude Opus 5.5
583ccba31c
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
69 lines
3.4 KiB
Python
69 lines
3.4 KiB
Python
"""Hybrid search (German full text + optional vector similarity)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import date
|
|
|
|
import httpx
|
|
from psycopg import Connection
|
|
|
|
from politrack.config import get_settings
|
|
from politrack.queries.common import VISIBLE, SourceFilter
|
|
|
|
|
|
def search(conn: Connection, q: str, *, sources: SourceFilter, since: date | None = None, until: date | None = None,
|
|
limit: int = 20) -> list[dict]:
|
|
"""Hybrid search: German full-text + (if configured) vector similarity, fused by reciprocal rank."""
|
|
settings = get_settings()
|
|
source_sql, params = sources.sql("s")
|
|
params.update({"q": q, "limit": limit * 3, "since": since, "until": until, "include_pending": False})
|
|
base = f"""
|
|
SELECT ch.id, ch.text, ch.claim_id, ch.document_id, c.summary_de, c.claim_type, c.occurred_at,
|
|
p.slug AS person_slug, p.full_name AS person_name, o.slug AS organization_slug,
|
|
dd.url AS document_url, dd.title AS document_title, dd.published_at AS document_date,
|
|
(SELECT jsonb_agg(DISTINCT jsonb_build_object('url', d.url, 'title', d.title, 'source', s.slug))
|
|
FROM claim_evidence e JOIN document d ON d.id = e.document_id JOIN source s ON s.id = d.source_id
|
|
WHERE e.claim_id = ch.claim_id AND {source_sql}) AS evidence
|
|
FROM chunk ch
|
|
LEFT JOIN claim c ON c.id = ch.claim_id
|
|
LEFT JOIN person p ON p.id = c.person_id
|
|
LEFT JOIN organization o ON o.id = c.organization_id
|
|
LEFT JOIN document dd ON dd.id = ch.document_id
|
|
LEFT JOIN source ds ON ds.id = dd.source_id
|
|
WHERE (ch.claim_id IS NULL OR ({VISIBLE}))
|
|
AND (ch.document_id IS NULL OR (NOT (ds.slug = ANY(%(exclude_sources)s)) AND ds.default_trust >= %(min_trust)s))
|
|
AND (%(since)s::date IS NULL OR COALESCE(c.occurred_at::date, c.valid_from, dd.published_at::date) >= %(since)s)
|
|
AND (%(until)s::date IS NULL OR COALESCE(c.occurred_at::date, c.valid_from, dd.published_at::date) <= %(until)s)
|
|
"""
|
|
fts = conn.execute(
|
|
base + " AND ch.tsv @@ websearch_to_tsquery('german', %(q)s)"
|
|
" ORDER BY ts_rank_cd(ch.tsv, websearch_to_tsquery('german', %(q)s)) DESC LIMIT %(limit)s",
|
|
params,
|
|
).fetchall()
|
|
ranked: dict[int, tuple[float, dict]] = {}
|
|
for rank, row in enumerate(fts):
|
|
ranked[row["id"]] = (1 / (60 + rank), row)
|
|
|
|
if settings.ollama_url:
|
|
try:
|
|
resp = httpx.post(f"{settings.ollama_url}/api/embed",
|
|
json={"model": settings.embedding_model, "input": [q]}, timeout=30)
|
|
resp.raise_for_status()
|
|
params["vec"] = str(resp.json()["embeddings"][0])
|
|
vec_rows = conn.execute(
|
|
base + " AND ch.embedding IS NOT NULL ORDER BY ch.embedding <=> %(vec)s::vector LIMIT %(limit)s",
|
|
params,
|
|
).fetchall()
|
|
for rank, row in enumerate(vec_rows):
|
|
score = ranked.get(row["id"], (0.0, row))[0] + 1 / (60 + rank)
|
|
ranked[row["id"]] = (score, row)
|
|
except httpx.HTTPError:
|
|
pass
|
|
|
|
results = [row | {"score": score} for score, row in sorted(ranked.values(), key=lambda x: -x[0])]
|
|
# Hide results whose only evidence comes from sources the user excluded.
|
|
return [r for r in results if r["evidence"] or r["claim_id"] is None][:limit]
|
|
|
|
|
|
# -- moderation -----------------------------------------------------------------
|