"""Hybrid search (German full text + optional vector similarity).""" from __future__ import annotations from datetime import date import httpx from psycopg import Connection from politrack.config import get_settings from politrack.queries.common import VISIBLE, SourceFilter def search(conn: Connection, q: str, *, sources: SourceFilter, since: date | None = None, until: date | None = None, limit: int = 20) -> list[dict]: """Hybrid search: German full-text + (if configured) vector similarity, fused by reciprocal rank.""" settings = get_settings() source_sql, params = sources.sql("s") params.update({"q": q, "limit": limit * 3, "since": since, "until": until, "include_pending": False}) base = f""" SELECT ch.id, ch.text, ch.claim_id, ch.document_id, c.summary_de, c.claim_type, c.occurred_at, p.slug AS person_slug, p.full_name AS person_name, o.slug AS organization_slug, dd.url AS document_url, dd.title AS document_title, dd.published_at AS document_date, (SELECT jsonb_agg(DISTINCT jsonb_build_object('url', d.url, 'title', d.title, 'source', s.slug)) FROM claim_evidence e JOIN document d ON d.id = e.document_id JOIN source s ON s.id = d.source_id WHERE e.claim_id = ch.claim_id AND {source_sql}) AS evidence FROM chunk ch LEFT JOIN claim c ON c.id = ch.claim_id LEFT JOIN person p ON p.id = c.person_id LEFT JOIN organization o ON o.id = c.organization_id LEFT JOIN document dd ON dd.id = ch.document_id LEFT JOIN source ds ON ds.id = dd.source_id WHERE (ch.claim_id IS NULL OR ({VISIBLE})) AND (ch.document_id IS NULL OR (NOT (ds.slug = ANY(%(exclude_sources)s)) AND ds.default_trust >= %(min_trust)s)) AND (%(since)s::date IS NULL OR COALESCE(c.occurred_at::date, c.valid_from, dd.published_at::date) >= %(since)s) AND (%(until)s::date IS NULL OR COALESCE(c.occurred_at::date, c.valid_from, dd.published_at::date) <= %(until)s) """ fts = conn.execute( base + " AND ch.tsv @@ websearch_to_tsquery('german', %(q)s)" " ORDER BY ts_rank_cd(ch.tsv, websearch_to_tsquery('german', %(q)s)) DESC LIMIT %(limit)s", params, ).fetchall() ranked: dict[int, tuple[float, dict]] = {} for rank, row in enumerate(fts): ranked[row["id"]] = (1 / (60 + rank), row) if settings.ollama_url: try: resp = httpx.post(f"{settings.ollama_url}/api/embed", json={"model": settings.embedding_model, "input": [q]}, timeout=30) resp.raise_for_status() params["vec"] = str(resp.json()["embeddings"][0]) vec_rows = conn.execute( base + " AND ch.embedding IS NOT NULL ORDER BY ch.embedding <=> %(vec)s::vector LIMIT %(limit)s", params, ).fetchall() for rank, row in enumerate(vec_rows): score = ranked.get(row["id"], (0.0, row))[0] + 1 / (60 + rank) ranked[row["id"]] = (score, row) except httpx.HTTPError: pass results = [row | {"score": score} for score, row in sorted(ranked.values(), key=lambda x: -x[0])] # Hide results whose only evidence comes from sources the user excluded. return [r for r in results if r["evidence"] or r["claim_id"] is None][:limit] # -- moderation -----------------------------------------------------------------