Files
simonandClaude Opus 5.5 b06f17e587 Add EP amendment and part votes, hidden from general lists by default
They come from the same responses as final votes. A sitting has over a
hundred, so they are flagged part_vote and only appear on request: the vote
lists have an option for them, feeds, counts and search chunks leave them out.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-09 13:17:10 +01:00

120 lines
4.6 KiB
Python

"""Chunking + embeddings for retrieval (RAG).
Every claim gets one chunk (its summary plus evidence titles) so agents can
retrieve facts *with citations*; documents with licensed full text are split
into ~1,200 character chunks. Embeddings are computed by an Ollama server
(model bge-m3, multilingual, 1024 dims) if POLITRACK_OLLAMA_URL is set;
otherwise only German full-text search is available.
"""
import httpx
from psycopg import Connection
from politrack.config import get_settings
from politrack.core import Enricher, RunContext, register
from politrack.core.spec import DERIVED
CHUNK_SIZE = 1200
UPSERT_CHUNK = """
INSERT INTO chunk (claim_id, document_id, chunk_index, text, lang) VALUES (%s, %s, %s, %s, 'de')
ON CONFLICT (COALESCE(document_id, 0), COALESCE(claim_id, 0), chunk_index)
DO UPDATE SET text = EXCLUDED.text, embedding = CASE WHEN chunk.text = EXCLUDED.text THEN chunk.embedding END
"""
@register
class Chunker(Enricher):
spec = DERIVED
name = "enrich.embed"
description = "Build search chunks for claims/documents and embed them (Ollama, optional)"
schedule = "*/15 * * * *"
def run(self, ctx: RunContext) -> None:
while (n := build_chunks(ctx.conn)):
ctx.stats["chunks"] += n
ctx.stats["embedded"] = embed_pending(ctx.conn)
return None
def split_text(text: str, size: int = CHUNK_SIZE) -> list[str]:
paragraphs, chunks, current = [p.strip() for p in text.split("\n") if p.strip()], [], ""
for p in paragraphs:
if current and len(current) + len(p) > size:
chunks.append(current)
current = ""
current = f"{current}\n{p}".strip()
while len(current) > size * 1.5:
chunks.append(current[:size])
current = current[size:]
if current:
chunks.append(current)
return chunks
def build_chunks(conn: Connection, batch: int = 5000) -> int:
claims = conn.execute(
"""
SELECT c.id, c.claim_type, c.summary_de, c.occurred_at, p.full_name, o.name AS org_name,
string_agg(DISTINCT s.name || ': ' || COALESCE(d.title, d.url), ' | ') AS sources
FROM claim c
LEFT JOIN person p ON p.id = c.person_id
LEFT JOIN organization o ON o.id = c.organization_id
JOIN claim_evidence e ON e.claim_id = c.id
JOIN document d ON d.id = e.document_id
JOIN source s ON s.id = d.source_id
WHERE c.merged_into IS NULL AND c.review_status = 'published'
AND NOT COALESCE((c.data->>'part_vote')::boolean, FALSE)
AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.claim_id = c.id)
GROUP BY c.id, p.full_name, o.name
LIMIT %s
""",
(batch,),
).fetchall()
for c in claims:
who = c["full_name"] or c["org_name"] or ""
when = c["occurred_at"].date().isoformat() if c["occurred_at"] else ""
text = f"[{c['claim_type']}] {when} {who}\n{c['summary_de']}\nQuellen: {c['sources']}"
conn.execute(UPSERT_CHUNK, (c["id"], None, 0, text.strip()))
docs = conn.execute(
r"""
SELECT d.id, d.title, d.full_text FROM document d
WHERE d.full_text ~ '\S' AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.document_id = d.id)
LIMIT %s
""",
(batch // 10,),
).fetchall()
for d in docs: # blank texts are excluded above: they yield no chunks and would be selected forever
for i, part in enumerate(split_text(d["full_text"])):
conn.execute(UPSERT_CHUNK, (None, d["id"], i, f"{d['title'] or ''}\n{part}".strip()))
conn.commit()
return len(claims) + len(docs)
def embed_texts(texts: list[str]) -> list[list[float]]:
settings = get_settings()
resp = httpx.post(f"{settings.ollama_url}/api/embed", json={"model": settings.embedding_model, "input": texts},
timeout=300)
resp.raise_for_status()
return resp.json()["embeddings"]
def embed_pending(conn: Connection, batch: int = 64, max_batches: int = 50) -> int:
settings = get_settings()
if not settings.ollama_url:
return 0
total = 0
for _ in range(max_batches):
rows = conn.execute("SELECT id, text FROM chunk WHERE embedding IS NULL ORDER BY id LIMIT %s", (batch,)).fetchall()
if not rows:
break
vectors = embed_texts([r["text"] for r in rows])
with conn.cursor() as cur:
cur.executemany(
"UPDATE chunk SET embedding = %s::vector, embedding_model = %s WHERE id = %s",
[(str(v), settings.embedding_model, r["id"]) for r, v in zip(rows, vectors)],
)
conn.commit()
total += len(rows)
return total