From 1f945b8baddd4af3c5f21a803c3c0fa7a0e75384 Mon Sep 17 00:00:00 2001 From: Simon Diesenreiter Date: Mon, 9 Nov 2026 09:56:58 +0100 Subject: [PATCH] Stop the chunker looping forever on blank documents A document whose text is only whitespace produces no chunks, so it was selected again on every pass and enrich.embed never finished. Co-Authored-By: Claude Opus 5.5 --- backend/politrack/enrich/embed.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/politrack/enrich/embed.py b/backend/politrack/enrich/embed.py index c7294fd..4d82f17 100644 --- a/backend/politrack/enrich/embed.py +++ b/backend/politrack/enrich/embed.py @@ -78,12 +78,12 @@ def build_chunks(conn: Connection, batch: int = 5000) -> int: docs = conn.execute( """ SELECT d.id, d.title, d.full_text FROM document d - WHERE d.full_text IS NOT NULL AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.document_id = d.id) + WHERE d.full_text ~ '\S' AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.document_id = d.id) LIMIT %s """, (batch // 10,), ).fetchall() - for d in docs: + for d in docs: # blank texts are excluded above: they yield no chunks and would be selected forever for i, part in enumerate(split_text(d["full_text"])): conn.execute(UPSERT_CHUNK, (None, d["id"], i, f"{d['title'] or ''}\n{part}".strip())) conn.commit()