Stop the chunker looping forever on blank documents

A document whose text is only whitespace produces no chunks, so it was
selected again on every pass and enrich.embed never finished.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
simonandClaude Opus 5.5 committed 2026-11-09 09:56:58 +01:00
1 parent 9c58bb3d8f
commit 1f945b8bad
1 file changed
+2 -2
+2 -2
View File
@@ -78,12 +78,12 @@ def build_chunks(conn: Connection, batch: int = 5000) -> int:
docs = conn.execute(
"""
SELECT d.id, d.title, d.full_text FROM document d
WHERE d.full_text IS NOT NULL AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.document_id = d.id)
WHERE d.full_text ~ '\S' AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.document_id = d.id)
LIMIT %s
""",
(batch // 10,),
).fetchall()
for d in docs:
for d in docs: # blank texts are excluded above: they yield no chunks and would be selected forever
for i, part in enumerate(split_text(d["full_text"])):
conn.execute(UPSERT_CHUNK, (None, d["id"], i, f"{d['title'] or ''}\n{part}".strip()))
conn.commit()