Stop the chunker looping forever on blank documents
A document whose text is only whitespace produces no chunks, so it was selected again on every pass and enrich.embed never finished. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9c58bb3d8f
commit
1f945b8bad
1 file changed
+2
-2
@@ -78,12 +78,12 @@ def build_chunks(conn: Connection, batch: int = 5000) -> int:
|
||||
docs = conn.execute(
|
||||
"""
|
||||
SELECT d.id, d.title, d.full_text FROM document d
|
||||
WHERE d.full_text IS NOT NULL AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.document_id = d.id)
|
||||
WHERE d.full_text ~ '\S' AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.document_id = d.id)
|
||||
LIMIT %s
|
||||
""",
|
||||
(batch // 10,),
|
||||
).fetchall()
|
||||
for d in docs:
|
||||
for d in docs: # blank texts are excluded above: they yield no chunks and would be selected forever
|
||||
for i, part in enumerate(split_text(d["full_text"])):
|
||||
conn.execute(UPSERT_CHUNK, (None, d["id"], i, f"{d['title'] or ''}\n{part}".strip()))
|
||||
conn.commit()
|
||||
|
||||
Reference in new issue
Block a user