Stop the chunker looping forever on blank documents
A document whose text is only whitespace produces no chunks, so it was selected again on every pass and enrich.embed never finished. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9c58bb3d8f
commit
1f945b8bad
1 file changed
+2
-2
@@ -78,12 +78,12 @@ def build_chunks(conn: Connection, batch: int = 5000) -> int:
|
|||||||
docs = conn.execute(
|
docs = conn.execute(
|
||||||
"""
|
"""
|
||||||
SELECT d.id, d.title, d.full_text FROM document d
|
SELECT d.id, d.title, d.full_text FROM document d
|
||||||
WHERE d.full_text IS NOT NULL AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.document_id = d.id)
|
WHERE d.full_text ~ '\S' AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.document_id = d.id)
|
||||||
LIMIT %s
|
LIMIT %s
|
||||||
""",
|
""",
|
||||||
(batch // 10,),
|
(batch // 10,),
|
||||||
).fetchall()
|
).fetchall()
|
||||||
for d in docs:
|
for d in docs: # blank texts are excluded above: they yield no chunks and would be selected forever
|
||||||
for i, part in enumerate(split_text(d["full_text"])):
|
for i, part in enumerate(split_text(d["full_text"])):
|
||||||
conn.execute(UPSERT_CHUNK, (None, d["id"], i, f"{d['title'] or ''}\n{part}".strip()))
|
conn.execute(UPSERT_CHUNK, (None, d["id"], i, f"{d['title'] or ''}\n{part}".strip()))
|
||||||
conn.commit()
|
conn.commit()
|
||||||
|
|||||||
Reference in new issue
Block a user