diff --git a/backend/politrack/enrich/embed.py b/backend/politrack/enrich/embed.py index c7294fd..4d82f17 100644 --- a/backend/politrack/enrich/embed.py +++ b/backend/politrack/enrich/embed.py @@ -78,12 +78,12 @@ def build_chunks(conn: Connection, batch: int = 5000) -> int: docs = conn.execute( """ SELECT d.id, d.title, d.full_text FROM document d - WHERE d.full_text IS NOT NULL AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.document_id = d.id) + WHERE d.full_text ~ '\S' AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.document_id = d.id) LIMIT %s """, (batch // 10,), ).fetchall() - for d in docs: + for d in docs: # blank texts are excluded above: they yield no chunks and would be selected forever for i, part in enumerate(split_text(d["full_text"])): conn.execute(UPSERT_CHUNK, (None, d["id"], i, f"{d['title'] or ''}\n{part}".strip())) conn.commit()