"""Keyword-based topic tagging for claims that did not get topics from their source. Parliament items carry official topics; news mentions, donations and posts are tagged here with the keyword lists of the `topic` table. An optional LLM classifier can replace this later (see docs/roadmap.md). """ import re from politrack.core import Enricher, RunContext, register from politrack.core.spec import DERIVED def keyword_pattern(keywords: list[str]) -> re.Pattern: """Keywords match as word prefixes ("Klima" -> "Klimaschutz"); short ones (<= 3 chars) only as whole words.""" parts = [re.escape(k) + (r"\b" if len(k) <= 3 else "") for k in keywords] return re.compile(r"\b(" + "|".join(parts) + ")", re.IGNORECASE) @register class TopicTagger(Enricher): spec = DERIVED name = "enrich.topics" description = "Keyword topics for claims without official topics" schedule = "8-59/10 * * * *" def run(self, ctx: RunContext) -> None: while (n := tag_claims(ctx.conn)): ctx.stats["claims_tagged"] += n return None def tag_claims(conn, batch: int = 2000) -> int: topics = [ (row["id"], keyword_pattern(row["keywords"])) for row in conn.execute("SELECT id, keywords FROM topic WHERE cardinality(keywords) > 0") ] claims = conn.execute( """ SELECT c.id, c.summary_de || ' ' || COALESCE(string_agg(d.title, ' '), '') AS text FROM claim c LEFT JOIN claim_evidence e ON e.claim_id = c.id LEFT JOIN document d ON d.id = e.document_id WHERE c.claim_type IN ('media_mention', 'statement', 'donation', 'side_job', 'appearance', 'meeting', 'travel', 'money_flow', 'government_decision', 'wiki_edit', 'career') AND NOT (c.data ? 'topics_tagged') GROUP BY c.id ORDER BY c.id LIMIT %s """, (batch,), ).fetchall() for claim in claims: for topic_id, pattern in topics: if pattern.search(claim["text"]): conn.execute( "INSERT INTO topic_assignment (topic_id, claim_id, confidence, method) " "VALUES (%s, %s, 0.5, 'keyword') ON CONFLICT DO NOTHING", (topic_id, claim["id"]), ) conn.execute("UPDATE claim SET data = data || '{\"topics_tagged\": true}' WHERE id = %s", (claim["id"],)) conn.commit() return len(claims)