simonandClaude Opus 5.5
583ccba31c
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
62 lines
2.4 KiB
Python
62 lines
2.4 KiB
Python
"""Keyword-based topic tagging for claims that did not get topics from their source.
|
|
|
|
Parliament items carry official topics; news mentions, donations and posts
|
|
are tagged here with the keyword lists of the `topic` table. An optional LLM
|
|
classifier can replace this later (see docs/roadmap.md).
|
|
"""
|
|
|
|
import re
|
|
|
|
from politrack.core import Enricher, RunContext, register
|
|
from politrack.core.spec import DERIVED
|
|
|
|
|
|
def keyword_pattern(keywords: list[str]) -> re.Pattern:
|
|
"""Keywords match as word prefixes ("Klima" -> "Klimaschutz"); short ones (<= 3 chars) only as whole words."""
|
|
parts = [re.escape(k) + (r"\b" if len(k) <= 3 else "") for k in keywords]
|
|
return re.compile(r"\b(" + "|".join(parts) + ")", re.IGNORECASE)
|
|
|
|
|
|
@register
|
|
class TopicTagger(Enricher):
|
|
spec = DERIVED
|
|
name = "enrich.topics"
|
|
description = "Keyword topics for claims without official topics"
|
|
schedule = "8-59/10 * * * *"
|
|
|
|
def run(self, ctx: RunContext) -> None:
|
|
while (n := tag_claims(ctx.conn)):
|
|
ctx.stats["claims_tagged"] += n
|
|
return None
|
|
|
|
|
|
def tag_claims(conn, batch: int = 2000) -> int:
|
|
topics = [
|
|
(row["id"], keyword_pattern(row["keywords"]))
|
|
for row in conn.execute("SELECT id, keywords FROM topic WHERE cardinality(keywords) > 0")
|
|
]
|
|
claims = conn.execute(
|
|
"""
|
|
SELECT c.id, c.summary_de || ' ' || COALESCE(string_agg(d.title, ' '), '') AS text
|
|
FROM claim c
|
|
LEFT JOIN claim_evidence e ON e.claim_id = c.id
|
|
LEFT JOIN document d ON d.id = e.document_id
|
|
WHERE c.claim_type IN ('media_mention', 'statement', 'donation', 'side_job', 'appearance', 'meeting', 'travel',
|
|
'money_flow', 'government_decision', 'wiki_edit', 'career')
|
|
AND NOT (c.data ? 'topics_tagged')
|
|
GROUP BY c.id ORDER BY c.id LIMIT %s
|
|
""",
|
|
(batch,),
|
|
).fetchall()
|
|
for claim in claims:
|
|
for topic_id, pattern in topics:
|
|
if pattern.search(claim["text"]):
|
|
conn.execute(
|
|
"INSERT INTO topic_assignment (topic_id, claim_id, confidence, method) "
|
|
"VALUES (%s, %s, 0.5, 'keyword') ON CONFLICT DO NOTHING",
|
|
(topic_id, claim["id"]),
|
|
)
|
|
conn.execute("UPDATE claim SET data = data || '{\"topics_tagged\": true}' WHERE id = %s", (claim["id"],))
|
|
conn.commit()
|
|
return len(claims)
|