Politrack: track Austrian politicians from public sources
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
583ccba31c
187 files changed
+39890
No files matched your search
@@ -0,0 +1,61 @@
|
||||
"""Keyword-based topic tagging for claims that did not get topics from their source.
|
||||
|
||||
Parliament items carry official topics; news mentions, donations and posts
|
||||
are tagged here with the keyword lists of the `topic` table. An optional LLM
|
||||
classifier can replace this later (see docs/roadmap.md).
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
from politrack.core import Enricher, RunContext, register
|
||||
from politrack.core.spec import DERIVED
|
||||
|
||||
|
||||
def keyword_pattern(keywords: list[str]) -> re.Pattern:
|
||||
"""Keywords match as word prefixes ("Klima" -> "Klimaschutz"); short ones (<= 3 chars) only as whole words."""
|
||||
parts = [re.escape(k) + (r"\b" if len(k) <= 3 else "") for k in keywords]
|
||||
return re.compile(r"\b(" + "|".join(parts) + ")", re.IGNORECASE)
|
||||
|
||||
|
||||
@register
|
||||
class TopicTagger(Enricher):
|
||||
spec = DERIVED
|
||||
name = "enrich.topics"
|
||||
description = "Keyword topics for claims without official topics"
|
||||
schedule = "8-59/10 * * * *"
|
||||
|
||||
def run(self, ctx: RunContext) -> None:
|
||||
while (n := tag_claims(ctx.conn)):
|
||||
ctx.stats["claims_tagged"] += n
|
||||
return None
|
||||
|
||||
|
||||
def tag_claims(conn, batch: int = 2000) -> int:
|
||||
topics = [
|
||||
(row["id"], keyword_pattern(row["keywords"]))
|
||||
for row in conn.execute("SELECT id, keywords FROM topic WHERE cardinality(keywords) > 0")
|
||||
]
|
||||
claims = conn.execute(
|
||||
"""
|
||||
SELECT c.id, c.summary_de || ' ' || COALESCE(string_agg(d.title, ' '), '') AS text
|
||||
FROM claim c
|
||||
LEFT JOIN claim_evidence e ON e.claim_id = c.id
|
||||
LEFT JOIN document d ON d.id = e.document_id
|
||||
WHERE c.claim_type IN ('media_mention', 'statement', 'donation', 'side_job', 'appearance', 'meeting', 'travel',
|
||||
'money_flow', 'government_decision', 'wiki_edit', 'career')
|
||||
AND NOT (c.data ? 'topics_tagged')
|
||||
GROUP BY c.id ORDER BY c.id LIMIT %s
|
||||
""",
|
||||
(batch,),
|
||||
).fetchall()
|
||||
for claim in claims:
|
||||
for topic_id, pattern in topics:
|
||||
if pattern.search(claim["text"]):
|
||||
conn.execute(
|
||||
"INSERT INTO topic_assignment (topic_id, claim_id, confidence, method) "
|
||||
"VALUES (%s, %s, 0.5, 'keyword') ON CONFLICT DO NOTHING",
|
||||
(topic_id, claim["id"]),
|
||||
)
|
||||
conn.execute("UPDATE claim SET data = data || '{\"topics_tagged\": true}' WHERE id = %s", (claim["id"],))
|
||||
conn.commit()
|
||||
return len(claims)
|
||||
Reference in new issue
Block a user