Politrack: track Austrian politicians from public sources

Initial import of Politrack, a self-hosted service that tracks Austrian
politicians from federal to municipal level - roles, votes, money flows,
statements, meetings, social media and legal proceedings - with every claim
linked to its sources.

- backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse ->
  load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive
  for reparsing, and a claim/evidence model with corroboration
- sources: Parliament, Wikidata, EU Parliament/Commission, Council of
  Ministers, Rechnungshof party reports and donations, procurement, RTR media
  transparency, political ad libraries, WKStA, news feeds, Wikipedia edits,
  Bluesky, Mastodon, public Telegram channels, YouTube and X (official API)
- web: Next.js frontend in German and English with typed API client
- mcp: MCP server for AI agents and RAG
- docs: architecture, data model, sources, legal notes, Unraid deployment and
  contributor guides; Docker Compose setup for Unraid

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
simonandClaude Opus 5.5 committed 2026-10-28 20:02:01 +01:00
commit 583ccba31c
187 files changed
+39890

No files matched your search

Whitespace-only changes.
+244
View File
@@ -0,0 +1,244 @@
"""Corroboration: find claims from different sources that state the same fact and merge them.
Merging keeps the oldest claim, copies the duplicate's evidence onto it and marks
the duplicate with `merged_into`. Afterwards `refresh_claim_status` marks the
survivor "corroborated" when at least two *independent sources* support it.
Matchers:
* roles: same person, equivalent body/office, overlapping or near-identical periods
(e.g. Wikidata "Abgeordneter zum Nationalrat" == Parliament mandate)
* stories: news articles about the same event (similar headlines, shared people,
within 72 hours) are grouped; mentions of a person in one story are merged
* events: trips/meetings of the same person to the same place/counterpart within 3 days
"""
from collections import defaultdict
from datetime import timedelta
from psycopg import Connection
from rapidfuzz import fuzz
from politrack.core import Enricher, RunContext, register
from politrack.core.spec import DERIVED
from politrack.enrich.events import counterpart_anchor
from politrack.text import normalize
REFRESH_STATUS = """
UPDATE claim SET status = CASE
WHEN status = 'retracted' THEN status
WHEN EXISTS (SELECT 1 FROM claim_evidence e WHERE e.claim_id = claim.id AND e.stance = 'contradicts') THEN 'disputed'
WHEN (SELECT count(DISTINCT d.source_id) FROM claim_evidence e JOIN document d ON d.id = e.document_id
WHERE e.claim_id = claim.id AND e.stance IN ('supports', 'mentions')) >= 2 THEN 'corroborated'
ELSE 'reported' END
WHERE id = %s
"""
def merge_claims(conn: Connection, keep_id: int, dup_id: int) -> None:
if keep_id == dup_id:
return
conn.execute(
"""
INSERT INTO claim_evidence (claim_id, document_id, stance, quote, locator, extracted_by, confidence)
SELECT %s, document_id, stance, quote, locator, extracted_by, confidence
FROM claim_evidence WHERE claim_id = %s
ON CONFLICT (claim_id, document_id, locator) DO NOTHING
""",
(keep_id, dup_id),
)
conn.execute(
"""
INSERT INTO topic_assignment (topic_id, claim_id, confidence, method)
SELECT topic_id, %s, confidence, method FROM topic_assignment WHERE claim_id = %s
ON CONFLICT DO NOTHING
""",
(keep_id, dup_id),
)
conn.execute("UPDATE claim SET merged_into = %s, updated_at = now() WHERE id = %s", (keep_id, dup_id))
conn.execute("UPDATE claim SET merged_into = %s WHERE merged_into = %s", (keep_id, dup_id))
conn.execute(REFRESH_STATUS, (keep_id,))
# -- roles ---------------------------------------------------------------------
BODY_KEYWORDS = {
"nationalrat": ("nationalrat",),
"bundesrat": ("bundesrat", "bundesrates"),
}
def _same_office(a: dict, b: dict) -> bool:
"""a: authoritative role (Parliament), b: other source (Wikidata)."""
if a["role"] == "mp" and b["role"] == "mp":
return a["body_slug"] == b["body_slug"]
label = normalize(b["role"])
if a["role"] == "mp" and a["body_slug"] in BODY_KEYWORDS:
return any(k in label for k in BODY_KEYWORDS[a["body_slug"]]) and "prasident" not in label
return fuzz.token_set_ratio(normalize(a["role"]), label) >= 80
def _periods_match(a: dict, b: dict) -> bool:
a_start, b_start = a["start_date"], b["start_date"]
a_end, b_end = a["end_date"], b["end_date"]
if a_start and b_start and abs((a_start - b_start).days) <= 60:
return True
# Wikidata often only records one term for several consecutive mandates: accept containment.
if a_start and b_start and b_start <= a_start and (b_end is None or a_end is None or a_end <= b_end + timedelta(days=60)):
return True
return False
def merge_roles(conn: Connection) -> int:
rows = conn.execute(
"""
SELECT r.claim_id, r.person_id, r.role, r.start_date, r.end_date, c.created_by, o.slug AS body_slug
FROM role r JOIN claim c ON c.id = r.claim_id LEFT JOIN organization o ON o.id = r.organization_id
WHERE c.merged_into IS NULL AND r.level <> 'party'
AND (c.created_by LIKE 'parlament%%' OR c.created_by LIKE 'wikidata%%' OR c.created_by LIKE 'europarl%%')
ORDER BY r.person_id, r.start_date NULLS LAST
"""
).fetchall()
by_person: dict[int, dict[str, list[dict]]] = defaultdict(lambda: defaultdict(list))
for r in rows:
family = "wikidata" if r["created_by"].startswith("wikidata") else "parlament"
by_person[r["person_id"]][family].append(r)
merged = 0
for roles in by_person.values():
for other in roles.get("wikidata", []):
match = next((a for a in roles.get("parlament", []) if _same_office(a, other) and _periods_match(a, other)),
None)
if match:
merge_claims(conn, match["claim_id"], other["claim_id"])
merged += 1
conn.commit()
return merged
# -- news stories -------------------------------------------------------------
def cluster_stories(conn: Connection, days: int = 14) -> int:
"""Group recent news documents about the same thing into stories."""
docs = conn.execute(
"""
SELECT d.id, d.title, d.published_at, d.story_id, d.source_id, d.meta->'people' AS people
FROM document d JOIN source s ON s.id = d.source_id
WHERE (s.kind = 'media' OR s.slug = 'gdelt') AND d.published_at > now() - make_interval(days => %s)
AND d.title IS NOT NULL
ORDER BY d.published_at
""",
(days,),
).fetchall()
stories: list[dict] = [] # {id, docs: [...]}
for doc in docs:
title = normalize(doc["title"])
people = set(doc["people"] or [])
best, best_score = None, 0
for story in stories[-400:]:
lead = story["docs"][0]
if doc["published_at"] - lead["published_at"] > timedelta(hours=72):
continue
score = fuzz.token_set_ratio(title, lead["norm"])
shared = people & lead["people"]
if score >= 82 or (shared and score >= 60):
if score > best_score:
best, best_score = story, score
entry = {"id": doc["id"], "norm": title, "people": people, "published_at": doc["published_at"],
"source_id": doc["source_id"], "story_id": doc["story_id"], "title": doc["title"]}
if best:
best["docs"].append(entry)
else:
stories.append({"docs": [entry]})
assigned = 0
for story in stories:
if len(story["docs"]) < 2:
continue
existing = next((d["story_id"] for d in story["docs"] if d["story_id"]), None)
lead = story["docs"][0]
if existing is None:
existing = conn.execute(
"INSERT INTO story (title, first_seen) VALUES (%s, %s) RETURNING id", (lead["title"], lead["published_at"])
).fetchone()["id"]
ids = [d["id"] for d in story["docs"]]
conn.execute("UPDATE document SET story_id = %s WHERE id = ANY(%s)", (existing, ids))
conn.execute(
"""
UPDATE story SET last_seen = sub.last_seen, document_count = sub.n, source_count = sub.sources
FROM (SELECT max(published_at) AS last_seen, count(*) AS n, count(DISTINCT source_id) AS sources
FROM document WHERE story_id = %s) sub
WHERE story.id = %s
""",
(existing, existing),
)
assigned += len(ids)
conn.commit()
return assigned
def merge_story_mentions(conn: Connection) -> int:
"""One media_mention claim per (person or party, story); other outlets become additional evidence."""
rows = conn.execute(
"""
SELECT c.id, c.person_id, c.organization_id, d.story_id, c.occurred_at
FROM claim c
JOIN claim_evidence e ON e.claim_id = c.id
JOIN document d ON d.id = e.document_id
WHERE c.claim_type = 'media_mention' AND c.merged_into IS NULL AND d.story_id IS NOT NULL
ORDER BY c.occurred_at NULLS LAST, c.id
"""
).fetchall()
keep: dict[tuple, int] = {}
merged = 0
for r in rows:
key = (r["person_id"], r["organization_id"], r["story_id"])
if key in keep:
if keep[key] != r["id"]:
merge_claims(conn, keep[key], r["id"])
merged += 1
else:
keep[key] = r["id"]
conn.commit()
return merged
# -- events --------------------------------------------------------------------
def merge_events(conn: Connection) -> int:
rows = conn.execute(
"""
SELECT c.id, c.person_id, c.claim_type, c.occurred_at, c.data->>'country' AS country,
c.data->>'counterpart' AS counterpart
FROM claim c
WHERE c.claim_type IN ('travel', 'meeting') AND c.merged_into IS NULL
ORDER BY c.person_id, c.claim_type, c.occurred_at
"""
).fetchall()
merged = 0
previous: dict[tuple, dict] = {}
for r in rows:
anchor = r["country"] if r["claim_type"] == "travel" else counterpart_anchor(r["counterpart"] or "")
key = (r["person_id"], r["claim_type"], anchor)
prev = previous.get(key)
if prev and r["occurred_at"] and prev["occurred_at"] and r["occurred_at"] - prev["occurred_at"] <= timedelta(days=3):
merge_claims(conn, prev["id"], r["id"])
merged += 1
continue
previous[key] = r
conn.commit()
return merged
@register
class Corroborator(Enricher):
spec = DERIVED
name = "enrich.corroborate"
description = "Merge near-duplicate claims from different sources; cluster news stories"
schedule = "9-59/10 * * * *"
def run(self, ctx: RunContext) -> None:
conn = ctx.conn
ctx.stats["roles_merged"] = merge_roles(conn)
ctx.stats["story_documents"] = cluster_stories(conn)
ctx.stats["story_mentions_merged"] = merge_story_mentions(conn)
ctx.stats["events_merged"] = merge_events(conn)
return None
+74
View File
@@ -0,0 +1,74 @@
"""Curated lists maintained by the operator in config/ (accounts that no official register lists).
* config.social_accounts - config/social_accounts.yaml -> social_account rows for persons/organisations,
e.g. party Telegram channels or Bluesky accounts linked from party websites. The social adapters then
collect their public posts.
"""
from __future__ import annotations
from collections.abc import Iterator
from typing import Any
import yaml
from politrack.config import get_settings
from politrack.core import Enricher, RunContext, SourceSpec, register
from politrack.core.records import OrgRef, PersonRef, SocialAccount
CURATED = SourceSpec(
slug="curated",
name="Kuratierte Listen (config/)",
kind="derived",
homepage="",
license="operator-maintained",
access="manual",
trust=60,
tos_notes="Entries maintained by the operator; each should name the public page that links the account.",
)
def person_ref(ctx: RunContext, entry: dict[str, Any]) -> PersonRef | None:
"""`person: <politrack slug>`, `person: {wikidata: Q..}` / `{parlament: ..}` or `name: <full name>`."""
person = entry.get("person")
if isinstance(person, dict):
return PersonRef(ids={k: str(v) for k, v in person.items()})
if isinstance(person, str):
row = ctx.conn.execute("SELECT id FROM person WHERE slug = %s", (person,)).fetchone()
return PersonRef(ids={"politrack": str(row["id"])}) if row else None
if entry.get("name"):
return PersonRef(name=entry["name"])
return None
def org_ref(entry: dict[str, Any]) -> OrgRef | None:
"""`org: <slug>` for existing organisations, or `org: {slug, name, kind}` to create one."""
org = entry.get("org")
if isinstance(org, str):
return OrgRef(slug=org, create=False)
if isinstance(org, dict):
return OrgRef(slug=org.get("slug"), name=org.get("name"), short_name=org.get("short_name"),
kind=org.get("kind", "other"), create=bool(org.get("name")))
return None
@register
class CuratedSocialAccounts(Enricher):
spec = CURATED
name = "config.social_accounts"
description = "Import social accounts from config/social_accounts.yaml"
schedule = "5 3 * * *"
def run(self, ctx: RunContext) -> Iterator[SocialAccount]:
path = get_settings().config_dir / "social_accounts.yaml"
if not path.exists():
return
for entry in (yaml.safe_load(path.read_text(encoding="utf-8")) or {}).get("accounts") or []:
person = person_ref(ctx, entry)
org = None if person else org_ref(entry)
if person is None and org is None:
ctx.stats["entries_without_owner"] += 1
ctx.log.warning("social_accounts.yaml: no person/org for %s", entry.get("url"))
continue
yield SocialAccount(platform=entry["platform"], url=entry["url"], handle=entry.get("handle"),
person=person, org=org, verified=bool(entry.get("verified")))
+118
View File
@@ -0,0 +1,118 @@
"""Chunking + embeddings for retrieval (RAG).
Every claim gets one chunk (its summary plus evidence titles) so agents can
retrieve facts *with citations*; documents with licensed full text are split
into ~1,200 character chunks. Embeddings are computed by an Ollama server
(model bge-m3, multilingual, 1024 dims) if POLITRACK_OLLAMA_URL is set;
otherwise only German full-text search is available.
"""
import httpx
from psycopg import Connection
from politrack.config import get_settings
from politrack.core import Enricher, RunContext, register
from politrack.core.spec import DERIVED
CHUNK_SIZE = 1200
UPSERT_CHUNK = """
INSERT INTO chunk (claim_id, document_id, chunk_index, text, lang) VALUES (%s, %s, %s, %s, 'de')
ON CONFLICT (COALESCE(document_id, 0), COALESCE(claim_id, 0), chunk_index)
DO UPDATE SET text = EXCLUDED.text, embedding = CASE WHEN chunk.text = EXCLUDED.text THEN chunk.embedding END
"""
@register
class Chunker(Enricher):
spec = DERIVED
name = "enrich.embed"
description = "Build search chunks for claims/documents and embed them (Ollama, optional)"
schedule = "*/15 * * * *"
def run(self, ctx: RunContext) -> None:
while (n := build_chunks(ctx.conn)):
ctx.stats["chunks"] += n
ctx.stats["embedded"] = embed_pending(ctx.conn)
return None
def split_text(text: str, size: int = CHUNK_SIZE) -> list[str]:
paragraphs, chunks, current = [p.strip() for p in text.split("\n") if p.strip()], [], ""
for p in paragraphs:
if current and len(current) + len(p) > size:
chunks.append(current)
current = ""
current = f"{current}\n{p}".strip()
while len(current) > size * 1.5:
chunks.append(current[:size])
current = current[size:]
if current:
chunks.append(current)
return chunks
def build_chunks(conn: Connection, batch: int = 5000) -> int:
claims = conn.execute(
"""
SELECT c.id, c.claim_type, c.summary_de, c.occurred_at, p.full_name, o.name AS org_name,
string_agg(DISTINCT s.name || ': ' || COALESCE(d.title, d.url), ' | ') AS sources
FROM claim c
LEFT JOIN person p ON p.id = c.person_id
LEFT JOIN organization o ON o.id = c.organization_id
JOIN claim_evidence e ON e.claim_id = c.id
JOIN document d ON d.id = e.document_id
JOIN source s ON s.id = d.source_id
WHERE c.merged_into IS NULL AND c.review_status = 'published'
AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.claim_id = c.id)
GROUP BY c.id, p.full_name, o.name
LIMIT %s
""",
(batch,),
).fetchall()
for c in claims:
who = c["full_name"] or c["org_name"] or ""
when = c["occurred_at"].date().isoformat() if c["occurred_at"] else ""
text = f"[{c['claim_type']}] {when} {who}\n{c['summary_de']}\nQuellen: {c['sources']}"
conn.execute(UPSERT_CHUNK, (c["id"], None, 0, text.strip()))
docs = conn.execute(
"""
SELECT d.id, d.title, d.full_text FROM document d
WHERE d.full_text IS NOT NULL AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.document_id = d.id)
LIMIT %s
""",
(batch // 10,),
).fetchall()
for d in docs:
for i, part in enumerate(split_text(d["full_text"])):
conn.execute(UPSERT_CHUNK, (None, d["id"], i, f"{d['title'] or ''}\n{part}".strip()))
conn.commit()
return len(claims) + len(docs)
def embed_texts(texts: list[str]) -> list[list[float]]:
settings = get_settings()
resp = httpx.post(f"{settings.ollama_url}/api/embed", json={"model": settings.embedding_model, "input": texts},
timeout=300)
resp.raise_for_status()
return resp.json()["embeddings"]
def embed_pending(conn: Connection, batch: int = 64, max_batches: int = 50) -> int:
settings = get_settings()
if not settings.ollama_url:
return 0
total = 0
for _ in range(max_batches):
rows = conn.execute("SELECT id, text FROM chunk WHERE embedding IS NULL ORDER BY id LIMIT %s", (batch,)).fetchall()
if not rows:
break
vectors = embed_texts([r["text"] for r in rows])
with conn.cursor() as cur:
cur.executemany(
"UPDATE chunk SET embedding = %s::vector, embedding_model = %s WHERE id = %s",
[(str(v), settings.embedding_model, r["id"]) for r, v in zip(rows, vectors)],
)
conn.commit()
total += len(rows)
return total
+179
View File
@@ -0,0 +1,179 @@
"""Extract trips and meetings from official press releases and news headlines.
Rule-based and conservative. A sentence becomes an event only if it names a
known politician and contains a travel or meeting trigger:
travel: "reist nach Brüssel", "Arbeitsbesuch in Israel", "beim Gipfel in Montenegro", "in China"
meeting: "empfängt den tschechischen Außenminister", "Treffen mit ...", "traf ... zu einem Gespräch"
Sources:
* Parlamentskorrespondenz and government news: title + full text (official works / CC BY),
* news headlines: title only (we do not store article text).
The event date is the publication date of the source (`data.date_precision =
"publication"`). The claim key is (kind, person, place/counterpart, ISO week),
so reports from different sources about the same trip attach to the same claim
as additional evidence and corroborate it. corroborate.merge_events() also
merges near-duplicates whose dates differ by a few days.
"""
from __future__ import annotations
import re
from collections.abc import Iterator
from datetime import datetime
from typing import Any
from psycopg.types.json import Jsonb
from politrack.core import Enricher, RunContext, register
from politrack.core.records import Claim, Event, Evidence, Participation, PersonRef
from politrack.core.spec import DERIVED
from politrack.enrich.gazetteer import AUSTRIA, PLACES
from politrack.enrich.linker import MentionLinker
from politrack.text import excerpt, fingerprint, normalize
# (?<!-)/(?!-): skip names inside compounds such as "Berlin-Prozess" or "EU-Brüssel-Treffen".
PLACE_RE = re.compile(
r"(?<![-\w])(" + "|".join(sorted((re.escape(p) for p in PLACES), key=len, reverse=True)) + r")(?![-\w])"
)
DESTINATION_RE = re.compile(r"\b(?:in|nach|aus)\s+(?:de[mnr]\s+|die\s+)?$")
STRONG_TRAVEL = re.compile(r"\b(reist\w*|Reise\w*|\w*[Bb]esuch\w*|\w*Gipfel\w*|Delegation\w*|Konferenz\w*|Tagung\w*)\b")
MEETING_RE = re.compile(
r"\b(?:empfängt|empfing|empfangen|trifft|traf|getroffen|Treffen mit|Gespräch mit|Arbeitsgespräch mit|"
r"Unterredung mit|tauschte sich mit|im Austausch mit|zusammengetroffen mit|kam mit)\s+"
r"(?P<counterpart>(?:de[mnr]|die|das|seine[mnr]?|ihre[mnr]?)?\s*[^.,;:]{3,90})",
re.IGNORECASE,
)
COUNTERPART_ROLE = re.compile(
r"(minister|präsident|kanzler|botschafter|delegation|amtskolleg|vertreter|bürgermeister|abgeordnet|chef|"
r"premier|könig|papst|kommissar|generalsekretär|regierungschef|staatssekretär|landeshauptm|gouverneur|"
r"bischof|kardinal|vorsitzend|ceo|gewerkschaft|sozialpartner|außenminister)",
re.IGNORECASE,
)
FULL_NAME = re.compile(r"\b[A-ZÄÖÜ][a-zäöüß]+(?:-[A-ZÄÖÜ][a-zäöüß]+)?\s+[A-ZÄÖÜ][a-zäöüß]+")
SENTENCE_SPLIT = re.compile(r"(?<=[.!?])\s+(?=[A-ZÄÖÜ„\"])|\n+")
FILLER = re.compile(r"^(?:(?:heute|gestern|morgen|am|vorgestern|Montag|Dienstag|Mittwoch|Donnerstag|Freitag|"
r"Samstag|Sonntag|de[mnr]|die|das|seine[mnr]?|ihre[mnr]?|eine[mnr]?)\s+)+", re.IGNORECASE)
def clean_counterpart(text: str) -> str:
return FILLER.sub("", text.strip()).strip()
def counterpart_anchor(text: str) -> str:
"""Stable key for a counterpart: its last capitalised word (usually the surname or country)."""
words = re.findall(r"[A-ZÄÖÜ][\wäöüß-]+", clean_counterpart(text))
return normalize(words[-1]) if words else normalize(text)[:25]
def find_places(text: str) -> list[tuple[str, str]]:
"""Places in order of preference: those introduced by "in/nach" first (the destination)."""
matches = list(PLACE_RE.finditer(text))
preferred = [m for m in matches if DESTINATION_RE.search(text[: m.start()])]
ordered = preferred + [m for m in matches if m not in preferred]
return [(m.group(1), PLACES[m.group(1)]) for m in ordered]
def detect(text: str) -> list[dict]:
"""Return candidate events found in one sentence (no person resolution)."""
events = []
places = find_places(text)
foreign = [(p, iso) for p, iso in places if iso != "AT"]
if foreign and STRONG_TRAVEL.search(text):
place, iso = foreign[0]
events.append({"kind": "foreign_trip", "claim_type": "travel", "place": place, "country": iso})
meeting = MEETING_RE.search(text)
if meeting:
counterpart = re.split(r"\s+(?:in|im|am|zu|über|bei|und|sowie)\s+", meeting.group("counterpart").strip())[0]
place, iso = (places[0] if places else (None, None))
# "traf eine Entscheidung" is not a meeting: require a role word or a person-like name.
if COUNTERPART_ROLE.search(counterpart) or FULL_NAME.search(counterpart):
events.append({"kind": "meeting", "claim_type": "meeting", "counterpart": clean_counterpart(counterpart),
"place": place, "country": iso})
return events
@register
class EventExtractor(Enricher):
spec = DERIVED
name = "enrich.events"
description = "Trips and meetings from press releases, government news and headlines"
schedule = "3-59/10 * * * *"
def run(self, ctx: RunContext) -> Iterator[Any]:
conn = ctx.conn
linker = MentionLinker(conn)
while True:
docs = conn.execute(
"""SELECT d.id, d.url, d.title, d.full_text, d.published_at, d.meta, s.slug AS source, s.kind AS source_kind
FROM document d JOIN source s ON s.id = d.source_id
WHERE NOT (d.meta ? 'events_at') AND d.published_at IS NOT NULL
AND (d.doc_type = 'press_release' OR s.kind = 'media' OR s.slug = 'gdelt')
ORDER BY d.id LIMIT 300"""
).fetchall()
if not docs:
return
for doc in docs:
found = 0
for record in self.document_events(conn, linker, doc):
found += isinstance(record, Claim)
yield record
conn.execute("UPDATE document SET meta = meta || %s WHERE id = %s",
(Jsonb({"events_at": datetime.now().isoformat(), "events": found}), doc["id"]))
ctx.stats["documents"] += 1
def document_events(self, conn, linker: MentionLinker, doc: dict) -> Iterator[Any]:
official = doc["source_kind"] == "official"
title = doc["title"] or ""
body = (doc["full_text"] or "")[:4000] if official else ""
pads = (doc["meta"] or {}).get("person_pads") or []
doc_people: set[int] = set()
if pads:
doc_people = {r["entity_id"] for r in conn.execute(
"SELECT entity_id FROM identifier WHERE entity_type = 'person' AND scheme = 'parlament' AND value = ANY(%s)",
([str(p) for p in pads],))}
sentences = [title] + [s for s in SENTENCE_SPLIT.split(body) if s][:25]
for i, sentence in enumerate(sentences):
candidates = detect(sentence)
if not candidates:
continue
people = linker.all_people(sentence)
# In the headline of an official release the actors are usually the people it links/names.
if not people and i == 0:
people = doc_people or linker.all_people(f"{title} {body[:600]}")
for person_id in people:
for ev in candidates:
yield from event_records(conn, doc, person_id, ev, sentence, official)
def event_records(conn, doc: dict, person_id: int, ev: dict, sentence: str, official: bool) -> Iterator[Any]:
published = doc["published_at"]
week = published.strftime("%G-W%V")
anchor = ev.get("country") if ev["kind"] == "foreign_trip" else counterpart_anchor(ev.get("counterpart") or "")
row = conn.execute("SELECT full_name, family_name FROM person WHERE id = %s", (person_id,)).fetchone()
# A sentence like "X empfing Y" yields the meeting for X and Y; never record "Y met Y".
if ev["kind"] == "meeting" and row["family_name"] and normalize(row["family_name"]) in normalize(ev["counterpart"]):
return
if ev["kind"] == "foreign_trip":
summary_de, summary_en = f"Reise/Aufenthalt in {ev['place']}", f"Trip to {ev['place']}"
else:
where = f" ({ev['place']})" if ev.get("place") else ""
summary_de = f"Treffen mit {ev['counterpart']}{where}"
summary_en = f"Meeting with {ev['counterpart']}{where}"
event_key = fingerprint("event", ev["kind"], person_id, anchor, week)
yield Event(key=event_key, kind=ev["kind"], title=f"{row['full_name']}: {summary_de}", starts_at=published,
location=ev.get("place"), country=ev.get("country"), url=doc["url"], description=excerpt(sentence, 400))
foreign = ev.get("country") and ev["country"] not in AUSTRIA.values()
yield Claim(
claim_type=ev["claim_type"], person=PersonRef(ids={"politrack": str(person_id)}), occurred_at=published,
summary_de=summary_de, summary_en=summary_en,
data={"place": ev.get("place"), "country": ev.get("country"), "counterpart": ev.get("counterpart"),
"date_precision": "publication", "extracted": "rules"},
evidence=[Evidence(url=doc["url"], source=doc["source"], title=doc["title"], published_at=published,
quote=excerpt(sentence, 400), confidence=0.8 if official else 0.6)],
topics=("Außenpolitik",) if foreign else (),
facet=Participation(event_key=event_key),
)
+52
View File
@@ -0,0 +1,52 @@
"""German place names -> ISO 3166-1 alpha-2, for detecting trips in German texts.
Countries in nominative form (as used after "nach"/"in"); cities that are
frequent destinations of Austrian politicians. Austrian places map to "AT" so
domestic visits are recognised but not counted as trips abroad.
"""
COUNTRIES = {
"Ägypten": "EG", "Albanien": "AL", "Algerien": "DZ", "Argentinien": "AR", "Armenien": "AM", "Aserbaidschan": "AZ",
"Äthiopien": "ET", "Australien": "AU", "Belgien": "BE", "Bosnien und Herzegowina": "BA", "Bosnien": "BA",
"Brasilien": "BR", "Bulgarien": "BG", "Chile": "CL", "China": "CN", "Dänemark": "DK", "Deutschland": "DE",
"Estland": "EE", "Finnland": "FI", "Frankreich": "FR", "Georgien": "GE", "Ghana": "GH", "Griechenland": "GR",
"Großbritannien": "GB", "Vereinigten Königreich": "GB", "Vereinigtes Königreich": "GB", "Indien": "IN",
"Indonesien": "ID", "Irak": "IQ", "Iran": "IR", "Irland": "IE", "Island": "IS", "Israel": "IL", "Italien": "IT",
"Japan": "JP", "Jordanien": "JO", "Kanada": "CA", "Kasachstan": "KZ", "Katar": "QA", "Kenia": "KE",
"Kolumbien": "CO", "Kosovo": "XK", "Kroatien": "HR", "Kuba": "CU", "Lettland": "LV", "Libanon": "LB",
"Libyen": "LY", "Liechtenstein": "LI", "Litauen": "LT", "Luxemburg": "LU", "Malta": "MT", "Marokko": "MA",
"Mexiko": "MX", "Moldau": "MD", "Moldawien": "MD", "Monaco": "MC", "Mongolei": "MN", "Montenegro": "ME",
"Neuseeland": "NZ", "Niederlande": "NL", "Nigeria": "NG", "Nordmazedonien": "MK", "Norwegen": "NO",
"Oman": "OM", "Pakistan": "PK", "Palästina": "PS", "Polen": "PL", "Portugal": "PT", "Ruanda": "RW",
"Rumänien": "RO", "Russland": "RU", "Saudi-Arabien": "SA", "Schweden": "SE", "Schweiz": "CH", "Senegal": "SN",
"Serbien": "RS", "Singapur": "SG", "Slowakei": "SK", "Slowenien": "SI", "Spanien": "ES", "Südafrika": "ZA",
"Südkorea": "KR", "Syrien": "SY", "Taiwan": "TW", "Thailand": "TH", "Tschechien": "CZ", "Tunesien": "TN",
"Türkei": "TR", "Ukraine": "UA", "Ungarn": "HU", "USA": "US", "Vereinigten Staaten": "US",
"Vereinigte Staaten": "US", "Usbekistan": "UZ", "Vatikan": "VA", "Venezuela": "VE",
"Vereinigten Arabischen Emiraten": "AE", "Vereinigte Arabische Emirate": "AE", "Vietnam": "VN",
"Weißrussland": "BY", "Belarus": "BY", "Zypern": "CY", "Andorra": "AD", "San Marino": "SM",
}
CITIES = {
"Brüssel": "BE", "Straßburg": "FR", "Luxemburg": "LU", "Berlin": "DE", "München": "DE", "Frankfurt": "DE",
"Paris": "FR", "Rom": "IT", "Mailand": "IT", "Madrid": "ES", "Lissabon": "PT", "London": "GB", "Dublin": "IE",
"Amsterdam": "NL", "Den Haag": "NL", "Kopenhagen": "DK", "Stockholm": "SE", "Oslo": "NO", "Helsinki": "FI",
"Warschau": "PL", "Prag": "CZ", "Bratislava": "SK", "Pressburg": "SK", "Budapest": "HU", "Ljubljana": "SI",
"Laibach": "SI", "Zagreb": "HR", "Belgrad": "RS", "Sarajevo": "BA", "Podgorica": "ME", "Skopje": "MK",
"Tirana": "AL", "Pristina": "XK", "Sofia": "BG", "Bukarest": "RO", "Chișinău": "MD", "Kiew": "UA",
"Kyjiw": "UA", "Moskau": "RU", "Minsk": "BY", "Athen": "GR", "Nikosia": "CY", "Valletta": "MT",
"Ankara": "TR", "Istanbul": "TR", "Bern": "CH", "Genf": "CH", "Zürich": "CH", "Davos": "CH", "Vaduz": "LI",
"Washington": "US", "New York": "US", "Ottawa": "CA", "Peking": "CN", "Beijing": "CN", "Shanghai": "CN",
"Tokio": "JP", "Neu-Delhi": "IN", "Jerusalem": "IL", "Tel Aviv": "IL", "Kairo": "EG", "Amman": "JO",
"Riad": "SA", "Doha": "QA", "Abu Dhabi": "AE", "Dubai": "AE", "Teheran": "IR", "Baku": "AZ", "Tiflis": "GE",
"Eriwan": "AM", "Astana": "KZ", "Taschkent": "UZ", "Brdo": "SI", "Göttweig": "AT",
}
AUSTRIA = {
"Österreich": "AT", "Wien": "AT", "Graz": "AT", "Linz": "AT", "Salzburg": "AT", "Innsbruck": "AT",
"Klagenfurt": "AT", "Villach": "AT", "Wels": "AT", "St. Pölten": "AT", "Dornbirn": "AT", "Bregenz": "AT",
"Eisenstadt": "AT", "Niederösterreich": "AT", "Oberösterreich": "AT", "Steiermark": "AT", "Kärnten": "AT",
"Tirol": "AT", "Vorarlberg": "AT", "Burgenland": "AT", "Alpbach": "AT",
}
PLACES = {**AUSTRIA, **COUNTRIES, **CITIES}
+91
View File
@@ -0,0 +1,91 @@
"""Detect reports about legal proceedings in news headlines (and WKStA press releases).
Everything found here is a *proposal*: claims are created with review=True (hidden until a human approves
them via `politrack review`). Headline wording is ambiguous, the data is specially protected (Art. 10 GDPR)
and wrong attributions can be defamatory. Official parliament immunity data does not go through this path.
Several outlets reporting the same person + stage in the same month attach to one claim (corroboration).
"""
from __future__ import annotations
import re
from collections.abc import Iterator
from datetime import datetime
from psycopg.types.json import Jsonb
from politrack.core import Enricher, RunContext, register
from politrack.core.records import Claim, Evidence, Legal, PersonRef
from politrack.core.spec import DERIVED
# Most specific first.
STAGES = [
# "nicht rechtskräftig verurteilt" must never be read as a final conviction.
("convicted_final", re.compile(r"(?<!nicht )\brechtskräftig\s+(verurteilt|schuldig)", re.I)),
("convicted_not_final", re.compile(r"\b(nicht rechtskräftig\b.*\bverurteilt|verurteilt\b.*nicht rechtskräftig|"
r"schuldig gesprochen|Schuldspruch|\bverurteilt\b)", re.I)),
("acquitted", re.compile(r"\b(freigesprochen|Freispruch)\b", re.I)),
("diversion", re.compile(r"\bDiversion\b", re.I)),
("proceedings_dropped", re.compile(r"\b(Verfahren|Ermittlungen)\b.*\beingestellt\b|\beingestellt\b.*\bVerfahren\b", re.I)),
("trial", re.compile(r"\b(Prozess|Strafprozess|vor Gericht|Prozessauftakt|Hauptverhandlung)\b", re.I)),
("indicted", re.compile(r"\b(angeklagt|Anklage|Strafantrag)\b", re.I)),
("investigation", re.compile(r"\b(ermittelt|Ermittlungen|Ermittlungsverfahren|Beschuldigte[rn]?|Hausdurchsuchung|"
r"WKStA|Staatsanwaltschaft|Razzia)\b", re.I)),
]
STAGE_DE = {
"convicted_final": "rechtskräftige Verurteilung (laut Bericht)",
"convicted_not_final": "nicht rechtskräftige Verurteilung (laut Bericht)",
"acquitted": "Freispruch (laut Bericht)",
"diversion": "Diversion (laut Bericht)",
"proceedings_dropped": "Verfahren eingestellt (laut Bericht)",
"trial": "Gerichtsverfahren (laut Bericht)",
"indicted": "Anklage (laut Bericht)",
"investigation": "Ermittlungen (laut Bericht)",
}
def classify(headline: str) -> str | None:
for stage, pattern in STAGES:
if pattern.search(headline):
return stage
return None
@register
class LegalDetector(Enricher):
spec = DERIVED
name = "enrich.legal"
description = "Propose legal-proceeding claims from headlines (human review required)"
schedule = "6-59/10 * * * *"
def run(self, ctx: RunContext) -> Iterator[Claim]:
conn = ctx.conn
while True:
rows = conn.execute(
"""SELECT c.id, c.person_id, c.occurred_at, d.url, d.title, d.published_at, s.slug AS source
FROM claim c
JOIN claim_evidence e ON e.claim_id = c.id
JOIN document d ON d.id = e.document_id
JOIN source s ON s.id = d.source_id
WHERE c.claim_type = 'media_mention' AND c.person_id IS NOT NULL AND NOT (c.data ? 'legal_checked')
AND (s.kind IN ('media', 'aggregator') OR s.slug = 'wksta')
ORDER BY c.id LIMIT 1000"""
).fetchall()
if not rows:
return
for row in rows:
stage = classify(row["title"] or "")
if stage:
when = row["published_at"] or row["occurred_at"] or datetime.now()
yield Claim(
claim_type="legal_proceeding", person=PersonRef(ids={"politrack": str(row["person_id"])}),
key=("legal-news", row["person_id"], stage, when.strftime("%Y-%m")), occurred_at=when,
summary_de=STAGE_DE[stage], summary_en=f"Reported legal matter ({stage.replace('_', ' ')})",
data={"detected_from": "headline"}, sensitive=True, review=True,
evidence=[Evidence(url=row["url"], source=row["source"], title=row["title"], published_at=when,
quote=row["title"], confidence=0.5)],
topics=("Inneres und Recht",),
facet=Legal(stage=stage, stage_date=when.date()),
)
ctx.stats["proposed"] += 1
conn.execute("UPDATE claim SET data = data || %s WHERE id = %s", (Jsonb({"legal_checked": True}), row["id"]))
+153
View File
@@ -0,0 +1,153 @@
"""Link documents (headlines, press releases, government news) to the politicians and parties they mention.
Conservative by design: full names (>= 2 tokens) only, plus "office title + surname" of *current* office
holders ("Bundeskanzler Stocker"); an ambiguous name is only linked if exactly one candidate is active.
False links are worse than missed ones on a site like this.
"""
from __future__ import annotations
import re
from collections import defaultdict
from collections.abc import Iterator
from datetime import datetime, timezone
from psycopg import Connection
from psycopg.types.json import Jsonb
from politrack.core import Enricher, RunContext, register
from politrack.core.records import Claim, Evidence, OrgRef, PersonRef
from politrack.core.spec import DERIVED
from politrack.text import normalize, strip_titles
TOKEN = re.compile(r"[\w'-]+")
OFFICE_TITLE = re.compile(
r"\b(?:Bundeskanzler(?:in)?|Vizekanzler(?:in)?|[A-ZÄÖÜ][a-zäöüß]*minister(?:in)?|Minister(?:in)?|"
r"Staatssekretär(?:in)?|Nationalratspräsident(?:in)?|Bundesratspräsident(?:in)?|Bundespräsident(?:in)?|"
r"Landeshauptmann|Landeshauptfrau|Landesrät(?:in)?|Landesrat|Bürgermeister(?:in)?|Klubobmann|Klubobfrau|"
r"Klubchef(?:in)?|Parteichef(?:in)?|Generalsekretär(?:in)?|Abgeordnete[rn]?|EU-Abgeordnete[rn]?|"
r"EU-Kommissar(?:in)?|Präsident(?:in)?)\s+(?:(?:Dr|Mag|DI|Ing)\.\s+)?(?P<surname>[A-ZÄÖÜ][a-zäöüß]+(?:-[A-ZÄÖÜ][a-zäöüß]+)?)\b"
# "Nationalratspräsident Walter Rosenkranz": a following capitalised word means we matched a first name.
r"(?!\s+(?:von\s+)?[A-ZÄÖÜ][a-zäöüß]+)"
)
CONTEXT_PATTERNS = [
("interview", re.compile(r"\binterview\b", re.I)),
("tv", re.compile(r"\b(ZIB ?2|ZiB ?2|Pressestunde|Im Zentrum|Sommergespräch|Report|Puls ?24|ServusTV|Talk im Hangar|"
r"Milborn|Pro und Contra)\b", re.I)),
("radio", re.compile(r"\b(Ö1|Morgenjournal|Mittagsjournal|Journal zu Gast)\b", re.I)),
("op_ed", re.compile(r"\b(Gastkommentar|Kommentar der anderen)\b", re.I)),
]
def tokens(text: str) -> list[str]:
return TOKEN.findall(normalize(text))
def detect_context(text: str) -> str | None:
for name, pattern in CONTEXT_PATTERNS:
if pattern.search(text):
return name
return None
class MentionLinker:
def __init__(self, conn: Connection) -> None:
self.person_index: dict[tuple[str, ...], set[int]] = defaultdict(set)
for row in conn.execute("SELECT entity_id, alias FROM alias WHERE entity_type = 'person'"):
key = tuple(tokens(strip_titles(row["alias"])))
if len(key) >= 2:
self.person_index[key].add(row["entity_id"])
self.active = {r["person_id"] for r in conn.execute("SELECT person_id FROM person_status WHERE status = 'active'")}
self.active_surnames: dict[str, set[int]] = defaultdict(set)
for row in conn.execute(
"SELECT p.id, p.family_name FROM person p JOIN person_status s ON s.person_id = p.id "
"WHERE s.status = 'active' AND p.family_name IS NOT NULL"
):
self.active_surnames[normalize(row["family_name"])].add(row["id"])
self.org_index: dict[tuple[str, ...], int] = {}
for row in conn.execute(
"""SELECT o.id, a.alias FROM organization o JOIN alias a ON a.entity_type = 'organization' AND a.entity_id = o.id
WHERE o.kind = 'party'
UNION SELECT id, short_name FROM organization WHERE kind = 'party' AND short_name IS NOT NULL"""
):
key = tuple(tokens(row["alias"]))
if key and len("".join(key)) >= 3:
self.org_index[key] = row["id"]
self.max_len = max((len(k) for k in self.person_index), default=2)
def people(self, text: str) -> set[int]:
toks = tokens(text)
found: set[int] = set()
for n in range(2, self.max_len + 1):
for i in range(len(toks) - n + 1):
candidates = self.person_index.get(tuple(toks[i : i + n]))
if not candidates:
continue
if len(candidates) == 1:
found |= candidates
elif len(active := candidates & self.active) == 1:
found |= active
return found
def people_by_title(self, text: str) -> set[int]:
found: set[int] = set()
for m in OFFICE_TITLE.finditer(text):
candidates = self.active_surnames.get(normalize(m.group("surname")), set())
if len(candidates) == 1:
found |= candidates
return found
def all_people(self, text: str) -> set[int]:
return self.people(text) | self.people_by_title(text)
def parties(self, text: str) -> set[int]:
toks = tokens(text)
found = set()
for n in (1, 2, 3):
for i in range(len(toks) - n + 1):
if org := self.org_index.get(tuple(toks[i : i + n])):
found.add(org)
return found
@register
class Linker(Enricher):
spec = DERIVED
name = "enrich.linker"
description = "Link news headlines, press releases and government news to politicians and parties"
schedule = "*/10 * * * *"
def run(self, ctx: RunContext) -> Iterator[Claim]:
conn = ctx.conn
linker = MentionLinker(conn)
while True:
docs = conn.execute(
"""SELECT d.id, d.url, d.title, d.excerpt, left(d.full_text, 3000) AS full_text, d.published_at,
s.slug AS source, s.name AS source_name, s.kind AS source_kind,
COALESCE((d.meta->>'tdm_reserved')::boolean, false) AS tdm_reserved
FROM document d JOIN source s ON s.id = d.source_id
WHERE (s.kind IN ('media', 'user') OR s.slug IN ('gdelt', 'bundesregierung', 'wksta'))
AND NOT (d.meta ? 'linked_at') AND d.doc_type IN ('article', 'press_release')
ORDER BY d.id LIMIT 500"""
).fetchall()
if not docs:
return
for doc in docs:
title = doc["title"] or ""
text = title if doc["tdm_reserved"] else f"{title} {doc['full_text'] or doc['excerpt'] or ''}"
context = detect_context(title)
people, parties = linker.all_people(text), linker.parties(title)
evidence = Evidence(url=doc["url"], source=doc["source"], title=title, quote=title, stance="mentions",
confidence=0.8, published_at=doc["published_at"])
for person_id in people:
yield Claim(claim_type="media_mention", person=PersonRef(ids={"politrack": str(person_id)}),
occurred_at=doc["published_at"], summary_de=f"{doc['source_name']}: {title}",
data={"context": context, "source": doc["source"]}, evidence=[evidence])
for org_id in parties:
yield Claim(claim_type="media_mention", org=OrgRef(ids={"politrack_org": str(org_id)}, create=False),
occurred_at=doc["published_at"], summary_de=f"{doc['source_name']}: {title}",
data={"context": context, "source": doc["source"]}, evidence=[evidence])
conn.execute("UPDATE document SET meta = meta || %s WHERE id = %s",
(Jsonb({"linked_at": datetime.now(timezone.utc).isoformat(), "people": sorted(people),
"parties": sorted(parties)}), doc["id"]))
ctx.stats["documents_linked"] += 1
+81
View File
@@ -0,0 +1,81 @@
"""Housekeeping enrichers that work directly on canonical rows (no new facts, only derived structure).
* enrich.roles - give open-ended offices the party of the holder's party membership at that time
(Wikidata offices carry no party), close open roles at the holder's date of death
* enrich.rollcall - derive each faction's majority position from individual roll-call votes, so party
pages show roll calls too ("ÖVP mostly for: ... 70 yes, 2 no")
"""
from __future__ import annotations
from collections import Counter, defaultdict
from collections.abc import Iterator
from politrack.core import Enricher, RunContext, register
from politrack.core.records import Claim, Evidence, OrgRef, VotePosition
from politrack.core.spec import DERIVED
@register
class RoleMaintenance(Enricher):
spec = DERIVED
name = "enrich.roles"
description = "Assign parties to offices from party memberships; close roles at death"
schedule = "50 5 * * *"
def run(self, ctx: RunContext) -> None:
conn = ctx.conn
ctx.stats["roles_with_party"] = conn.execute(
"""UPDATE role r SET party_id = pm.party_id
FROM role pm JOIN claim c ON c.id = pm.claim_id
WHERE r.party_id IS NULL AND r.level <> 'party' AND pm.role = 'party_member' AND c.merged_into IS NULL
AND pm.person_id = r.person_id
AND (pm.start_date IS NULL OR r.end_date IS NULL OR pm.start_date <= r.end_date)
AND (pm.end_date IS NULL OR r.start_date IS NULL OR pm.end_date >= r.start_date)"""
).rowcount
ctx.stats["roles_closed_at_death"] = conn.execute(
"""UPDATE role r SET end_date = p.death_date FROM person p
WHERE p.id = r.person_id AND r.end_date IS NULL AND p.death_date IS NOT NULL"""
).rowcount
return None
@register
class RollCallFactions(Enricher):
spec = DERIVED
name = "enrich.rollcall"
description = "Faction majority positions derived from individual roll-call votes"
schedule = "55 4 * * *"
def run(self, ctx: RunContext) -> Iterator[Claim]:
rows = ctx.conn.execute(
"""SELECT ve.event_key, ve.item_title, ve.voted_on, ve.url, ch.slug AS chamber, vp.party_id, vp.position,
o.short_name, o.name
FROM vote_event ve JOIN vote_position vp ON vp.vote_event_id = ve.id
JOIN organization ch ON ch.id = ve.chamber_id
JOIN organization o ON o.id = vp.party_id
JOIN claim c ON c.id = vp.claim_id AND c.merged_into IS NULL
WHERE ve.roll_call AND vp.person_id IS NOT NULL"""
).fetchall()
tally: dict[tuple, Counter] = defaultdict(Counter)
info: dict[tuple, dict] = {}
for r in rows:
key = (r["event_key"], r["party_id"])
tally[key][r["position"]] += 1
info[key] = r
for key, counts in tally.items():
r = info[key]
position, _ = counts.most_common(1)[0]
party = r["short_name"] or r["name"]
yield Claim(
claim_type="vote", org=OrgRef(ids={"politrack_org": str(r["party_id"])}, create=False),
counterpart_org=OrgRef(slug=r["chamber"]), occurred_at=r["voted_on"],
summary_de=f"{party} stimmte mehrheitlich {'für' if position == 'for' else 'gegen'}: {r['item_title']} "
f"(namentliche Abstimmung, {counts['for']} Ja, {counts['against']} Nein)",
summary_en=f"{party} mostly voted {'for' if position == 'for' else 'against'}: {r['item_title']} "
f"(roll call, {counts['for']} yes / {counts['against']} no)",
data={"position": position, "roll_call": True, "yes": counts["for"], "no": counts["against"]},
evidence=[Evidence(url=r["url"], source="parlament", doc_type="api_record", title=r["item_title"],
published_at=r["voted_on"], locator=f"rollcall:{r['voted_on']}")],
facet=VotePosition(event_key=r["event_key"], position=position),
)
+61
View File
@@ -0,0 +1,61 @@
"""Keyword-based topic tagging for claims that did not get topics from their source.
Parliament items carry official topics; news mentions, donations and posts
are tagged here with the keyword lists of the `topic` table. An optional LLM
classifier can replace this later (see docs/roadmap.md).
"""
import re
from politrack.core import Enricher, RunContext, register
from politrack.core.spec import DERIVED
def keyword_pattern(keywords: list[str]) -> re.Pattern:
"""Keywords match as word prefixes ("Klima" -> "Klimaschutz"); short ones (<= 3 chars) only as whole words."""
parts = [re.escape(k) + (r"\b" if len(k) <= 3 else "") for k in keywords]
return re.compile(r"\b(" + "|".join(parts) + ")", re.IGNORECASE)
@register
class TopicTagger(Enricher):
spec = DERIVED
name = "enrich.topics"
description = "Keyword topics for claims without official topics"
schedule = "8-59/10 * * * *"
def run(self, ctx: RunContext) -> None:
while (n := tag_claims(ctx.conn)):
ctx.stats["claims_tagged"] += n
return None
def tag_claims(conn, batch: int = 2000) -> int:
topics = [
(row["id"], keyword_pattern(row["keywords"]))
for row in conn.execute("SELECT id, keywords FROM topic WHERE cardinality(keywords) > 0")
]
claims = conn.execute(
"""
SELECT c.id, c.summary_de || ' ' || COALESCE(string_agg(d.title, ' '), '') AS text
FROM claim c
LEFT JOIN claim_evidence e ON e.claim_id = c.id
LEFT JOIN document d ON d.id = e.document_id
WHERE c.claim_type IN ('media_mention', 'statement', 'donation', 'side_job', 'appearance', 'meeting', 'travel',
'money_flow', 'government_decision', 'wiki_edit', 'career')
AND NOT (c.data ? 'topics_tagged')
GROUP BY c.id ORDER BY c.id LIMIT %s
""",
(batch,),
).fetchall()
for claim in claims:
for topic_id, pattern in topics:
if pattern.search(claim["text"]):
conn.execute(
"INSERT INTO topic_assignment (topic_id, claim_id, confidence, method) "
"VALUES (%s, %s, 0.5, 'keyword') ON CONFLICT DO NOTHING",
(topic_id, claim["id"]),
)
conn.execute("UPDATE claim SET data = data || '{\"topics_tagged\": true}' WHERE id = %s", (claim["id"],))
conn.commit()
return len(claims)