Politrack: track Austrian politicians from public sources
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
583ccba31c
187 files changed
+39890
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,244 @@
|
||||
"""Corroboration: find claims from different sources that state the same fact and merge them.
|
||||
|
||||
Merging keeps the oldest claim, copies the duplicate's evidence onto it and marks
|
||||
the duplicate with `merged_into`. Afterwards `refresh_claim_status` marks the
|
||||
survivor "corroborated" when at least two *independent sources* support it.
|
||||
|
||||
Matchers:
|
||||
* roles: same person, equivalent body/office, overlapping or near-identical periods
|
||||
(e.g. Wikidata "Abgeordneter zum Nationalrat" == Parliament mandate)
|
||||
* stories: news articles about the same event (similar headlines, shared people,
|
||||
within 72 hours) are grouped; mentions of a person in one story are merged
|
||||
* events: trips/meetings of the same person to the same place/counterpart within 3 days
|
||||
"""
|
||||
|
||||
from collections import defaultdict
|
||||
from datetime import timedelta
|
||||
|
||||
from psycopg import Connection
|
||||
from rapidfuzz import fuzz
|
||||
|
||||
from politrack.core import Enricher, RunContext, register
|
||||
from politrack.core.spec import DERIVED
|
||||
from politrack.enrich.events import counterpart_anchor
|
||||
from politrack.text import normalize
|
||||
|
||||
REFRESH_STATUS = """
|
||||
UPDATE claim SET status = CASE
|
||||
WHEN status = 'retracted' THEN status
|
||||
WHEN EXISTS (SELECT 1 FROM claim_evidence e WHERE e.claim_id = claim.id AND e.stance = 'contradicts') THEN 'disputed'
|
||||
WHEN (SELECT count(DISTINCT d.source_id) FROM claim_evidence e JOIN document d ON d.id = e.document_id
|
||||
WHERE e.claim_id = claim.id AND e.stance IN ('supports', 'mentions')) >= 2 THEN 'corroborated'
|
||||
ELSE 'reported' END
|
||||
WHERE id = %s
|
||||
"""
|
||||
|
||||
|
||||
def merge_claims(conn: Connection, keep_id: int, dup_id: int) -> None:
|
||||
if keep_id == dup_id:
|
||||
return
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO claim_evidence (claim_id, document_id, stance, quote, locator, extracted_by, confidence)
|
||||
SELECT %s, document_id, stance, quote, locator, extracted_by, confidence
|
||||
FROM claim_evidence WHERE claim_id = %s
|
||||
ON CONFLICT (claim_id, document_id, locator) DO NOTHING
|
||||
""",
|
||||
(keep_id, dup_id),
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO topic_assignment (topic_id, claim_id, confidence, method)
|
||||
SELECT topic_id, %s, confidence, method FROM topic_assignment WHERE claim_id = %s
|
||||
ON CONFLICT DO NOTHING
|
||||
""",
|
||||
(keep_id, dup_id),
|
||||
)
|
||||
conn.execute("UPDATE claim SET merged_into = %s, updated_at = now() WHERE id = %s", (keep_id, dup_id))
|
||||
conn.execute("UPDATE claim SET merged_into = %s WHERE merged_into = %s", (keep_id, dup_id))
|
||||
conn.execute(REFRESH_STATUS, (keep_id,))
|
||||
|
||||
|
||||
# -- roles ---------------------------------------------------------------------
|
||||
|
||||
BODY_KEYWORDS = {
|
||||
"nationalrat": ("nationalrat",),
|
||||
"bundesrat": ("bundesrat", "bundesrates"),
|
||||
}
|
||||
|
||||
|
||||
def _same_office(a: dict, b: dict) -> bool:
|
||||
"""a: authoritative role (Parliament), b: other source (Wikidata)."""
|
||||
if a["role"] == "mp" and b["role"] == "mp":
|
||||
return a["body_slug"] == b["body_slug"]
|
||||
label = normalize(b["role"])
|
||||
if a["role"] == "mp" and a["body_slug"] in BODY_KEYWORDS:
|
||||
return any(k in label for k in BODY_KEYWORDS[a["body_slug"]]) and "prasident" not in label
|
||||
return fuzz.token_set_ratio(normalize(a["role"]), label) >= 80
|
||||
|
||||
|
||||
def _periods_match(a: dict, b: dict) -> bool:
|
||||
a_start, b_start = a["start_date"], b["start_date"]
|
||||
a_end, b_end = a["end_date"], b["end_date"]
|
||||
if a_start and b_start and abs((a_start - b_start).days) <= 60:
|
||||
return True
|
||||
# Wikidata often only records one term for several consecutive mandates: accept containment.
|
||||
if a_start and b_start and b_start <= a_start and (b_end is None or a_end is None or a_end <= b_end + timedelta(days=60)):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def merge_roles(conn: Connection) -> int:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT r.claim_id, r.person_id, r.role, r.start_date, r.end_date, c.created_by, o.slug AS body_slug
|
||||
FROM role r JOIN claim c ON c.id = r.claim_id LEFT JOIN organization o ON o.id = r.organization_id
|
||||
WHERE c.merged_into IS NULL AND r.level <> 'party'
|
||||
AND (c.created_by LIKE 'parlament%%' OR c.created_by LIKE 'wikidata%%' OR c.created_by LIKE 'europarl%%')
|
||||
ORDER BY r.person_id, r.start_date NULLS LAST
|
||||
"""
|
||||
).fetchall()
|
||||
by_person: dict[int, dict[str, list[dict]]] = defaultdict(lambda: defaultdict(list))
|
||||
for r in rows:
|
||||
family = "wikidata" if r["created_by"].startswith("wikidata") else "parlament"
|
||||
by_person[r["person_id"]][family].append(r)
|
||||
merged = 0
|
||||
for roles in by_person.values():
|
||||
for other in roles.get("wikidata", []):
|
||||
match = next((a for a in roles.get("parlament", []) if _same_office(a, other) and _periods_match(a, other)),
|
||||
None)
|
||||
if match:
|
||||
merge_claims(conn, match["claim_id"], other["claim_id"])
|
||||
merged += 1
|
||||
conn.commit()
|
||||
return merged
|
||||
|
||||
|
||||
# -- news stories -------------------------------------------------------------
|
||||
|
||||
def cluster_stories(conn: Connection, days: int = 14) -> int:
|
||||
"""Group recent news documents about the same thing into stories."""
|
||||
docs = conn.execute(
|
||||
"""
|
||||
SELECT d.id, d.title, d.published_at, d.story_id, d.source_id, d.meta->'people' AS people
|
||||
FROM document d JOIN source s ON s.id = d.source_id
|
||||
WHERE (s.kind = 'media' OR s.slug = 'gdelt') AND d.published_at > now() - make_interval(days => %s)
|
||||
AND d.title IS NOT NULL
|
||||
ORDER BY d.published_at
|
||||
""",
|
||||
(days,),
|
||||
).fetchall()
|
||||
stories: list[dict] = [] # {id, docs: [...]}
|
||||
for doc in docs:
|
||||
title = normalize(doc["title"])
|
||||
people = set(doc["people"] or [])
|
||||
best, best_score = None, 0
|
||||
for story in stories[-400:]:
|
||||
lead = story["docs"][0]
|
||||
if doc["published_at"] - lead["published_at"] > timedelta(hours=72):
|
||||
continue
|
||||
score = fuzz.token_set_ratio(title, lead["norm"])
|
||||
shared = people & lead["people"]
|
||||
if score >= 82 or (shared and score >= 60):
|
||||
if score > best_score:
|
||||
best, best_score = story, score
|
||||
entry = {"id": doc["id"], "norm": title, "people": people, "published_at": doc["published_at"],
|
||||
"source_id": doc["source_id"], "story_id": doc["story_id"], "title": doc["title"]}
|
||||
if best:
|
||||
best["docs"].append(entry)
|
||||
else:
|
||||
stories.append({"docs": [entry]})
|
||||
|
||||
assigned = 0
|
||||
for story in stories:
|
||||
if len(story["docs"]) < 2:
|
||||
continue
|
||||
existing = next((d["story_id"] for d in story["docs"] if d["story_id"]), None)
|
||||
lead = story["docs"][0]
|
||||
if existing is None:
|
||||
existing = conn.execute(
|
||||
"INSERT INTO story (title, first_seen) VALUES (%s, %s) RETURNING id", (lead["title"], lead["published_at"])
|
||||
).fetchone()["id"]
|
||||
ids = [d["id"] for d in story["docs"]]
|
||||
conn.execute("UPDATE document SET story_id = %s WHERE id = ANY(%s)", (existing, ids))
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE story SET last_seen = sub.last_seen, document_count = sub.n, source_count = sub.sources
|
||||
FROM (SELECT max(published_at) AS last_seen, count(*) AS n, count(DISTINCT source_id) AS sources
|
||||
FROM document WHERE story_id = %s) sub
|
||||
WHERE story.id = %s
|
||||
""",
|
||||
(existing, existing),
|
||||
)
|
||||
assigned += len(ids)
|
||||
conn.commit()
|
||||
return assigned
|
||||
|
||||
|
||||
def merge_story_mentions(conn: Connection) -> int:
|
||||
"""One media_mention claim per (person or party, story); other outlets become additional evidence."""
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT c.id, c.person_id, c.organization_id, d.story_id, c.occurred_at
|
||||
FROM claim c
|
||||
JOIN claim_evidence e ON e.claim_id = c.id
|
||||
JOIN document d ON d.id = e.document_id
|
||||
WHERE c.claim_type = 'media_mention' AND c.merged_into IS NULL AND d.story_id IS NOT NULL
|
||||
ORDER BY c.occurred_at NULLS LAST, c.id
|
||||
"""
|
||||
).fetchall()
|
||||
keep: dict[tuple, int] = {}
|
||||
merged = 0
|
||||
for r in rows:
|
||||
key = (r["person_id"], r["organization_id"], r["story_id"])
|
||||
if key in keep:
|
||||
if keep[key] != r["id"]:
|
||||
merge_claims(conn, keep[key], r["id"])
|
||||
merged += 1
|
||||
else:
|
||||
keep[key] = r["id"]
|
||||
conn.commit()
|
||||
return merged
|
||||
|
||||
|
||||
# -- events --------------------------------------------------------------------
|
||||
|
||||
def merge_events(conn: Connection) -> int:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT c.id, c.person_id, c.claim_type, c.occurred_at, c.data->>'country' AS country,
|
||||
c.data->>'counterpart' AS counterpart
|
||||
FROM claim c
|
||||
WHERE c.claim_type IN ('travel', 'meeting') AND c.merged_into IS NULL
|
||||
ORDER BY c.person_id, c.claim_type, c.occurred_at
|
||||
"""
|
||||
).fetchall()
|
||||
merged = 0
|
||||
previous: dict[tuple, dict] = {}
|
||||
for r in rows:
|
||||
anchor = r["country"] if r["claim_type"] == "travel" else counterpart_anchor(r["counterpart"] or "")
|
||||
key = (r["person_id"], r["claim_type"], anchor)
|
||||
prev = previous.get(key)
|
||||
if prev and r["occurred_at"] and prev["occurred_at"] and r["occurred_at"] - prev["occurred_at"] <= timedelta(days=3):
|
||||
merge_claims(conn, prev["id"], r["id"])
|
||||
merged += 1
|
||||
continue
|
||||
previous[key] = r
|
||||
conn.commit()
|
||||
return merged
|
||||
|
||||
|
||||
@register
|
||||
class Corroborator(Enricher):
|
||||
spec = DERIVED
|
||||
name = "enrich.corroborate"
|
||||
description = "Merge near-duplicate claims from different sources; cluster news stories"
|
||||
schedule = "9-59/10 * * * *"
|
||||
|
||||
def run(self, ctx: RunContext) -> None:
|
||||
conn = ctx.conn
|
||||
ctx.stats["roles_merged"] = merge_roles(conn)
|
||||
ctx.stats["story_documents"] = cluster_stories(conn)
|
||||
ctx.stats["story_mentions_merged"] = merge_story_mentions(conn)
|
||||
ctx.stats["events_merged"] = merge_events(conn)
|
||||
return None
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Curated lists maintained by the operator in config/ (accounts that no official register lists).
|
||||
|
||||
* config.social_accounts - config/social_accounts.yaml -> social_account rows for persons/organisations,
|
||||
e.g. party Telegram channels or Bluesky accounts linked from party websites. The social adapters then
|
||||
collect their public posts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Enricher, RunContext, SourceSpec, register
|
||||
from politrack.core.records import OrgRef, PersonRef, SocialAccount
|
||||
|
||||
CURATED = SourceSpec(
|
||||
slug="curated",
|
||||
name="Kuratierte Listen (config/)",
|
||||
kind="derived",
|
||||
homepage="",
|
||||
license="operator-maintained",
|
||||
access="manual",
|
||||
trust=60,
|
||||
tos_notes="Entries maintained by the operator; each should name the public page that links the account.",
|
||||
)
|
||||
|
||||
|
||||
def person_ref(ctx: RunContext, entry: dict[str, Any]) -> PersonRef | None:
|
||||
"""`person: <politrack slug>`, `person: {wikidata: Q..}` / `{parlament: ..}` or `name: <full name>`."""
|
||||
person = entry.get("person")
|
||||
if isinstance(person, dict):
|
||||
return PersonRef(ids={k: str(v) for k, v in person.items()})
|
||||
if isinstance(person, str):
|
||||
row = ctx.conn.execute("SELECT id FROM person WHERE slug = %s", (person,)).fetchone()
|
||||
return PersonRef(ids={"politrack": str(row["id"])}) if row else None
|
||||
if entry.get("name"):
|
||||
return PersonRef(name=entry["name"])
|
||||
return None
|
||||
|
||||
|
||||
def org_ref(entry: dict[str, Any]) -> OrgRef | None:
|
||||
"""`org: <slug>` for existing organisations, or `org: {slug, name, kind}` to create one."""
|
||||
org = entry.get("org")
|
||||
if isinstance(org, str):
|
||||
return OrgRef(slug=org, create=False)
|
||||
if isinstance(org, dict):
|
||||
return OrgRef(slug=org.get("slug"), name=org.get("name"), short_name=org.get("short_name"),
|
||||
kind=org.get("kind", "other"), create=bool(org.get("name")))
|
||||
return None
|
||||
|
||||
|
||||
@register
|
||||
class CuratedSocialAccounts(Enricher):
|
||||
spec = CURATED
|
||||
name = "config.social_accounts"
|
||||
description = "Import social accounts from config/social_accounts.yaml"
|
||||
schedule = "5 3 * * *"
|
||||
|
||||
def run(self, ctx: RunContext) -> Iterator[SocialAccount]:
|
||||
path = get_settings().config_dir / "social_accounts.yaml"
|
||||
if not path.exists():
|
||||
return
|
||||
for entry in (yaml.safe_load(path.read_text(encoding="utf-8")) or {}).get("accounts") or []:
|
||||
person = person_ref(ctx, entry)
|
||||
org = None if person else org_ref(entry)
|
||||
if person is None and org is None:
|
||||
ctx.stats["entries_without_owner"] += 1
|
||||
ctx.log.warning("social_accounts.yaml: no person/org for %s", entry.get("url"))
|
||||
continue
|
||||
yield SocialAccount(platform=entry["platform"], url=entry["url"], handle=entry.get("handle"),
|
||||
person=person, org=org, verified=bool(entry.get("verified")))
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Chunking + embeddings for retrieval (RAG).
|
||||
|
||||
Every claim gets one chunk (its summary plus evidence titles) so agents can
|
||||
retrieve facts *with citations*; documents with licensed full text are split
|
||||
into ~1,200 character chunks. Embeddings are computed by an Ollama server
|
||||
(model bge-m3, multilingual, 1024 dims) if POLITRACK_OLLAMA_URL is set;
|
||||
otherwise only German full-text search is available.
|
||||
"""
|
||||
|
||||
import httpx
|
||||
from psycopg import Connection
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Enricher, RunContext, register
|
||||
from politrack.core.spec import DERIVED
|
||||
|
||||
CHUNK_SIZE = 1200
|
||||
UPSERT_CHUNK = """
|
||||
INSERT INTO chunk (claim_id, document_id, chunk_index, text, lang) VALUES (%s, %s, %s, %s, 'de')
|
||||
ON CONFLICT (COALESCE(document_id, 0), COALESCE(claim_id, 0), chunk_index)
|
||||
DO UPDATE SET text = EXCLUDED.text, embedding = CASE WHEN chunk.text = EXCLUDED.text THEN chunk.embedding END
|
||||
"""
|
||||
|
||||
|
||||
@register
|
||||
class Chunker(Enricher):
|
||||
spec = DERIVED
|
||||
name = "enrich.embed"
|
||||
description = "Build search chunks for claims/documents and embed them (Ollama, optional)"
|
||||
schedule = "*/15 * * * *"
|
||||
|
||||
def run(self, ctx: RunContext) -> None:
|
||||
while (n := build_chunks(ctx.conn)):
|
||||
ctx.stats["chunks"] += n
|
||||
ctx.stats["embedded"] = embed_pending(ctx.conn)
|
||||
return None
|
||||
|
||||
|
||||
def split_text(text: str, size: int = CHUNK_SIZE) -> list[str]:
|
||||
paragraphs, chunks, current = [p.strip() for p in text.split("\n") if p.strip()], [], ""
|
||||
for p in paragraphs:
|
||||
if current and len(current) + len(p) > size:
|
||||
chunks.append(current)
|
||||
current = ""
|
||||
current = f"{current}\n{p}".strip()
|
||||
while len(current) > size * 1.5:
|
||||
chunks.append(current[:size])
|
||||
current = current[size:]
|
||||
if current:
|
||||
chunks.append(current)
|
||||
return chunks
|
||||
|
||||
|
||||
def build_chunks(conn: Connection, batch: int = 5000) -> int:
|
||||
claims = conn.execute(
|
||||
"""
|
||||
SELECT c.id, c.claim_type, c.summary_de, c.occurred_at, p.full_name, o.name AS org_name,
|
||||
string_agg(DISTINCT s.name || ': ' || COALESCE(d.title, d.url), ' | ') AS sources
|
||||
FROM claim c
|
||||
LEFT JOIN person p ON p.id = c.person_id
|
||||
LEFT JOIN organization o ON o.id = c.organization_id
|
||||
JOIN claim_evidence e ON e.claim_id = c.id
|
||||
JOIN document d ON d.id = e.document_id
|
||||
JOIN source s ON s.id = d.source_id
|
||||
WHERE c.merged_into IS NULL AND c.review_status = 'published'
|
||||
AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.claim_id = c.id)
|
||||
GROUP BY c.id, p.full_name, o.name
|
||||
LIMIT %s
|
||||
""",
|
||||
(batch,),
|
||||
).fetchall()
|
||||
for c in claims:
|
||||
who = c["full_name"] or c["org_name"] or ""
|
||||
when = c["occurred_at"].date().isoformat() if c["occurred_at"] else ""
|
||||
text = f"[{c['claim_type']}] {when} {who}\n{c['summary_de']}\nQuellen: {c['sources']}"
|
||||
conn.execute(UPSERT_CHUNK, (c["id"], None, 0, text.strip()))
|
||||
|
||||
docs = conn.execute(
|
||||
"""
|
||||
SELECT d.id, d.title, d.full_text FROM document d
|
||||
WHERE d.full_text IS NOT NULL AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.document_id = d.id)
|
||||
LIMIT %s
|
||||
""",
|
||||
(batch // 10,),
|
||||
).fetchall()
|
||||
for d in docs:
|
||||
for i, part in enumerate(split_text(d["full_text"])):
|
||||
conn.execute(UPSERT_CHUNK, (None, d["id"], i, f"{d['title'] or ''}\n{part}".strip()))
|
||||
conn.commit()
|
||||
return len(claims) + len(docs)
|
||||
|
||||
|
||||
def embed_texts(texts: list[str]) -> list[list[float]]:
|
||||
settings = get_settings()
|
||||
resp = httpx.post(f"{settings.ollama_url}/api/embed", json={"model": settings.embedding_model, "input": texts},
|
||||
timeout=300)
|
||||
resp.raise_for_status()
|
||||
return resp.json()["embeddings"]
|
||||
|
||||
|
||||
def embed_pending(conn: Connection, batch: int = 64, max_batches: int = 50) -> int:
|
||||
settings = get_settings()
|
||||
if not settings.ollama_url:
|
||||
return 0
|
||||
total = 0
|
||||
for _ in range(max_batches):
|
||||
rows = conn.execute("SELECT id, text FROM chunk WHERE embedding IS NULL ORDER BY id LIMIT %s", (batch,)).fetchall()
|
||||
if not rows:
|
||||
break
|
||||
vectors = embed_texts([r["text"] for r in rows])
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"UPDATE chunk SET embedding = %s::vector, embedding_model = %s WHERE id = %s",
|
||||
[(str(v), settings.embedding_model, r["id"]) for r, v in zip(rows, vectors)],
|
||||
)
|
||||
conn.commit()
|
||||
total += len(rows)
|
||||
return total
|
||||
@@ -0,0 +1,179 @@
|
||||
"""Extract trips and meetings from official press releases and news headlines.
|
||||
|
||||
Rule-based and conservative. A sentence becomes an event only if it names a
|
||||
known politician and contains a travel or meeting trigger:
|
||||
|
||||
travel: "reist nach Brüssel", "Arbeitsbesuch in Israel", "beim Gipfel in Montenegro", "in China"
|
||||
meeting: "empfängt den tschechischen Außenminister", "Treffen mit ...", "traf ... zu einem Gespräch"
|
||||
|
||||
Sources:
|
||||
* Parlamentskorrespondenz and government news: title + full text (official works / CC BY),
|
||||
* news headlines: title only (we do not store article text).
|
||||
|
||||
The event date is the publication date of the source (`data.date_precision =
|
||||
"publication"`). The claim key is (kind, person, place/counterpart, ISO week),
|
||||
so reports from different sources about the same trip attach to the same claim
|
||||
as additional evidence and corroborate it. corroborate.merge_events() also
|
||||
merges near-duplicates whose dates differ by a few days.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from politrack.core import Enricher, RunContext, register
|
||||
from politrack.core.records import Claim, Event, Evidence, Participation, PersonRef
|
||||
from politrack.core.spec import DERIVED
|
||||
from politrack.enrich.gazetteer import AUSTRIA, PLACES
|
||||
from politrack.enrich.linker import MentionLinker
|
||||
from politrack.text import excerpt, fingerprint, normalize
|
||||
|
||||
# (?<!-)/(?!-): skip names inside compounds such as "Berlin-Prozess" or "EU-Brüssel-Treffen".
|
||||
PLACE_RE = re.compile(
|
||||
r"(?<![-\w])(" + "|".join(sorted((re.escape(p) for p in PLACES), key=len, reverse=True)) + r")(?![-\w])"
|
||||
)
|
||||
DESTINATION_RE = re.compile(r"\b(?:in|nach|aus)\s+(?:de[mnr]\s+|die\s+)?$")
|
||||
STRONG_TRAVEL = re.compile(r"\b(reist\w*|Reise\w*|\w*[Bb]esuch\w*|\w*Gipfel\w*|Delegation\w*|Konferenz\w*|Tagung\w*)\b")
|
||||
MEETING_RE = re.compile(
|
||||
r"\b(?:empfängt|empfing|empfangen|trifft|traf|getroffen|Treffen mit|Gespräch mit|Arbeitsgespräch mit|"
|
||||
r"Unterredung mit|tauschte sich mit|im Austausch mit|zusammengetroffen mit|kam mit)\s+"
|
||||
r"(?P<counterpart>(?:de[mnr]|die|das|seine[mnr]?|ihre[mnr]?)?\s*[^.,;:]{3,90})",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
COUNTERPART_ROLE = re.compile(
|
||||
r"(minister|präsident|kanzler|botschafter|delegation|amtskolleg|vertreter|bürgermeister|abgeordnet|chef|"
|
||||
r"premier|könig|papst|kommissar|generalsekretär|regierungschef|staatssekretär|landeshauptm|gouverneur|"
|
||||
r"bischof|kardinal|vorsitzend|ceo|gewerkschaft|sozialpartner|außenminister)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
FULL_NAME = re.compile(r"\b[A-ZÄÖÜ][a-zäöüß]+(?:-[A-ZÄÖÜ][a-zäöüß]+)?\s+[A-ZÄÖÜ][a-zäöüß]+")
|
||||
SENTENCE_SPLIT = re.compile(r"(?<=[.!?])\s+(?=[A-ZÄÖÜ„\"])|\n+")
|
||||
|
||||
|
||||
FILLER = re.compile(r"^(?:(?:heute|gestern|morgen|am|vorgestern|Montag|Dienstag|Mittwoch|Donnerstag|Freitag|"
|
||||
r"Samstag|Sonntag|de[mnr]|die|das|seine[mnr]?|ihre[mnr]?|eine[mnr]?)\s+)+", re.IGNORECASE)
|
||||
|
||||
|
||||
def clean_counterpart(text: str) -> str:
|
||||
return FILLER.sub("", text.strip()).strip()
|
||||
|
||||
|
||||
def counterpart_anchor(text: str) -> str:
|
||||
"""Stable key for a counterpart: its last capitalised word (usually the surname or country)."""
|
||||
words = re.findall(r"[A-ZÄÖÜ][\wäöüß-]+", clean_counterpart(text))
|
||||
return normalize(words[-1]) if words else normalize(text)[:25]
|
||||
|
||||
|
||||
def find_places(text: str) -> list[tuple[str, str]]:
|
||||
"""Places in order of preference: those introduced by "in/nach" first (the destination)."""
|
||||
matches = list(PLACE_RE.finditer(text))
|
||||
preferred = [m for m in matches if DESTINATION_RE.search(text[: m.start()])]
|
||||
ordered = preferred + [m for m in matches if m not in preferred]
|
||||
return [(m.group(1), PLACES[m.group(1)]) for m in ordered]
|
||||
|
||||
|
||||
def detect(text: str) -> list[dict]:
|
||||
"""Return candidate events found in one sentence (no person resolution)."""
|
||||
events = []
|
||||
places = find_places(text)
|
||||
foreign = [(p, iso) for p, iso in places if iso != "AT"]
|
||||
if foreign and STRONG_TRAVEL.search(text):
|
||||
place, iso = foreign[0]
|
||||
events.append({"kind": "foreign_trip", "claim_type": "travel", "place": place, "country": iso})
|
||||
meeting = MEETING_RE.search(text)
|
||||
if meeting:
|
||||
counterpart = re.split(r"\s+(?:in|im|am|zu|über|bei|und|sowie)\s+", meeting.group("counterpart").strip())[0]
|
||||
place, iso = (places[0] if places else (None, None))
|
||||
# "traf eine Entscheidung" is not a meeting: require a role word or a person-like name.
|
||||
if COUNTERPART_ROLE.search(counterpart) or FULL_NAME.search(counterpart):
|
||||
events.append({"kind": "meeting", "claim_type": "meeting", "counterpart": clean_counterpart(counterpart),
|
||||
"place": place, "country": iso})
|
||||
return events
|
||||
|
||||
|
||||
@register
|
||||
class EventExtractor(Enricher):
|
||||
spec = DERIVED
|
||||
name = "enrich.events"
|
||||
description = "Trips and meetings from press releases, government news and headlines"
|
||||
schedule = "3-59/10 * * * *"
|
||||
|
||||
def run(self, ctx: RunContext) -> Iterator[Any]:
|
||||
conn = ctx.conn
|
||||
linker = MentionLinker(conn)
|
||||
while True:
|
||||
docs = conn.execute(
|
||||
"""SELECT d.id, d.url, d.title, d.full_text, d.published_at, d.meta, s.slug AS source, s.kind AS source_kind
|
||||
FROM document d JOIN source s ON s.id = d.source_id
|
||||
WHERE NOT (d.meta ? 'events_at') AND d.published_at IS NOT NULL
|
||||
AND (d.doc_type = 'press_release' OR s.kind = 'media' OR s.slug = 'gdelt')
|
||||
ORDER BY d.id LIMIT 300"""
|
||||
).fetchall()
|
||||
if not docs:
|
||||
return
|
||||
for doc in docs:
|
||||
found = 0
|
||||
for record in self.document_events(conn, linker, doc):
|
||||
found += isinstance(record, Claim)
|
||||
yield record
|
||||
conn.execute("UPDATE document SET meta = meta || %s WHERE id = %s",
|
||||
(Jsonb({"events_at": datetime.now().isoformat(), "events": found}), doc["id"]))
|
||||
ctx.stats["documents"] += 1
|
||||
|
||||
def document_events(self, conn, linker: MentionLinker, doc: dict) -> Iterator[Any]:
|
||||
official = doc["source_kind"] == "official"
|
||||
title = doc["title"] or ""
|
||||
body = (doc["full_text"] or "")[:4000] if official else ""
|
||||
pads = (doc["meta"] or {}).get("person_pads") or []
|
||||
doc_people: set[int] = set()
|
||||
if pads:
|
||||
doc_people = {r["entity_id"] for r in conn.execute(
|
||||
"SELECT entity_id FROM identifier WHERE entity_type = 'person' AND scheme = 'parlament' AND value = ANY(%s)",
|
||||
([str(p) for p in pads],))}
|
||||
sentences = [title] + [s for s in SENTENCE_SPLIT.split(body) if s][:25]
|
||||
for i, sentence in enumerate(sentences):
|
||||
candidates = detect(sentence)
|
||||
if not candidates:
|
||||
continue
|
||||
people = linker.all_people(sentence)
|
||||
# In the headline of an official release the actors are usually the people it links/names.
|
||||
if not people and i == 0:
|
||||
people = doc_people or linker.all_people(f"{title} {body[:600]}")
|
||||
for person_id in people:
|
||||
for ev in candidates:
|
||||
yield from event_records(conn, doc, person_id, ev, sentence, official)
|
||||
|
||||
|
||||
def event_records(conn, doc: dict, person_id: int, ev: dict, sentence: str, official: bool) -> Iterator[Any]:
|
||||
published = doc["published_at"]
|
||||
week = published.strftime("%G-W%V")
|
||||
anchor = ev.get("country") if ev["kind"] == "foreign_trip" else counterpart_anchor(ev.get("counterpart") or "")
|
||||
row = conn.execute("SELECT full_name, family_name FROM person WHERE id = %s", (person_id,)).fetchone()
|
||||
# A sentence like "X empfing Y" yields the meeting for X and Y; never record "Y met Y".
|
||||
if ev["kind"] == "meeting" and row["family_name"] and normalize(row["family_name"]) in normalize(ev["counterpart"]):
|
||||
return
|
||||
if ev["kind"] == "foreign_trip":
|
||||
summary_de, summary_en = f"Reise/Aufenthalt in {ev['place']}", f"Trip to {ev['place']}"
|
||||
else:
|
||||
where = f" ({ev['place']})" if ev.get("place") else ""
|
||||
summary_de = f"Treffen mit {ev['counterpart']}{where}"
|
||||
summary_en = f"Meeting with {ev['counterpart']}{where}"
|
||||
event_key = fingerprint("event", ev["kind"], person_id, anchor, week)
|
||||
yield Event(key=event_key, kind=ev["kind"], title=f"{row['full_name']}: {summary_de}", starts_at=published,
|
||||
location=ev.get("place"), country=ev.get("country"), url=doc["url"], description=excerpt(sentence, 400))
|
||||
foreign = ev.get("country") and ev["country"] not in AUSTRIA.values()
|
||||
yield Claim(
|
||||
claim_type=ev["claim_type"], person=PersonRef(ids={"politrack": str(person_id)}), occurred_at=published,
|
||||
summary_de=summary_de, summary_en=summary_en,
|
||||
data={"place": ev.get("place"), "country": ev.get("country"), "counterpart": ev.get("counterpart"),
|
||||
"date_precision": "publication", "extracted": "rules"},
|
||||
evidence=[Evidence(url=doc["url"], source=doc["source"], title=doc["title"], published_at=published,
|
||||
quote=excerpt(sentence, 400), confidence=0.8 if official else 0.6)],
|
||||
topics=("Außenpolitik",) if foreign else (),
|
||||
facet=Participation(event_key=event_key),
|
||||
)
|
||||
@@ -0,0 +1,52 @@
|
||||
"""German place names -> ISO 3166-1 alpha-2, for detecting trips in German texts.
|
||||
|
||||
Countries in nominative form (as used after "nach"/"in"); cities that are
|
||||
frequent destinations of Austrian politicians. Austrian places map to "AT" so
|
||||
domestic visits are recognised but not counted as trips abroad.
|
||||
"""
|
||||
|
||||
COUNTRIES = {
|
||||
"Ägypten": "EG", "Albanien": "AL", "Algerien": "DZ", "Argentinien": "AR", "Armenien": "AM", "Aserbaidschan": "AZ",
|
||||
"Äthiopien": "ET", "Australien": "AU", "Belgien": "BE", "Bosnien und Herzegowina": "BA", "Bosnien": "BA",
|
||||
"Brasilien": "BR", "Bulgarien": "BG", "Chile": "CL", "China": "CN", "Dänemark": "DK", "Deutschland": "DE",
|
||||
"Estland": "EE", "Finnland": "FI", "Frankreich": "FR", "Georgien": "GE", "Ghana": "GH", "Griechenland": "GR",
|
||||
"Großbritannien": "GB", "Vereinigten Königreich": "GB", "Vereinigtes Königreich": "GB", "Indien": "IN",
|
||||
"Indonesien": "ID", "Irak": "IQ", "Iran": "IR", "Irland": "IE", "Island": "IS", "Israel": "IL", "Italien": "IT",
|
||||
"Japan": "JP", "Jordanien": "JO", "Kanada": "CA", "Kasachstan": "KZ", "Katar": "QA", "Kenia": "KE",
|
||||
"Kolumbien": "CO", "Kosovo": "XK", "Kroatien": "HR", "Kuba": "CU", "Lettland": "LV", "Libanon": "LB",
|
||||
"Libyen": "LY", "Liechtenstein": "LI", "Litauen": "LT", "Luxemburg": "LU", "Malta": "MT", "Marokko": "MA",
|
||||
"Mexiko": "MX", "Moldau": "MD", "Moldawien": "MD", "Monaco": "MC", "Mongolei": "MN", "Montenegro": "ME",
|
||||
"Neuseeland": "NZ", "Niederlande": "NL", "Nigeria": "NG", "Nordmazedonien": "MK", "Norwegen": "NO",
|
||||
"Oman": "OM", "Pakistan": "PK", "Palästina": "PS", "Polen": "PL", "Portugal": "PT", "Ruanda": "RW",
|
||||
"Rumänien": "RO", "Russland": "RU", "Saudi-Arabien": "SA", "Schweden": "SE", "Schweiz": "CH", "Senegal": "SN",
|
||||
"Serbien": "RS", "Singapur": "SG", "Slowakei": "SK", "Slowenien": "SI", "Spanien": "ES", "Südafrika": "ZA",
|
||||
"Südkorea": "KR", "Syrien": "SY", "Taiwan": "TW", "Thailand": "TH", "Tschechien": "CZ", "Tunesien": "TN",
|
||||
"Türkei": "TR", "Ukraine": "UA", "Ungarn": "HU", "USA": "US", "Vereinigten Staaten": "US",
|
||||
"Vereinigte Staaten": "US", "Usbekistan": "UZ", "Vatikan": "VA", "Venezuela": "VE",
|
||||
"Vereinigten Arabischen Emiraten": "AE", "Vereinigte Arabische Emirate": "AE", "Vietnam": "VN",
|
||||
"Weißrussland": "BY", "Belarus": "BY", "Zypern": "CY", "Andorra": "AD", "San Marino": "SM",
|
||||
}
|
||||
|
||||
CITIES = {
|
||||
"Brüssel": "BE", "Straßburg": "FR", "Luxemburg": "LU", "Berlin": "DE", "München": "DE", "Frankfurt": "DE",
|
||||
"Paris": "FR", "Rom": "IT", "Mailand": "IT", "Madrid": "ES", "Lissabon": "PT", "London": "GB", "Dublin": "IE",
|
||||
"Amsterdam": "NL", "Den Haag": "NL", "Kopenhagen": "DK", "Stockholm": "SE", "Oslo": "NO", "Helsinki": "FI",
|
||||
"Warschau": "PL", "Prag": "CZ", "Bratislava": "SK", "Pressburg": "SK", "Budapest": "HU", "Ljubljana": "SI",
|
||||
"Laibach": "SI", "Zagreb": "HR", "Belgrad": "RS", "Sarajevo": "BA", "Podgorica": "ME", "Skopje": "MK",
|
||||
"Tirana": "AL", "Pristina": "XK", "Sofia": "BG", "Bukarest": "RO", "Chișinău": "MD", "Kiew": "UA",
|
||||
"Kyjiw": "UA", "Moskau": "RU", "Minsk": "BY", "Athen": "GR", "Nikosia": "CY", "Valletta": "MT",
|
||||
"Ankara": "TR", "Istanbul": "TR", "Bern": "CH", "Genf": "CH", "Zürich": "CH", "Davos": "CH", "Vaduz": "LI",
|
||||
"Washington": "US", "New York": "US", "Ottawa": "CA", "Peking": "CN", "Beijing": "CN", "Shanghai": "CN",
|
||||
"Tokio": "JP", "Neu-Delhi": "IN", "Jerusalem": "IL", "Tel Aviv": "IL", "Kairo": "EG", "Amman": "JO",
|
||||
"Riad": "SA", "Doha": "QA", "Abu Dhabi": "AE", "Dubai": "AE", "Teheran": "IR", "Baku": "AZ", "Tiflis": "GE",
|
||||
"Eriwan": "AM", "Astana": "KZ", "Taschkent": "UZ", "Brdo": "SI", "Göttweig": "AT",
|
||||
}
|
||||
|
||||
AUSTRIA = {
|
||||
"Österreich": "AT", "Wien": "AT", "Graz": "AT", "Linz": "AT", "Salzburg": "AT", "Innsbruck": "AT",
|
||||
"Klagenfurt": "AT", "Villach": "AT", "Wels": "AT", "St. Pölten": "AT", "Dornbirn": "AT", "Bregenz": "AT",
|
||||
"Eisenstadt": "AT", "Niederösterreich": "AT", "Oberösterreich": "AT", "Steiermark": "AT", "Kärnten": "AT",
|
||||
"Tirol": "AT", "Vorarlberg": "AT", "Burgenland": "AT", "Alpbach": "AT",
|
||||
}
|
||||
|
||||
PLACES = {**AUSTRIA, **COUNTRIES, **CITIES}
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Detect reports about legal proceedings in news headlines (and WKStA press releases).
|
||||
|
||||
Everything found here is a *proposal*: claims are created with review=True (hidden until a human approves
|
||||
them via `politrack review`). Headline wording is ambiguous, the data is specially protected (Art. 10 GDPR)
|
||||
and wrong attributions can be defamatory. Official parliament immunity data does not go through this path.
|
||||
Several outlets reporting the same person + stage in the same month attach to one claim (corroboration).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime
|
||||
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from politrack.core import Enricher, RunContext, register
|
||||
from politrack.core.records import Claim, Evidence, Legal, PersonRef
|
||||
from politrack.core.spec import DERIVED
|
||||
|
||||
# Most specific first.
|
||||
STAGES = [
|
||||
# "nicht rechtskräftig verurteilt" must never be read as a final conviction.
|
||||
("convicted_final", re.compile(r"(?<!nicht )\brechtskräftig\s+(verurteilt|schuldig)", re.I)),
|
||||
("convicted_not_final", re.compile(r"\b(nicht rechtskräftig\b.*\bverurteilt|verurteilt\b.*nicht rechtskräftig|"
|
||||
r"schuldig gesprochen|Schuldspruch|\bverurteilt\b)", re.I)),
|
||||
("acquitted", re.compile(r"\b(freigesprochen|Freispruch)\b", re.I)),
|
||||
("diversion", re.compile(r"\bDiversion\b", re.I)),
|
||||
("proceedings_dropped", re.compile(r"\b(Verfahren|Ermittlungen)\b.*\beingestellt\b|\beingestellt\b.*\bVerfahren\b", re.I)),
|
||||
("trial", re.compile(r"\b(Prozess|Strafprozess|vor Gericht|Prozessauftakt|Hauptverhandlung)\b", re.I)),
|
||||
("indicted", re.compile(r"\b(angeklagt|Anklage|Strafantrag)\b", re.I)),
|
||||
("investigation", re.compile(r"\b(ermittelt|Ermittlungen|Ermittlungsverfahren|Beschuldigte[rn]?|Hausdurchsuchung|"
|
||||
r"WKStA|Staatsanwaltschaft|Razzia)\b", re.I)),
|
||||
]
|
||||
STAGE_DE = {
|
||||
"convicted_final": "rechtskräftige Verurteilung (laut Bericht)",
|
||||
"convicted_not_final": "nicht rechtskräftige Verurteilung (laut Bericht)",
|
||||
"acquitted": "Freispruch (laut Bericht)",
|
||||
"diversion": "Diversion (laut Bericht)",
|
||||
"proceedings_dropped": "Verfahren eingestellt (laut Bericht)",
|
||||
"trial": "Gerichtsverfahren (laut Bericht)",
|
||||
"indicted": "Anklage (laut Bericht)",
|
||||
"investigation": "Ermittlungen (laut Bericht)",
|
||||
}
|
||||
|
||||
|
||||
def classify(headline: str) -> str | None:
|
||||
for stage, pattern in STAGES:
|
||||
if pattern.search(headline):
|
||||
return stage
|
||||
return None
|
||||
|
||||
|
||||
@register
|
||||
class LegalDetector(Enricher):
|
||||
spec = DERIVED
|
||||
name = "enrich.legal"
|
||||
description = "Propose legal-proceeding claims from headlines (human review required)"
|
||||
schedule = "6-59/10 * * * *"
|
||||
|
||||
def run(self, ctx: RunContext) -> Iterator[Claim]:
|
||||
conn = ctx.conn
|
||||
while True:
|
||||
rows = conn.execute(
|
||||
"""SELECT c.id, c.person_id, c.occurred_at, d.url, d.title, d.published_at, s.slug AS source
|
||||
FROM claim c
|
||||
JOIN claim_evidence e ON e.claim_id = c.id
|
||||
JOIN document d ON d.id = e.document_id
|
||||
JOIN source s ON s.id = d.source_id
|
||||
WHERE c.claim_type = 'media_mention' AND c.person_id IS NOT NULL AND NOT (c.data ? 'legal_checked')
|
||||
AND (s.kind IN ('media', 'aggregator') OR s.slug = 'wksta')
|
||||
ORDER BY c.id LIMIT 1000"""
|
||||
).fetchall()
|
||||
if not rows:
|
||||
return
|
||||
for row in rows:
|
||||
stage = classify(row["title"] or "")
|
||||
if stage:
|
||||
when = row["published_at"] or row["occurred_at"] or datetime.now()
|
||||
yield Claim(
|
||||
claim_type="legal_proceeding", person=PersonRef(ids={"politrack": str(row["person_id"])}),
|
||||
key=("legal-news", row["person_id"], stage, when.strftime("%Y-%m")), occurred_at=when,
|
||||
summary_de=STAGE_DE[stage], summary_en=f"Reported legal matter ({stage.replace('_', ' ')})",
|
||||
data={"detected_from": "headline"}, sensitive=True, review=True,
|
||||
evidence=[Evidence(url=row["url"], source=row["source"], title=row["title"], published_at=when,
|
||||
quote=row["title"], confidence=0.5)],
|
||||
topics=("Inneres und Recht",),
|
||||
facet=Legal(stage=stage, stage_date=when.date()),
|
||||
)
|
||||
ctx.stats["proposed"] += 1
|
||||
conn.execute("UPDATE claim SET data = data || %s WHERE id = %s", (Jsonb({"legal_checked": True}), row["id"]))
|
||||
@@ -0,0 +1,153 @@
|
||||
"""Link documents (headlines, press releases, government news) to the politicians and parties they mention.
|
||||
|
||||
Conservative by design: full names (>= 2 tokens) only, plus "office title + surname" of *current* office
|
||||
holders ("Bundeskanzler Stocker"); an ambiguous name is only linked if exactly one candidate is active.
|
||||
False links are worse than missed ones on a site like this.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from psycopg import Connection
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from politrack.core import Enricher, RunContext, register
|
||||
from politrack.core.records import Claim, Evidence, OrgRef, PersonRef
|
||||
from politrack.core.spec import DERIVED
|
||||
from politrack.text import normalize, strip_titles
|
||||
|
||||
TOKEN = re.compile(r"[\w'-]+")
|
||||
OFFICE_TITLE = re.compile(
|
||||
r"\b(?:Bundeskanzler(?:in)?|Vizekanzler(?:in)?|[A-ZÄÖÜ][a-zäöüß]*minister(?:in)?|Minister(?:in)?|"
|
||||
r"Staatssekretär(?:in)?|Nationalratspräsident(?:in)?|Bundesratspräsident(?:in)?|Bundespräsident(?:in)?|"
|
||||
r"Landeshauptmann|Landeshauptfrau|Landesrät(?:in)?|Landesrat|Bürgermeister(?:in)?|Klubobmann|Klubobfrau|"
|
||||
r"Klubchef(?:in)?|Parteichef(?:in)?|Generalsekretär(?:in)?|Abgeordnete[rn]?|EU-Abgeordnete[rn]?|"
|
||||
r"EU-Kommissar(?:in)?|Präsident(?:in)?)\s+(?:(?:Dr|Mag|DI|Ing)\.\s+)?(?P<surname>[A-ZÄÖÜ][a-zäöüß]+(?:-[A-ZÄÖÜ][a-zäöüß]+)?)\b"
|
||||
# "Nationalratspräsident Walter Rosenkranz": a following capitalised word means we matched a first name.
|
||||
r"(?!\s+(?:von\s+)?[A-ZÄÖÜ][a-zäöüß]+)"
|
||||
)
|
||||
CONTEXT_PATTERNS = [
|
||||
("interview", re.compile(r"\binterview\b", re.I)),
|
||||
("tv", re.compile(r"\b(ZIB ?2|ZiB ?2|Pressestunde|Im Zentrum|Sommergespräch|Report|Puls ?24|ServusTV|Talk im Hangar|"
|
||||
r"Milborn|Pro und Contra)\b", re.I)),
|
||||
("radio", re.compile(r"\b(Ö1|Morgenjournal|Mittagsjournal|Journal zu Gast)\b", re.I)),
|
||||
("op_ed", re.compile(r"\b(Gastkommentar|Kommentar der anderen)\b", re.I)),
|
||||
]
|
||||
|
||||
|
||||
def tokens(text: str) -> list[str]:
|
||||
return TOKEN.findall(normalize(text))
|
||||
|
||||
|
||||
def detect_context(text: str) -> str | None:
|
||||
for name, pattern in CONTEXT_PATTERNS:
|
||||
if pattern.search(text):
|
||||
return name
|
||||
return None
|
||||
|
||||
|
||||
class MentionLinker:
|
||||
def __init__(self, conn: Connection) -> None:
|
||||
self.person_index: dict[tuple[str, ...], set[int]] = defaultdict(set)
|
||||
for row in conn.execute("SELECT entity_id, alias FROM alias WHERE entity_type = 'person'"):
|
||||
key = tuple(tokens(strip_titles(row["alias"])))
|
||||
if len(key) >= 2:
|
||||
self.person_index[key].add(row["entity_id"])
|
||||
self.active = {r["person_id"] for r in conn.execute("SELECT person_id FROM person_status WHERE status = 'active'")}
|
||||
self.active_surnames: dict[str, set[int]] = defaultdict(set)
|
||||
for row in conn.execute(
|
||||
"SELECT p.id, p.family_name FROM person p JOIN person_status s ON s.person_id = p.id "
|
||||
"WHERE s.status = 'active' AND p.family_name IS NOT NULL"
|
||||
):
|
||||
self.active_surnames[normalize(row["family_name"])].add(row["id"])
|
||||
self.org_index: dict[tuple[str, ...], int] = {}
|
||||
for row in conn.execute(
|
||||
"""SELECT o.id, a.alias FROM organization o JOIN alias a ON a.entity_type = 'organization' AND a.entity_id = o.id
|
||||
WHERE o.kind = 'party'
|
||||
UNION SELECT id, short_name FROM organization WHERE kind = 'party' AND short_name IS NOT NULL"""
|
||||
):
|
||||
key = tuple(tokens(row["alias"]))
|
||||
if key and len("".join(key)) >= 3:
|
||||
self.org_index[key] = row["id"]
|
||||
self.max_len = max((len(k) for k in self.person_index), default=2)
|
||||
|
||||
def people(self, text: str) -> set[int]:
|
||||
toks = tokens(text)
|
||||
found: set[int] = set()
|
||||
for n in range(2, self.max_len + 1):
|
||||
for i in range(len(toks) - n + 1):
|
||||
candidates = self.person_index.get(tuple(toks[i : i + n]))
|
||||
if not candidates:
|
||||
continue
|
||||
if len(candidates) == 1:
|
||||
found |= candidates
|
||||
elif len(active := candidates & self.active) == 1:
|
||||
found |= active
|
||||
return found
|
||||
|
||||
def people_by_title(self, text: str) -> set[int]:
|
||||
found: set[int] = set()
|
||||
for m in OFFICE_TITLE.finditer(text):
|
||||
candidates = self.active_surnames.get(normalize(m.group("surname")), set())
|
||||
if len(candidates) == 1:
|
||||
found |= candidates
|
||||
return found
|
||||
|
||||
def all_people(self, text: str) -> set[int]:
|
||||
return self.people(text) | self.people_by_title(text)
|
||||
|
||||
def parties(self, text: str) -> set[int]:
|
||||
toks = tokens(text)
|
||||
found = set()
|
||||
for n in (1, 2, 3):
|
||||
for i in range(len(toks) - n + 1):
|
||||
if org := self.org_index.get(tuple(toks[i : i + n])):
|
||||
found.add(org)
|
||||
return found
|
||||
|
||||
|
||||
@register
|
||||
class Linker(Enricher):
|
||||
spec = DERIVED
|
||||
name = "enrich.linker"
|
||||
description = "Link news headlines, press releases and government news to politicians and parties"
|
||||
schedule = "*/10 * * * *"
|
||||
|
||||
def run(self, ctx: RunContext) -> Iterator[Claim]:
|
||||
conn = ctx.conn
|
||||
linker = MentionLinker(conn)
|
||||
while True:
|
||||
docs = conn.execute(
|
||||
"""SELECT d.id, d.url, d.title, d.excerpt, left(d.full_text, 3000) AS full_text, d.published_at,
|
||||
s.slug AS source, s.name AS source_name, s.kind AS source_kind,
|
||||
COALESCE((d.meta->>'tdm_reserved')::boolean, false) AS tdm_reserved
|
||||
FROM document d JOIN source s ON s.id = d.source_id
|
||||
WHERE (s.kind IN ('media', 'user') OR s.slug IN ('gdelt', 'bundesregierung', 'wksta'))
|
||||
AND NOT (d.meta ? 'linked_at') AND d.doc_type IN ('article', 'press_release')
|
||||
ORDER BY d.id LIMIT 500"""
|
||||
).fetchall()
|
||||
if not docs:
|
||||
return
|
||||
for doc in docs:
|
||||
title = doc["title"] or ""
|
||||
text = title if doc["tdm_reserved"] else f"{title} {doc['full_text'] or doc['excerpt'] or ''}"
|
||||
context = detect_context(title)
|
||||
people, parties = linker.all_people(text), linker.parties(title)
|
||||
evidence = Evidence(url=doc["url"], source=doc["source"], title=title, quote=title, stance="mentions",
|
||||
confidence=0.8, published_at=doc["published_at"])
|
||||
for person_id in people:
|
||||
yield Claim(claim_type="media_mention", person=PersonRef(ids={"politrack": str(person_id)}),
|
||||
occurred_at=doc["published_at"], summary_de=f"{doc['source_name']}: {title}",
|
||||
data={"context": context, "source": doc["source"]}, evidence=[evidence])
|
||||
for org_id in parties:
|
||||
yield Claim(claim_type="media_mention", org=OrgRef(ids={"politrack_org": str(org_id)}, create=False),
|
||||
occurred_at=doc["published_at"], summary_de=f"{doc['source_name']}: {title}",
|
||||
data={"context": context, "source": doc["source"]}, evidence=[evidence])
|
||||
conn.execute("UPDATE document SET meta = meta || %s WHERE id = %s",
|
||||
(Jsonb({"linked_at": datetime.now(timezone.utc).isoformat(), "people": sorted(people),
|
||||
"parties": sorted(parties)}), doc["id"]))
|
||||
ctx.stats["documents_linked"] += 1
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Housekeeping enrichers that work directly on canonical rows (no new facts, only derived structure).
|
||||
|
||||
* enrich.roles - give open-ended offices the party of the holder's party membership at that time
|
||||
(Wikidata offices carry no party), close open roles at the holder's date of death
|
||||
* enrich.rollcall - derive each faction's majority position from individual roll-call votes, so party
|
||||
pages show roll calls too ("ÖVP mostly for: ... 70 yes, 2 no")
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter, defaultdict
|
||||
from collections.abc import Iterator
|
||||
|
||||
from politrack.core import Enricher, RunContext, register
|
||||
from politrack.core.records import Claim, Evidence, OrgRef, VotePosition
|
||||
from politrack.core.spec import DERIVED
|
||||
|
||||
|
||||
@register
|
||||
class RoleMaintenance(Enricher):
|
||||
spec = DERIVED
|
||||
name = "enrich.roles"
|
||||
description = "Assign parties to offices from party memberships; close roles at death"
|
||||
schedule = "50 5 * * *"
|
||||
|
||||
def run(self, ctx: RunContext) -> None:
|
||||
conn = ctx.conn
|
||||
ctx.stats["roles_with_party"] = conn.execute(
|
||||
"""UPDATE role r SET party_id = pm.party_id
|
||||
FROM role pm JOIN claim c ON c.id = pm.claim_id
|
||||
WHERE r.party_id IS NULL AND r.level <> 'party' AND pm.role = 'party_member' AND c.merged_into IS NULL
|
||||
AND pm.person_id = r.person_id
|
||||
AND (pm.start_date IS NULL OR r.end_date IS NULL OR pm.start_date <= r.end_date)
|
||||
AND (pm.end_date IS NULL OR r.start_date IS NULL OR pm.end_date >= r.start_date)"""
|
||||
).rowcount
|
||||
ctx.stats["roles_closed_at_death"] = conn.execute(
|
||||
"""UPDATE role r SET end_date = p.death_date FROM person p
|
||||
WHERE p.id = r.person_id AND r.end_date IS NULL AND p.death_date IS NOT NULL"""
|
||||
).rowcount
|
||||
return None
|
||||
|
||||
|
||||
@register
|
||||
class RollCallFactions(Enricher):
|
||||
spec = DERIVED
|
||||
name = "enrich.rollcall"
|
||||
description = "Faction majority positions derived from individual roll-call votes"
|
||||
schedule = "55 4 * * *"
|
||||
|
||||
def run(self, ctx: RunContext) -> Iterator[Claim]:
|
||||
rows = ctx.conn.execute(
|
||||
"""SELECT ve.event_key, ve.item_title, ve.voted_on, ve.url, ch.slug AS chamber, vp.party_id, vp.position,
|
||||
o.short_name, o.name
|
||||
FROM vote_event ve JOIN vote_position vp ON vp.vote_event_id = ve.id
|
||||
JOIN organization ch ON ch.id = ve.chamber_id
|
||||
JOIN organization o ON o.id = vp.party_id
|
||||
JOIN claim c ON c.id = vp.claim_id AND c.merged_into IS NULL
|
||||
WHERE ve.roll_call AND vp.person_id IS NOT NULL"""
|
||||
).fetchall()
|
||||
tally: dict[tuple, Counter] = defaultdict(Counter)
|
||||
info: dict[tuple, dict] = {}
|
||||
for r in rows:
|
||||
key = (r["event_key"], r["party_id"])
|
||||
tally[key][r["position"]] += 1
|
||||
info[key] = r
|
||||
for key, counts in tally.items():
|
||||
r = info[key]
|
||||
position, _ = counts.most_common(1)[0]
|
||||
party = r["short_name"] or r["name"]
|
||||
yield Claim(
|
||||
claim_type="vote", org=OrgRef(ids={"politrack_org": str(r["party_id"])}, create=False),
|
||||
counterpart_org=OrgRef(slug=r["chamber"]), occurred_at=r["voted_on"],
|
||||
summary_de=f"{party} stimmte mehrheitlich {'für' if position == 'for' else 'gegen'}: {r['item_title']} "
|
||||
f"(namentliche Abstimmung, {counts['for']} Ja, {counts['against']} Nein)",
|
||||
summary_en=f"{party} mostly voted {'for' if position == 'for' else 'against'}: {r['item_title']} "
|
||||
f"(roll call, {counts['for']} yes / {counts['against']} no)",
|
||||
data={"position": position, "roll_call": True, "yes": counts["for"], "no": counts["against"]},
|
||||
evidence=[Evidence(url=r["url"], source="parlament", doc_type="api_record", title=r["item_title"],
|
||||
published_at=r["voted_on"], locator=f"rollcall:{r['voted_on']}")],
|
||||
facet=VotePosition(event_key=r["event_key"], position=position),
|
||||
)
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Keyword-based topic tagging for claims that did not get topics from their source.
|
||||
|
||||
Parliament items carry official topics; news mentions, donations and posts
|
||||
are tagged here with the keyword lists of the `topic` table. An optional LLM
|
||||
classifier can replace this later (see docs/roadmap.md).
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
from politrack.core import Enricher, RunContext, register
|
||||
from politrack.core.spec import DERIVED
|
||||
|
||||
|
||||
def keyword_pattern(keywords: list[str]) -> re.Pattern:
|
||||
"""Keywords match as word prefixes ("Klima" -> "Klimaschutz"); short ones (<= 3 chars) only as whole words."""
|
||||
parts = [re.escape(k) + (r"\b" if len(k) <= 3 else "") for k in keywords]
|
||||
return re.compile(r"\b(" + "|".join(parts) + ")", re.IGNORECASE)
|
||||
|
||||
|
||||
@register
|
||||
class TopicTagger(Enricher):
|
||||
spec = DERIVED
|
||||
name = "enrich.topics"
|
||||
description = "Keyword topics for claims without official topics"
|
||||
schedule = "8-59/10 * * * *"
|
||||
|
||||
def run(self, ctx: RunContext) -> None:
|
||||
while (n := tag_claims(ctx.conn)):
|
||||
ctx.stats["claims_tagged"] += n
|
||||
return None
|
||||
|
||||
|
||||
def tag_claims(conn, batch: int = 2000) -> int:
|
||||
topics = [
|
||||
(row["id"], keyword_pattern(row["keywords"]))
|
||||
for row in conn.execute("SELECT id, keywords FROM topic WHERE cardinality(keywords) > 0")
|
||||
]
|
||||
claims = conn.execute(
|
||||
"""
|
||||
SELECT c.id, c.summary_de || ' ' || COALESCE(string_agg(d.title, ' '), '') AS text
|
||||
FROM claim c
|
||||
LEFT JOIN claim_evidence e ON e.claim_id = c.id
|
||||
LEFT JOIN document d ON d.id = e.document_id
|
||||
WHERE c.claim_type IN ('media_mention', 'statement', 'donation', 'side_job', 'appearance', 'meeting', 'travel',
|
||||
'money_flow', 'government_decision', 'wiki_edit', 'career')
|
||||
AND NOT (c.data ? 'topics_tagged')
|
||||
GROUP BY c.id ORDER BY c.id LIMIT %s
|
||||
""",
|
||||
(batch,),
|
||||
).fetchall()
|
||||
for claim in claims:
|
||||
for topic_id, pattern in topics:
|
||||
if pattern.search(claim["text"]):
|
||||
conn.execute(
|
||||
"INSERT INTO topic_assignment (topic_id, claim_id, confidence, method) "
|
||||
"VALUES (%s, %s, 0.5, 'keyword') ON CONFLICT DO NOTHING",
|
||||
(topic_id, claim["id"]),
|
||||
)
|
||||
conn.execute("UPDATE claim SET data = data || '{\"topics_tagged\": true}' WHERE id = %s", (claim["id"],))
|
||||
conn.commit()
|
||||
return len(claims)
|
||||
Reference in new issue
Block a user