politics-tracking/backend/politrack/enrich/events.py

"""Extract trips and meetings from official press releases and news headlines.

Rule-based and conservative. A sentence becomes an event only if it names a
known politician and contains a travel or meeting trigger:

  travel:  "reist nach Brüssel", "Arbeitsbesuch in Israel", "beim Gipfel in Montenegro", "in China"
  meeting: "empfängt den tschechischen Außenminister", "Treffen mit ...", "traf ... zu einem Gespräch"

Sources:
  * Parlamentskorrespondenz and government news: title + full text (official works / CC BY),
  * news headlines: title only (we do not store article text).

The event date is the publication date of the source (`data.date_precision =
"publication"`). The claim key is (kind, person, place/counterpart, ISO week),
so reports from different sources about the same trip attach to the same claim
as additional evidence and corroborate it. corroborate.merge_events() also
merges near-duplicates whose dates differ by a few days.
"""

from __future__ import annotations

import re
from collections.abc import Iterator
from datetime import datetime
from typing import Any

from psycopg.types.json import Jsonb

from politrack.core import Enricher, RunContext, register
from politrack.core.records import Claim, Event, Evidence, Participation, PersonRef
from politrack.core.spec import DERIVED
from politrack.enrich.gazetteer import AUSTRIA, PLACES
from politrack.enrich.linker import MentionLinker
from politrack.text import excerpt, fingerprint, normalize

# (?<!-)/(?!-): skip names inside compounds such as "Berlin-Prozess" or "EU-Brüssel-Treffen".
PLACE_RE = re.compile(
    r"(?<![-\w])(" + "|".join(sorted((re.escape(p) for p in PLACES), key=len, reverse=True)) + r")(?![-\w])"
)
DESTINATION_RE = re.compile(r"\b(?:in|nach|aus)\s+(?:de[mnr]\s+|die\s+)?$")
STRONG_TRAVEL = re.compile(r"\b(reist\w*|Reise\w*|\w*[Bb]esuch\w*|\w*Gipfel\w*|Delegation\w*|Konferenz\w*|Tagung\w*)\b")
MEETING_RE = re.compile(
    r"\b(?:empfängt|empfing|empfangen|trifft|traf|getroffen|Treffen mit|Gespräch mit|Arbeitsgespräch mit|"
    r"Unterredung mit|tauschte sich mit|im Austausch mit|zusammengetroffen mit|kam mit)\s+"
    r"(?P<counterpart>(?:de[mnr]|die|das|seine[mnr]?|ihre[mnr]?)?\s*[^.,;:]{3,90})",
    re.IGNORECASE,
)
COUNTERPART_ROLE = re.compile(
    r"(minister|präsident|kanzler|botschafter|delegation|amtskolleg|vertreter|bürgermeister|abgeordnet|chef|"
    r"premier|könig|papst|kommissar|generalsekretär|regierungschef|staatssekretär|landeshauptm|gouverneur|"
    r"bischof|kardinal|vorsitzend|ceo|gewerkschaft|sozialpartner|außenminister)",
    re.IGNORECASE,
)
FULL_NAME = re.compile(r"\b[A-ZÄÖÜ][a-zäöüß]+(?:-[A-ZÄÖÜ][a-zäöüß]+)?\s+[A-ZÄÖÜ][a-zäöüß]+")
SENTENCE_SPLIT = re.compile(r"(?<=[.!?])\s+(?=[A-ZÄÖÜ„\"])|\n+")


FILLER = re.compile(r"^(?:(?:heute|gestern|morgen|am|vorgestern|Montag|Dienstag|Mittwoch|Donnerstag|Freitag|"
                    r"Samstag|Sonntag|de[mnr]|die|das|seine[mnr]?|ihre[mnr]?|eine[mnr]?)\s+)+", re.IGNORECASE)


def clean_counterpart(text: str) -> str:
    return FILLER.sub("", text.strip()).strip()


def counterpart_anchor(text: str) -> str:
    """Stable key for a counterpart: its last capitalised word (usually the surname or country)."""
    words = re.findall(r"[A-ZÄÖÜ][\wäöüß-]+", clean_counterpart(text))
    return normalize(words[-1]) if words else normalize(text)[:25]


def find_places(text: str) -> list[tuple[str, str]]:
    """Places in order of preference: those introduced by "in/nach" first (the destination)."""
    matches = list(PLACE_RE.finditer(text))
    preferred = [m for m in matches if DESTINATION_RE.search(text[: m.start()])]
    ordered = preferred + [m for m in matches if m not in preferred]
    return [(m.group(1), PLACES[m.group(1)]) for m in ordered]


def detect(text: str) -> list[dict]:
    """Return candidate events found in one sentence (no person resolution)."""
    events = []
    places = find_places(text)
    foreign = [(p, iso) for p, iso in places if iso != "AT"]
    if foreign and STRONG_TRAVEL.search(text):
        place, iso = foreign[0]
        events.append({"kind": "foreign_trip", "claim_type": "travel", "place": place, "country": iso})
    meeting = MEETING_RE.search(text)
    if meeting:
        counterpart = re.split(r"\s+(?:in|im|am|zu|über|bei|und|sowie)\s+", meeting.group("counterpart").strip())[0]
        place, iso = (places[0] if places else (None, None))
        # "traf eine Entscheidung" is not a meeting: require a role word or a person-like name.
        if COUNTERPART_ROLE.search(counterpart) or FULL_NAME.search(counterpart):
            events.append({"kind": "meeting", "claim_type": "meeting", "counterpart": clean_counterpart(counterpart),
                           "place": place, "country": iso})
    return events


@register
class EventExtractor(Enricher):
    spec = DERIVED
    name = "enrich.events"
    description = "Trips and meetings from press releases, government news and headlines"
    schedule = "3-59/10 * * * *"

    def run(self, ctx: RunContext) -> Iterator[Any]:
        conn = ctx.conn
        linker = MentionLinker(conn)
        while True:
            docs = conn.execute(
                """SELECT d.id, d.url, d.title, d.full_text, d.published_at, d.meta, s.slug AS source, s.kind AS source_kind
                   FROM document d JOIN source s ON s.id = d.source_id
                   WHERE NOT (d.meta ? 'events_at') AND d.published_at IS NOT NULL
                     AND (d.doc_type = 'press_release' OR s.kind = 'media' OR s.slug = 'gdelt')
                   ORDER BY d.id LIMIT 300"""
            ).fetchall()
            if not docs:
                return
            for doc in docs:
                found = 0
                for record in self.document_events(conn, linker, doc):
                    found += isinstance(record, Claim)
                    yield record
                conn.execute("UPDATE document SET meta = meta || %s WHERE id = %s",
                             (Jsonb({"events_at": datetime.now().isoformat(), "events": found}), doc["id"]))
                ctx.stats["documents"] += 1

    def document_events(self, conn, linker: MentionLinker, doc: dict) -> Iterator[Any]:
        official = doc["source_kind"] == "official"
        title = doc["title"] or ""
        body = (doc["full_text"] or "")[:4000] if official else ""
        pads = (doc["meta"] or {}).get("person_pads") or []
        doc_people: set[int] = set()
        if pads:
            doc_people = {r["entity_id"] for r in conn.execute(
                "SELECT entity_id FROM identifier WHERE entity_type = 'person' AND scheme = 'parlament' AND value = ANY(%s)",
                ([str(p) for p in pads],))}
        sentences = [title] + [s for s in SENTENCE_SPLIT.split(body) if s][:25]
        for i, sentence in enumerate(sentences):
            candidates = detect(sentence)
            if not candidates:
                continue
            people = linker.all_people(sentence)
            # In the headline of an official release the actors are usually the people it links/names.
            if not people and i == 0:
                people = doc_people or linker.all_people(f"{title} {body[:600]}")
            for person_id in people:
                for ev in candidates:
                    yield from event_records(conn, doc, person_id, ev, sentence, official)


def event_records(conn, doc: dict, person_id: int, ev: dict, sentence: str, official: bool) -> Iterator[Any]:
    published = doc["published_at"]
    week = published.strftime("%G-W%V")
    anchor = ev.get("country") if ev["kind"] == "foreign_trip" else counterpart_anchor(ev.get("counterpart") or "")
    row = conn.execute("SELECT full_name, family_name FROM person WHERE id = %s", (person_id,)).fetchone()
    # A sentence like "X empfing Y" yields the meeting for X and Y; never record "Y met Y".
    if ev["kind"] == "meeting" and row["family_name"] and normalize(row["family_name"]) in normalize(ev["counterpart"]):
        return
    if ev["kind"] == "foreign_trip":
        summary_de, summary_en = f"Reise/Aufenthalt in {ev['place']}", f"Trip to {ev['place']}"
    else:
        where = f" ({ev['place']})" if ev.get("place") else ""
        summary_de = f"Treffen mit {ev['counterpart']}{where}"
        summary_en = f"Meeting with {ev['counterpart']}{where}"
    event_key = fingerprint("event", ev["kind"], person_id, anchor, week)
    yield Event(key=event_key, kind=ev["kind"], title=f"{row['full_name']}: {summary_de}", starts_at=published,
                location=ev.get("place"), country=ev.get("country"), url=doc["url"], description=excerpt(sentence, 400))
    foreign = ev.get("country") and ev["country"] not in AUSTRIA.values()
    yield Claim(
        claim_type=ev["claim_type"], person=PersonRef(ids={"politrack": str(person_id)}), occurred_at=published,
        summary_de=summary_de, summary_en=summary_en,
        data={"place": ev.get("place"), "country": ev.get("country"), "counterpart": ev.get("counterpart"),
              "date_precision": "publication", "extracted": "rules"},
        evidence=[Evidence(url=doc["url"], source=doc["source"], title=doc["title"], published_at=published,
                           quote=excerpt(sentence, 400), confidence=0.8 if official else 0.6)],
        topics=("Außenpolitik",) if foreign else (),
        facet=Participation(event_key=event_key),
    )