"""Extract trips and meetings from official press releases and news headlines. Rule-based and conservative. A sentence becomes an event only if it names a known politician and contains a travel or meeting trigger: travel: "reist nach Brüssel", "Arbeitsbesuch in Israel", "beim Gipfel in Montenegro", "in China" meeting: "empfängt den tschechischen Außenminister", "Treffen mit ...", "traf ... zu einem Gespräch" Sources: * Parlamentskorrespondenz and government news: title + full text (official works / CC BY), * news headlines: title only (we do not store article text). The event date is the publication date of the source (`data.date_precision = "publication"`). The claim key is (kind, person, place/counterpart, ISO week), so reports from different sources about the same trip attach to the same claim as additional evidence and corroborate it. corroborate.merge_events() also merges near-duplicates whose dates differ by a few days. """ from __future__ import annotations import re from collections.abc import Iterator from datetime import datetime from typing import Any from psycopg.types.json import Jsonb from politrack.core import Enricher, RunContext, register from politrack.core.records import Claim, Event, Evidence, Participation, PersonRef from politrack.core.spec import DERIVED from politrack.enrich.gazetteer import AUSTRIA, PLACES from politrack.enrich.linker import MentionLinker from politrack.text import excerpt, fingerprint, normalize # (?(?:de[mnr]|die|das|seine[mnr]?|ihre[mnr]?)?\s*[^.,;:]{3,90})", re.IGNORECASE, ) COUNTERPART_ROLE = re.compile( r"(minister|präsident|kanzler|botschafter|delegation|amtskolleg|vertreter|bürgermeister|abgeordnet|chef|" r"premier|könig|papst|kommissar|generalsekretär|regierungschef|staatssekretär|landeshauptm|gouverneur|" r"bischof|kardinal|vorsitzend|ceo|gewerkschaft|sozialpartner|außenminister)", re.IGNORECASE, ) FULL_NAME = re.compile(r"\b[A-ZÄÖÜ][a-zäöüß]+(?:-[A-ZÄÖÜ][a-zäöüß]+)?\s+[A-ZÄÖÜ][a-zäöüß]+") SENTENCE_SPLIT = re.compile(r"(?<=[.!?])\s+(?=[A-ZÄÖÜ„\"])|\n+") FILLER = re.compile(r"^(?:(?:heute|gestern|morgen|am|vorgestern|Montag|Dienstag|Mittwoch|Donnerstag|Freitag|" r"Samstag|Sonntag|de[mnr]|die|das|seine[mnr]?|ihre[mnr]?|eine[mnr]?)\s+)+", re.IGNORECASE) def clean_counterpart(text: str) -> str: return FILLER.sub("", text.strip()).strip() def counterpart_anchor(text: str) -> str: """Stable key for a counterpart: its last capitalised word (usually the surname or country).""" words = re.findall(r"[A-ZÄÖÜ][\wäöüß-]+", clean_counterpart(text)) return normalize(words[-1]) if words else normalize(text)[:25] def find_places(text: str) -> list[tuple[str, str]]: """Places in order of preference: those introduced by "in/nach" first (the destination).""" matches = list(PLACE_RE.finditer(text)) preferred = [m for m in matches if DESTINATION_RE.search(text[: m.start()])] ordered = preferred + [m for m in matches if m not in preferred] return [(m.group(1), PLACES[m.group(1)]) for m in ordered] def detect(text: str) -> list[dict]: """Return candidate events found in one sentence (no person resolution).""" events = [] places = find_places(text) foreign = [(p, iso) for p, iso in places if iso != "AT"] if foreign and STRONG_TRAVEL.search(text): place, iso = foreign[0] events.append({"kind": "foreign_trip", "claim_type": "travel", "place": place, "country": iso}) meeting = MEETING_RE.search(text) if meeting: counterpart = re.split(r"\s+(?:in|im|am|zu|über|bei|und|sowie)\s+", meeting.group("counterpart").strip())[0] place, iso = (places[0] if places else (None, None)) # "traf eine Entscheidung" is not a meeting: require a role word or a person-like name. if COUNTERPART_ROLE.search(counterpart) or FULL_NAME.search(counterpart): events.append({"kind": "meeting", "claim_type": "meeting", "counterpart": clean_counterpart(counterpart), "place": place, "country": iso}) return events @register class EventExtractor(Enricher): spec = DERIVED name = "enrich.events" description = "Trips and meetings from press releases, government news and headlines" schedule = "3-59/10 * * * *" def run(self, ctx: RunContext) -> Iterator[Any]: conn = ctx.conn linker = MentionLinker(conn) while True: docs = conn.execute( """SELECT d.id, d.url, d.title, d.full_text, d.published_at, d.meta, s.slug AS source, s.kind AS source_kind FROM document d JOIN source s ON s.id = d.source_id WHERE NOT (d.meta ? 'events_at') AND d.published_at IS NOT NULL AND (d.doc_type = 'press_release' OR s.kind = 'media' OR s.slug = 'gdelt') ORDER BY d.id LIMIT 300""" ).fetchall() if not docs: return for doc in docs: found = 0 for record in self.document_events(conn, linker, doc): found += isinstance(record, Claim) yield record conn.execute("UPDATE document SET meta = meta || %s WHERE id = %s", (Jsonb({"events_at": datetime.now().isoformat(), "events": found}), doc["id"])) ctx.stats["documents"] += 1 def document_events(self, conn, linker: MentionLinker, doc: dict) -> Iterator[Any]: official = doc["source_kind"] == "official" title = doc["title"] or "" body = (doc["full_text"] or "")[:4000] if official else "" pads = (doc["meta"] or {}).get("person_pads") or [] doc_people: set[int] = set() if pads: doc_people = {r["entity_id"] for r in conn.execute( "SELECT entity_id FROM identifier WHERE entity_type = 'person' AND scheme = 'parlament' AND value = ANY(%s)", ([str(p) for p in pads],))} sentences = [title] + [s for s in SENTENCE_SPLIT.split(body) if s][:25] for i, sentence in enumerate(sentences): candidates = detect(sentence) if not candidates: continue people = linker.all_people(sentence) # In the headline of an official release the actors are usually the people it links/names. if not people and i == 0: people = doc_people or linker.all_people(f"{title} {body[:600]}") for person_id in people: for ev in candidates: yield from event_records(conn, doc, person_id, ev, sentence, official) def event_records(conn, doc: dict, person_id: int, ev: dict, sentence: str, official: bool) -> Iterator[Any]: published = doc["published_at"] week = published.strftime("%G-W%V") anchor = ev.get("country") if ev["kind"] == "foreign_trip" else counterpart_anchor(ev.get("counterpart") or "") row = conn.execute("SELECT full_name, family_name FROM person WHERE id = %s", (person_id,)).fetchone() # A sentence like "X empfing Y" yields the meeting for X and Y; never record "Y met Y". if ev["kind"] == "meeting" and row["family_name"] and normalize(row["family_name"]) in normalize(ev["counterpart"]): return if ev["kind"] == "foreign_trip": summary_de, summary_en = f"Reise/Aufenthalt in {ev['place']}", f"Trip to {ev['place']}" else: where = f" ({ev['place']})" if ev.get("place") else "" summary_de = f"Treffen mit {ev['counterpart']}{where}" summary_en = f"Meeting with {ev['counterpart']}{where}" event_key = fingerprint("event", ev["kind"], person_id, anchor, week) yield Event(key=event_key, kind=ev["kind"], title=f"{row['full_name']}: {summary_de}", starts_at=published, location=ev.get("place"), country=ev.get("country"), url=doc["url"], description=excerpt(sentence, 400)) foreign = ev.get("country") and ev["country"] not in AUSTRIA.values() yield Claim( claim_type=ev["claim_type"], person=PersonRef(ids={"politrack": str(person_id)}), occurred_at=published, summary_de=summary_de, summary_en=summary_en, data={"place": ev.get("place"), "country": ev.get("country"), "counterpart": ev.get("counterpart"), "date_precision": "publication", "extracted": "rules"}, evidence=[Evidence(url=doc["url"], source=doc["source"], title=doc["title"], published_at=published, quote=excerpt(sentence, 400), confidence=0.8 if official else 0.6)], topics=("Außenpolitik",) if foreign else (), facet=Participation(event_key=event_key), )