diff --git a/backend/politrack/core/loader.py b/backend/politrack/core/loader.py index dfdfba8..bc8fde1 100644 --- a/backend/politrack/core/loader.py +++ b/backend/politrack/core/loader.py @@ -109,7 +109,7 @@ class Loader: retrieved_at = now() RETURNING id """, - (src["id"], ev.url, ev.title, ev.doc_type, ev.published_at, short, full_text, fetch_id, + (src["id"], ev.url, ev.title, ev.doc_type, ev.published_at or None, short, full_text, fetch_id, Jsonb({**ev.meta, **({"tdm_reserved": True} if tdm else {})})), ).fetchone() self.ctx.stats["documents"] += 1 @@ -315,7 +315,8 @@ class Loader: updated_at = now() RETURNING id, (xmax = 0) AS inserted """, - (c.claim_type, fp, person_id, org_id, cp_person, cp_org, c.occurred_at, c.valid_from, c.valid_to, + (c.claim_type, fp, person_id, org_id, cp_person, cp_org, c.occurred_at or None, c.valid_from or None, + c.valid_to or None, c.summary_de, c.summary_en, Jsonb(c.data), c.sensitive or c.claim_type == "legal_proceeding", "pending" if c.review else "published", self.ctx.adapter), ).fetchone() diff --git a/backend/politrack/enrich/embed.py b/backend/politrack/enrich/embed.py index 27ea5fa..6b9325e 100644 --- a/backend/politrack/enrich/embed.py +++ b/backend/politrack/enrich/embed.py @@ -7,6 +7,8 @@ into ~1,200 character chunks. Embeddings are computed by an Ollama server otherwise only German full-text search is available. """ +from datetime import date + import httpx from psycopg import Connection @@ -30,7 +32,17 @@ class Chunker(Enricher): schedule = "*/15 * * * *" def run(self, ctx: RunContext) -> None: - while (n := build_chunks(ctx.conn)): + # Claims are chunked in id order: a cursor saves scanning the >1 M claims that already have one (or never + # get one, like EP part votes) on every batch. Once a day the cursor starts over, so claims that became + # visible later (e.g. approved after review) are picked up too. + today = date.today().isoformat() + after = ctx.state.get("claim_cursor", 0) if ctx.state.get("cursor_day") == today else 0 + ctx.state.set("cursor_day", today) + while True: + n, after = build_chunks(ctx.conn, after=after) + ctx.state.set("claim_cursor", after) + if not n: + break ctx.stats["chunks"] += n ctx.stats["embedded"] = embed_pending(ctx.conn) return None @@ -51,7 +63,9 @@ def split_text(text: str, size: int = CHUNK_SIZE) -> list[str]: return chunks -def build_chunks(conn: Connection, batch: int = 5000) -> int: +def build_chunks(conn: Connection, batch: int = 5000, after: int = 0) -> tuple[int, int]: + """Chunks for up to `batch` claims with an id above `after` and for unchunked documents. + Returns (number of claims and documents chunked, highest claim id looked at).""" claims = conn.execute( """ SELECT c.id, c.claim_type, c.summary_de, c.occurred_at, p.full_name, o.name AS org_name, @@ -65,10 +79,12 @@ def build_chunks(conn: Connection, batch: int = 5000) -> int: WHERE c.merged_into IS NULL AND c.review_status = 'published' AND NOT COALESCE((c.data->>'part_vote')::boolean, FALSE) AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.claim_id = c.id) + AND c.id > %s GROUP BY c.id, p.full_name, o.name + ORDER BY c.id LIMIT %s """, - (batch,), + (after, batch), ).fetchall() for c in claims: who = c["full_name"] or c["org_name"] or "" @@ -88,7 +104,7 @@ def build_chunks(conn: Connection, batch: int = 5000) -> int: for i, part in enumerate(split_text(d["full_text"])): conn.execute(UPSERT_CHUNK, (None, d["id"], i, f"{d['title'] or ''}\n{part}".strip())) conn.commit() - return len(claims) + len(docs) + return len(claims) + len(docs), max((c["id"] for c in claims), default=after) def embed_texts(texts: list[str]) -> list[list[float]]: diff --git a/backend/politrack/enrich/events.py b/backend/politrack/enrich/events.py index 7a1cd65..a2c02f0 100644 --- a/backend/politrack/enrich/events.py +++ b/backend/politrack/enrich/events.py @@ -10,8 +10,9 @@ Sources: * Parlamentskorrespondenz and government news: title + full text (official works / CC BY), * news headlines: title only (we do not store article text). -The event date is the publication date of the source (`data.date_precision = -"publication"`). The claim key is (kind, person, place/counterpart, ISO week), +The event date comes from the sentence when it names one ("gestern", "am Montag", "am 12. November", +"12.11.2026"; `data.date_precision = "day"`), otherwise it is the publication date of the source +(`"publication"`). The claim key is (kind, person, place/counterpart, ISO week), so reports from different sources about the same trip attach to the same claim as additional evidence and corroborate it. corroborate.merge_events() also merges near-duplicates whose dates differ by a few days. @@ -21,7 +22,7 @@ from __future__ import annotations import re from collections.abc import Iterator -from datetime import datetime +from datetime import date, datetime, timedelta from typing import Any from psycopg.types.json import Jsonb @@ -69,6 +70,64 @@ def counterpart_anchor(text: str) -> str: return normalize(words[-1]) if words else normalize(text)[:25] +MONTHS = {m: i for i, m in enumerate(["jänner", "februar", "märz", "april", "mai", "juni", "juli", "august", + "september", "oktober", "november", "dezember"], start=1)} | {"januar": 1} +WEEKDAYS = {d: i for i, d in enumerate(["montag", "dienstag", "mittwoch", "donnerstag", "freitag", "samstag", + "sonntag"])} +RELATIVE = {"vorgestern": -2, "gestern": -1, "heute": 0, "morgen": 1, "übermorgen": 2} +NAMED_DATE = re.compile(r"\b(\d{1,2})\.\s*(" + "|".join(MONTHS) + r")(?:\s+(\d{4}))?", re.IGNORECASE) +NUMERIC_DATE = re.compile(r"\b(\d{1,2})\.(\d{1,2})\.(\d{4}|\d{2})?(?!\d)") +WEEKDAY_RE = re.compile(r"\b(?:am|kommenden|vergangenen|letzten)\s+(" + "|".join(WEEKDAYS) + r")\b", re.IGNORECASE) +RELATIVE_RE = re.compile(r"\b(" + "|".join(RELATIVE) + r")\b(?!\s*(?:früh|abend|nacht)?\s*-)", re.IGNORECASE) +MAX_DISTANCE = timedelta(days=60) + + +def event_date(text: str, published: date) -> date | None: + """The date a sentence gives for its event, if any and plausible (within 60 days of publication). + + Explicit dates win over relative words; a date without a year takes the year that puts it closest to the + publication date; a weekday is the nearest one to the publication date (reports cover the days around it). + """ + def plausible(d: date | None) -> date | None: + return d if d and abs(d - published) <= MAX_DISTANCE else None + + def closest_year(day: int, month: int) -> date | None: + options = [] + for year in (published.year - 1, published.year, published.year + 1): + try: + options.append(date(year, month, day)) + except ValueError: + continue + return min(options, key=lambda d: abs(d - published), default=None) + + if (m := NAMED_DATE.search(text)): + day, month = int(m.group(1)), MONTHS[m.group(2).lower()] + try: + d = date(int(m.group(3)), month, day) if m.group(3) else closest_year(day, month) + except ValueError: + d = None + if plausible(d): + return d + if (m := NUMERIC_DATE.search(text)): + day, month = int(m.group(1)), int(m.group(2)) + if 1 <= month <= 12: + year = m.group(3) + try: + d = (date(int(year) + (2000 if len(year) == 2 else 0), month, day) if year + else closest_year(day, month)) + except ValueError: + d = None + if plausible(d): + return d + if (m := WEEKDAY_RE.search(text)): + target = WEEKDAYS[m.group(1).lower()] + diff = (target - published.weekday()) % 7 + return published + timedelta(days=diff if diff <= 3 else diff - 7) + if (m := RELATIVE_RE.search(text)): + return published + timedelta(days=RELATIVE[m.group(1).lower()]) + return None + + def find_places(text: str) -> list[tuple[str, str]]: """Places in order of preference: those introduced by "in/nach" first (the destination).""" matches = list(PLACE_RE.finditer(text)) @@ -105,6 +164,12 @@ class EventExtractor(Enricher): def run(self, ctx: RunContext) -> Iterator[Any]: conn = ctx.conn + if ctx.params.get("redo"): # -p redo=true: rebuild all extracted trips/meetings (after a rule change) + ctx.stats["claims_removed"] = conn.execute("DELETE FROM claim WHERE created_by = %s", (self.name,)).rowcount + conn.execute("DELETE FROM event e WHERE NOT EXISTS (SELECT 1 FROM event_participant p WHERE p.event_id = e.id) " + "AND e.source_id = (SELECT id FROM source WHERE slug = %s)", (self.spec.slug,)) + conn.execute("UPDATE document SET meta = meta - 'events_at' - 'events' WHERE meta ? 'events_at'") + conn.commit() linker = MentionLinker(conn) while True: docs = conn.execute( @@ -151,7 +216,9 @@ class EventExtractor(Enricher): def event_records(conn, doc: dict, person_id: int, ev: dict, sentence: str, official: bool) -> Iterator[Any]: published = doc["published_at"] - week = published.strftime("%G-W%V") + on = event_date(sentence, published.date()) + when = on or published + week = when.strftime("%G-W%V") anchor = ev.get("country") if ev["kind"] == "foreign_trip" else counterpart_anchor(ev.get("counterpart") or "") row = conn.execute("SELECT full_name, family_name FROM person WHERE id = %s", (person_id,)).fetchone() # A sentence like "X empfing Y" yields the meeting for X and Y; never record "Y met Y". @@ -164,14 +231,14 @@ def event_records(conn, doc: dict, person_id: int, ev: dict, sentence: str, offi summary_de = f"Treffen mit {ev['counterpart']}{where}" summary_en = f"Meeting with {ev['counterpart']}{where}" event_key = fingerprint("event", ev["kind"], person_id, anchor, week) - yield Event(key=event_key, kind=ev["kind"], title=f"{row['full_name']}: {summary_de}", starts_at=published, + yield Event(key=event_key, kind=ev["kind"], title=f"{row['full_name']}: {summary_de}", starts_at=when, location=ev.get("place"), country=ev.get("country"), url=doc["url"], description=excerpt(sentence, 400)) foreign = ev.get("country") and ev["country"] not in AUSTRIA.values() yield Claim( - claim_type=ev["claim_type"], person=PersonRef(ids={"politrack": str(person_id)}), occurred_at=published, + claim_type=ev["claim_type"], person=PersonRef(ids={"politrack": str(person_id)}), occurred_at=when, summary_de=summary_de, summary_en=summary_en, data={"place": ev.get("place"), "country": ev.get("country"), "counterpart": ev.get("counterpart"), - "date_precision": "publication", "extracted": "rules"}, + "date_precision": "day" if on else "publication", "extracted": "rules"}, evidence=[Evidence(url=doc["url"], source=doc["source"], title=doc["title"], published_at=published, quote=excerpt(sentence, 400), confidence=0.8 if official else 0.6)], topics=("Außenpolitik",) if foreign else (), diff --git a/backend/tests/test_extraction.py b/backend/tests/test_extraction.py index b28122c..9e43d92 100644 --- a/backend/tests/test_extraction.py +++ b/backend/tests/test_extraction.py @@ -1,4 +1,6 @@ -from politrack.enrich.events import detect +from datetime import date + +from politrack.enrich.events import detect, event_date from politrack.enrich.legal import classify from politrack.enrich.linker import OFFICE_TITLE from politrack.sources.parlament.common import parse_immunity_report, parse_roll_call @@ -96,3 +98,40 @@ def test_office_title_surname(): assert names == ["Stocker", "Bauer", "Mikl-Leitner"] # A first name after the title must not be taken for a surname. assert [m.group("surname") for m in OFFICE_TITLE.finditer("Nationalratspräsident Walter Rosenkranz")] == [] + + +def test_event_date_from_the_sentence(): + published = date(2026, 11, 12) # a Thursday + assert event_date("Die Ministerin reiste gestern nach Brüssel", published) == date(2026, 11, 11) + assert event_date("Der Kanzler empfängt heute den Premier", published) == published + assert event_date("Am Montag traf sie den Botschafter", published) == date(2026, 11, 9) + assert event_date("am Samstag reist er nach Rom", published) == date(2026, 11, 14) + assert event_date("Treffen am 3. November in Paris", published) == date(2026, 11, 3) + assert event_date("Gipfel am 28. Dezember 2025", published) is None # too far from publication + assert event_date("Besuch in Wien am 10.11.2026", published) == date(2026, 11, 10) + assert event_date("Treffen am 2. Jänner", date(2026, 12, 20)) == date(2027, 1, 2) # year closest to publication + assert event_date("Arbeitsbesuch in Israel", published) is None + assert event_date("die Morgen-Ausgabe", published) is None + + +def test_event_records_use_the_date_from_the_sentence(): + from datetime import datetime, timezone + + from politrack.core.records import Claim, Event + from politrack.enrich.events import event_records + + class Conn: + def execute(self, *_): + class Row: + def fetchone(self): + return {"full_name": "Beate Meinl-Reisinger", "family_name": "Meinl-Reisinger"} + return Row() + + doc = {"published_at": datetime(2026, 11, 12, 9, tzinfo=timezone.utc), "url": "https://example.org/x", + "source": "bundesregierung", "title": "Besuch"} + ev = {"kind": "foreign_trip", "claim_type": "travel", "place": "Brüssel", "country": "BE"} + records = list(event_records(Conn(), doc, 1, ev, "Die Außenministerin reiste gestern nach Brüssel", True)) + event = next(r for r in records if isinstance(r, Event)) + claim = next(r for r in records if isinstance(r, Claim)) + assert event.starts_at == claim.occurred_at == date(2026, 11, 11) + assert claim.data["date_precision"] == "day" diff --git a/docker-compose.yml b/docker-compose.yml index d6ee993..edfb80f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -76,7 +76,7 @@ services: worker-enrich: <<: *worker hostname: worker-enrich - command: ["procrastinate", "--app=politrack.worker.app", "worker", "--queues=enrich", "--concurrency=1"] + command: ["procrastinate", "--app=politrack.worker.app", "worker", "--queues=enrich", "--concurrency=2"] # Nightly pg_dump at 02:30 UTC into ${BACKUP_DIR}, kept for ${BACKUP_KEEP_DAYS} days (see backup.sh). backup: