Event dates from the text, faster chunking, fixes

- enrich.events: take the date a sentence names (gestern, am Montag,
  am 12. November, 12.11.2026) within 60 days of publication; otherwise the
  publication date as before. -p redo=true rebuilds all extracted events.
- enrich.embed: claim cursor instead of rescanning every claim per batch;
  a full sweep once a day
- worker-enrich: two concurrent jobs, so one slow enricher can't block the rest
- Loader: empty date strings are stored as no date (a news sitemap with an
  empty publication date failed every 20 minutes)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
simonandClaude Opus 5.5 committed 2026-11-17 21:22:24 +01:00
1 parent 7bd0142734
commit 0f7e003879
5 files changed
+138 -15

No files matched your search

+3 -2
View File
@@ -109,7 +109,7 @@ class Loader:
retrieved_at = now() retrieved_at = now()
RETURNING id RETURNING id
""", """,
(src["id"], ev.url, ev.title, ev.doc_type, ev.published_at, short, full_text, fetch_id, (src["id"], ev.url, ev.title, ev.doc_type, ev.published_at or None, short, full_text, fetch_id,
Jsonb({**ev.meta, **({"tdm_reserved": True} if tdm else {})})), Jsonb({**ev.meta, **({"tdm_reserved": True} if tdm else {})})),
).fetchone() ).fetchone()
self.ctx.stats["documents"] += 1 self.ctx.stats["documents"] += 1
@@ -315,7 +315,8 @@ class Loader:
updated_at = now() updated_at = now()
RETURNING id, (xmax = 0) AS inserted RETURNING id, (xmax = 0) AS inserted
""", """,
(c.claim_type, fp, person_id, org_id, cp_person, cp_org, c.occurred_at, c.valid_from, c.valid_to, (c.claim_type, fp, person_id, org_id, cp_person, cp_org, c.occurred_at or None, c.valid_from or None,
c.valid_to or None,
c.summary_de, c.summary_en, Jsonb(c.data), c.sensitive or c.claim_type == "legal_proceeding", c.summary_de, c.summary_en, Jsonb(c.data), c.sensitive or c.claim_type == "legal_proceeding",
"pending" if c.review else "published", self.ctx.adapter), "pending" if c.review else "published", self.ctx.adapter),
).fetchone() ).fetchone()
+20 -4
View File
@@ -7,6 +7,8 @@ into ~1,200 character chunks. Embeddings are computed by an Ollama server
otherwise only German full-text search is available. otherwise only German full-text search is available.
""" """
from datetime import date
import httpx import httpx
from psycopg import Connection from psycopg import Connection
@@ -30,7 +32,17 @@ class Chunker(Enricher):
schedule = "*/15 * * * *" schedule = "*/15 * * * *"
def run(self, ctx: RunContext) -> None: def run(self, ctx: RunContext) -> None:
while (n := build_chunks(ctx.conn)): # Claims are chunked in id order: a cursor saves scanning the >1 M claims that already have one (or never
# get one, like EP part votes) on every batch. Once a day the cursor starts over, so claims that became
# visible later (e.g. approved after review) are picked up too.
today = date.today().isoformat()
after = ctx.state.get("claim_cursor", 0) if ctx.state.get("cursor_day") == today else 0
ctx.state.set("cursor_day", today)
while True:
n, after = build_chunks(ctx.conn, after=after)
ctx.state.set("claim_cursor", after)
if not n:
break
ctx.stats["chunks"] += n ctx.stats["chunks"] += n
ctx.stats["embedded"] = embed_pending(ctx.conn) ctx.stats["embedded"] = embed_pending(ctx.conn)
return None return None
@@ -51,7 +63,9 @@ def split_text(text: str, size: int = CHUNK_SIZE) -> list[str]:
return chunks return chunks
def build_chunks(conn: Connection, batch: int = 5000) -> int: def build_chunks(conn: Connection, batch: int = 5000, after: int = 0) -> tuple[int, int]:
"""Chunks for up to `batch` claims with an id above `after` and for unchunked documents.
Returns (number of claims and documents chunked, highest claim id looked at)."""
claims = conn.execute( claims = conn.execute(
""" """
SELECT c.id, c.claim_type, c.summary_de, c.occurred_at, p.full_name, o.name AS org_name, SELECT c.id, c.claim_type, c.summary_de, c.occurred_at, p.full_name, o.name AS org_name,
@@ -65,10 +79,12 @@ def build_chunks(conn: Connection, batch: int = 5000) -> int:
WHERE c.merged_into IS NULL AND c.review_status = 'published' WHERE c.merged_into IS NULL AND c.review_status = 'published'
AND NOT COALESCE((c.data->>'part_vote')::boolean, FALSE) AND NOT COALESCE((c.data->>'part_vote')::boolean, FALSE)
AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.claim_id = c.id) AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.claim_id = c.id)
AND c.id > %s
GROUP BY c.id, p.full_name, o.name GROUP BY c.id, p.full_name, o.name
ORDER BY c.id
LIMIT %s LIMIT %s
""", """,
(batch,), (after, batch),
).fetchall() ).fetchall()
for c in claims: for c in claims:
who = c["full_name"] or c["org_name"] or "" who = c["full_name"] or c["org_name"] or ""
@@ -88,7 +104,7 @@ def build_chunks(conn: Connection, batch: int = 5000) -> int:
for i, part in enumerate(split_text(d["full_text"])): for i, part in enumerate(split_text(d["full_text"])):
conn.execute(UPSERT_CHUNK, (None, d["id"], i, f"{d['title'] or ''}\n{part}".strip())) conn.execute(UPSERT_CHUNK, (None, d["id"], i, f"{d['title'] or ''}\n{part}".strip()))
conn.commit() conn.commit()
return len(claims) + len(docs) return len(claims) + len(docs), max((c["id"] for c in claims), default=after)
def embed_texts(texts: list[str]) -> list[list[float]]: def embed_texts(texts: list[str]) -> list[list[float]]:
+74 -7
View File
@@ -10,8 +10,9 @@ Sources:
* Parlamentskorrespondenz and government news: title + full text (official works / CC BY), * Parlamentskorrespondenz and government news: title + full text (official works / CC BY),
* news headlines: title only (we do not store article text). * news headlines: title only (we do not store article text).
The event date is the publication date of the source (`data.date_precision = The event date comes from the sentence when it names one ("gestern", "am Montag", "am 12. November",
"publication"`). The claim key is (kind, person, place/counterpart, ISO week), "12.11.2026"; `data.date_precision = "day"`), otherwise it is the publication date of the source
(`"publication"`). The claim key is (kind, person, place/counterpart, ISO week),
so reports from different sources about the same trip attach to the same claim so reports from different sources about the same trip attach to the same claim
as additional evidence and corroborate it. corroborate.merge_events() also as additional evidence and corroborate it. corroborate.merge_events() also
merges near-duplicates whose dates differ by a few days. merges near-duplicates whose dates differ by a few days.
@@ -21,7 +22,7 @@ from __future__ import annotations
import re import re
from collections.abc import Iterator from collections.abc import Iterator
from datetime import datetime from datetime import date, datetime, timedelta
from typing import Any from typing import Any
from psycopg.types.json import Jsonb from psycopg.types.json import Jsonb
@@ -69,6 +70,64 @@ def counterpart_anchor(text: str) -> str:
return normalize(words[-1]) if words else normalize(text)[:25] return normalize(words[-1]) if words else normalize(text)[:25]
MONTHS = {m: i for i, m in enumerate(["jänner", "februar", "märz", "april", "mai", "juni", "juli", "august",
"september", "oktober", "november", "dezember"], start=1)} | {"januar": 1}
WEEKDAYS = {d: i for i, d in enumerate(["montag", "dienstag", "mittwoch", "donnerstag", "freitag", "samstag",
"sonntag"])}
RELATIVE = {"vorgestern": -2, "gestern": -1, "heute": 0, "morgen": 1, "übermorgen": 2}
NAMED_DATE = re.compile(r"\b(\d{1,2})\.\s*(" + "|".join(MONTHS) + r")(?:\s+(\d{4}))?", re.IGNORECASE)
NUMERIC_DATE = re.compile(r"\b(\d{1,2})\.(\d{1,2})\.(\d{4}|\d{2})?(?!\d)")
WEEKDAY_RE = re.compile(r"\b(?:am|kommenden|vergangenen|letzten)\s+(" + "|".join(WEEKDAYS) + r")\b", re.IGNORECASE)
RELATIVE_RE = re.compile(r"\b(" + "|".join(RELATIVE) + r")\b(?!\s*(?:früh|abend|nacht)?\s*-)", re.IGNORECASE)
MAX_DISTANCE = timedelta(days=60)
def event_date(text: str, published: date) -> date | None:
"""The date a sentence gives for its event, if any and plausible (within 60 days of publication).
Explicit dates win over relative words; a date without a year takes the year that puts it closest to the
publication date; a weekday is the nearest one to the publication date (reports cover the days around it).
"""
def plausible(d: date | None) -> date | None:
return d if d and abs(d - published) <= MAX_DISTANCE else None
def closest_year(day: int, month: int) -> date | None:
options = []
for year in (published.year - 1, published.year, published.year + 1):
try:
options.append(date(year, month, day))
except ValueError:
continue
return min(options, key=lambda d: abs(d - published), default=None)
if (m := NAMED_DATE.search(text)):
day, month = int(m.group(1)), MONTHS[m.group(2).lower()]
try:
d = date(int(m.group(3)), month, day) if m.group(3) else closest_year(day, month)
except ValueError:
d = None
if plausible(d):
return d
if (m := NUMERIC_DATE.search(text)):
day, month = int(m.group(1)), int(m.group(2))
if 1 <= month <= 12:
year = m.group(3)
try:
d = (date(int(year) + (2000 if len(year) == 2 else 0), month, day) if year
else closest_year(day, month))
except ValueError:
d = None
if plausible(d):
return d
if (m := WEEKDAY_RE.search(text)):
target = WEEKDAYS[m.group(1).lower()]
diff = (target - published.weekday()) % 7
return published + timedelta(days=diff if diff <= 3 else diff - 7)
if (m := RELATIVE_RE.search(text)):
return published + timedelta(days=RELATIVE[m.group(1).lower()])
return None
def find_places(text: str) -> list[tuple[str, str]]: def find_places(text: str) -> list[tuple[str, str]]:
"""Places in order of preference: those introduced by "in/nach" first (the destination).""" """Places in order of preference: those introduced by "in/nach" first (the destination)."""
matches = list(PLACE_RE.finditer(text)) matches = list(PLACE_RE.finditer(text))
@@ -105,6 +164,12 @@ class EventExtractor(Enricher):
def run(self, ctx: RunContext) -> Iterator[Any]: def run(self, ctx: RunContext) -> Iterator[Any]:
conn = ctx.conn conn = ctx.conn
if ctx.params.get("redo"): # -p redo=true: rebuild all extracted trips/meetings (after a rule change)
ctx.stats["claims_removed"] = conn.execute("DELETE FROM claim WHERE created_by = %s", (self.name,)).rowcount
conn.execute("DELETE FROM event e WHERE NOT EXISTS (SELECT 1 FROM event_participant p WHERE p.event_id = e.id) "
"AND e.source_id = (SELECT id FROM source WHERE slug = %s)", (self.spec.slug,))
conn.execute("UPDATE document SET meta = meta - 'events_at' - 'events' WHERE meta ? 'events_at'")
conn.commit()
linker = MentionLinker(conn) linker = MentionLinker(conn)
while True: while True:
docs = conn.execute( docs = conn.execute(
@@ -151,7 +216,9 @@ class EventExtractor(Enricher):
def event_records(conn, doc: dict, person_id: int, ev: dict, sentence: str, official: bool) -> Iterator[Any]: def event_records(conn, doc: dict, person_id: int, ev: dict, sentence: str, official: bool) -> Iterator[Any]:
published = doc["published_at"] published = doc["published_at"]
week = published.strftime("%G-W%V") on = event_date(sentence, published.date())
when = on or published
week = when.strftime("%G-W%V")
anchor = ev.get("country") if ev["kind"] == "foreign_trip" else counterpart_anchor(ev.get("counterpart") or "") anchor = ev.get("country") if ev["kind"] == "foreign_trip" else counterpart_anchor(ev.get("counterpart") or "")
row = conn.execute("SELECT full_name, family_name FROM person WHERE id = %s", (person_id,)).fetchone() row = conn.execute("SELECT full_name, family_name FROM person WHERE id = %s", (person_id,)).fetchone()
# A sentence like "X empfing Y" yields the meeting for X and Y; never record "Y met Y". # A sentence like "X empfing Y" yields the meeting for X and Y; never record "Y met Y".
@@ -164,14 +231,14 @@ def event_records(conn, doc: dict, person_id: int, ev: dict, sentence: str, offi
summary_de = f"Treffen mit {ev['counterpart']}{where}" summary_de = f"Treffen mit {ev['counterpart']}{where}"
summary_en = f"Meeting with {ev['counterpart']}{where}" summary_en = f"Meeting with {ev['counterpart']}{where}"
event_key = fingerprint("event", ev["kind"], person_id, anchor, week) event_key = fingerprint("event", ev["kind"], person_id, anchor, week)
yield Event(key=event_key, kind=ev["kind"], title=f"{row['full_name']}: {summary_de}", starts_at=published, yield Event(key=event_key, kind=ev["kind"], title=f"{row['full_name']}: {summary_de}", starts_at=when,
location=ev.get("place"), country=ev.get("country"), url=doc["url"], description=excerpt(sentence, 400)) location=ev.get("place"), country=ev.get("country"), url=doc["url"], description=excerpt(sentence, 400))
foreign = ev.get("country") and ev["country"] not in AUSTRIA.values() foreign = ev.get("country") and ev["country"] not in AUSTRIA.values()
yield Claim( yield Claim(
claim_type=ev["claim_type"], person=PersonRef(ids={"politrack": str(person_id)}), occurred_at=published, claim_type=ev["claim_type"], person=PersonRef(ids={"politrack": str(person_id)}), occurred_at=when,
summary_de=summary_de, summary_en=summary_en, summary_de=summary_de, summary_en=summary_en,
data={"place": ev.get("place"), "country": ev.get("country"), "counterpart": ev.get("counterpart"), data={"place": ev.get("place"), "country": ev.get("country"), "counterpart": ev.get("counterpart"),
"date_precision": "publication", "extracted": "rules"}, "date_precision": "day" if on else "publication", "extracted": "rules"},
evidence=[Evidence(url=doc["url"], source=doc["source"], title=doc["title"], published_at=published, evidence=[Evidence(url=doc["url"], source=doc["source"], title=doc["title"], published_at=published,
quote=excerpt(sentence, 400), confidence=0.8 if official else 0.6)], quote=excerpt(sentence, 400), confidence=0.8 if official else 0.6)],
topics=("Außenpolitik",) if foreign else (), topics=("Außenpolitik",) if foreign else (),
+40 -1
View File
@@ -1,4 +1,6 @@
from politrack.enrich.events import detect from datetime import date
from politrack.enrich.events import detect, event_date
from politrack.enrich.legal import classify from politrack.enrich.legal import classify
from politrack.enrich.linker import OFFICE_TITLE from politrack.enrich.linker import OFFICE_TITLE
from politrack.sources.parlament.common import parse_immunity_report, parse_roll_call from politrack.sources.parlament.common import parse_immunity_report, parse_roll_call
@@ -96,3 +98,40 @@ def test_office_title_surname():
assert names == ["Stocker", "Bauer", "Mikl-Leitner"] assert names == ["Stocker", "Bauer", "Mikl-Leitner"]
# A first name after the title must not be taken for a surname. # A first name after the title must not be taken for a surname.
assert [m.group("surname") for m in OFFICE_TITLE.finditer("Nationalratspräsident Walter Rosenkranz")] == [] assert [m.group("surname") for m in OFFICE_TITLE.finditer("Nationalratspräsident Walter Rosenkranz")] == []
def test_event_date_from_the_sentence():
published = date(2026, 11, 12) # a Thursday
assert event_date("Die Ministerin reiste gestern nach Brüssel", published) == date(2026, 11, 11)
assert event_date("Der Kanzler empfängt heute den Premier", published) == published
assert event_date("Am Montag traf sie den Botschafter", published) == date(2026, 11, 9)
assert event_date("am Samstag reist er nach Rom", published) == date(2026, 11, 14)
assert event_date("Treffen am 3. November in Paris", published) == date(2026, 11, 3)
assert event_date("Gipfel am 28. Dezember 2025", published) is None # too far from publication
assert event_date("Besuch in Wien am 10.11.2026", published) == date(2026, 11, 10)
assert event_date("Treffen am 2. Jänner", date(2026, 12, 20)) == date(2027, 1, 2) # year closest to publication
assert event_date("Arbeitsbesuch in Israel", published) is None
assert event_date("die Morgen-Ausgabe", published) is None
def test_event_records_use_the_date_from_the_sentence():
from datetime import datetime, timezone
from politrack.core.records import Claim, Event
from politrack.enrich.events import event_records
class Conn:
def execute(self, *_):
class Row:
def fetchone(self):
return {"full_name": "Beate Meinl-Reisinger", "family_name": "Meinl-Reisinger"}
return Row()
doc = {"published_at": datetime(2026, 11, 12, 9, tzinfo=timezone.utc), "url": "https://example.org/x",
"source": "bundesregierung", "title": "Besuch"}
ev = {"kind": "foreign_trip", "claim_type": "travel", "place": "Brüssel", "country": "BE"}
records = list(event_records(Conn(), doc, 1, ev, "Die Außenministerin reiste gestern nach Brüssel", True))
event = next(r for r in records if isinstance(r, Event))
claim = next(r for r in records if isinstance(r, Claim))
assert event.starts_at == claim.occurred_at == date(2026, 11, 11)
assert claim.data["date_precision"] == "day"
+1 -1
View File
@@ -76,7 +76,7 @@ services:
worker-enrich: worker-enrich:
<<: *worker <<: *worker
hostname: worker-enrich hostname: worker-enrich
command: ["procrastinate", "--app=politrack.worker.app", "worker", "--queues=enrich", "--concurrency=1"] command: ["procrastinate", "--app=politrack.worker.app", "worker", "--queues=enrich", "--concurrency=2"]
# Nightly pg_dump at 02:30 UTC into ${BACKUP_DIR}, kept for ${BACKUP_KEEP_DAYS} days (see backup.sh). # Nightly pg_dump at 02:30 UTC into ${BACKUP_DIR}, kept for ${BACKUP_KEEP_DAYS} days (see backup.sh).
backup: backup: