Event dates from the text, faster chunking, fixes

- enrich.events: take the date a sentence names (gestern, am Montag,
  am 12. November, 12.11.2026) within 60 days of publication; otherwise the
  publication date as before. -p redo=true rebuilds all extracted events.
- enrich.embed: claim cursor instead of rescanning every claim per batch;
  a full sweep once a day
- worker-enrich: two concurrent jobs, so one slow enricher can't block the rest
- Loader: empty date strings are stored as no date (a news sitemap with an
  empty publication date failed every 20 minutes)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
simonandClaude Opus 5.5 committed 2026-11-17 21:22:24 +01:00
1 parent 7bd0142734
commit 0f7e003879
5 files changed
+138 -15

No files matched your search

+3 -2
View File
@@ -109,7 +109,7 @@ class Loader:
retrieved_at = now()
RETURNING id
""",
(src["id"], ev.url, ev.title, ev.doc_type, ev.published_at, short, full_text, fetch_id,
(src["id"], ev.url, ev.title, ev.doc_type, ev.published_at or None, short, full_text, fetch_id,
Jsonb({**ev.meta, **({"tdm_reserved": True} if tdm else {})})),
).fetchone()
self.ctx.stats["documents"] += 1
@@ -315,7 +315,8 @@ class Loader:
updated_at = now()
RETURNING id, (xmax = 0) AS inserted
""",
(c.claim_type, fp, person_id, org_id, cp_person, cp_org, c.occurred_at, c.valid_from, c.valid_to,
(c.claim_type, fp, person_id, org_id, cp_person, cp_org, c.occurred_at or None, c.valid_from or None,
c.valid_to or None,
c.summary_de, c.summary_en, Jsonb(c.data), c.sensitive or c.claim_type == "legal_proceeding",
"pending" if c.review else "published", self.ctx.adapter),
).fetchone()
+20 -4
View File
@@ -7,6 +7,8 @@ into ~1,200 character chunks. Embeddings are computed by an Ollama server
otherwise only German full-text search is available.
"""
from datetime import date
import httpx
from psycopg import Connection
@@ -30,7 +32,17 @@ class Chunker(Enricher):
schedule = "*/15 * * * *"
def run(self, ctx: RunContext) -> None:
while (n := build_chunks(ctx.conn)):
# Claims are chunked in id order: a cursor saves scanning the >1 M claims that already have one (or never
# get one, like EP part votes) on every batch. Once a day the cursor starts over, so claims that became
# visible later (e.g. approved after review) are picked up too.
today = date.today().isoformat()
after = ctx.state.get("claim_cursor", 0) if ctx.state.get("cursor_day") == today else 0
ctx.state.set("cursor_day", today)
while True:
n, after = build_chunks(ctx.conn, after=after)
ctx.state.set("claim_cursor", after)
if not n:
break
ctx.stats["chunks"] += n
ctx.stats["embedded"] = embed_pending(ctx.conn)
return None
@@ -51,7 +63,9 @@ def split_text(text: str, size: int = CHUNK_SIZE) -> list[str]:
return chunks
def build_chunks(conn: Connection, batch: int = 5000) -> int:
def build_chunks(conn: Connection, batch: int = 5000, after: int = 0) -> tuple[int, int]:
"""Chunks for up to `batch` claims with an id above `after` and for unchunked documents.
Returns (number of claims and documents chunked, highest claim id looked at)."""
claims = conn.execute(
"""
SELECT c.id, c.claim_type, c.summary_de, c.occurred_at, p.full_name, o.name AS org_name,
@@ -65,10 +79,12 @@ def build_chunks(conn: Connection, batch: int = 5000) -> int:
WHERE c.merged_into IS NULL AND c.review_status = 'published'
AND NOT COALESCE((c.data->>'part_vote')::boolean, FALSE)
AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.claim_id = c.id)
AND c.id > %s
GROUP BY c.id, p.full_name, o.name
ORDER BY c.id
LIMIT %s
""",
(batch,),
(after, batch),
).fetchall()
for c in claims:
who = c["full_name"] or c["org_name"] or ""
@@ -88,7 +104,7 @@ def build_chunks(conn: Connection, batch: int = 5000) -> int:
for i, part in enumerate(split_text(d["full_text"])):
conn.execute(UPSERT_CHUNK, (None, d["id"], i, f"{d['title'] or ''}\n{part}".strip()))
conn.commit()
return len(claims) + len(docs)
return len(claims) + len(docs), max((c["id"] for c in claims), default=after)
def embed_texts(texts: list[str]) -> list[list[float]]:
+74 -7
View File
@@ -10,8 +10,9 @@ Sources:
* Parlamentskorrespondenz and government news: title + full text (official works / CC BY),
* news headlines: title only (we do not store article text).
The event date is the publication date of the source (`data.date_precision =
"publication"`). The claim key is (kind, person, place/counterpart, ISO week),
The event date comes from the sentence when it names one ("gestern", "am Montag", "am 12. November",
"12.11.2026"; `data.date_precision = "day"`), otherwise it is the publication date of the source
(`"publication"`). The claim key is (kind, person, place/counterpart, ISO week),
so reports from different sources about the same trip attach to the same claim
as additional evidence and corroborate it. corroborate.merge_events() also
merges near-duplicates whose dates differ by a few days.
@@ -21,7 +22,7 @@ from __future__ import annotations
import re
from collections.abc import Iterator
from datetime import datetime
from datetime import date, datetime, timedelta
from typing import Any
from psycopg.types.json import Jsonb
@@ -69,6 +70,64 @@ def counterpart_anchor(text: str) -> str:
return normalize(words[-1]) if words else normalize(text)[:25]
MONTHS = {m: i for i, m in enumerate(["jänner", "februar", "märz", "april", "mai", "juni", "juli", "august",
"september", "oktober", "november", "dezember"], start=1)} | {"januar": 1}
WEEKDAYS = {d: i for i, d in enumerate(["montag", "dienstag", "mittwoch", "donnerstag", "freitag", "samstag",
"sonntag"])}
RELATIVE = {"vorgestern": -2, "gestern": -1, "heute": 0, "morgen": 1, "übermorgen": 2}
NAMED_DATE = re.compile(r"\b(\d{1,2})\.\s*(" + "|".join(MONTHS) + r")(?:\s+(\d{4}))?", re.IGNORECASE)
NUMERIC_DATE = re.compile(r"\b(\d{1,2})\.(\d{1,2})\.(\d{4}|\d{2})?(?!\d)")
WEEKDAY_RE = re.compile(r"\b(?:am|kommenden|vergangenen|letzten)\s+(" + "|".join(WEEKDAYS) + r")\b", re.IGNORECASE)
RELATIVE_RE = re.compile(r"\b(" + "|".join(RELATIVE) + r")\b(?!\s*(?:früh|abend|nacht)?\s*-)", re.IGNORECASE)
MAX_DISTANCE = timedelta(days=60)
def event_date(text: str, published: date) -> date | None:
"""The date a sentence gives for its event, if any and plausible (within 60 days of publication).
Explicit dates win over relative words; a date without a year takes the year that puts it closest to the
publication date; a weekday is the nearest one to the publication date (reports cover the days around it).
"""
def plausible(d: date | None) -> date | None:
return d if d and abs(d - published) <= MAX_DISTANCE else None
def closest_year(day: int, month: int) -> date | None:
options = []
for year in (published.year - 1, published.year, published.year + 1):
try:
options.append(date(year, month, day))
except ValueError:
continue
return min(options, key=lambda d: abs(d - published), default=None)
if (m := NAMED_DATE.search(text)):
day, month = int(m.group(1)), MONTHS[m.group(2).lower()]
try:
d = date(int(m.group(3)), month, day) if m.group(3) else closest_year(day, month)
except ValueError:
d = None
if plausible(d):
return d
if (m := NUMERIC_DATE.search(text)):
day, month = int(m.group(1)), int(m.group(2))
if 1 <= month <= 12:
year = m.group(3)
try:
d = (date(int(year) + (2000 if len(year) == 2 else 0), month, day) if year
else closest_year(day, month))
except ValueError:
d = None
if plausible(d):
return d
if (m := WEEKDAY_RE.search(text)):
target = WEEKDAYS[m.group(1).lower()]
diff = (target - published.weekday()) % 7
return published + timedelta(days=diff if diff <= 3 else diff - 7)
if (m := RELATIVE_RE.search(text)):
return published + timedelta(days=RELATIVE[m.group(1).lower()])
return None
def find_places(text: str) -> list[tuple[str, str]]:
"""Places in order of preference: those introduced by "in/nach" first (the destination)."""
matches = list(PLACE_RE.finditer(text))
@@ -105,6 +164,12 @@ class EventExtractor(Enricher):
def run(self, ctx: RunContext) -> Iterator[Any]:
conn = ctx.conn
if ctx.params.get("redo"): # -p redo=true: rebuild all extracted trips/meetings (after a rule change)
ctx.stats["claims_removed"] = conn.execute("DELETE FROM claim WHERE created_by = %s", (self.name,)).rowcount
conn.execute("DELETE FROM event e WHERE NOT EXISTS (SELECT 1 FROM event_participant p WHERE p.event_id = e.id) "
"AND e.source_id = (SELECT id FROM source WHERE slug = %s)", (self.spec.slug,))
conn.execute("UPDATE document SET meta = meta - 'events_at' - 'events' WHERE meta ? 'events_at'")
conn.commit()
linker = MentionLinker(conn)
while True:
docs = conn.execute(
@@ -151,7 +216,9 @@ class EventExtractor(Enricher):
def event_records(conn, doc: dict, person_id: int, ev: dict, sentence: str, official: bool) -> Iterator[Any]:
published = doc["published_at"]
week = published.strftime("%G-W%V")
on = event_date(sentence, published.date())
when = on or published
week = when.strftime("%G-W%V")
anchor = ev.get("country") if ev["kind"] == "foreign_trip" else counterpart_anchor(ev.get("counterpart") or "")
row = conn.execute("SELECT full_name, family_name FROM person WHERE id = %s", (person_id,)).fetchone()
# A sentence like "X empfing Y" yields the meeting for X and Y; never record "Y met Y".
@@ -164,14 +231,14 @@ def event_records(conn, doc: dict, person_id: int, ev: dict, sentence: str, offi
summary_de = f"Treffen mit {ev['counterpart']}{where}"
summary_en = f"Meeting with {ev['counterpart']}{where}"
event_key = fingerprint("event", ev["kind"], person_id, anchor, week)
yield Event(key=event_key, kind=ev["kind"], title=f"{row['full_name']}: {summary_de}", starts_at=published,
yield Event(key=event_key, kind=ev["kind"], title=f"{row['full_name']}: {summary_de}", starts_at=when,
location=ev.get("place"), country=ev.get("country"), url=doc["url"], description=excerpt(sentence, 400))
foreign = ev.get("country") and ev["country"] not in AUSTRIA.values()
yield Claim(
claim_type=ev["claim_type"], person=PersonRef(ids={"politrack": str(person_id)}), occurred_at=published,
claim_type=ev["claim_type"], person=PersonRef(ids={"politrack": str(person_id)}), occurred_at=when,
summary_de=summary_de, summary_en=summary_en,
data={"place": ev.get("place"), "country": ev.get("country"), "counterpart": ev.get("counterpart"),
"date_precision": "publication", "extracted": "rules"},
"date_precision": "day" if on else "publication", "extracted": "rules"},
evidence=[Evidence(url=doc["url"], source=doc["source"], title=doc["title"], published_at=published,
quote=excerpt(sentence, 400), confidence=0.8 if official else 0.6)],
topics=("Außenpolitik",) if foreign else (),
+40 -1
View File
@@ -1,4 +1,6 @@
from politrack.enrich.events import detect
from datetime import date
from politrack.enrich.events import detect, event_date
from politrack.enrich.legal import classify
from politrack.enrich.linker import OFFICE_TITLE
from politrack.sources.parlament.common import parse_immunity_report, parse_roll_call
@@ -96,3 +98,40 @@ def test_office_title_surname():
assert names == ["Stocker", "Bauer", "Mikl-Leitner"]
# A first name after the title must not be taken for a surname.
assert [m.group("surname") for m in OFFICE_TITLE.finditer("Nationalratspräsident Walter Rosenkranz")] == []
def test_event_date_from_the_sentence():
published = date(2026, 11, 12) # a Thursday
assert event_date("Die Ministerin reiste gestern nach Brüssel", published) == date(2026, 11, 11)
assert event_date("Der Kanzler empfängt heute den Premier", published) == published
assert event_date("Am Montag traf sie den Botschafter", published) == date(2026, 11, 9)
assert event_date("am Samstag reist er nach Rom", published) == date(2026, 11, 14)
assert event_date("Treffen am 3. November in Paris", published) == date(2026, 11, 3)
assert event_date("Gipfel am 28. Dezember 2025", published) is None # too far from publication
assert event_date("Besuch in Wien am 10.11.2026", published) == date(2026, 11, 10)
assert event_date("Treffen am 2. Jänner", date(2026, 12, 20)) == date(2027, 1, 2) # year closest to publication
assert event_date("Arbeitsbesuch in Israel", published) is None
assert event_date("die Morgen-Ausgabe", published) is None
def test_event_records_use_the_date_from_the_sentence():
from datetime import datetime, timezone
from politrack.core.records import Claim, Event
from politrack.enrich.events import event_records
class Conn:
def execute(self, *_):
class Row:
def fetchone(self):
return {"full_name": "Beate Meinl-Reisinger", "family_name": "Meinl-Reisinger"}
return Row()
doc = {"published_at": datetime(2026, 11, 12, 9, tzinfo=timezone.utc), "url": "https://example.org/x",
"source": "bundesregierung", "title": "Besuch"}
ev = {"kind": "foreign_trip", "claim_type": "travel", "place": "Brüssel", "country": "BE"}
records = list(event_records(Conn(), doc, 1, ev, "Die Außenministerin reiste gestern nach Brüssel", True))
event = next(r for r in records if isinstance(r, Event))
claim = next(r for r in records if isinstance(r, Claim))
assert event.starts_at == claim.occurred_at == date(2026, 11, 11)
assert claim.data["date_precision"] == "day"
+1 -1
View File
@@ -76,7 +76,7 @@ services:
worker-enrich:
<<: *worker
hostname: worker-enrich
command: ["procrastinate", "--app=politrack.worker.app", "worker", "--queues=enrich", "--concurrency=1"]
command: ["procrastinate", "--app=politrack.worker.app", "worker", "--queues=enrich", "--concurrency=2"]
# Nightly pg_dump at 02:30 UTC into ${BACKUP_DIR}, kept for ${BACKUP_KEEP_DAYS} days (see backup.sh).
backup: