Event dates from the text, faster chunking, fixes
- enrich.events: take the date a sentence names (gestern, am Montag, am 12. November, 12.11.2026) within 60 days of publication; otherwise the publication date as before. -p redo=true rebuilds all extracted events. - enrich.embed: claim cursor instead of rescanning every claim per batch; a full sweep once a day - worker-enrich: two concurrent jobs, so one slow enricher can't block the rest - Loader: empty date strings are stored as no date (a news sitemap with an empty publication date failed every 20 minutes) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7bd0142734
commit
0f7e003879
5 files changed
+138
-15
No files matched your search
@@ -109,7 +109,7 @@ class Loader:
|
||||
retrieved_at = now()
|
||||
RETURNING id
|
||||
""",
|
||||
(src["id"], ev.url, ev.title, ev.doc_type, ev.published_at, short, full_text, fetch_id,
|
||||
(src["id"], ev.url, ev.title, ev.doc_type, ev.published_at or None, short, full_text, fetch_id,
|
||||
Jsonb({**ev.meta, **({"tdm_reserved": True} if tdm else {})})),
|
||||
).fetchone()
|
||||
self.ctx.stats["documents"] += 1
|
||||
@@ -315,7 +315,8 @@ class Loader:
|
||||
updated_at = now()
|
||||
RETURNING id, (xmax = 0) AS inserted
|
||||
""",
|
||||
(c.claim_type, fp, person_id, org_id, cp_person, cp_org, c.occurred_at, c.valid_from, c.valid_to,
|
||||
(c.claim_type, fp, person_id, org_id, cp_person, cp_org, c.occurred_at or None, c.valid_from or None,
|
||||
c.valid_to or None,
|
||||
c.summary_de, c.summary_en, Jsonb(c.data), c.sensitive or c.claim_type == "legal_proceeding",
|
||||
"pending" if c.review else "published", self.ctx.adapter),
|
||||
).fetchone()
|
||||
|
||||
@@ -7,6 +7,8 @@ into ~1,200 character chunks. Embeddings are computed by an Ollama server
|
||||
otherwise only German full-text search is available.
|
||||
"""
|
||||
|
||||
from datetime import date
|
||||
|
||||
import httpx
|
||||
from psycopg import Connection
|
||||
|
||||
@@ -30,7 +32,17 @@ class Chunker(Enricher):
|
||||
schedule = "*/15 * * * *"
|
||||
|
||||
def run(self, ctx: RunContext) -> None:
|
||||
while (n := build_chunks(ctx.conn)):
|
||||
# Claims are chunked in id order: a cursor saves scanning the >1 M claims that already have one (or never
|
||||
# get one, like EP part votes) on every batch. Once a day the cursor starts over, so claims that became
|
||||
# visible later (e.g. approved after review) are picked up too.
|
||||
today = date.today().isoformat()
|
||||
after = ctx.state.get("claim_cursor", 0) if ctx.state.get("cursor_day") == today else 0
|
||||
ctx.state.set("cursor_day", today)
|
||||
while True:
|
||||
n, after = build_chunks(ctx.conn, after=after)
|
||||
ctx.state.set("claim_cursor", after)
|
||||
if not n:
|
||||
break
|
||||
ctx.stats["chunks"] += n
|
||||
ctx.stats["embedded"] = embed_pending(ctx.conn)
|
||||
return None
|
||||
@@ -51,7 +63,9 @@ def split_text(text: str, size: int = CHUNK_SIZE) -> list[str]:
|
||||
return chunks
|
||||
|
||||
|
||||
def build_chunks(conn: Connection, batch: int = 5000) -> int:
|
||||
def build_chunks(conn: Connection, batch: int = 5000, after: int = 0) -> tuple[int, int]:
|
||||
"""Chunks for up to `batch` claims with an id above `after` and for unchunked documents.
|
||||
Returns (number of claims and documents chunked, highest claim id looked at)."""
|
||||
claims = conn.execute(
|
||||
"""
|
||||
SELECT c.id, c.claim_type, c.summary_de, c.occurred_at, p.full_name, o.name AS org_name,
|
||||
@@ -65,10 +79,12 @@ def build_chunks(conn: Connection, batch: int = 5000) -> int:
|
||||
WHERE c.merged_into IS NULL AND c.review_status = 'published'
|
||||
AND NOT COALESCE((c.data->>'part_vote')::boolean, FALSE)
|
||||
AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.claim_id = c.id)
|
||||
AND c.id > %s
|
||||
GROUP BY c.id, p.full_name, o.name
|
||||
ORDER BY c.id
|
||||
LIMIT %s
|
||||
""",
|
||||
(batch,),
|
||||
(after, batch),
|
||||
).fetchall()
|
||||
for c in claims:
|
||||
who = c["full_name"] or c["org_name"] or ""
|
||||
@@ -88,7 +104,7 @@ def build_chunks(conn: Connection, batch: int = 5000) -> int:
|
||||
for i, part in enumerate(split_text(d["full_text"])):
|
||||
conn.execute(UPSERT_CHUNK, (None, d["id"], i, f"{d['title'] or ''}\n{part}".strip()))
|
||||
conn.commit()
|
||||
return len(claims) + len(docs)
|
||||
return len(claims) + len(docs), max((c["id"] for c in claims), default=after)
|
||||
|
||||
|
||||
def embed_texts(texts: list[str]) -> list[list[float]]:
|
||||
|
||||
@@ -10,8 +10,9 @@ Sources:
|
||||
* Parlamentskorrespondenz and government news: title + full text (official works / CC BY),
|
||||
* news headlines: title only (we do not store article text).
|
||||
|
||||
The event date is the publication date of the source (`data.date_precision =
|
||||
"publication"`). The claim key is (kind, person, place/counterpart, ISO week),
|
||||
The event date comes from the sentence when it names one ("gestern", "am Montag", "am 12. November",
|
||||
"12.11.2026"; `data.date_precision = "day"`), otherwise it is the publication date of the source
|
||||
(`"publication"`). The claim key is (kind, person, place/counterpart, ISO week),
|
||||
so reports from different sources about the same trip attach to the same claim
|
||||
as additional evidence and corroborate it. corroborate.merge_events() also
|
||||
merges near-duplicates whose dates differ by a few days.
|
||||
@@ -21,7 +22,7 @@ from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime
|
||||
from datetime import date, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from psycopg.types.json import Jsonb
|
||||
@@ -69,6 +70,64 @@ def counterpart_anchor(text: str) -> str:
|
||||
return normalize(words[-1]) if words else normalize(text)[:25]
|
||||
|
||||
|
||||
MONTHS = {m: i for i, m in enumerate(["jänner", "februar", "märz", "april", "mai", "juni", "juli", "august",
|
||||
"september", "oktober", "november", "dezember"], start=1)} | {"januar": 1}
|
||||
WEEKDAYS = {d: i for i, d in enumerate(["montag", "dienstag", "mittwoch", "donnerstag", "freitag", "samstag",
|
||||
"sonntag"])}
|
||||
RELATIVE = {"vorgestern": -2, "gestern": -1, "heute": 0, "morgen": 1, "übermorgen": 2}
|
||||
NAMED_DATE = re.compile(r"\b(\d{1,2})\.\s*(" + "|".join(MONTHS) + r")(?:\s+(\d{4}))?", re.IGNORECASE)
|
||||
NUMERIC_DATE = re.compile(r"\b(\d{1,2})\.(\d{1,2})\.(\d{4}|\d{2})?(?!\d)")
|
||||
WEEKDAY_RE = re.compile(r"\b(?:am|kommenden|vergangenen|letzten)\s+(" + "|".join(WEEKDAYS) + r")\b", re.IGNORECASE)
|
||||
RELATIVE_RE = re.compile(r"\b(" + "|".join(RELATIVE) + r")\b(?!\s*(?:früh|abend|nacht)?\s*-)", re.IGNORECASE)
|
||||
MAX_DISTANCE = timedelta(days=60)
|
||||
|
||||
|
||||
def event_date(text: str, published: date) -> date | None:
|
||||
"""The date a sentence gives for its event, if any and plausible (within 60 days of publication).
|
||||
|
||||
Explicit dates win over relative words; a date without a year takes the year that puts it closest to the
|
||||
publication date; a weekday is the nearest one to the publication date (reports cover the days around it).
|
||||
"""
|
||||
def plausible(d: date | None) -> date | None:
|
||||
return d if d and abs(d - published) <= MAX_DISTANCE else None
|
||||
|
||||
def closest_year(day: int, month: int) -> date | None:
|
||||
options = []
|
||||
for year in (published.year - 1, published.year, published.year + 1):
|
||||
try:
|
||||
options.append(date(year, month, day))
|
||||
except ValueError:
|
||||
continue
|
||||
return min(options, key=lambda d: abs(d - published), default=None)
|
||||
|
||||
if (m := NAMED_DATE.search(text)):
|
||||
day, month = int(m.group(1)), MONTHS[m.group(2).lower()]
|
||||
try:
|
||||
d = date(int(m.group(3)), month, day) if m.group(3) else closest_year(day, month)
|
||||
except ValueError:
|
||||
d = None
|
||||
if plausible(d):
|
||||
return d
|
||||
if (m := NUMERIC_DATE.search(text)):
|
||||
day, month = int(m.group(1)), int(m.group(2))
|
||||
if 1 <= month <= 12:
|
||||
year = m.group(3)
|
||||
try:
|
||||
d = (date(int(year) + (2000 if len(year) == 2 else 0), month, day) if year
|
||||
else closest_year(day, month))
|
||||
except ValueError:
|
||||
d = None
|
||||
if plausible(d):
|
||||
return d
|
||||
if (m := WEEKDAY_RE.search(text)):
|
||||
target = WEEKDAYS[m.group(1).lower()]
|
||||
diff = (target - published.weekday()) % 7
|
||||
return published + timedelta(days=diff if diff <= 3 else diff - 7)
|
||||
if (m := RELATIVE_RE.search(text)):
|
||||
return published + timedelta(days=RELATIVE[m.group(1).lower()])
|
||||
return None
|
||||
|
||||
|
||||
def find_places(text: str) -> list[tuple[str, str]]:
|
||||
"""Places in order of preference: those introduced by "in/nach" first (the destination)."""
|
||||
matches = list(PLACE_RE.finditer(text))
|
||||
@@ -105,6 +164,12 @@ class EventExtractor(Enricher):
|
||||
|
||||
def run(self, ctx: RunContext) -> Iterator[Any]:
|
||||
conn = ctx.conn
|
||||
if ctx.params.get("redo"): # -p redo=true: rebuild all extracted trips/meetings (after a rule change)
|
||||
ctx.stats["claims_removed"] = conn.execute("DELETE FROM claim WHERE created_by = %s", (self.name,)).rowcount
|
||||
conn.execute("DELETE FROM event e WHERE NOT EXISTS (SELECT 1 FROM event_participant p WHERE p.event_id = e.id) "
|
||||
"AND e.source_id = (SELECT id FROM source WHERE slug = %s)", (self.spec.slug,))
|
||||
conn.execute("UPDATE document SET meta = meta - 'events_at' - 'events' WHERE meta ? 'events_at'")
|
||||
conn.commit()
|
||||
linker = MentionLinker(conn)
|
||||
while True:
|
||||
docs = conn.execute(
|
||||
@@ -151,7 +216,9 @@ class EventExtractor(Enricher):
|
||||
|
||||
def event_records(conn, doc: dict, person_id: int, ev: dict, sentence: str, official: bool) -> Iterator[Any]:
|
||||
published = doc["published_at"]
|
||||
week = published.strftime("%G-W%V")
|
||||
on = event_date(sentence, published.date())
|
||||
when = on or published
|
||||
week = when.strftime("%G-W%V")
|
||||
anchor = ev.get("country") if ev["kind"] == "foreign_trip" else counterpart_anchor(ev.get("counterpart") or "")
|
||||
row = conn.execute("SELECT full_name, family_name FROM person WHERE id = %s", (person_id,)).fetchone()
|
||||
# A sentence like "X empfing Y" yields the meeting for X and Y; never record "Y met Y".
|
||||
@@ -164,14 +231,14 @@ def event_records(conn, doc: dict, person_id: int, ev: dict, sentence: str, offi
|
||||
summary_de = f"Treffen mit {ev['counterpart']}{where}"
|
||||
summary_en = f"Meeting with {ev['counterpart']}{where}"
|
||||
event_key = fingerprint("event", ev["kind"], person_id, anchor, week)
|
||||
yield Event(key=event_key, kind=ev["kind"], title=f"{row['full_name']}: {summary_de}", starts_at=published,
|
||||
yield Event(key=event_key, kind=ev["kind"], title=f"{row['full_name']}: {summary_de}", starts_at=when,
|
||||
location=ev.get("place"), country=ev.get("country"), url=doc["url"], description=excerpt(sentence, 400))
|
||||
foreign = ev.get("country") and ev["country"] not in AUSTRIA.values()
|
||||
yield Claim(
|
||||
claim_type=ev["claim_type"], person=PersonRef(ids={"politrack": str(person_id)}), occurred_at=published,
|
||||
claim_type=ev["claim_type"], person=PersonRef(ids={"politrack": str(person_id)}), occurred_at=when,
|
||||
summary_de=summary_de, summary_en=summary_en,
|
||||
data={"place": ev.get("place"), "country": ev.get("country"), "counterpart": ev.get("counterpart"),
|
||||
"date_precision": "publication", "extracted": "rules"},
|
||||
"date_precision": "day" if on else "publication", "extracted": "rules"},
|
||||
evidence=[Evidence(url=doc["url"], source=doc["source"], title=doc["title"], published_at=published,
|
||||
quote=excerpt(sentence, 400), confidence=0.8 if official else 0.6)],
|
||||
topics=("Außenpolitik",) if foreign else (),
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
from politrack.enrich.events import detect
|
||||
from datetime import date
|
||||
|
||||
from politrack.enrich.events import detect, event_date
|
||||
from politrack.enrich.legal import classify
|
||||
from politrack.enrich.linker import OFFICE_TITLE
|
||||
from politrack.sources.parlament.common import parse_immunity_report, parse_roll_call
|
||||
@@ -96,3 +98,40 @@ def test_office_title_surname():
|
||||
assert names == ["Stocker", "Bauer", "Mikl-Leitner"]
|
||||
# A first name after the title must not be taken for a surname.
|
||||
assert [m.group("surname") for m in OFFICE_TITLE.finditer("Nationalratspräsident Walter Rosenkranz")] == []
|
||||
|
||||
|
||||
def test_event_date_from_the_sentence():
|
||||
published = date(2026, 11, 12) # a Thursday
|
||||
assert event_date("Die Ministerin reiste gestern nach Brüssel", published) == date(2026, 11, 11)
|
||||
assert event_date("Der Kanzler empfängt heute den Premier", published) == published
|
||||
assert event_date("Am Montag traf sie den Botschafter", published) == date(2026, 11, 9)
|
||||
assert event_date("am Samstag reist er nach Rom", published) == date(2026, 11, 14)
|
||||
assert event_date("Treffen am 3. November in Paris", published) == date(2026, 11, 3)
|
||||
assert event_date("Gipfel am 28. Dezember 2025", published) is None # too far from publication
|
||||
assert event_date("Besuch in Wien am 10.11.2026", published) == date(2026, 11, 10)
|
||||
assert event_date("Treffen am 2. Jänner", date(2026, 12, 20)) == date(2027, 1, 2) # year closest to publication
|
||||
assert event_date("Arbeitsbesuch in Israel", published) is None
|
||||
assert event_date("die Morgen-Ausgabe", published) is None
|
||||
|
||||
|
||||
def test_event_records_use_the_date_from_the_sentence():
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from politrack.core.records import Claim, Event
|
||||
from politrack.enrich.events import event_records
|
||||
|
||||
class Conn:
|
||||
def execute(self, *_):
|
||||
class Row:
|
||||
def fetchone(self):
|
||||
return {"full_name": "Beate Meinl-Reisinger", "family_name": "Meinl-Reisinger"}
|
||||
return Row()
|
||||
|
||||
doc = {"published_at": datetime(2026, 11, 12, 9, tzinfo=timezone.utc), "url": "https://example.org/x",
|
||||
"source": "bundesregierung", "title": "Besuch"}
|
||||
ev = {"kind": "foreign_trip", "claim_type": "travel", "place": "Brüssel", "country": "BE"}
|
||||
records = list(event_records(Conn(), doc, 1, ev, "Die Außenministerin reiste gestern nach Brüssel", True))
|
||||
event = next(r for r in records if isinstance(r, Event))
|
||||
claim = next(r for r in records if isinstance(r, Claim))
|
||||
assert event.starts_at == claim.occurred_at == date(2026, 11, 11)
|
||||
assert claim.data["date_precision"] == "day"
|
||||
+1
-1
@@ -76,7 +76,7 @@ services:
|
||||
worker-enrich:
|
||||
<<: *worker
|
||||
hostname: worker-enrich
|
||||
command: ["procrastinate", "--app=politrack.worker.app", "worker", "--queues=enrich", "--concurrency=1"]
|
||||
command: ["procrastinate", "--app=politrack.worker.app", "worker", "--queues=enrich", "--concurrency=2"]
|
||||
|
||||
# Nightly pg_dump at 02:30 UTC into ${BACKUP_DIR}, kept for ${BACKUP_KEEP_DAYS} days (see backup.sh).
|
||||
backup:
|
||||
|
||||
Reference in new issue
Block a user