Files

130 lines
6.7 KiB
Python

"""Parliament press releases (Parlamentskorrespondenz, list 110) and calendar (Termine, list 600). CC BY 4.0.
Press releases are stored with full text; the people they link (/person/<id>) get a mention claim.
The event extractor later turns visits, receptions and trips described in them into meeting/travel
claims. The calendar fills the calendar view with sessions, committee meetings, hearings and events.
"""
from __future__ import annotations
import json
import re
from collections.abc import Iterator
from datetime import date, timedelta
from typing import Any
from politrack.core import Adapter, Request, Response, RunContext, register
from politrack.core.records import Claim, Document, Event, Evidence, OrgRef, PersonRef
from politrack.sources.parlament.common import BASE, CHAMBER_SLUG, SPEC, list_request, list_rows, parse_date
from politrack.text import excerpt, fingerprint, html_to_text
ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}")
SKIP_TERMIN = re.compile(r"Guided Tour|Führung|Galeriebesuch|Besuch einer Plenarsitzung|Demokratiebildung", re.I)
TERMIN_KIND = [
(re.compile(r"Plenarsitzung|Bundesversammlung|Fest-|Gedenksitzung", re.I), "parliament_session"),
(re.compile(r"Untersuchungsausschuss", re.I), "hearing"),
(re.compile(r"Ausschuss|Enquete", re.I), "committee"),
(re.compile(r"Pressekonferenz", re.I), "press_conference"),
(re.compile(r"Internationales|Konferenz", re.I), "meeting"),
]
def release_body(content: list) -> str:
"""The detail JSON is a list of render blocks; the article is the one wrapped in <div class=pk>."""
for block in content:
text = (block.get("data") or {}).get("text") if isinstance(block, dict) else None
if isinstance(text, str) and re.search(r"class=\"?pk\b", text):
return text
return ""
@register
class ParlamentPress(Adapter):
spec = SPEC
name = "parlament.press"
description = "Parlamentskorrespondenz: official press releases with linked people"
schedule = "5 */2 * * *"
def start(self, ctx: RunContext) -> Iterator[Request]:
last = ctx.state.get("last_date")
backfill = int(ctx.params.get("backfill_days", 365))
# an explicit backfill_days (history backfill) wins over the cursor of the regular runs
if last and "backfill_days" not in ctx.params:
since = date.fromisoformat(last) - timedelta(days=2)
else:
since = date.today() - timedelta(days=backfill)
yield list_request(110, {"DATERANGE": [since.isoformat(), date.today().isoformat()]}, kind="list",
meta={"since": since.isoformat()})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if response.request.kind == "list":
newest = response.meta["since"]
for row in list_rows(response.json()):
if not row.get("Link"):
continue
published = parse_date(row.get("Datum")) # "23.10.2026" (German) in list 110
if published:
newest = max(newest, published.isoformat())
yield Request(url=f"{BASE}{row['Link']}", params={"json": "TRUE"}, kind="release",
meta={k: row.get(k) for k in ("Titel", "Untertitel", "Datum", "Nr.", "Abothemen",
"Stichworte")})
if ctx.mode != "test":
ctx.state.set("last_date", newest)
return
yield from self.parse_release(response)
def parse_release(self, response: Response) -> Iterator[Any]:
meta = response.meta
url = response.request.url
body_html = release_body(response.json().get("content") or [])
pads = sorted({p for p in re.findall(r"/person/(\d+)", body_html)})
text = html_to_text(body_html)
title, subtitle = meta.get("Titel") or "", meta.get("Untertitel") or ""
topics_raw = meta.get("Abothemen")
topics = tuple(json.loads(topics_raw)) if isinstance(topics_raw, str) and topics_raw.startswith("[") else ()
published = parse_date(meta.get("Datum"))
yield Document(Evidence(url=url, doc_type="press_release", title=title, published_at=published,
excerpt=subtitle or excerpt(text, 300), full_text=f"{title}\n{subtitle}\n{text}",
meta={"pk_number": meta.get("Nr."), "person_pads": pads, "topics": list(topics)}))
for pad in pads:
yield Claim(
claim_type="media_mention", person=PersonRef(ids={"parlament": pad}), occurred_at=published,
summary_de=f"Parlamentskorrespondenz: {title}",
data={"context": "parlamentskorrespondenz"},
evidence=[Evidence(url=url, doc_type="press_release", title=title, published_at=published,
quote=subtitle or None, stance="mentions")],
topics=topics,
)
@register
class ParlamentCalendar(Adapter):
spec = SPEC
name = "parlament.calendar"
description = "Parliament calendar: plenary sessions, committees, hearings, events (tours skipped)"
schedule = "30 2 * * *"
parser_version = 2 # 2: titles are HTML-decoded ("84.&nbsp;Sitzung")
def start(self, ctx: RunContext) -> Iterator[Request]:
yield list_request(600, {}, kind="list")
def parse(self, ctx: RunContext, response: Response) -> Iterator[Event]:
since = str(ctx.params.get("since_year", date.today().year - 2))
for row in list_rows(response.json()):
kind_label = html_to_text(row.get("Terminart"))
starts = next((v for k, v in row.items() if k.startswith("Datum") and k != "Datum Bis"
and isinstance(v, str) and ISO_DATE.match(v)), None)
if not starts or starts[:4] < since or SKIP_TERMIN.search(f"{kind_label} {row.get('Bezeichnung') or ''}"):
continue
title = html_to_text(row.get("Bezeichnung")) or kind_label
kind = next((k for pattern, k in TERMIN_KIND if pattern.search(f"{kind_label} {title}")), "public_event")
chamber = {"Nationalrat": "NR", "Bundesrat": "BR"}.get(row.get("Gremium") or "")
ends = next((v for k, v in row.items() if k.startswith("Datum Bis") and isinstance(v, str)
and ISO_DATE.match(v)), None)
yield Event(
key=fingerprint("termin", row.get("WENTRY_ID") or row.get("Link")), kind=kind, title=title,
starts_at=starts, ends_at=ends, location=(row.get("Ort") or "").replace(" | ", ", ") or "Parlament, Wien",
country="AT", url=f"{BASE}{row['Link']}" if row.get("Link") else None, description=kind_label,
body=OrgRef(slug=CHAMBER_SLUG[chamber]) if chamber else None,
)