politics-tracking/backend/politrack/sources/parlament/news.py

"""Parliament press releases (Parlamentskorrespondenz, list 110) and calendar (Termine, list 600). CC BY 4.0.

Press releases are stored with full text; the people they link (/person/<id>) get a mention claim.
The event extractor later turns visits, receptions and trips described in them into meeting/travel
claims. The calendar fills the calendar view with sessions, committee meetings, hearings and events.
"""

from __future__ import annotations

import json
import re
from collections.abc import Iterator
from datetime import date, timedelta
from typing import Any

from politrack.core import Adapter, Request, Response, RunContext, register
from politrack.core.records import Claim, Document, Event, Evidence, OrgRef, PersonRef
from politrack.sources.parlament.common import BASE, CHAMBER_SLUG, SPEC, list_request, list_rows, parse_date
from politrack.text import excerpt, fingerprint, html_to_text

ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}")
SKIP_TERMIN = re.compile(r"Guided Tour|Führung|Galeriebesuch|Besuch einer Plenarsitzung|Demokratiebildung", re.I)
TERMIN_KIND = [
    (re.compile(r"Plenarsitzung|Bundesversammlung|Fest-|Gedenksitzung", re.I), "parliament_session"),
    (re.compile(r"Untersuchungsausschuss", re.I), "hearing"),
    (re.compile(r"Ausschuss|Enquete", re.I), "committee"),
    (re.compile(r"Pressekonferenz", re.I), "press_conference"),
    (re.compile(r"Internationales|Konferenz", re.I), "meeting"),
]


def release_body(content: list) -> str:
    """The detail JSON is a list of render blocks; the article is the one wrapped in <div class=pk>."""
    for block in content:
        text = (block.get("data") or {}).get("text") if isinstance(block, dict) else None
        if isinstance(text, str) and re.search(r"class=\"?pk\b", text):
            return text
    return ""


@register
class ParlamentPress(Adapter):
    spec = SPEC
    name = "parlament.press"
    description = "Parlamentskorrespondenz: official press releases with linked people"
    schedule = "5 */2 * * *"

    def start(self, ctx: RunContext) -> Iterator[Request]:
        last = ctx.state.get("last_date")
        backfill = int(ctx.params.get("backfill_days", 365))
        since = (date.fromisoformat(last) - timedelta(days=2)) if last else date.today() - timedelta(days=backfill)
        yield list_request(110, {"DATERANGE": [since.isoformat(), date.today().isoformat()]}, kind="list",
                           meta={"since": since.isoformat()})

    def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
        if response.request.kind == "list":
            newest = response.meta["since"]
            for row in list_rows(response.json()):
                if not row.get("Link"):
                    continue
                published = parse_date(row.get("Datum"))  # "23.10.2026" (German) in list 110
                if published:
                    newest = max(newest, published.isoformat())
                yield Request(url=f"{BASE}{row['Link']}", params={"json": "TRUE"}, kind="release",
                              meta={k: row.get(k) for k in ("Titel", "Untertitel", "Datum", "Nr.", "Abothemen",
                                                            "Stichworte")})
            if ctx.mode != "test":
                ctx.state.set("last_date", newest)
            return
        yield from self.parse_release(response)

    def parse_release(self, response: Response) -> Iterator[Any]:
        meta = response.meta
        url = response.request.url
        body_html = release_body(response.json().get("content") or [])
        pads = sorted({p for p in re.findall(r"/person/(\d+)", body_html)})
        text = html_to_text(body_html)
        title, subtitle = meta.get("Titel") or "", meta.get("Untertitel") or ""
        topics_raw = meta.get("Abothemen")
        topics = tuple(json.loads(topics_raw)) if isinstance(topics_raw, str) and topics_raw.startswith("[") else ()
        published = parse_date(meta.get("Datum"))
        yield Document(Evidence(url=url, doc_type="press_release", title=title, published_at=published,
                                excerpt=subtitle or excerpt(text, 300), full_text=f"{title}\n{subtitle}\n{text}",
                                meta={"pk_number": meta.get("Nr."), "person_pads": pads, "topics": list(topics)}))
        for pad in pads:
            yield Claim(
                claim_type="media_mention", person=PersonRef(ids={"parlament": pad}), occurred_at=published,
                summary_de=f"Parlamentskorrespondenz: {title}",
                data={"context": "parlamentskorrespondenz"},
                evidence=[Evidence(url=url, doc_type="press_release", title=title, published_at=published,
                                   quote=subtitle or None, stance="mentions")],
                topics=topics,
            )


@register
class ParlamentCalendar(Adapter):
    spec = SPEC
    name = "parlament.calendar"
    description = "Parliament calendar: plenary sessions, committees, hearings, events (tours skipped)"
    schedule = "30 2 * * *"
    parser_version = 2  # 2: titles are HTML-decoded ("84.&nbsp;Sitzung")

    def start(self, ctx: RunContext) -> Iterator[Request]:
        yield list_request(600, {}, kind="list")

    def parse(self, ctx: RunContext, response: Response) -> Iterator[Event]:
        since = str(ctx.params.get("since_year", date.today().year - 2))
        for row in list_rows(response.json()):
            kind_label = html_to_text(row.get("Terminart"))
            starts = next((v for k, v in row.items() if k.startswith("Datum") and k != "Datum Bis"
                           and isinstance(v, str) and ISO_DATE.match(v)), None)
            if not starts or starts[:4] < since or SKIP_TERMIN.search(f"{kind_label} {row.get('Bezeichnung') or ''}"):
                continue
            title = html_to_text(row.get("Bezeichnung")) or kind_label
            kind = next((k for pattern, k in TERMIN_KIND if pattern.search(f"{kind_label} {title}")), "public_event")
            chamber = {"Nationalrat": "NR", "Bundesrat": "BR"}.get(row.get("Gremium") or "")
            ends = next((v for k, v in row.items() if k.startswith("Datum Bis") and isinstance(v, str)
                         and ISO_DATE.match(v)), None)
            yield Event(
                key=fingerprint("termin", row.get("WENTRY_ID") or row.get("Link")), kind=kind, title=title,
                starts_at=starts, ends_at=ends, location=(row.get("Ort") or "").replace("  |  ", ", ") or "Parlament, Wien",
                country="AT", url=f"{BASE}{row['Link']}" if row.get("Link") else None, description=kind_label,
                body=OrgRef(slug=CHAMBER_SLUG[chamber]) if chamber else None,
            )