politics-tracking/backend/politrack/core/records.py

"""Typed records: the only way adapters and enrichers hand data to the Loader.

Records are plain dataclasses so parsers stay pure and testable (see tests/golden). They refer to
entities through PersonRef/OrgRef, which the Loader resolves (identifiers first, then hints, then
names). Data minimisation is enforced by schema: there is, for example, no address field anywhere,
so a parser cannot store a donor's street address even by accident.
"""

from __future__ import annotations

from dataclasses import dataclass, field, fields, is_dataclass
from datetime import date, datetime
from decimal import Decimal
from typing import Any, Literal, Union

# --------------------------------------------------------------------------------------------
# References to entities
# --------------------------------------------------------------------------------------------


@dataclass
class PersonRef:
    """How a record points at a person.

    ids:   strong identifiers, e.g. {"parlament": "2339"} or {"wikidata": "Q123"}
    name:  display/full name; used for alias matching if no id matches
    hints: resolution strategies, e.g.
           {"surname_in_body": "nationalrat", "on": date(...)}  -> roll-call surname lists
           {"office": "Bundesministerin für Justiz", "on": date(...)} -> office holder on a date
    """

    ids: dict[str, str] = field(default_factory=dict)
    name: str | None = None
    hints: dict[str, Any] = field(default_factory=dict)


@dataclass
class OrgRef:
    """How a record points at an organization. `slug` addresses seeded orgs (e.g. "nationalrat")."""

    ids: dict[str, str] = field(default_factory=dict)
    name: str | None = None
    short_name: str | None = None
    kind: str = "other"
    slug: str | None = None
    create: bool = True


# --------------------------------------------------------------------------------------------
# Evidence & documents
# --------------------------------------------------------------------------------------------

DocType = Literal["article", "press_release", "api_record", "protocol", "social_post", "register_entry",
                  "web_page", "pdf", "dataset_row"]


@dataclass
class Evidence:
    url: str
    title: str | None = None
    doc_type: str = "web_page"
    published_at: datetime | date | str | None = None
    excerpt: str | None = None
    full_text: str | None = None
    quote: str | None = None
    locator: str = ""
    stance: Literal["supports", "contradicts", "mentions"] = "supports"
    confidence: float = 1.0
    source: str | None = None          # defaults to the adapter's source
    meta: dict[str, Any] = field(default_factory=dict)


@dataclass
class Document:
    """A document without a claim yet (e.g. a headline); enrichers derive claims from it later."""

    evidence: Evidence


# --------------------------------------------------------------------------------------------
# Entities
# --------------------------------------------------------------------------------------------


@dataclass
class PersonRecord:
    ref: PersonRef
    full_name: str
    given_name: str | None = None
    family_name: str | None = None
    honorific_prefix: str | None = None
    honorific_suffix: str | None = None
    birth_date: date | None = None
    death_date: date | None = None
    gender: str | None = None
    photo_url: str | None = None
    aliases: tuple[str, ...] = ()
    #: if no identifier matches, merge with an existing person of exactly this (unambiguous) name before
    #: creating a new one - for authoritative sources that share no id with others (e.g. EP)
    match_by_name: bool = False


@dataclass
class OrgRecord:
    ref: OrgRef
    aliases: tuple[str, ...] = ()
    color: str | None = None
    parent: OrgRef | None = None


@dataclass
class SocialAccount:
    platform: str                      # bluesky | mastodon | telegram | youtube | twitter | facebook | ...
    url: str
    handle: str | None = None
    person: PersonRef | None = None
    org: OrgRef | None = None
    verified: bool = False             # True only for official sources (parliament profile, official website)


# --------------------------------------------------------------------------------------------
# Facets (structured payload of a claim)
# --------------------------------------------------------------------------------------------


@dataclass
class Role:
    role: str                          # "mp" for chamber members, otherwise office title
    level: Literal["eu", "federal", "state", "district", "municipal", "party", "other"]
    body: OrgRef | None = None         # chamber/government/party the role is in
    party: OrgRef | None = None        # party/faction of the holder in this role
    region: str | None = None
    start: date | None = None
    end: date | None = None


@dataclass
class VoteEvent:
    """A vote (not a claim itself). Positions reference it by `key`."""

    key: str
    chamber: OrgRef | None
    title: str
    voted_on: date | None
    result: Literal["accepted", "rejected", "unknown"] = "unknown"
    stage: str | None = None
    item_ref: str | None = None
    roll_call: bool = False
    url: str | None = None
    protocol_url: str | None = None
    raw_text: str | None = None


@dataclass
class VotePosition:
    event_key: str
    position: Literal["for", "against", "abstain", "absent"]
    party: OrgRef | None = None        # faction of an individual voter at that time


@dataclass
class Money:
    kind: Literal["donation", "contract", "government_ad", "political_ad", "subsidy", "party_funding", "party_ad",
                  "loan", "side_income", "sponsorship", "other"]
    amount: Decimal | float | None = None
    currency: str = "EUR"
    payer: OrgRef | PersonRef | None = None
    payee: OrgRef | PersonRef | None = None
    payer_name: str | None = None
    payee_name: str | None = None
    paid_on: date | None = None
    period: str | None = None
    reference: str | None = None       # notice id, campaign, ...
    category: str | None = None        # CPV code, ad category, ...
    amount_category: str | None = None # when only a bracket is published


@dataclass
class Event:
    """Something with time and place (not a claim). Participation claims reference it by `key`."""

    key: str
    kind: str
    title: str
    starts_at: datetime | date | str | None
    ends_at: datetime | date | str | None = None
    location: str | None = None
    country: str | None = None
    url: str | None = None
    description: str | None = None
    body: OrgRef | None = None


@dataclass
class Participation:
    event_key: str
    participation: str = "participant"


@dataclass
class Statement:
    context: Literal["parliament_speech", "interview", "tv", "radio", "social_post", "press_release",
                     "event_speech", "op_ed", "testimony", "other"]
    url: str
    said_at: datetime | date | str | None = None
    platform: str | None = None
    medium: str | None = None
    title: str | None = None
    excerpt: str | None = None


@dataclass
class Legal:
    stage: str
    offense: str | None = None
    authority: str | None = None
    case_ref: str | None = None
    stage_date: date | None = None


Facet = Union[Role, VotePosition, Money, Participation, Statement, Legal]


# --------------------------------------------------------------------------------------------
# Claims
# --------------------------------------------------------------------------------------------


@dataclass
class Claim:
    claim_type: str
    summary_de: str
    evidence: list[Evidence]
    person: PersonRef | None = None
    org: OrgRef | None = None
    counterpart_person: PersonRef | None = None
    counterpart_org: OrgRef | None = None
    occurred_at: datetime | date | str | None = None
    valid_from: date | None = None
    valid_to: date | None = None
    summary_en: str | None = None
    data: dict[str, Any] = field(default_factory=dict)
    topics: tuple[str, ...] = ()
    sensitive: bool = False
    review: bool = False               # True -> published only after human review
    key: tuple | None = None           # explicit identity (otherwise computed by the Loader)
    facet: Facet | None = None


Record = Union[PersonRecord, OrgRecord, SocialAccount, VoteEvent, Event, Claim, Document]


def to_jsonable(obj: Any) -> Any:
    """Serialise records for golden tests and debugging (stable, sorted keys)."""
    if is_dataclass(obj) and not isinstance(obj, type):
        return {"__type__": type(obj).__name__, **{f.name: to_jsonable(getattr(obj, f.name)) for f in fields(obj)}}
    if isinstance(obj, dict):
        return {str(k): to_jsonable(v) for k, v in sorted(obj.items(), key=lambda kv: str(kv[0]))}
    if isinstance(obj, (list, tuple)):
        return [to_jsonable(v) for v in obj]
    if isinstance(obj, (datetime, date)):
        return obj.isoformat()
    if isinstance(obj, Decimal):
        return str(obj)
    return obj