"""Typed records: the only way adapters and enrichers hand data to the Loader. Records are plain dataclasses so parsers stay pure and testable (see tests/golden). They refer to entities through PersonRef/OrgRef, which the Loader resolves (identifiers first, then hints, then names). Data minimisation is enforced by schema: there is, for example, no address field anywhere, so a parser cannot store a donor's street address even by accident. """ from __future__ import annotations from dataclasses import dataclass, field, fields, is_dataclass from datetime import date, datetime from decimal import Decimal from typing import Any, Literal, Union # -------------------------------------------------------------------------------------------- # References to entities # -------------------------------------------------------------------------------------------- @dataclass class PersonRef: """How a record points at a person. ids: strong identifiers, e.g. {"parlament": "2339"} or {"wikidata": "Q123"} name: display/full name; used for alias matching if no id matches hints: resolution strategies, e.g. {"surname_in_body": "nationalrat", "on": date(...)} -> roll-call surname lists {"office": "Bundesministerin für Justiz", "on": date(...)} -> office holder on a date """ ids: dict[str, str] = field(default_factory=dict) name: str | None = None hints: dict[str, Any] = field(default_factory=dict) @dataclass class OrgRef: """How a record points at an organization. `slug` addresses seeded orgs (e.g. "nationalrat").""" ids: dict[str, str] = field(default_factory=dict) name: str | None = None short_name: str | None = None kind: str = "other" slug: str | None = None create: bool = True # -------------------------------------------------------------------------------------------- # Evidence & documents # -------------------------------------------------------------------------------------------- DocType = Literal["article", "press_release", "api_record", "protocol", "social_post", "register_entry", "web_page", "pdf", "dataset_row"] @dataclass class Evidence: url: str title: str | None = None doc_type: str = "web_page" published_at: datetime | date | str | None = None excerpt: str | None = None full_text: str | None = None quote: str | None = None locator: str = "" stance: Literal["supports", "contradicts", "mentions"] = "supports" confidence: float = 1.0 source: str | None = None # defaults to the adapter's source meta: dict[str, Any] = field(default_factory=dict) @dataclass class Document: """A document without a claim yet (e.g. a headline); enrichers derive claims from it later.""" evidence: Evidence # -------------------------------------------------------------------------------------------- # Entities # -------------------------------------------------------------------------------------------- @dataclass class PersonRecord: ref: PersonRef full_name: str given_name: str | None = None family_name: str | None = None honorific_prefix: str | None = None honorific_suffix: str | None = None birth_date: date | None = None death_date: date | None = None gender: str | None = None photo_url: str | None = None aliases: tuple[str, ...] = () #: if no identifier matches, merge with an existing person of exactly this (unambiguous) name before #: creating a new one - for authoritative sources that share no id with others (e.g. EP) match_by_name: bool = False @dataclass class OrgRecord: ref: OrgRef aliases: tuple[str, ...] = () color: str | None = None parent: OrgRef | None = None @dataclass class SocialAccount: platform: str # bluesky | mastodon | telegram | youtube | twitter | facebook | ... url: str handle: str | None = None person: PersonRef | None = None org: OrgRef | None = None verified: bool = False # True only for official sources (parliament profile, official website) # -------------------------------------------------------------------------------------------- # Facets (structured payload of a claim) # -------------------------------------------------------------------------------------------- @dataclass class Role: role: str # "mp" for chamber members, otherwise office title level: Literal["eu", "federal", "state", "district", "municipal", "party", "other"] body: OrgRef | None = None # chamber/government/party the role is in party: OrgRef | None = None # party/faction of the holder in this role region: str | None = None start: date | None = None end: date | None = None @dataclass class VoteEvent: """A vote (not a claim itself). Positions reference it by `key`.""" key: str chamber: OrgRef | None title: str voted_on: date | None result: Literal["accepted", "rejected", "unknown"] = "unknown" stage: str | None = None item_ref: str | None = None roll_call: bool = False url: str | None = None protocol_url: str | None = None raw_text: str | None = None @dataclass class VotePosition: event_key: str position: Literal["for", "against", "abstain", "absent"] party: OrgRef | None = None # faction of an individual voter at that time @dataclass class Money: kind: Literal["donation", "contract", "government_ad", "political_ad", "subsidy", "party_funding", "party_ad", "loan", "side_income", "sponsorship", "other"] amount: Decimal | float | None = None currency: str = "EUR" payer: OrgRef | PersonRef | None = None payee: OrgRef | PersonRef | None = None payer_name: str | None = None payee_name: str | None = None paid_on: date | None = None period: str | None = None reference: str | None = None # notice id, campaign, ... category: str | None = None # CPV code, ad category, ... amount_category: str | None = None # when only a bracket is published @dataclass class Event: """Something with time and place (not a claim). Participation claims reference it by `key`.""" key: str kind: str title: str starts_at: datetime | date | str | None ends_at: datetime | date | str | None = None location: str | None = None country: str | None = None url: str | None = None description: str | None = None body: OrgRef | None = None @dataclass class Participation: event_key: str participation: str = "participant" @dataclass class Statement: context: Literal["parliament_speech", "interview", "tv", "radio", "social_post", "press_release", "event_speech", "op_ed", "testimony", "other"] url: str said_at: datetime | date | str | None = None platform: str | None = None medium: str | None = None title: str | None = None excerpt: str | None = None @dataclass class Legal: stage: str offense: str | None = None authority: str | None = None case_ref: str | None = None stage_date: date | None = None Facet = Union[Role, VotePosition, Money, Participation, Statement, Legal] # -------------------------------------------------------------------------------------------- # Claims # -------------------------------------------------------------------------------------------- @dataclass class Claim: claim_type: str summary_de: str evidence: list[Evidence] person: PersonRef | None = None org: OrgRef | None = None counterpart_person: PersonRef | None = None counterpart_org: OrgRef | None = None occurred_at: datetime | date | str | None = None valid_from: date | None = None valid_to: date | None = None summary_en: str | None = None data: dict[str, Any] = field(default_factory=dict) topics: tuple[str, ...] = () sensitive: bool = False review: bool = False # True -> published only after human review key: tuple | None = None # explicit identity (otherwise computed by the Loader) facet: Facet | None = None Record = Union[PersonRecord, OrgRecord, SocialAccount, VoteEvent, Event, Claim, Document] def to_jsonable(obj: Any) -> Any: """Serialise records for golden tests and debugging (stable, sorted keys).""" if is_dataclass(obj) and not isinstance(obj, type): return {"__type__": type(obj).__name__, **{f.name: to_jsonable(getattr(obj, f.name)) for f in fields(obj)}} if isinstance(obj, dict): return {str(k): to_jsonable(v) for k, v in sorted(obj.items(), key=lambda kv: str(kv[0]))} if isinstance(obj, (list, tuple)): return [to_jsonable(v) for v in obj] if isinstance(obj, (datetime, date)): return obj.isoformat() if isinstance(obj, Decimal): return str(obj) return obj