"""SourceSpec: the single source of truth about a publisher we collect from. Specs live next to the adapters that use them (backend/politrack/sources/*.py) and are synced into the `source` table by `politrack migrate` / `politrack sources sync`. """ from dataclasses import dataclass from typing import Literal SourceKind = Literal["official", "ngo", "media", "social", "aggregator", "user", "derived"] AccessMethod = Literal["api", "bulk", "rss", "scrape", "manual", "derived"] @dataclass(frozen=True) class SourceSpec: slug: str name: str kind: SourceKind homepage: str license: str access: AccessMethod trust: int #: may we keep body text (articles, posts, protocols)? False -> title, short excerpt and link only store_fulltext: bool = True #: may raw responses be archived? False for TDM-reserved publishers or restrictive terms keep_raw: bool = True #: minimum seconds between two requests to the same host (robots.txt crawl-delay can raise it) min_interval: float = 2.0 #: URL prefixes of documented APIs whose operators allow programmatic use although robots.txt #: lists them (justify in tos_notes). Everything else always obeys robots.txt. robots_exempt: tuple[str, ...] = () tos_notes: str = "" def __post_init__(self) -> None: if not 0 <= self.trust <= 100: raise ValueError(f"{self.slug}: trust must be 0-100") # Sources used by the built-in enrichers for derived documents/claims. DERIVED = SourceSpec( slug="politrack", name="Politrack (abgeleitet)", kind="derived", homepage="", license="derived from cited sources", access="derived", trust=50, tos_notes="Claims computed by Politrack from other sources; each cites its original evidence.", )