simonandClaude Opus 5.5
583ccba31c
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
96 lines
3.5 KiB
Python
96 lines
3.5 KiB
Python
"""Registry of adapters, enrichers and source specs.
|
|
|
|
Built-in modules register themselves with @register when imported (see load_builtin()).
|
|
External packages can add adapters via the entry-point group `politrack.adapters`
|
|
(value: a module path whose import registers adapters), e.g. in their pyproject.toml:
|
|
|
|
[project.entry-points."politrack.adapters"]
|
|
my_sources = "my_package.sources"
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import logging
|
|
import pkgutil
|
|
from importlib.metadata import entry_points
|
|
|
|
from politrack.core.adapter import Adapter, Enricher
|
|
from politrack.core.spec import DERIVED, SourceSpec
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
ADAPTERS: dict[str, type[Adapter]] = {}
|
|
ENRICHERS: dict[str, type[Enricher]] = {}
|
|
SPECS: dict[str, SourceSpec] = {DERIVED.slug: DERIVED}
|
|
_loaded = False
|
|
|
|
|
|
def register(cls):
|
|
"""Class decorator for Adapter and Enricher subclasses."""
|
|
if issubclass(cls, Adapter):
|
|
target = ADAPTERS
|
|
elif issubclass(cls, Enricher):
|
|
target = ENRICHERS
|
|
else:
|
|
raise TypeError(f"{cls} is neither an Adapter nor an Enricher")
|
|
if cls.name in ADAPTERS or cls.name in ENRICHERS:
|
|
raise ValueError(f"duplicate adapter name {cls.name!r}")
|
|
target[cls.name] = cls
|
|
register_spec(cls.spec)
|
|
return cls
|
|
|
|
|
|
def register_spec(spec: SourceSpec) -> SourceSpec:
|
|
existing = SPECS.get(spec.slug)
|
|
if existing is not None and existing != spec:
|
|
raise ValueError(f"conflicting SourceSpec definitions for {spec.slug!r}")
|
|
SPECS[spec.slug] = spec
|
|
return spec
|
|
|
|
|
|
def load_builtin() -> None:
|
|
"""Import all built-in source and enricher modules (and plugins) exactly once."""
|
|
global _loaded
|
|
if _loaded:
|
|
return
|
|
_loaded = True
|
|
for package_name in ("politrack.sources", "politrack.enrich"):
|
|
package = importlib.import_module(package_name)
|
|
for module in pkgutil.walk_packages(package.__path__, prefix=f"{package_name}."):
|
|
importlib.import_module(module.name)
|
|
for ep in entry_points(group="politrack.adapters"):
|
|
try:
|
|
importlib.import_module(ep.value)
|
|
except Exception: # a broken plugin must not take the platform down
|
|
log.exception("failed to load adapter plugin %s", ep.value)
|
|
|
|
|
|
def get(name: str) -> type[Adapter] | type[Enricher]:
|
|
load_builtin()
|
|
if name in ADAPTERS:
|
|
return ADAPTERS[name]
|
|
if name in ENRICHERS:
|
|
return ENRICHERS[name]
|
|
raise KeyError(f"unknown adapter/enricher {name!r}; known: {', '.join(sorted(ADAPTERS) + sorted(ENRICHERS))}")
|
|
|
|
|
|
def sync_sources(conn) -> int:
|
|
"""Upsert all registered SourceSpecs into the `source` table."""
|
|
load_builtin()
|
|
for spec in SPECS.values():
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO source (slug, name, kind, homepage, license, access_method, store_fulltext, keep_raw,
|
|
default_trust, tos_notes)
|
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
|
ON CONFLICT (slug) DO UPDATE SET name = EXCLUDED.name, kind = EXCLUDED.kind, homepage = EXCLUDED.homepage,
|
|
license = EXCLUDED.license, access_method = EXCLUDED.access_method,
|
|
store_fulltext = EXCLUDED.store_fulltext, keep_raw = EXCLUDED.keep_raw,
|
|
default_trust = EXCLUDED.default_trust, tos_notes = EXCLUDED.tos_notes, updated_at = now()
|
|
""",
|
|
(spec.slug, spec.name, spec.kind, spec.homepage, spec.license, spec.access, spec.store_fulltext,
|
|
spec.keep_raw, spec.trust, spec.tos_notes),
|
|
)
|
|
return len(SPECS)
|