Politrack: track Austrian politicians from public sources
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
583ccba31c
187 files changed
+39890
No files matched your search
@@ -0,0 +1,95 @@
|
||||
"""Registry of adapters, enrichers and source specs.
|
||||
|
||||
Built-in modules register themselves with @register when imported (see load_builtin()).
|
||||
External packages can add adapters via the entry-point group `politrack.adapters`
|
||||
(value: a module path whose import registers adapters), e.g. in their pyproject.toml:
|
||||
|
||||
[project.entry-points."politrack.adapters"]
|
||||
my_sources = "my_package.sources"
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import logging
|
||||
import pkgutil
|
||||
from importlib.metadata import entry_points
|
||||
|
||||
from politrack.core.adapter import Adapter, Enricher
|
||||
from politrack.core.spec import DERIVED, SourceSpec
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
ADAPTERS: dict[str, type[Adapter]] = {}
|
||||
ENRICHERS: dict[str, type[Enricher]] = {}
|
||||
SPECS: dict[str, SourceSpec] = {DERIVED.slug: DERIVED}
|
||||
_loaded = False
|
||||
|
||||
|
||||
def register(cls):
|
||||
"""Class decorator for Adapter and Enricher subclasses."""
|
||||
if issubclass(cls, Adapter):
|
||||
target = ADAPTERS
|
||||
elif issubclass(cls, Enricher):
|
||||
target = ENRICHERS
|
||||
else:
|
||||
raise TypeError(f"{cls} is neither an Adapter nor an Enricher")
|
||||
if cls.name in ADAPTERS or cls.name in ENRICHERS:
|
||||
raise ValueError(f"duplicate adapter name {cls.name!r}")
|
||||
target[cls.name] = cls
|
||||
register_spec(cls.spec)
|
||||
return cls
|
||||
|
||||
|
||||
def register_spec(spec: SourceSpec) -> SourceSpec:
|
||||
existing = SPECS.get(spec.slug)
|
||||
if existing is not None and existing != spec:
|
||||
raise ValueError(f"conflicting SourceSpec definitions for {spec.slug!r}")
|
||||
SPECS[spec.slug] = spec
|
||||
return spec
|
||||
|
||||
|
||||
def load_builtin() -> None:
|
||||
"""Import all built-in source and enricher modules (and plugins) exactly once."""
|
||||
global _loaded
|
||||
if _loaded:
|
||||
return
|
||||
_loaded = True
|
||||
for package_name in ("politrack.sources", "politrack.enrich"):
|
||||
package = importlib.import_module(package_name)
|
||||
for module in pkgutil.walk_packages(package.__path__, prefix=f"{package_name}."):
|
||||
importlib.import_module(module.name)
|
||||
for ep in entry_points(group="politrack.adapters"):
|
||||
try:
|
||||
importlib.import_module(ep.value)
|
||||
except Exception: # a broken plugin must not take the platform down
|
||||
log.exception("failed to load adapter plugin %s", ep.value)
|
||||
|
||||
|
||||
def get(name: str) -> type[Adapter] | type[Enricher]:
|
||||
load_builtin()
|
||||
if name in ADAPTERS:
|
||||
return ADAPTERS[name]
|
||||
if name in ENRICHERS:
|
||||
return ENRICHERS[name]
|
||||
raise KeyError(f"unknown adapter/enricher {name!r}; known: {', '.join(sorted(ADAPTERS) + sorted(ENRICHERS))}")
|
||||
|
||||
|
||||
def sync_sources(conn) -> int:
|
||||
"""Upsert all registered SourceSpecs into the `source` table."""
|
||||
load_builtin()
|
||||
for spec in SPECS.values():
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO source (slug, name, kind, homepage, license, access_method, store_fulltext, keep_raw,
|
||||
default_trust, tos_notes)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (slug) DO UPDATE SET name = EXCLUDED.name, kind = EXCLUDED.kind, homepage = EXCLUDED.homepage,
|
||||
license = EXCLUDED.license, access_method = EXCLUDED.access_method,
|
||||
store_fulltext = EXCLUDED.store_fulltext, keep_raw = EXCLUDED.keep_raw,
|
||||
default_trust = EXCLUDED.default_trust, tos_notes = EXCLUDED.tos_notes, updated_at = now()
|
||||
""",
|
||||
(spec.slug, spec.name, spec.kind, spec.homepage, spec.license, spec.access, spec.store_fulltext,
|
||||
spec.keep_raw, spec.trust, spec.tos_notes),
|
||||
)
|
||||
return len(SPECS)
|
||||
Reference in new issue
Block a user