"""Background jobs on Procrastinate (Postgres-backed queue; no Redis). One periodic task is generated per registered adapter/enricher from its `schedule` (cron, UTC) and `queue`. Run workers per queue so heavy jobs don't block light ones: procrastinate --app=politrack.worker.app worker --queues=scrape,social procrastinate --app=politrack.worker.app worker --queues=bulk procrastinate --app=politrack.worker.app worker --queues=enrich Periodic tasks are deferred by every running worker and de-duplicated by Procrastinate; queueing locks prevent two runs of the same adapter at once. Workers can run on other machines that reach the DB. """ from __future__ import annotations import logging from procrastinate import App, PsycopgConnector from politrack.config import get_settings from politrack.core import registry from politrack.core.runner import run logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") app = App(connector=PsycopgConnector(conninfo=get_settings().database_url)) def _make_task(name: str, queue: str, schedule: str | None) -> None: def job(timestamp: int | None = None, **params) -> dict: return run(name, params=params or None) job.__name__ = f"run_{name.replace('.', '_')}" task = app.task(name=f"politrack.run.{name}", queue=queue, queueing_lock=name, pass_context=False)(job) if schedule: app.periodic(cron=schedule, periodic_id=name)(task) registry.load_builtin() for _name, _cls in {**registry.ADAPTERS, **registry.ENRICHERS}.items(): _make_task(_name, _cls.queue, _cls.schedule)