Politrack: track Austrian politicians from public sources

Initial import of Politrack, a self-hosted service that tracks Austrian
politicians from federal to municipal level - roles, votes, money flows,
statements, meetings, social media and legal proceedings - with every claim
linked to its sources.

- backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse ->
  load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive
  for reparsing, and a claim/evidence model with corroboration
- sources: Parliament, Wikidata, EU Parliament/Commission, Council of
  Ministers, Rechnungshof party reports and donations, procurement, RTR media
  transparency, political ad libraries, WKStA, news feeds, Wikipedia edits,
  Bluesky, Mastodon, public Telegram channels, YouTube and X (official API)
- web: Next.js frontend in German and English with typed API client
- mcp: MCP server for AI agents and RAG
- docs: architecture, data model, sources, legal notes, Unraid deployment and
  contributor guides; Docker Compose setup for Unraid

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
simonandClaude Opus 5.5 committed 2026-10-28 20:02:01 +01:00
commit 583ccba31c
187 files changed
+39890

No files matched your search

View File
Whitespace-only changes.
View File
Whitespace-only changes.
+253
View File
@@ -0,0 +1,253 @@
"""Public read API (JSON). OpenAPI docs at /api/docs; schema at /api/openapi.json.
Conventions: every list endpoint takes limit/offset; claim-like endpoints take the source filter
(exclude_sources, min_trust) and topic/date filters; slugs identify persons, parties and organisations.
"""
from __future__ import annotations
from collections.abc import Iterator
from datetime import date
from typing import Annotated
from fastapi import Depends, FastAPI, Header, HTTPException, Query
from fastapi.middleware.cors import CORSMiddleware
from psycopg import Connection
from politrack import queries
from politrack.api import schemas as s
from politrack.config import get_settings
from politrack.db import connection
from politrack.queries import SourceFilter
app = FastAPI(
title="Politrack API",
version="0.2.0",
description="Evidence-backed information on Austrian politicians and parties. Every claim links to its sources.",
docs_url="/api/docs",
openapi_url="/api/openapi.json",
)
app.add_middleware(CORSMiddleware, allow_origins=get_settings().api_cors_origins, allow_methods=["GET", "POST"],
allow_headers=["*"])
def db() -> Iterator[Connection]:
with connection() as conn:
yield conn
def csv_list(value: str | None) -> list[str] | None:
items = [v.strip() for v in (value or "").split(",") if v.strip()]
return items or None
def source_filter(
exclude_sources: Annotated[str | None, Query(description="Comma-separated source slugs to ignore")] = None,
min_trust: Annotated[int, Query(ge=0, le=100, description="Ignore sources below this trust score")] = 0,
) -> SourceFilter:
return SourceFilter(exclude_sources=csv_list(exclude_sources) or [], min_trust=min_trust)
DB = Annotated[Connection, Depends(db)]
Sources = Annotated[SourceFilter, Depends(source_filter)]
Limit = Annotated[int, Query(ge=1, le=500)]
Offset = Annotated[int, Query(ge=0)]
def not_found(value, what: str):
if not value:
raise HTTPException(404, f"{what} not found")
return value
@app.get("/healthz", include_in_schema=False)
def healthz(conn: DB) -> dict:
conn.execute("SELECT 1")
return {"ok": True}
# -- reference data -----------------------------------------------------------------------
@app.get("/api/v1/stats", response_model=s.Stats)
def get_stats(conn: DB):
return queries.stats(conn)
@app.get("/api/v1/sources", response_model=list[s.Source])
def get_sources(conn: DB):
return queries.list_sources(conn)
@app.get("/api/v1/topics", response_model=list[s.Topic])
def get_topics(conn: DB):
return queries.list_topics(conn)
# -- parties & organisations ----------------------------------------------------------------
@app.get("/api/v1/parties", response_model=list[s.Party])
def get_parties(conn: DB):
return queries.list_parties(conn)
@app.get("/api/v1/parties/{slug}", response_model=s.PartyDetail)
def get_party(slug: str, conn: DB):
return not_found(queries.get_party(conn, slug), "party")
@app.get("/api/v1/organizations", response_model=list[s.OrganizationListItem])
def get_organizations(conn: DB, q: str | None = None, kind: str | None = None, limit: Limit = 50, offset: Offset = 0):
return queries.list_organizations(conn, q=q, kind=kind, limit=limit, offset=offset)
@app.get("/api/v1/organizations/{slug}", response_model=s.OrganizationDetail)
def get_organization(slug: str, conn: DB):
return not_found(queries.get_organization(conn, slug), "organization")
# -- persons --------------------------------------------------------------------------------
@app.get("/api/v1/persons", response_model=list[s.PersonListItem])
def get_persons(
conn: DB,
q: str | None = None,
party: str | None = None,
level: Annotated[str | None, Query(pattern="^(eu|federal|state|district|municipal|party|other)$")] = None,
status: Annotated[str, Query(pattern="^(active|former|party_member|deceased|all)$")] = "active",
limit: Limit = 50,
offset: Offset = 0,
):
return queries.list_persons(conn, q=q, party=party, level=level, status=None if status == "all" else status,
limit=limit, offset=offset)
@app.get("/api/v1/former", response_model=list[s.FormerPerson])
def get_former(conn: DB, party: str | None = None, limit: Limit = 50, offset: Offset = 0):
"""Former politicians, most recently in the news first."""
return queries.former_overview(conn, party=party, limit=limit, offset=offset)
@app.get("/api/v1/persons/{slug}", response_model=s.PersonDetail)
def get_person(slug: str, conn: DB):
return not_found(queries.get_person(conn, slug), "person")
@app.get("/api/v1/persons/{slug}/votes", response_model=list[s.PersonVote])
def get_person_votes(slug: str, conn: DB, topics: str | None = None, limit: Limit = 100, offset: Offset = 0):
return queries.person_votes(conn, slug, topics=csv_list(topics), limit=limit, offset=offset)
@app.get("/api/v1/persons/{slug}/network", response_model=s.Network)
def get_person_network(slug: str, conn: DB):
"""Organisations connected to the person (side jobs, careers, party, meetings) and their money flows."""
return queries.person_network(conn, slug)
# -- claims, timeline, calendar ----------------------------------------------------------------
@app.get("/api/v1/claims", response_model=list[s.Claim])
def get_claims(
conn: DB,
sources: Sources,
person: str | None = None,
party: str | None = None,
organization: str | None = None,
types: Annotated[str | None, Query(description="Comma-separated claim types")] = None,
topics: Annotated[str | None, Query(description="Comma-separated topic slugs")] = None,
since: date | None = None,
until: date | None = None,
q: str | None = None,
limit: Limit = 50,
offset: Offset = 0,
):
return queries.list_claims(conn, sources=sources, person=person, party=party, organization=organization,
claim_types=csv_list(types), topics=csv_list(topics), since=since, until=until, q=q,
limit=limit, offset=offset)
@app.get("/api/v1/timeline", response_model=list[s.TimelineBucket])
def get_timeline(conn: DB, sources: Sources, person: str | None = None, party: str | None = None):
return queries.timeline_histogram(conn, sources=sources, person=person, party=party)
@app.get("/api/v1/calendar", response_model=s.CalendarData)
def get_calendar(conn: DB, sources: Sources, start: date, end: date, person: str | None = None,
party: str | None = None, kinds: str | None = None):
if (end - start).days > 400:
raise HTTPException(400, "range too large (max 400 days)")
return queries.calendar(conn, sources=sources, start=start, end=end, person=person, party=party,
kinds=csv_list(kinds))
# -- money, social, wikipedia ---------------------------------------------------------------------
@app.get("/api/v1/money", response_model=list[s.MoneyFlow])
def get_money(conn: DB, sources: Sources, kinds: str | None = None, org: str | None = None, person: str | None = None,
q: str | None = None, since: date | None = None, until: date | None = None,
min_amount: float | None = None, limit: Limit = 50, offset: Offset = 0):
"""Donations, contracts, government ads, party funding, sponsoring, political ads."""
return queries.list_money(conn, sources=sources, kinds=csv_list(kinds), org=org, person=person, q=q, since=since,
until=until, min_amount=min_amount, limit=limit, offset=offset)
@app.get("/api/v1/money/summary", response_model=list[s.MoneySummaryRow])
def get_money_summary(conn: DB, kinds: str | None = None, since: date | None = None):
return queries.money_summary(conn, kinds=csv_list(kinds), since=since)
@app.get("/api/v1/social", response_model=list[s.SocialPost])
def get_social(conn: DB, sources: Sources, person: str | None = None, party: str | None = None,
platforms: str | None = None, limit: Limit = 50, offset: Offset = 0):
return queries.social_feed(conn, sources=sources, person=person, party=party, platforms=csv_list(platforms),
limit=limit, offset=offset)
@app.get("/api/v1/wiki-edits", response_model=list[s.WikiEdit])
def get_wiki_edits(conn: DB, limit: Limit = 100, offset: Offset = 0):
return queries.wiki_edits(conn, limit=limit, offset=offset)
# -- votes, search ------------------------------------------------------------------------------
@app.get("/api/v1/votes", response_model=list[s.VoteItem])
def get_votes(conn: DB, party: str | None = None, topics: str | None = None, roll_call: bool | None = None,
since: date | None = None, until: date | None = None, limit: Limit = 50, offset: Offset = 0):
return queries.list_votes(conn, party=party, topics=csv_list(topics), roll_call=roll_call, since=since,
until=until, limit=limit, offset=offset)
@app.get("/api/v1/votes/{vote_id}", response_model=s.VoteDetail)
def get_vote(vote_id: int, conn: DB):
return not_found(queries.vote_detail(conn, vote_id), "vote")
@app.get("/api/v1/search", response_model=list[s.SearchHit])
def get_search(conn: DB, sources: Sources, q: Annotated[str, Query(min_length=2)], since: date | None = None,
until: date | None = None, limit: Limit = 20):
return queries.search(conn, q, sources=sources, since=since, until=until, limit=limit)
# -- admin (POLITRACK_ADMIN_TOKEN) ------------------------------------------------------------------
def require_admin(authorization: Annotated[str | None, Header()] = None) -> str:
token = get_settings().admin_token
if not token or authorization != f"Bearer {token}":
raise HTTPException(401, "admin token required")
return "admin-api"
@app.get("/api/v1/admin/runs", response_model=list[s.Run], include_in_schema=False)
def get_runs(conn: DB, _: Annotated[str, Depends(require_admin)], limit: Limit = 50):
return queries.recent_runs(conn, limit=limit)
@app.get("/api/v1/review", response_model=list[s.PendingClaim], include_in_schema=False)
def get_review_queue(conn: DB, _: Annotated[str, Depends(require_admin)], limit: Limit = 50):
return queries.pending_claims(conn, limit=limit)
@app.post("/api/v1/review/{claim_id}", include_in_schema=False)
def post_review(claim_id: int, decision: Annotated[str, Query(pattern="^(published|rejected)$")], conn: DB,
reviewer: Annotated[str, Depends(require_admin)]) -> dict:
queries.set_review(conn, claim_id, decision, reviewer)
return {"ok": True}
+535
View File
@@ -0,0 +1,535 @@
"""Response models: the API contract. `politrack openapi --ts ../web/src/lib/api-types.ts` turns them into
TypeScript types for the web app, so frontend and backend cannot silently drift apart."""
from __future__ import annotations
from datetime import date, datetime
from typing import Any, Literal
from pydantic import BaseModel, ConfigDict
ClaimType = Literal["role", "vote", "donation", "money_flow", "statement", "appearance", "meeting", "travel",
"side_job", "legal_proceeding", "lobbying", "media_mention", "career", "parliamentary_question",
"government_decision", "wiki_edit", "declaration", "ownership", "affiliation", "party_finance",
"other"]
PersonStatus = Literal["active", "former", "party_member", "deceased"]
class Model(BaseModel):
# Fields with defaults are always present in responses, so mark them required in the (serialization)
# schema - the generated TypeScript types then have no needless optional fields.
model_config = ConfigDict(from_attributes=True, json_schema_serialization_defaults_required=True)
class Stats(Model):
persons: int
active_persons: int
parties: int
claims: int
corroborated: int
pending_review: int
documents: int
vote_events: int
roll_call_votes: int
events: int
money_flows: int
last_update: datetime | None = None
class Source(Model):
slug: str
name: str
kind: str
homepage: str | None = None
license: str | None = None
access_method: str
default_trust: int
tos_notes: str | None = None
documents: int
class Topic(Model):
slug: str
name_de: str
name_en: str
class Evidence(Model):
url: str
title: str | None = None
source: str
source_name: str | None = None
source_kind: str | None = None
published_at: datetime | None = None
quote: str | None = None
stance: Literal["supports", "contradicts", "mentions"] = "supports"
archive_url: str | None = None
class MoneyFacet(Model):
kind: str
amount: float | None = None
amount_category: str | None = None
currency: str | None = None
paid_on: date | None = None
period: str | None = None
reference: str | None = None
category: str | None = None
payer_name: str | None = None
payee_name: str | None = None
class Claim(Model):
id: int
claim_type: ClaimType
summary_de: str
summary_en: str | None = None
occurred_at: datetime | None = None
valid_from: date | None = None
valid_to: date | None = None
status: Literal["reported", "corroborated", "disputed", "retracted"]
sensitive: bool
review_status: Literal["pending", "published", "rejected"]
data: dict[str, Any] = {}
person_slug: str | None = None
person_name: str | None = None
organization_slug: str | None = None
organization_short: str | None = None
organization_name: str | None = None
counterpart_slug: str | None = None
counterpart_name: str | None = None
counterpart_kind: str | None = None
counterpart_person_slug: str | None = None
counterpart_person_name: str | None = None
legal_stage: str | None = None
legal_offense: str | None = None
legal_authority: str | None = None
money: MoneyFacet | None = None
platform: str | None = None
topics: list[Topic] | None = None
evidence: list[Evidence]
source_count: int
class PartyRef(Model):
slug: str
short_name: str | None = None
color: str | None = None
class PersonListItem(Model):
slug: str
full_name: str
photo_url: str | None = None
status: PersonStatus
status_verified: bool
last_office_end: date | None = None
current_roles: list[str] | None = None
parties: list[PartyRef] | None = None
class RoleEvidence(Model):
url: str
source: str
class Role(Model):
role: str
level: str
region: str | None = None
start_date: date | None = None
end_date: date | None = None
body: str | None = None
body_slug: str | None = None
party_slug: str | None = None
party_short: str | None = None
party_color: str | None = None
claim_status: str
evidence: list[RoleEvidence] | None = None
class SocialAccountOut(Model):
platform: str
handle: str | None = None
url: str
verified: bool
class PersonDetail(Model):
slug: str
full_name: str
given_name: str | None = None
family_name: str | None = None
birth_date: date | None = None
death_date: date | None = None
photo_url: str | None = None
wikidata_qid: str | None = None
parlament_pad: str | None = None
ep_id: str | None = None
status: PersonStatus
status_verified: bool
last_office_end: date | None = None
roles: list[Role]
social_accounts: list[SocialAccountOut]
claim_counts: dict[str, int]
class FormerPerson(Model):
slug: str
full_name: str
photo_url: str | None = None
status: PersonStatus
last_office_end: date | None = None
last_role: str | None = None
latest_news: str | None = None
latest_news_at: datetime | None = None
latest_news_url: str | None = None
latest_career: str | None = None
class NetworkOrg(Model):
id: int
slug: str
name: str
kind: str
via: list[str]
label: str | None = None
money: dict[str, Any] | None = None
class Network(Model):
organizations: list[NetworkOrg]
class Party(Model):
slug: str
name: str
short_name: str | None = None
color: str | None = None
logo_url: str | None = None
active_officials: int = 0
total_people: int = 0
class PartyMember(Model):
slug: str
full_name: str
photo_url: str | None = None
role: str
level: str
region: str | None = None
body: str | None = None
class FormerMember(Model):
slug: str
full_name: str
last_office_end: date | None = None
last_mention: datetime | None = None
class MoneyByYear(Model):
year: int
kind: str
total: float | None = None
count: int
class Finance(Model):
data: dict[str, Any]
valid_from: date | None = None
class CountByPosition(Model):
position: str
count: int
class Dissenter(Model):
slug: str
full_name: str
dissenting_votes: int
class RelatedOrg(Model):
claim_type: str
slug: str
name: str
kind: str
class PartyDetail(Model):
slug: str
name: str
short_name: str | None = None
color: str | None = None
logo_url: str | None = None
kind: str
members: list[PartyMember]
former_members: list[FormerMember]
money_by_year: list[MoneyByYear]
finance: list[Finance]
vote_stats: list[CountByPosition]
dissent: list[Dissenter]
related: list[RelatedOrg]
class OrganizationListItem(Model):
slug: str
name: str
short_name: str | None = None
kind: str
paid: float | None = None
received: float | None = None
class OrgMoney(Model):
kind: str
direction: Literal["paid", "received"]
total: float | None = None
count: int
first: str | None = None
last: str | None = None
class Counterparty(Model):
slug: str
name: str
kind: str
money_kind: str
direction: Literal["paid_to", "received_from"]
total: float | None = None
count: int
class OrgPerson(Model):
slug: str
full_name: str
via: str
summary_de: str
class OrganizationDetail(Model):
slug: str
name: str
short_name: str | None = None
kind: str
parent_slug: str | None = None
parent_name: str | None = None
identifiers: dict[str, str] | None = None
money: list[OrgMoney]
counterparties: list[Counterparty]
people: list[OrgPerson]
class MoneyFlow(Model):
id: int
summary_de: str
summary_en: str | None = None
occurred_at: datetime | None = None
status: str
data: dict[str, Any] = {}
kind: str
amount: float | None = None
amount_category: str | None = None
currency: str | None = None
paid_on: date | None = None
period: str | None = None
reference: str | None = None
category: str | None = None
payer_name: str | None = None
payee_name: str | None = None
payer_slug: str | None = None
payer_kind: str | None = None
payee_slug: str | None = None
payee_kind: str | None = None
evidence: list[Evidence]
source_count: int
class MoneySummaryRow(Model):
kind: str
slug: str
name: str
org_kind: str
side: Literal["payer", "payee"]
total: float | None = None
count: int
class SocialPost(Model):
id: int
platform: str | None = None
said_at: datetime | None = None
excerpt: str | None = None
url: str
title: str | None = None
data: dict[str, Any] = {}
person_slug: str | None = None
person_name: str | None = None
org_slug: str | None = None
org_name: str | None = None
evidence: list[Evidence]
class WikiEdit(Model):
id: int
occurred_at: datetime | None = None
summary_de: str
summary_en: str | None = None
data: dict[str, Any] = {}
org_slug: str
org_name: str
person_slug: str | None = None
person_name: str | None = None
url: str | None = None
class TimelineBucket(Model):
month: str
claim_type: str
count: int
class Participant(Model):
slug: str
name: str
class CalendarEvent(Model):
id: int
kind: str
title: str
starts_at: datetime | None = None
ends_at: datetime | None = None
location_name: str | None = None
country: str | None = None
url: str | None = None
description: str | None = None
body: str | None = None
participants: list[Participant]
class DayCount(Model):
day: date
claim_type: str
count: int
class CalendarData(Model):
events: list[CalendarEvent]
days: list[DayCount]
class PartyPosition(Model):
party: str | None = None
slug: str
color: str | None = None
position: str
class Speech(Model):
url: str
title: str | None = None
class VoteItem(Model):
id: int
item_title: str
item_ref: str | None = None
stage: str | None = None
voted_on: date | None = None
result: str | None = None
url: str | None = None
protocol_url: str | None = None
roll_call: bool
raw_text: str | None = None
chamber: str
positions: list[PartyPosition] = []
individual_votes: int = 0
class PersonVote(Model):
id: int
item_title: str
item_ref: str | None = None
stage: str | None = None
voted_on: date | None = None
result: str | None = None
url: str | None = None
protocol_url: str | None = None
roll_call: bool
position: str
faction: str | None = None
chamber: str
individual: bool
dissent: bool
spoke: Speech | None = None
topics: list[str] | None = None
class IndividualVote(Model):
slug: str
full_name: str
position: str
faction: str | None = None
dissent: bool
class VoteDetail(Model):
id: int
item_title: str
item_ref: str | None = None
stage: str | None = None
voted_on: date | None = None
result: str | None = None
url: str | None = None
protocol_url: str | None = None
roll_call: bool
raw_text: str | None = None
chamber: str
positions: list[PartyPosition]
individual: list[IndividualVote]
class SearchEvidence(Model):
url: str
title: str | None = None
source: str
class SearchHit(Model):
id: int
text: str
claim_id: int | None = None
document_id: int | None = None
summary_de: str | None = None
claim_type: str | None = None
occurred_at: datetime | None = None
person_slug: str | None = None
person_name: str | None = None
organization_slug: str | None = None
document_url: str | None = None
document_title: str | None = None
document_date: datetime | None = None
evidence: list[SearchEvidence] | None = None
score: float
class Run(Model):
adapter: str
mode: str
status: str
started_at: datetime
finished_at: datetime | None = None
stats: dict[str, Any] = {}
error: str | None = None
class PendingClaim(Model):
id: int
claim_type: str
summary_de: str
occurred_at: datetime | None = None
full_name: str | None = None
evidence: list[dict[str, Any]] | None = None
+191
View File
@@ -0,0 +1,191 @@
"""`politrack` command line. Run `politrack --help` for all commands."""
from __future__ import annotations
import json
import logging
from datetime import datetime
from pathlib import Path
import typer
app = typer.Typer(help="Politrack command line", no_args_is_help=True)
sources_app = typer.Typer(help="Source specs and adapters")
app.add_typer(sources_app, name="sources")
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
def _params(values: list[str]) -> dict:
"""--param key=value (value parsed as JSON when possible)."""
out = {}
for item in values:
key, _, raw = item.partition("=")
try:
out[key] = json.loads(raw)
except json.JSONDecodeError:
out[key] = raw
return out
@app.command()
def migrate() -> None:
"""Apply SQL migrations, sync source specs and install the job-queue schema."""
from politrack.core.registry import sync_sources
from politrack.db import connection
from politrack.migrate import migrate as run_migrations
run_migrations()
with connection() as conn:
typer.echo(f"synced {sync_sources(conn)} sources")
installed = conn.execute("SELECT to_regclass('procrastinate_jobs') AS t").fetchone()["t"]
if not installed:
from politrack.worker import app as jobs
with jobs.open():
jobs.schema_manager.apply_schema()
@sources_app.command("list")
def sources_list() -> None:
"""List adapters, enrichers and their schedules."""
from politrack.core import registry
registry.load_builtin()
for name, cls in sorted({**registry.ADAPTERS, **registry.ENRICHERS}.items()):
typer.echo(f"{name:34} {cls.queue:7} {cls.schedule or 'manual':16} {cls.spec.slug:22} {cls.description}")
@sources_app.command("sync")
def sources_sync() -> None:
"""Upsert SourceSpecs into the database."""
from politrack.core.registry import sync_sources
from politrack.db import connection
with connection() as conn:
typer.echo(f"synced {sync_sources(conn)} sources")
@app.command("run")
def run_cmd(
name: str,
param: list[str] = typer.Option([], "--param", "-p", help="key=value, e.g. -p periods='[\"XXVIII\"]'"),
since: str = typer.Option(None, help="ISO date for incremental adapters"),
max_requests: int = typer.Option(None, help="Stop after N requests (testing)"),
) -> None:
"""Run one adapter or enricher now, e.g. `politrack run parlament.persons`."""
from politrack.core.runner import run
result = run(name, params=_params(param), since=datetime.fromisoformat(since) if since else None,
max_requests=max_requests)
typer.echo(json.dumps(result, indent=2, default=str))
@app.command()
def reparse(name: str, param: list[str] = typer.Option([], "--param", "-p")) -> None:
"""Re-run parse() on the archived responses of an adapter (after a parser fix, no network)."""
from politrack.core.runner import run
typer.echo(json.dumps(run(name, mode="reparse", params=_params(param)), indent=2, default=str))
@app.command()
def enrich() -> None:
"""Run all enrichers once (linking, events, legal, topics, roll calls, corroboration, embeddings)."""
from politrack.core import registry
from politrack.core.runner import run
registry.load_builtin()
for name in ("enrich.linker", "enrich.events", "enrich.legal", "enrich.topics", "enrich.roles",
"enrich.rollcall", "enrich.corroborate", "enrich.embed"):
typer.echo(f"{name}: {run(name)}")
@app.command()
def status(limit: int = 30) -> None:
"""Show recent runs."""
from politrack.db import connection
with connection() as conn:
for r in conn.execute("SELECT adapter, mode, status, started_at, finished_at, stats FROM run "
"ORDER BY id DESC LIMIT %s", (limit,)):
took = (r["finished_at"] - r["started_at"]).seconds if r["finished_at"] else "…"
typer.echo(f"{r['started_at']:%Y-%m-%d %H:%M} {r['adapter']:32} {r['status']:8} {took}s "
f"{json.dumps(r['stats'])[:120]}")
@app.command()
def review(
approve: list[int] = typer.Option([], help="Claim ids to publish"),
reject: list[int] = typer.Option([], help="Claim ids to reject"),
reviewer: str = typer.Option("cli", help="Your name, stored with the decision"),
) -> None:
"""List claims waiting for review (legal matters detected in headlines) or approve/reject them."""
from politrack import queries
from politrack.db import connection
with connection() as conn:
for claim_id in approve:
queries.set_review(conn, claim_id, "published", reviewer)
for claim_id in reject:
queries.set_review(conn, claim_id, "rejected", reviewer)
if approve or reject:
typer.echo(f"published {len(approve)}, rejected {len(reject)}")
return
for c in queries.pending_claims(conn, limit=100):
typer.echo(f"[{c['id']}] {c['occurred_at']:%Y-%m-%d} {c['full_name']}: {c['summary_de']}")
for ev in c["evidence"] or []:
typer.echo(f" {ev['title']} {ev['url']}")
@app.command()
def fixture(name: str, url: str, case: str, kind: str = "default", meta: str = "{}", method: str = "GET",
body: str = typer.Option(None, help="JSON request body for POST")) -> None:
"""Capture a live response as a golden-test fixture (tests/fixtures/<adapter>/<case>.*)."""
import httpx
from politrack.config import get_settings
out = Path(__file__).resolve().parent.parent / "tests" / "fixtures" / name
out.mkdir(parents=True, exist_ok=True)
resp = httpx.request(method, url, json=json.loads(body) if body else None, timeout=120, follow_redirects=True,
headers={"User-Agent": get_settings().user_agent})
resp.raise_for_status()
suffix = {"application/json": "json", "text/html": "html", "text/xml": "xml", "application/xml": "xml"}.get(
resp.headers.get("content-type", "").split(";")[0], "bin")
(out / f"{case}.{suffix}").write_bytes(resp.content)
(out / f"{case}.request.json").write_text(json.dumps(
{"url": url, "kind": kind, "meta": json.loads(meta), "method": method, "json": json.loads(body) if body else None,
"content_type": resp.headers.get("content-type", ""), "body_file": f"{case}.{suffix}"}, indent=2), encoding="utf-8")
typer.echo(f"saved {out / case}.* - now run: UPDATE_GOLDEN=1 pytest tests/test_golden.py -k {case}")
@app.command()
def openapi(json_out: Path = Path("openapi.json"), ts_out: Path = typer.Option(None, "--ts")) -> None:
"""Export the OpenAPI schema and (optionally) TypeScript types for the web app."""
from politrack.api.main import app as api_app
from politrack.openapi_ts import to_typescript
schema = api_app.openapi()
json_out.write_text(json.dumps(schema, indent=2, ensure_ascii=False), encoding="utf-8")
if ts_out:
ts_out.write_text(to_typescript(schema), encoding="utf-8")
typer.echo(f"wrote {json_out}" + (f" and {ts_out}" if ts_out else ""))
@app.command()
def api(host: str = "0.0.0.0", port: int = 8000) -> None:
import uvicorn
uvicorn.run("politrack.api.main:app", host=host, port=port, proxy_headers=True)
@app.command()
def mcp(host: str = "0.0.0.0", port: int = 8001) -> None:
"""Serve the MCP endpoint for AI agents (streamable HTTP at /mcp)."""
from politrack.mcp_server import serve
serve(host, port)
if __name__ == "__main__":
app()
+48
View File
@@ -0,0 +1,48 @@
from functools import lru_cache
from pathlib import Path
from pydantic_settings import BaseSettings, SettingsConfigDict
class Settings(BaseSettings):
"""All settings come from environment variables prefixed POLITRACK_ (see .env.example)."""
model_config = SettingsConfigDict(env_prefix="POLITRACK_", env_file=".env", extra="ignore")
database_url: str = "postgresql://politrack:politrack@localhost:5432/politrack"
reader_password: str | None = None
# Crawling identity & politeness
user_agent: str = "PolitrackBot/0.2 (+https://github.com/your-user/politics-tracking; contact: admin@example.org)"
min_request_interval: float = 2.0
http_timeout: float = 60.0
# Raw archive (fetched responses, gzip, content-addressed)
raw_dir: Path = Path("/data/raw")
config_dir: Path = Path(__file__).resolve().parent.parent / "config"
# Parliament: legislative periods (GP) ingested in detail
parlament_periods: list[str] = ["XXVII", "XXVIII"]
# Embeddings via Ollama (optional)
ollama_url: str | None = None
embedding_model: str = "bge-m3"
# Optional credentials for sources that require them (adapters skip themselves when unset)
youtube_api_key: str | None = None
meta_ad_library_token: str | None = None
x_bearer_token: str | None = None
# Bulk sources: how far back to backfill on the first run
procurement_since: str = "2024-01-01"
ad_archive_since: str = "2023-01-01"
api_cors_origins: list[str] = ["http://localhost:3000"]
# Bearer token for moderation/admin endpoints. Empty = disabled (use the CLI).
admin_token: str | None = None
@lru_cache
def get_settings() -> Settings:
return Settings()
+11
View File
@@ -0,0 +1,11 @@
"""The adapter framework. Most adapter modules only need:
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Evidence, PersonRef, OrgRef, ...
"""
from politrack.core.adapter import Adapter, Enricher, Request, Response, RunContext
from politrack.core.registry import register, register_spec
from politrack.core.spec import SourceSpec
__all__ = ["Adapter", "Enricher", "Request", "Response", "RunContext", "SourceSpec", "register", "register_spec"]
+177
View File
@@ -0,0 +1,177 @@
"""Adapter / Enricher base classes and the request/response/context types they work with.
An Adapter turns one publisher's data into Records in two steps:
start(ctx) -> Request... entry points (lists, feeds, index pages)
parse(ctx, resp) -> Record | Request... PURE: no DB access, no network
The runner fetches requests (policy-checked, archived), calls parse() and hands records to the
Loader. See docs/contributing/adapters.md for a walkthrough.
"""
from __future__ import annotations
import hashlib
import json
import logging
from collections import Counter
from collections.abc import Iterable, Iterator
from dataclasses import dataclass, field
from datetime import datetime
from functools import cached_property
from typing import TYPE_CHECKING, Any, ClassVar
from bs4 import BeautifulSoup
from politrack.core.spec import SourceSpec
if TYPE_CHECKING:
from psycopg import Connection
from politrack.core.records import Record
SECRET_PARAMS = {"key", "api_key", "apikey", "token", "access_token"}
def redact_url(url: str) -> str:
"""Remove credentials from a URL before it is logged or stored."""
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
parts = urlsplit(url)
query = [(k, "<redacted>" if k.lower() in SECRET_PARAMS else v) for k, v in parse_qsl(parts.query, keep_blank_values=True)]
return urlunsplit(parts._replace(query=urlencode(query)))
@dataclass(frozen=True)
class Request:
url: str
kind: str = "default" # parse() dispatch key
method: str = "GET"
params: dict[str, Any] | None = None
json: Any = None
headers: dict[str, str] | None = None
meta: dict[str, Any] = field(default_factory=dict, compare=False, hash=False)
conditional: bool = True # send If-None-Match / If-Modified-Since
skip_unchanged: bool = True # body + parser_version unchanged: follow-ups only
@property
def fingerprint(self) -> str:
raw = json.dumps([self.method, self.url, self.params or {}, self.json], sort_keys=True, default=str)
return hashlib.sha256(raw.encode()).hexdigest()[:32]
def describe(self) -> dict[str, Any]:
"""What gets stored in the archive index (secrets removed)."""
params = {k: ("<redacted>" if k.lower() in SECRET_PARAMS else v) for k, v in (self.params or {}).items()}
return {"kind": self.kind, "params": params or None, "json": self.json, "meta": self.meta}
@dataclass
class Response:
request: Request
status: int
url: str
headers: dict[str, str]
body: bytes
fetched_at: datetime
sha256: str
fetch_id: int | None = None
tdm_reserved: bool = False
from_archive: bool = False
@cached_property
def text(self) -> str:
encoding = "utf-8"
content_type = self.headers.get("content-type", "")
if "charset=" in content_type:
encoding = content_type.split("charset=", 1)[1].split(";")[0].strip() or "utf-8"
try:
return self.body.decode(encoding)
except (LookupError, UnicodeDecodeError):
return self.body.decode("utf-8", errors="replace")
def json(self) -> Any:
return json.loads(self.text, strict=False)
@cached_property
def soup(self) -> BeautifulSoup:
return BeautifulSoup(self.text, "html.parser")
@property
def meta(self) -> dict[str, Any]:
return self.request.meta
class StateStore:
"""Cursors of incremental adapters (persisted in the `state` table)."""
def __init__(self, conn: Connection | None, adapter: str) -> None:
self.conn = conn
self.adapter = adapter
self._local: dict[str, Any] = {}
def get(self, key: str, default: Any = None) -> Any:
if self.conn is None:
return self._local.get(key, default)
row = self.conn.execute("SELECT value FROM state WHERE adapter = %s AND key = %s",
(self.adapter, key)).fetchone()
return row["value"] if row else default
def set(self, key: str, value: Any) -> None:
if self.conn is None:
self._local[key] = value
return
from psycopg.types.json import Jsonb
self.conn.execute(
"""INSERT INTO state (adapter, key, value) VALUES (%s, %s, %s)
ON CONFLICT (adapter, key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()""",
(self.adapter, key, Jsonb(value)),
)
@dataclass
class RunContext:
adapter: str
spec: SourceSpec
state: StateStore
stats: Counter = field(default_factory=Counter)
since: datetime | None = None
params: dict[str, Any] = field(default_factory=dict) # CLI/worker parameters, e.g. {"gp": "XXVIII"}
mode: str = "fetch" # fetch | reparse | test
conn: Connection | None = None # only for enrichers / the loader, never in parse()
response: Response | None = None # set by the runner while parsing
creates_persons: bool = False
log: logging.Logger = field(default_factory=lambda: logging.getLogger("politrack"))
class Adapter:
"""Base class for source adapters. Subclass, set the class attributes, implement start()/parse()."""
spec: ClassVar[SourceSpec]
name: ClassVar[str]
description: ClassVar[str] = ""
schedule: ClassVar[str | None] = None # cron, UTC; None = manual only
queue: ClassVar[str] = "scrape" # scrape | bulk | social | enrich
parser_version: ClassVar[int] = 1 # bump when parse() output changes -> `politrack reparse`
creates_persons: ClassVar[bool] = False # may this adapter create new person entities?
def start(self, ctx: RunContext) -> Iterable[Request]:
raise NotImplementedError
def parse(self, ctx: RunContext, response: Response) -> Iterator[Record | Request]:
raise NotImplementedError
class Enricher:
"""Derives records from data already in the database (linking, events, corroboration...)."""
spec: ClassVar[SourceSpec]
name: ClassVar[str]
description: ClassVar[str] = ""
schedule: ClassVar[str | None] = None
queue: ClassVar[str] = "enrich"
creates_persons: ClassVar[bool] = False
def run(self, ctx: RunContext) -> Iterable[Record] | None:
raise NotImplementedError
+94
View File
@@ -0,0 +1,94 @@
"""Raw archive: every fetch is logged in the `fetch_log` table; bodies are stored gzip-compressed and
content-addressed (sha256) below POLITRACK_RAW_DIR when the source allows it.
Used for: conditional requests (ETag/Last-Modified), change detection (skip re-parsing identical
bodies), `politrack reparse` after parser fixes, and as an audit trail.
"""
from __future__ import annotations
import gzip
import hashlib
import os
from pathlib import Path
from typing import Any
from psycopg import Connection
from psycopg.types.json import Jsonb
from politrack.config import get_settings
class Archive:
def __init__(self, raw_dir: Path | None = None) -> None:
self.raw_dir = Path(raw_dir or get_settings().raw_dir)
# -- blobs ----------------------------------------------------------------------------
def _path(self, sha256: str) -> Path:
return self.raw_dir / sha256[:2] / f"{sha256}.gz"
def write_blob(self, body: bytes) -> str:
sha = hashlib.sha256(body).hexdigest()
path = self._path(sha)
if not path.exists():
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(f".tmp{os.getpid()}")
with gzip.open(tmp, "wb") as fh:
fh.write(body)
tmp.replace(path)
return sha
def read_blob(self, sha256: str) -> bytes | None:
path = self._path(sha256)
if not path.exists():
return None
with gzip.open(path, "rb") as fh:
return fh.read()
# -- index ----------------------------------------------------------------------------
def last(self, conn: Connection, request_fp: str) -> dict[str, Any] | None:
return conn.execute(
"""SELECT id, sha256, etag, last_modified, stored, parsed_sha256, parser_version, status, url,
content_type, tdm_reserved, fetched_at
FROM fetch_log WHERE request_fp = %s AND status IS NOT NULL AND status < 400
ORDER BY fetched_at DESC LIMIT 1""",
(request_fp,),
).fetchone()
def log(self, conn: Connection, *, adapter: str, source_id: int | None, request_fp: str, url: str, method: str,
request: dict, status: int | None, body: bytes | None, headers: dict[str, str] | None,
keep: bool, tdm_reserved: bool, error: str | None = None) -> tuple[int, str | None]:
sha = hashlib.sha256(body).hexdigest() if body is not None else None
stored = False
if body is not None and keep and not tdm_reserved:
self.write_blob(body)
stored = True
headers = headers or {}
row = conn.execute(
"""INSERT INTO fetch_log (adapter, source_id, request_fp, url, method, request, status, sha256, size,
content_type, etag, last_modified, stored, tdm_reserved, error)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id""",
(adapter, source_id, request_fp, url, method, Jsonb(request), status, sha,
len(body) if body is not None else None, headers.get("content-type"), headers.get("etag"),
headers.get("last-modified"), stored, tdm_reserved, error),
).fetchone()
return row["id"], sha
def mark_parsed(self, conn: Connection, fetch_id: int, sha256: str, parser_version: int) -> None:
conn.execute("UPDATE fetch_log SET parsed_sha256 = %s, parser_version = %s WHERE id = %s",
(sha256, parser_version, fetch_id))
def mark_error(self, conn: Connection, fetch_id: int, error: str) -> None:
conn.execute("UPDATE fetch_log SET error = %s WHERE id = %s", (error[:2000], fetch_id))
def latest_stored(self, conn: Connection, adapter: str) -> list[dict[str, Any]]:
"""Latest stored fetch per request of an adapter (input for `reparse`)."""
return conn.execute(
"""SELECT DISTINCT ON (request_fp) id, request_fp, url, method, request, status, sha256, fetched_at,
content_type, tdm_reserved
FROM fetch_log WHERE adapter = %s AND stored AND status < 400
ORDER BY request_fp, fetched_at DESC""",
(adapter,),
).fetchall()
+158
View File
@@ -0,0 +1,158 @@
"""The Fetcher: the only component that talks to the network. It enforces the crawling policy:
* honest User-Agent with contact address,
* robots.txt (cached per origin); exemptions only for documented APIs listed in SourceSpec.robots_exempt,
* per-host throttling (max of SourceSpec.min_interval and robots.txt crawl-delay),
* conditional requests (ETag / Last-Modified from the raw archive),
* retries with exponential back-off for 429/5xx, honouring Retry-After,
* TDM reservations (W3C TDMRep: `tdm-reservation` header, /.well-known/tdmrep.json, /tdmrep.json),
* archiving of raw bodies (unless SourceSpec.keep_raw is False or TDM is reserved).
"""
from __future__ import annotations
import logging
import threading
import time
from datetime import datetime, timezone
from urllib.parse import urlsplit
import httpx
from protego import Protego
from psycopg import Connection
from politrack.config import get_settings
from politrack.core.adapter import Request, Response, redact_url
from politrack.core.archive import Archive
from politrack.core.spec import SourceSpec
log = logging.getLogger(__name__)
RETRY_STATUS = {429, 500, 502, 503, 504}
class Disallowed(Exception):
"""robots.txt forbids fetching this URL."""
class Fetcher:
def __init__(self, archive: Archive | None = None, client: httpx.Client | None = None) -> None:
settings = get_settings()
self.user_agent = settings.user_agent
self.default_interval = settings.min_request_interval
self.archive = archive or Archive()
self.client = client or httpx.Client(
headers={"User-Agent": self.user_agent, "Accept-Language": "de-AT,de;q=0.9,en;q=0.5"},
timeout=settings.http_timeout,
follow_redirects=True,
)
self._robots: dict[str, Protego | None] = {}
self._tdm: dict[str, bool] = {}
self._last: dict[str, float] = {}
self._lock = threading.Lock()
# -- policy -------------------------------------------------------------------------
@staticmethod
def origin(url: str) -> str:
parts = urlsplit(url)
return f"{parts.scheme}://{parts.netloc}"
def robots(self, url: str) -> Protego | None:
origin = self.origin(url)
if origin not in self._robots:
try:
resp = self.client.get(f"{origin}/robots.txt")
self._robots[origin] = Protego.parse(resp.text) if resp.status_code == 200 else None
except httpx.HTTPError:
self._robots[origin] = None
return self._robots[origin]
def allowed(self, url: str, spec: SourceSpec) -> bool:
if any(url.startswith(prefix) for prefix in spec.robots_exempt):
return True
rules = self.robots(url)
return rules is None or rules.can_fetch(url, self.user_agent)
def tdm_reserved(self, url: str) -> bool:
origin = self.origin(url)
if origin not in self._tdm:
reserved = False
for path in ("/.well-known/tdmrep.json", "/tdmrep.json"):
try:
resp = self.client.get(origin + path)
except httpx.HTTPError:
continue
if resp.status_code == 200:
try:
rules = resp.json()
except ValueError:
continue
reserved = any(str(r.get("tdm-reservation")) == "1" for r in rules if isinstance(r, dict))
break
self._tdm[origin] = reserved
return self._tdm[origin]
def _throttle(self, url: str, spec: SourceSpec) -> None:
rules = self._robots.get(self.origin(url))
delay = rules.crawl_delay(self.user_agent) if rules else None
interval = max(spec.min_interval or self.default_interval, float(delay or 0))
host = urlsplit(url).netloc
with self._lock:
wait = self._last.get(host, 0) + interval - time.monotonic()
if wait > 0:
time.sleep(wait)
self._last[host] = time.monotonic()
# -- fetching ---------------------------------------------------------------------
def fetch(self, conn: Connection, req: Request, *, adapter: str, spec: SourceSpec, source_id: int | None,
attempts: int = 5) -> Response:
if not self.allowed(req.url, spec):
raise Disallowed(req.url)
last = self.archive.last(conn, req.fingerprint) if req.conditional else None
headers = dict(req.headers or {})
if last and last["stored"]:
if last["etag"]:
headers["If-None-Match"] = last["etag"]
if last["last_modified"]:
headers["If-Modified-Since"] = last["last_modified"]
resp: httpx.Response | None = None
for attempt in range(attempts):
self._throttle(req.url, spec)
try:
resp = self.client.request(req.method, req.url, params=req.params, json=req.json, headers=headers)
except httpx.TransportError as exc:
if attempt == attempts - 1:
raise
log.warning("transport error %s (%s), retrying", req.url, exc)
time.sleep(min(5 * 2 ** attempt, 120))
continue
if resp.status_code in RETRY_STATUS and attempt < attempts - 1:
retry_after = resp.headers.get("Retry-After", "")
time.sleep(min(float(retry_after) if retry_after.isdigit() else 5 * 2 ** attempt, 300))
continue
break
assert resp is not None
tdm = resp.headers.get("tdm-reservation") == "1" or self.tdm_reserved(req.url)
now = datetime.now(timezone.utc)
if resp.status_code == 304 and last:
body = self.archive.read_blob(last["sha256"]) or b""
return Response(request=req, status=200, url=last["url"], headers={"content-type": last["content_type"] or ""},
body=body, fetched_at=now, sha256=last["sha256"], fetch_id=last["id"],
tdm_reserved=last["tdm_reserved"], from_archive=True)
lowered = {k.lower(): v for k, v in resp.headers.items()}
fetch_id, sha = self.archive.log(
conn, adapter=adapter, source_id=source_id, request_fp=req.fingerprint, url=redact_url(str(resp.url)),
method=req.method, request=req.describe(), status=resp.status_code, body=resp.content, headers=lowered,
keep=spec.keep_raw, tdm_reserved=tdm,
)
conn.commit()
resp.raise_for_status()
return Response(request=req, status=resp.status_code, url=redact_url(str(resp.url)), headers=lowered, body=resp.content,
fetched_at=now, sha256=sha or "", fetch_id=fetch_id, tdm_reserved=tdm)
def close(self) -> None:
self.client.close()
+448
View File
@@ -0,0 +1,448 @@
"""The Loader: the only code that writes canonical rows.
Invariants enforced here (not in adapters):
* no claim without evidence; claims whose person cannot be resolved are skipped (and counted),
* full text is kept only for sources with store_fulltext and responses without TDM reservation,
* claim identity is computed from *resolved* entity ids + type-specific fields, so the same fact
from different sources becomes one claim with several evidence rows ("corroborated"),
* claims marked `review=True` start as review_status='pending' (hidden until a human approves),
* persons are only created by adapters with creates_persons=True.
"""
from __future__ import annotations
from datetime import date, datetime
from decimal import Decimal, InvalidOperation
from typing import Any
from psycopg import Connection
from psycopg.types.json import Jsonb
from politrack.core.adapter import RunContext
from politrack.core.records import (
Claim,
Document,
Event,
Evidence,
Legal,
Money,
OrgRecord,
OrgRef,
Participation,
PersonRecord,
PersonRef,
Role,
SocialAccount,
Statement,
VoteEvent,
VotePosition,
)
from politrack.core.resolve import Resolver
from politrack.text import excerpt, fingerprint, make_slug, normalize, normalize_org_name, strip_titles
# Source policy is read from the DB once per process (synced from SourceSpecs).
_SOURCES: dict[str, dict[str, Any]] = {}
def source_row(conn: Connection, slug: str) -> dict[str, Any]:
if slug not in _SOURCES:
row = conn.execute("SELECT id, store_fulltext, kind FROM source WHERE slug = %s", (slug,)).fetchone()
if row is None:
raise KeyError(f"unknown source {slug!r}: run `politrack sources sync`")
_SOURCES[slug] = row
return _SOURCES[slug]
class Loader:
def __init__(self, conn: Connection, ctx: RunContext) -> None:
self.conn = conn
self.ctx = ctx
self.resolver = Resolver(conn)
# -- dispatch ---------------------------------------------------------------------
def load(self, record: Any) -> int | None:
if isinstance(record, Claim):
return self.claim(record)
if isinstance(record, Document):
return self.document(record.evidence)
if isinstance(record, PersonRecord):
return self.person(record)
if isinstance(record, OrgRecord):
return self.organization(record)
if isinstance(record, SocialAccount):
return self.social_account(record)
if isinstance(record, VoteEvent):
return self.vote_event(record)
if isinstance(record, Event):
return self.event(record)
raise TypeError(f"unknown record type {type(record).__name__}")
# -- documents --------------------------------------------------------------------
def document(self, ev: Evidence) -> int:
slug = ev.source or self.ctx.spec.slug
src = source_row(self.conn, slug)
response = self.ctx.response
tdm = bool(response and response.tdm_reserved)
full_text = ev.full_text if (src["store_fulltext"] and not tdm) else None
short = None if tdm else (excerpt(ev.excerpt, 400) if ev.excerpt else None)
# Link the document to the raw response only if it *is* that response (not e.g. a protocol it cites).
fetch_id = response.fetch_id if response and ev.url in (response.url, response.request.url) else None
row = self.conn.execute(
"""
INSERT INTO document (source_id, url, title, doc_type, published_at, excerpt, full_text, fetch_id, meta)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (url) DO UPDATE SET
title = COALESCE(EXCLUDED.title, document.title),
published_at = COALESCE(EXCLUDED.published_at, document.published_at),
excerpt = COALESCE(EXCLUDED.excerpt, document.excerpt),
full_text = COALESCE(EXCLUDED.full_text, document.full_text),
fetch_id = COALESCE(EXCLUDED.fetch_id, document.fetch_id),
meta = document.meta || EXCLUDED.meta,
retrieved_at = now()
RETURNING id
""",
(src["id"], ev.url, ev.title, ev.doc_type, ev.published_at, short, full_text, fetch_id,
Jsonb({**ev.meta, **({"tdm_reserved": True} if tdm else {})})),
).fetchone()
self.ctx.stats["documents"] += 1
return row["id"]
# -- entities ---------------------------------------------------------------------
def person(self, rec: PersonRecord) -> int | None:
person_id = self.resolver.person_by_ids(rec.ref.ids)
if person_id is None and rec.match_by_name:
person_id = self.resolver.person_by_name(rec.full_name)
if person_id is None:
if not self.ctx.creates_persons:
self.ctx.stats["persons_not_created"] += 1
return None
base = make_slug(rec.given_name or "", rec.family_name or strip_titles(rec.full_name)) or "person"
slug, n = base, 1
while self.conn.execute("SELECT 1 FROM person WHERE slug = %s", (slug,)).fetchone():
n += 1
slug = f"{base}-{n}"
person_id = self.conn.execute(
"""INSERT INTO person (slug, full_name, given_name, family_name, honorific_prefix, honorific_suffix,
birth_date, death_date, gender, photo_url)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id""",
(slug, rec.full_name, rec.given_name, rec.family_name, rec.honorific_prefix, rec.honorific_suffix,
rec.birth_date, rec.death_date, rec.gender, rec.photo_url),
).fetchone()["id"]
self.ctx.stats["persons_created"] += 1
else:
self.conn.execute(
"""UPDATE person SET full_name = %s,
given_name = COALESCE(%s, given_name), family_name = COALESCE(%s, family_name),
honorific_prefix = COALESCE(%s, honorific_prefix), honorific_suffix = COALESCE(%s, honorific_suffix),
birth_date = COALESCE(%s, birth_date), death_date = COALESCE(%s, death_date),
gender = COALESCE(%s, gender), photo_url = COALESCE(%s, photo_url), updated_at = now()
WHERE id = %s""",
(rec.full_name, rec.given_name, rec.family_name, rec.honorific_prefix, rec.honorific_suffix,
rec.birth_date, rec.death_date, rec.gender, rec.photo_url, person_id),
)
for scheme, value in rec.ref.ids.items():
if value:
self.conn.execute(
"""INSERT INTO identifier (entity_type, entity_id, scheme, value, source_slug)
VALUES ('person', %s, %s, %s, %s) ON CONFLICT DO NOTHING""",
(person_id, scheme, str(value), self.ctx.spec.slug),
)
names = {rec.full_name, strip_titles(rec.full_name), *rec.aliases}
if rec.given_name and rec.family_name:
names.add(f"{rec.given_name} {rec.family_name}")
for name in names:
if name:
self.conn.execute(
"""INSERT INTO alias (entity_type, entity_id, alias, alias_norm) VALUES ('person', %s, %s, %s)
ON CONFLICT DO NOTHING""",
(person_id, name, normalize(strip_titles(name))),
)
self.ctx.stats["persons"] += 1
return person_id
def organization(self, rec: OrgRecord) -> int | None:
org_id = self.resolver.org(rec.ref)
if org_id is None:
return None
if rec.color:
self.conn.execute("UPDATE organization SET color = COALESCE(color, %s) WHERE id = %s", (rec.color, org_id))
if rec.parent:
parent = self.resolver.org(rec.parent)
if parent and parent != org_id:
self.conn.execute("UPDATE organization SET parent_id = %s WHERE id = %s", (parent, org_id))
self.resolver.add_org_aliases(org_id, list(rec.aliases))
return org_id
def social_account(self, rec: SocialAccount) -> int | None:
person_id = self.resolver.person(rec.person)
org_id = self.resolver.org(rec.org) if rec.org else None # OrgRef.create decides
if person_id is None and org_id is None:
self.ctx.stats["social_unresolved"] += 1
return None
row = self.conn.execute(
"""INSERT INTO social_account (person_id, organization_id, platform, handle, url, verified, source_slug)
VALUES (%s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (platform, url) DO UPDATE SET
handle = COALESCE(EXCLUDED.handle, social_account.handle),
verified = social_account.verified OR EXCLUDED.verified,
person_id = COALESCE(social_account.person_id, EXCLUDED.person_id),
organization_id = COALESCE(social_account.organization_id, EXCLUDED.organization_id)
RETURNING id""",
(person_id, org_id, rec.platform.lower(), rec.handle, rec.url, rec.verified, self.ctx.spec.slug),
).fetchone()
self.ctx.stats["social_accounts"] += 1
return row["id"]
# -- standalone facets ------------------------------------------------------------
def vote_event(self, rec: VoteEvent) -> int:
chamber = self.resolver.org(rec.chamber, create=False) if rec.chamber else None
row = self.conn.execute(
"""INSERT INTO vote_event (event_key, chamber_id, item_ref, item_title, stage, voted_on, result, roll_call,
url, protocol_url, raw_text)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (event_key) DO UPDATE SET result = EXCLUDED.result, raw_text = EXCLUDED.raw_text,
roll_call = EXCLUDED.roll_call, item_title = EXCLUDED.item_title,
protocol_url = COALESCE(EXCLUDED.protocol_url, vote_event.protocol_url)
RETURNING id""",
(rec.key, chamber, rec.item_ref, rec.title, rec.stage, rec.voted_on, rec.result, rec.roll_call, rec.url,
rec.protocol_url, rec.raw_text),
).fetchone()
self.ctx.stats["vote_events"] += 1
return row["id"]
def event(self, rec: Event) -> int:
body = self.resolver.org(rec.body, create=False) if rec.body else None
source_id = source_row(self.conn, self.ctx.spec.slug)["id"]
row = self.conn.execute(
"""INSERT INTO event (event_key, kind, title, starts_at, ends_at, location_name, country, url, description,
organization_id, source_id)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (event_key) DO UPDATE SET title = EXCLUDED.title,
starts_at = COALESCE(EXCLUDED.starts_at, event.starts_at), ends_at = EXCLUDED.ends_at,
location_name = COALESCE(EXCLUDED.location_name, event.location_name),
description = COALESCE(EXCLUDED.description, event.description)
RETURNING id""",
(rec.key, rec.kind, rec.title, rec.starts_at, rec.ends_at, rec.location, rec.country, rec.url,
rec.description, body, source_id),
).fetchone()
self.ctx.stats["events"] += 1
return row["id"]
# -- claims -----------------------------------------------------------------------
def claim(self, c: Claim) -> int | None:
if not c.evidence:
raise ValueError("a claim needs at least one piece of evidence")
person_id = self.resolver.person(c.person)
if c.person is not None and person_id is None:
self.ctx.stats["unresolved_person"] += 1
self.ctx.log.debug("unresolved person %s for %s", c.person, c.summary_de[:80])
return None
org_id = self.resolver.org(c.org) if c.org else None
cp_person = self.resolver.person(c.counterpart_person)
cp_org = self.resolver.org(c.counterpart_org) if c.counterpart_org else None
if person_id is None and org_id is None and cp_org is None:
self.ctx.stats["claims_without_subject"] += 1
return None
fp = fingerprint(*self.identity(c, person_id, org_id, cp_org))
row = self.conn.execute(
"""
INSERT INTO claim (claim_type, fingerprint, person_id, organization_id, counterpart_person_id,
counterpart_org_id, occurred_at, valid_from, valid_to, summary_de, summary_en,
data, sensitive, review_status, created_by)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (fingerprint) DO UPDATE SET
summary_de = EXCLUDED.summary_de,
summary_en = COALESCE(EXCLUDED.summary_en, claim.summary_en),
valid_to = COALESCE(EXCLUDED.valid_to, claim.valid_to),
occurred_at = COALESCE(claim.occurred_at, EXCLUDED.occurred_at),
data = claim.data || EXCLUDED.data,
sensitive = claim.sensitive OR EXCLUDED.sensitive,
updated_at = now()
RETURNING id, (xmax = 0) AS inserted
""",
(c.claim_type, fp, person_id, org_id, cp_person, cp_org, c.occurred_at, c.valid_from, c.valid_to,
c.summary_de, c.summary_en, Jsonb(c.data), c.sensitive or c.claim_type == "legal_proceeding",
"pending" if c.review else "published", self.ctx.adapter),
).fetchone()
claim_id = row["id"]
self.ctx.stats["claims_new" if row["inserted"] else "claims_updated"] += 1
for ev in c.evidence:
doc_id = self.document(ev)
self.conn.execute(
"""INSERT INTO claim_evidence (claim_id, document_id, stance, quote, locator, extracted_by, confidence)
VALUES (%s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (claim_id, document_id, locator) DO UPDATE SET
quote = COALESCE(EXCLUDED.quote, claim_evidence.quote),
confidence = GREATEST(claim_evidence.confidence, EXCLUDED.confidence)""",
(claim_id, doc_id, ev.stance, ev.quote and excerpt(ev.quote, 600), ev.locator, self.ctx.adapter,
ev.confidence),
)
for topic in c.topics:
self.assign_topic(claim_id, topic)
if c.facet is not None:
self.facet(claim_id, c, person_id, org_id)
self.refresh_status(claim_id)
return claim_id
def identity(self, c: Claim, person_id: int | None, org_id: int | None, cp_org: int | None) -> tuple:
"""Semantic identity of a claim (see docs/architecture/01-design-from-scratch.md §3.3)."""
if c.key is not None:
return (c.claim_type, "key", *c.key)
f = c.facet
if isinstance(f, Role):
body = self.resolver.org(f.body) if f.body else cp_org
return ("role", person_id, body, normalize(f.role), f.start)
if isinstance(f, VotePosition):
return ("vote", f.event_key, person_id, org_id)
if isinstance(f, Money):
def side(ref: Any, name: str | None) -> Any:
if isinstance(ref, PersonRef):
return ("p", self.resolver.person(ref))
if isinstance(ref, OrgRef):
return ("o", self.resolver.org(ref))
return ("n", normalize_org_name(name or ""))
return ("money", f.kind, side(f.payer, f.payer_name), side(f.payee, f.payee_name), _amount(f.amount),
f.paid_on, f.period, f.reference)
if isinstance(f, Statement):
return ("statement", person_id, org_id, f.url)
if isinstance(f, Participation):
return ("participation", f.event_key, person_id, org_id)
if c.claim_type == "media_mention":
return ("mention", c.evidence[0].url, person_id, org_id)
raise ValueError(f"claim of type {c.claim_type} needs an explicit key")
def facet(self, claim_id: int, c: Claim, person_id: int | None, org_id: int | None) -> None:
f = c.facet
if isinstance(f, Role):
body = self.resolver.org(f.body) if f.body else None
party = self.resolver.org(f.party) if f.party else None
self.conn.execute(
"""INSERT INTO role (claim_id, person_id, organization_id, party_id, role, level, region, start_date, end_date)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (claim_id) DO UPDATE SET end_date = COALESCE(EXCLUDED.end_date, role.end_date),
party_id = COALESCE(EXCLUDED.party_id, role.party_id), region = COALESCE(EXCLUDED.region, role.region),
organization_id = COALESCE(EXCLUDED.organization_id, role.organization_id)""",
(claim_id, person_id, body, party, f.role, f.level, f.region, f.start, f.end),
)
elif isinstance(f, VotePosition):
event = self.conn.execute(
"""SELECT ve.id, ve.voted_on, o.slug AS chamber FROM vote_event ve
LEFT JOIN organization o ON o.id = ve.chamber_id WHERE ve.event_key = %s""",
(f.event_key,),
).fetchone()
if event is None:
raise ValueError(f"vote event {f.event_key} must be loaded before its positions")
party = self.resolver.org(f.party) if f.party else None
if person_id is not None and party is None and event["chamber"]:
# individual (roll-call) vote: remember the voter's faction on that day (dissent detection)
party = self.resolver.faction_of(person_id, event["chamber"], event["voted_on"])
self.conn.execute(
"""INSERT INTO vote_position (claim_id, vote_event_id, organization_id, person_id, party_id, position)
VALUES (%s, %s, %s, %s, %s, %s)
ON CONFLICT (claim_id) DO UPDATE SET position = EXCLUDED.position,
party_id = COALESCE(EXCLUDED.party_id, vote_position.party_id)""",
(claim_id, event["id"], org_id if person_id is None else None, person_id, party, f.position),
)
elif isinstance(f, Money):
payer_org = self.resolver.org(f.payer) if isinstance(f.payer, OrgRef) else None
payee_org = self.resolver.org(f.payee) if isinstance(f.payee, OrgRef) else None
payer_person = self.resolver.person(f.payer) if isinstance(f.payer, PersonRef) else None
payee_person = self.resolver.person(f.payee) if isinstance(f.payee, PersonRef) else None
self.conn.execute(
"""INSERT INTO money_flow (claim_id, kind, payer_name, payer_org_id, payer_person_id, payee_name,
payee_org_id, payee_person_id, amount, amount_category, currency, paid_on,
period, reference, category)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (claim_id) DO UPDATE SET amount = EXCLUDED.amount,
payer_org_id = COALESCE(EXCLUDED.payer_org_id, money_flow.payer_org_id),
payee_org_id = COALESCE(EXCLUDED.payee_org_id, money_flow.payee_org_id)""",
(claim_id, f.kind, f.payer_name, payer_org, payer_person, f.payee_name, payee_org, payee_person,
_amount(f.amount), f.amount_category, f.currency, f.paid_on, f.period, f.reference, f.category),
)
elif isinstance(f, Participation):
event = self.conn.execute("SELECT id FROM event WHERE event_key = %s", (f.event_key,)).fetchone()
if event is None:
raise ValueError(f"event {f.event_key} must be loaded before participations")
self.conn.execute(
"""INSERT INTO event_participant (claim_id, event_id, person_id, organization_id, participation)
VALUES (%s, %s, %s, %s, %s) ON CONFLICT (claim_id) DO NOTHING""",
(claim_id, event["id"], person_id, org_id, f.participation),
)
elif isinstance(f, Statement):
self.conn.execute(
"""INSERT INTO statement (claim_id, person_id, context, platform, medium, title, said_at, excerpt, url)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) ON CONFLICT (claim_id) DO UPDATE SET
excerpt = COALESCE(EXCLUDED.excerpt, statement.excerpt)""",
(claim_id, person_id, f.context, f.platform, f.medium, f.title, f.said_at,
f.excerpt and excerpt(f.excerpt, 600), f.url),
)
elif isinstance(f, Legal):
self.conn.execute(
"""INSERT INTO legal_proceeding (claim_id, person_id, organization_id, stage, offense, authority, case_ref,
stage_date)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (claim_id) DO UPDATE SET stage = EXCLUDED.stage,
offense = COALESCE(EXCLUDED.offense, legal_proceeding.offense),
authority = COALESCE(EXCLUDED.authority, legal_proceeding.authority),
case_ref = COALESCE(EXCLUDED.case_ref, legal_proceeding.case_ref)""",
(claim_id, person_id, org_id, f.stage, f.offense, f.authority, f.case_ref, f.stage_date),
)
# -- helpers ----------------------------------------------------------------------
def assign_topic(self, claim_id: int, topic_name: str, method: str | None = None, confidence: float = 1.0) -> None:
norm = normalize(topic_name)
row = self.conn.execute(
"SELECT id FROM topic WHERE lower(unaccent(name_de)) = %s OR slug = %s", (norm, topic_name)
).fetchone()
if row is None:
row = self.conn.execute(
"INSERT INTO topic (slug, name_de, name_en) VALUES (%s, %s, %s) ON CONFLICT (slug) DO UPDATE "
"SET name_de = topic.name_de RETURNING id",
(make_slug(topic_name), topic_name, topic_name),
).fetchone()
self.conn.execute(
"INSERT INTO topic_assignment (topic_id, claim_id, confidence, method) VALUES (%s, %s, %s, %s) "
"ON CONFLICT DO NOTHING",
(row["id"], claim_id, confidence, method or self.ctx.adapter),
)
def refresh_status(self, claim_id: int) -> None:
"""corroborated = backed by >= 2 different sources; disputed = any contradicting evidence."""
self.conn.execute(
"""
UPDATE claim SET status = CASE
WHEN status = 'retracted' THEN status
WHEN EXISTS (SELECT 1 FROM claim_evidence e WHERE e.claim_id = claim.id AND e.stance = 'contradicts')
THEN 'disputed'
WHEN (SELECT count(DISTINCT d.source_id) FROM claim_evidence e JOIN document d ON d.id = e.document_id
WHERE e.claim_id = claim.id AND e.stance IN ('supports', 'mentions')) >= 2 THEN 'corroborated'
ELSE 'reported' END
WHERE id = %s
""",
(claim_id,),
)
def _amount(value: Any) -> Decimal | None:
if value is None or value == "":
return None
try:
return Decimal(str(value)).quantize(Decimal("0.01"))
except InvalidOperation:
return None
def as_date(value: Any) -> date | None:
if isinstance(value, datetime):
return value.date()
if isinstance(value, date):
return value
return None
+265
View File
@@ -0,0 +1,265 @@
"""Typed records: the only way adapters and enrichers hand data to the Loader.
Records are plain dataclasses so parsers stay pure and testable (see tests/golden). They refer to
entities through PersonRef/OrgRef, which the Loader resolves (identifiers first, then hints, then
names). Data minimisation is enforced by schema: there is, for example, no address field anywhere,
so a parser cannot store a donor's street address even by accident.
"""
from __future__ import annotations
from dataclasses import dataclass, field, fields, is_dataclass
from datetime import date, datetime
from decimal import Decimal
from typing import Any, Literal, Union
# --------------------------------------------------------------------------------------------
# References to entities
# --------------------------------------------------------------------------------------------
@dataclass
class PersonRef:
"""How a record points at a person.
ids: strong identifiers, e.g. {"parlament": "2339"} or {"wikidata": "Q123"}
name: display/full name; used for alias matching if no id matches
hints: resolution strategies, e.g.
{"surname_in_body": "nationalrat", "on": date(...)} -> roll-call surname lists
{"office": "Bundesministerin für Justiz", "on": date(...)} -> office holder on a date
"""
ids: dict[str, str] = field(default_factory=dict)
name: str | None = None
hints: dict[str, Any] = field(default_factory=dict)
@dataclass
class OrgRef:
"""How a record points at an organization. `slug` addresses seeded orgs (e.g. "nationalrat")."""
ids: dict[str, str] = field(default_factory=dict)
name: str | None = None
short_name: str | None = None
kind: str = "other"
slug: str | None = None
create: bool = True
# --------------------------------------------------------------------------------------------
# Evidence & documents
# --------------------------------------------------------------------------------------------
DocType = Literal["article", "press_release", "api_record", "protocol", "social_post", "register_entry",
"web_page", "pdf", "dataset_row"]
@dataclass
class Evidence:
url: str
title: str | None = None
doc_type: str = "web_page"
published_at: datetime | date | str | None = None
excerpt: str | None = None
full_text: str | None = None
quote: str | None = None
locator: str = ""
stance: Literal["supports", "contradicts", "mentions"] = "supports"
confidence: float = 1.0
source: str | None = None # defaults to the adapter's source
meta: dict[str, Any] = field(default_factory=dict)
@dataclass
class Document:
"""A document without a claim yet (e.g. a headline); enrichers derive claims from it later."""
evidence: Evidence
# --------------------------------------------------------------------------------------------
# Entities
# --------------------------------------------------------------------------------------------
@dataclass
class PersonRecord:
ref: PersonRef
full_name: str
given_name: str | None = None
family_name: str | None = None
honorific_prefix: str | None = None
honorific_suffix: str | None = None
birth_date: date | None = None
death_date: date | None = None
gender: str | None = None
photo_url: str | None = None
aliases: tuple[str, ...] = ()
#: if no identifier matches, merge with an existing person of exactly this (unambiguous) name before
#: creating a new one - for authoritative sources that share no id with others (e.g. EP)
match_by_name: bool = False
@dataclass
class OrgRecord:
ref: OrgRef
aliases: tuple[str, ...] = ()
color: str | None = None
parent: OrgRef | None = None
@dataclass
class SocialAccount:
platform: str # bluesky | mastodon | telegram | youtube | twitter | facebook | ...
url: str
handle: str | None = None
person: PersonRef | None = None
org: OrgRef | None = None
verified: bool = False # True only for official sources (parliament profile, official website)
# --------------------------------------------------------------------------------------------
# Facets (structured payload of a claim)
# --------------------------------------------------------------------------------------------
@dataclass
class Role:
role: str # "mp" for chamber members, otherwise office title
level: Literal["eu", "federal", "state", "district", "municipal", "party", "other"]
body: OrgRef | None = None # chamber/government/party the role is in
party: OrgRef | None = None # party/faction of the holder in this role
region: str | None = None
start: date | None = None
end: date | None = None
@dataclass
class VoteEvent:
"""A vote (not a claim itself). Positions reference it by `key`."""
key: str
chamber: OrgRef | None
title: str
voted_on: date | None
result: Literal["accepted", "rejected", "unknown"] = "unknown"
stage: str | None = None
item_ref: str | None = None
roll_call: bool = False
url: str | None = None
protocol_url: str | None = None
raw_text: str | None = None
@dataclass
class VotePosition:
event_key: str
position: Literal["for", "against", "abstain", "absent"]
party: OrgRef | None = None # faction of an individual voter at that time
@dataclass
class Money:
kind: Literal["donation", "contract", "government_ad", "political_ad", "subsidy", "party_funding", "party_ad",
"loan", "side_income", "sponsorship", "other"]
amount: Decimal | float | None = None
currency: str = "EUR"
payer: OrgRef | PersonRef | None = None
payee: OrgRef | PersonRef | None = None
payer_name: str | None = None
payee_name: str | None = None
paid_on: date | None = None
period: str | None = None
reference: str | None = None # notice id, campaign, ...
category: str | None = None # CPV code, ad category, ...
amount_category: str | None = None # when only a bracket is published
@dataclass
class Event:
"""Something with time and place (not a claim). Participation claims reference it by `key`."""
key: str
kind: str
title: str
starts_at: datetime | date | str | None
ends_at: datetime | date | str | None = None
location: str | None = None
country: str | None = None
url: str | None = None
description: str | None = None
body: OrgRef | None = None
@dataclass
class Participation:
event_key: str
participation: str = "participant"
@dataclass
class Statement:
context: Literal["parliament_speech", "interview", "tv", "radio", "social_post", "press_release",
"event_speech", "op_ed", "testimony", "other"]
url: str
said_at: datetime | date | str | None = None
platform: str | None = None
medium: str | None = None
title: str | None = None
excerpt: str | None = None
@dataclass
class Legal:
stage: str
offense: str | None = None
authority: str | None = None
case_ref: str | None = None
stage_date: date | None = None
Facet = Union[Role, VotePosition, Money, Participation, Statement, Legal]
# --------------------------------------------------------------------------------------------
# Claims
# --------------------------------------------------------------------------------------------
@dataclass
class Claim:
claim_type: str
summary_de: str
evidence: list[Evidence]
person: PersonRef | None = None
org: OrgRef | None = None
counterpart_person: PersonRef | None = None
counterpart_org: OrgRef | None = None
occurred_at: datetime | date | str | None = None
valid_from: date | None = None
valid_to: date | None = None
summary_en: str | None = None
data: dict[str, Any] = field(default_factory=dict)
topics: tuple[str, ...] = ()
sensitive: bool = False
review: bool = False # True -> published only after human review
key: tuple | None = None # explicit identity (otherwise computed by the Loader)
facet: Facet | None = None
Record = Union[PersonRecord, OrgRecord, SocialAccount, VoteEvent, Event, Claim, Document]
def to_jsonable(obj: Any) -> Any:
"""Serialise records for golden tests and debugging (stable, sorted keys)."""
if is_dataclass(obj) and not isinstance(obj, type):
return {"__type__": type(obj).__name__, **{f.name: to_jsonable(getattr(obj, f.name)) for f in fields(obj)}}
if isinstance(obj, dict):
return {str(k): to_jsonable(v) for k, v in sorted(obj.items(), key=lambda kv: str(kv[0]))}
if isinstance(obj, (list, tuple)):
return [to_jsonable(v) for v in obj]
if isinstance(obj, (datetime, date)):
return obj.isoformat()
if isinstance(obj, Decimal):
return str(obj)
return obj
+95
View File
@@ -0,0 +1,95 @@
"""Registry of adapters, enrichers and source specs.
Built-in modules register themselves with @register when imported (see load_builtin()).
External packages can add adapters via the entry-point group `politrack.adapters`
(value: a module path whose import registers adapters), e.g. in their pyproject.toml:
[project.entry-points."politrack.adapters"]
my_sources = "my_package.sources"
"""
from __future__ import annotations
import importlib
import logging
import pkgutil
from importlib.metadata import entry_points
from politrack.core.adapter import Adapter, Enricher
from politrack.core.spec import DERIVED, SourceSpec
log = logging.getLogger(__name__)
ADAPTERS: dict[str, type[Adapter]] = {}
ENRICHERS: dict[str, type[Enricher]] = {}
SPECS: dict[str, SourceSpec] = {DERIVED.slug: DERIVED}
_loaded = False
def register(cls):
"""Class decorator for Adapter and Enricher subclasses."""
if issubclass(cls, Adapter):
target = ADAPTERS
elif issubclass(cls, Enricher):
target = ENRICHERS
else:
raise TypeError(f"{cls} is neither an Adapter nor an Enricher")
if cls.name in ADAPTERS or cls.name in ENRICHERS:
raise ValueError(f"duplicate adapter name {cls.name!r}")
target[cls.name] = cls
register_spec(cls.spec)
return cls
def register_spec(spec: SourceSpec) -> SourceSpec:
existing = SPECS.get(spec.slug)
if existing is not None and existing != spec:
raise ValueError(f"conflicting SourceSpec definitions for {spec.slug!r}")
SPECS[spec.slug] = spec
return spec
def load_builtin() -> None:
"""Import all built-in source and enricher modules (and plugins) exactly once."""
global _loaded
if _loaded:
return
_loaded = True
for package_name in ("politrack.sources", "politrack.enrich"):
package = importlib.import_module(package_name)
for module in pkgutil.walk_packages(package.__path__, prefix=f"{package_name}."):
importlib.import_module(module.name)
for ep in entry_points(group="politrack.adapters"):
try:
importlib.import_module(ep.value)
except Exception: # a broken plugin must not take the platform down
log.exception("failed to load adapter plugin %s", ep.value)
def get(name: str) -> type[Adapter] | type[Enricher]:
load_builtin()
if name in ADAPTERS:
return ADAPTERS[name]
if name in ENRICHERS:
return ENRICHERS[name]
raise KeyError(f"unknown adapter/enricher {name!r}; known: {', '.join(sorted(ADAPTERS) + sorted(ENRICHERS))}")
def sync_sources(conn) -> int:
"""Upsert all registered SourceSpecs into the `source` table."""
load_builtin()
for spec in SPECS.values():
conn.execute(
"""
INSERT INTO source (slug, name, kind, homepage, license, access_method, store_fulltext, keep_raw,
default_trust, tos_notes)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (slug) DO UPDATE SET name = EXCLUDED.name, kind = EXCLUDED.kind, homepage = EXCLUDED.homepage,
license = EXCLUDED.license, access_method = EXCLUDED.access_method,
store_fulltext = EXCLUDED.store_fulltext, keep_raw = EXCLUDED.keep_raw,
default_trust = EXCLUDED.default_trust, tos_notes = EXCLUDED.tos_notes, updated_at = now()
""",
(spec.slug, spec.name, spec.kind, spec.homepage, spec.license, spec.access, spec.store_fulltext,
spec.keep_raw, spec.trust, spec.tos_notes),
)
return len(SPECS)
+269
View File
@@ -0,0 +1,269 @@
"""Entity resolution: PersonRef/OrgRef -> database ids.
Order (first hit wins, ambiguity never guesses):
persons: identifiers -> hint strategies -> exact alias (titles stripped; prefer active)
organizations: identifiers -> slug -> exact alias -> normalised legal name -> create (if allowed)
Hint strategies for persons (documented in docs/data-model.md):
surname_in_body + on members of a chamber on a date with that surname (roll-call lists)
office + on holder of an office title on a date (Council of Ministers minutes)
"""
from __future__ import annotations
import re
from datetime import date
from typing import Any
from psycopg import Connection
from rapidfuzz import fuzz
from politrack.core.records import OrgRef, PersonRef
from politrack.text import make_slug, normalize, normalize_org_name, strip_titles
OFFICE_FILLER = re.compile(r"\b(der republik osterreich|der bundesrepublik osterreich|im bundeskanzleramt)\b|"
r"\(osterreich\)")
def office_key(title: str) -> str:
"""'Bundesministerin für Justiz' / 'Bundeskanzler der Republik Österreich' -> 'bundesminister fur justiz' /
'bundeskanzler' (gender-neutral, without filler, punctuation-free) for comparing office titles."""
text = normalize(title)
text = re.sub(r"\b(\w*(?:minister|kanzler|sekretar|prasident|rat))in\b", r"\1", text)
text = OFFICE_FILLER.sub(" ", text)
return re.sub(r"\s+", " ", re.sub(r"[^\w\s]", " ", text)).strip()
# Fallback for colloquial or English office labels (Wikidata: "Justizminister Österreichs", "Federal Minister for
# Arts, Culture, the Civil Service and Sport"): same kind of office + at least one common portfolio.
OFFICE_KINDS = [("vizekanzler", r"vizekanzler|vice.chancellor"), ("kanzler", r"kanzler|chancellor"),
("staatssekretar", r"staatssekretar|state secretary"), ("minister", r"minist")]
PORTFOLIOS = {
"justiz": r"justiz|justice", "finanzen": r"finanz|financ", "inneres": r"\binnere|\binnenmin|interior",
"aussen": r"au(ss|ß)en|auswartig|europaische und internationale|foreign|european and international",
"verteidigung": r"verteidigung|defen[cs]e", "soziales": r"sozial|\barbeit|labou?r|social",
"gesundheit": r"gesundheit|health", "bildung": r"bildung|education", "wissenschaft": r"wissenschaft|science",
"frauen": r"frauen|women", "landwirtschaft": r"landwirtschaft|forstwirtschaft|agricultur",
"umwelt": r"umwelt|klima|environment|climate", "kunst": r"kunst|kultur|\barts\b|culture|\bsport",
"mobilitat": r"mobilitat|verkehr|infrastruktur|mobility|transport|infrastructure",
"wirtschaft": r"\bwirtschaft|econom", "europa": r"\beuropa\b|integration|familie|famil",
"digitalisierung": r"digital", "energie": r"energie|energy", "tourismus": r"touris",
}
def office_kind(key: str) -> str | None:
return next((kind for kind, pattern in OFFICE_KINDS if re.search(pattern, key)), None)
def portfolios(key: str) -> set[str]:
return {name for name, pattern in PORTFOLIOS.items() if re.search(pattern, key)}
def lead_portfolio(key: str) -> str | None:
"""The portfolio named first ("Wirtschaft, Energie und Tourismus" -> wirtschaft): a role must cover it."""
found = [(m.start(), name) for name, pattern in PORTFOLIOS.items() if (m := re.search(pattern, key))]
return min(found)[1] if found else None
class Resolver:
def __init__(self, conn: Connection) -> None:
self.conn = conn
self._person_cache: dict[tuple, int | None] = {}
self._org_cache: dict[tuple, int | None] = {}
self._members_cache: dict[tuple, list[dict]] = {}
self._office_cache: dict[tuple, int | None] = {}
# -- persons ------------------------------------------------------------------------
def person_by_ids(self, ids: dict[str, str]) -> int | None:
for scheme, value in ids.items():
if value is None:
continue
if scheme == "politrack": # internal id, used by adapters that plan requests from the database
row = self.conn.execute("SELECT id FROM person WHERE id = %s", (int(value),)).fetchone()
if row:
return row["id"]
continue
row = self.conn.execute(
"SELECT entity_id FROM identifier WHERE entity_type = 'person' AND scheme = %s AND value = %s",
(scheme, str(value)),
).fetchone()
if row:
return row["entity_id"]
return None
def person(self, ref: PersonRef | None) -> int | None:
if ref is None:
return None
cache_key = (tuple(sorted(ref.ids.items())), ref.name, tuple(sorted((k, str(v)) for k, v in ref.hints.items())))
if cache_key in self._person_cache:
return self._person_cache[cache_key]
found = self.person_by_ids(ref.ids)
if found is None and ref.hints:
found = self._person_by_hints(ref)
if found is None and ref.name and not ref.hints.get("no_name_match"):
found = self.person_by_name(ref.name)
self._person_cache[cache_key] = found
return found
def person_by_name(self, name: str) -> int | None:
norm = normalize(strip_titles(name))
rows = self.conn.execute(
"""SELECT DISTINCT a.entity_id AS id, ps.status
FROM alias a LEFT JOIN person_status ps ON ps.person_id = a.entity_id
WHERE a.entity_type = 'person' AND a.alias_norm = %s""",
(norm,),
).fetchall()
if len(rows) == 1:
return rows[0]["id"]
active = [r for r in rows if r["status"] == "active"]
return active[0]["id"] if len(active) == 1 else None
def _person_by_hints(self, ref: PersonRef) -> int | None:
hints = ref.hints
on: date | None = hints.get("on")
if "surname_in_body" in hints and ref.name:
members = self.members(hints["surname_in_body"], on)
norm = normalize(strip_titles(ref.name))
exact = [m for m in members if m["family_name"] and normalize(m["family_name"]) == norm]
if len(exact) == 1:
return exact[0]["id"]
suffix = [m for m in members if normalize(strip_titles(m["full_name"])).endswith(norm)]
return suffix[0]["id"] if len(suffix) == 1 else None
if "office" in hints:
return self.office_holder(hints["office"], on)
return None
def office_holder(self, office: str, on: date | None) -> int | None:
"""Holder of a (single-holder) office on a date. Titles are compared whole, gender-neutral and without
filler ("Bundesministerin für Justiz" == "Bundesminister für Justiz"), so a generic "Bundesminister"
never matches a specific ministry. Several matches (open-ended roles from Wikidata) -> the most recent
appointment wins; no dated match -> None."""
key = (office, on)
if key not in self._office_cache:
rows = self.conn.execute(
"""SELECT DISTINCT r.person_id, r.role, r.start_date FROM role r JOIN claim c ON c.id = r.claim_id
WHERE c.merged_into IS NULL AND r.level IN ('federal', 'state', 'eu')
AND (%(on)s::date IS NULL OR ((r.start_date IS NULL OR r.start_date <= %(on)s)
AND (r.end_date IS NULL OR r.end_date >= %(on)s)))""",
{"on": on},
).fetchall()
target = office_key(office)
candidates = [(r["start_date"] or date.min, r["person_id"]) for r in rows
if fuzz.ratio(office_key(r["role"]), target) >= 92]
kind, lead = office_kind(target), lead_portfolio(target)
if not candidates and kind and (lead or kind in ("kanzler", "vizekanzler")):
# Fuzzy labels only for appointments since the current government took office (the latest
# chancellor appointment): stale open-ended roles of former ministers must not win.
formed = max((r["start_date"] for r in rows if r["start_date"]
and office_kind(office_key(r["role"])) == "kanzler"), default=date.min)
candidates = [(r["start_date"], r["person_id"]) for r in rows
if r["start_date"] and r["start_date"] >= formed
and office_kind(office_key(r["role"])) == kind
and (kind in ("kanzler", "vizekanzler") or lead in portfolios(office_key(r["role"])))]
persons = {p for _, p in candidates}
if len(persons) <= 1:
self._office_cache[key] = persons.pop() if persons else None
else:
latest = max(start for start, _ in candidates)
newest = {p for start, p in candidates if start == latest}
self._office_cache[key] = newest.pop() if len(newest) == 1 and latest != date.min else None
return self._office_cache[key]
def members(self, body_slug: str, on: date | None) -> list[dict[str, Any]]:
key = (body_slug, on)
if key not in self._members_cache:
self._members_cache[key] = self.conn.execute(
"""SELECT DISTINCT ON (p.id) p.id, p.full_name, p.family_name, r.party_id
FROM role r JOIN person p ON p.id = r.person_id JOIN organization o ON o.id = r.organization_id
JOIN claim c ON c.id = r.claim_id
WHERE o.slug = %(body)s AND r.role = 'mp' AND c.merged_into IS NULL
AND (%(on)s::date IS NULL OR (r.start_date <= %(on)s AND COALESCE(r.end_date, 'infinity') >= %(on)s))
ORDER BY p.id, r.start_date DESC""",
{"body": body_slug, "on": on},
).fetchall()
return self._members_cache[key]
def faction_of(self, person_id: int, body_slug: str, on: date | None) -> int | None:
return next((m["party_id"] for m in self.members(body_slug, on) if m["id"] == person_id), None)
# -- organizations ------------------------------------------------------------------
def org(self, ref: OrgRef | None, create: bool | None = None) -> int | None:
if ref is None:
return None
allow_create = ref.create if create is None else create
cache_key = (tuple(sorted(ref.ids.items())), ref.slug, ref.name, ref.short_name)
if cache_key in self._org_cache and (self._org_cache[cache_key] is not None or not allow_create):
return self._org_cache[cache_key]
found = self._org_lookup(ref)
if found is None and allow_create and (ref.name or ref.short_name):
found = self._org_create(ref)
elif found is not None:
self._add_org_ids(found, ref)
self._org_cache[cache_key] = found
return found
def _org_lookup(self, ref: OrgRef) -> int | None:
for scheme, value in ref.ids.items():
if scheme == "politrack_org": # internal id (enrichers)
row = self.conn.execute("SELECT id FROM organization WHERE id = %s", (int(value),)).fetchone()
if row:
return row["id"]
continue
row = self.conn.execute(
"SELECT entity_id FROM identifier WHERE entity_type = 'organization' AND scheme = %s AND value = %s",
(scheme, str(value)),
).fetchone()
if row:
return row["entity_id"]
if ref.slug:
row = self.conn.execute("SELECT id FROM organization WHERE slug = %s", (ref.slug,)).fetchone()
if row:
return row["id"]
for label in (ref.short_name, ref.name):
if not label:
continue
for norm in {normalize(label), normalize_org_name(label)} - {""}:
rows = self.conn.execute(
"SELECT DISTINCT entity_id FROM alias WHERE entity_type = 'organization' AND alias_norm = %s",
(norm,),
).fetchall()
if len(rows) == 1:
return rows[0]["entity_id"]
return None
def _org_create(self, ref: OrgRef) -> int:
base = ref.slug or make_slug(ref.short_name or ref.name or "org")[:60] or "org"
slug, n = base, 1
while self.conn.execute("SELECT 1 FROM organization WHERE slug = %s", (slug,)).fetchone():
n += 1
slug = f"{base}-{n}"
org_id = self.conn.execute(
"INSERT INTO organization (slug, name, short_name, kind) VALUES (%s, %s, %s, %s) RETURNING id",
(slug, ref.name or ref.short_name, ref.short_name, ref.kind if ref.kind else "other"),
).fetchone()["id"]
self.add_org_aliases(org_id, [ref.name, ref.short_name])
self._add_org_ids(org_id, ref)
return org_id
def add_org_aliases(self, org_id: int, names: list[str | None]) -> None:
for name in names:
if not name:
continue
for norm in {normalize(name), normalize_org_name(name)} - {""}:
self.conn.execute(
"INSERT INTO alias (entity_type, entity_id, alias, alias_norm) VALUES ('organization', %s, %s, %s) "
"ON CONFLICT DO NOTHING",
(org_id, name, norm),
)
def _add_org_ids(self, org_id: int, ref: OrgRef) -> None:
for scheme, value in ref.ids.items():
if value:
self.conn.execute(
"INSERT INTO identifier (entity_type, entity_id, scheme, value) VALUES ('organization', %s, %s, %s) "
"ON CONFLICT DO NOTHING",
(org_id, scheme, str(value)),
)
+232
View File
@@ -0,0 +1,232 @@
"""Runs adapters and enrichers.
fetch mode: start() -> fetch (policy, archive) -> parse() -> Loader, one transaction per response
reparse mode: replay the latest archived response of every request the adapter ever made
test mode: parse a given response without DB (used by golden tests)
"""
from __future__ import annotations
import logging
import traceback
from collections import deque
from collections.abc import Iterable
from datetime import datetime, timezone
from typing import Any
from psycopg import Connection
from psycopg.types.json import Jsonb
from politrack.core import registry
from politrack.core.adapter import Adapter, Enricher, Request, Response, RunContext, StateStore
from politrack.core.archive import Archive
from politrack.core.fetch import Disallowed, Fetcher
from politrack.core.loader import Loader, source_row
from politrack.db import connection
log = logging.getLogger(__name__)
ENRICH_COMMIT_EVERY = 200
def run(name: str, *, mode: str = "fetch", since: datetime | None = None, params: dict[str, Any] | None = None,
max_requests: int | None = None) -> dict:
cls = registry.get(name)
with connection() as conn:
run_id = conn.execute("INSERT INTO run (adapter, mode) VALUES (%s, %s) RETURNING id", (name, mode)).fetchone()["id"]
conn.commit()
ctx = RunContext(adapter=name, spec=cls.spec, state=StateStore(conn, name), since=since, params=params or {},
mode=mode, conn=conn, creates_persons=cls.creates_persons,
log=logging.getLogger(f"politrack.{name}"))
status, error = "ok", None
try:
if issubclass(cls, Enricher):
_run_enricher(cls(), ctx, conn)
elif mode == "reparse":
_reparse(cls(), ctx, conn)
else:
_fetch_run(cls(), ctx, conn, max_requests)
if ctx.stats.get("errors"):
status = "partial"
except Exception:
conn.rollback()
status, error = "error", traceback.format_exc()
ctx.log.exception("%s failed", name)
conn.execute("UPDATE run SET finished_at = now(), status = %s, stats = %s, error = %s WHERE id = %s",
(status, Jsonb(dict(ctx.stats)), error, run_id))
conn.commit()
ctx.log.info("%s %s: %s %s", name, mode, status, dict(ctx.stats))
if status == "error":
raise RuntimeError(f"{name} failed, see run {run_id}")
return {"status": status, **dict(ctx.stats)}
def _fetch_run(adapter: Adapter, ctx: RunContext, conn: Connection, max_requests: int | None) -> None:
fetcher = Fetcher()
loader = Loader(conn, ctx)
queue: deque[Request] = deque(adapter.start(ctx))
seen: set[str] = set()
try:
while queue:
req = queue.popleft()
if req.fingerprint in seen:
continue
seen.add(req.fingerprint)
if max_requests is not None and ctx.stats["requests"] >= max_requests:
ctx.stats["stopped_at_max_requests"] = 1
break
ctx.stats["requests"] += 1
# Multi-publisher adapters (news feeds, GDELT) name the publisher per request.
spec = registry.SPECS.get(req.meta.get("source") or adapter.spec.slug, adapter.spec)
try:
resp = fetcher.fetch(conn, req, adapter=adapter.name, spec=spec,
source_id=source_row(conn, spec.slug)["id"])
except Disallowed:
ctx.stats["disallowed_by_robots"] += 1
ctx.log.warning("robots.txt disallows %s", req.url)
continue
except Exception as exc:
conn.rollback()
ctx.stats["fetch_errors"] += 1
ctx.stats["errors"] += 1
ctx.log.warning("fetch failed %s: %s", req.url, exc)
continue
if req.skip_unchanged and _unchanged(conn, resp, adapter.parser_version):
# Same body, same parser: its records are already loaded. Still collect follow-up requests,
# so children an interrupted run never fetched are picked up (they have their own checks).
ctx.stats["unchanged"] += 1
queue.extend(_follow_ups(adapter, ctx, resp))
continue
queue.extend(_parse_and_load(adapter, ctx, conn, loader, resp))
finally:
fetcher.close()
def _unchanged(conn: Connection, resp: Response, parser_version: int) -> bool:
if resp.fetch_id is None:
return False
row = conn.execute(
"""SELECT 1 FROM fetch_log WHERE request_fp = %s AND parsed_sha256 = %s AND parser_version = %s LIMIT 1""",
(resp.request.fingerprint, resp.sha256, parser_version),
).fetchone()
return row is not None
def _follow_ups(adapter: Adapter, ctx: RunContext, resp: Response) -> list[Request]:
ctx.response = resp
try:
return [item for item in adapter.parse(ctx, resp) if isinstance(item, Request)]
except Exception:
ctx.stats["parse_errors"] += 1
ctx.log.exception("parse failed for unchanged %s", resp.url)
return []
finally:
ctx.response = None
def _parse_and_load(adapter: Adapter, ctx: RunContext, conn: Connection, loader: Loader,
resp: Response) -> list[Request]:
follow: list[Request] = []
ctx.response = resp
try:
for item in adapter.parse(ctx, resp):
if isinstance(item, Request):
follow.append(item)
else:
loader.load(item)
if resp.fetch_id is not None:
Archive().mark_parsed(conn, resp.fetch_id, resp.sha256, adapter.parser_version)
conn.commit()
except Exception as exc:
conn.rollback()
ctx.stats["errors"] += 1
ctx.stats["parse_errors"] += 1
ctx.log.exception("parse/load failed for %s", resp.url)
if resp.fetch_id is not None:
Archive().mark_error(conn, resp.fetch_id, f"{type(exc).__name__}: {exc}")
conn.commit()
loader.resolver = type(loader.resolver)(conn) # drop caches that may reference rolled-back rows
finally:
ctx.response = None
return follow
def _reparse(adapter: Adapter, ctx: RunContext, conn: Connection) -> None:
archive = Archive()
loader = Loader(conn, ctx)
started = conn.execute("SELECT now() AS t").fetchone()["t"]
rows = archive.latest_stored(conn, adapter.name)
ctx.log.info("reparsing %s archived responses", len(rows))
# Parse list/index responses before detail responses: records referencing others (vote positions ->
# vote events) are emitted by the parent's parse() in order, so the original order is preserved
# by sorting on fetch time.
for row in sorted(rows, key=lambda r: r["fetched_at"]):
body = archive.read_blob(row["sha256"])
if body is None:
ctx.stats["missing_blobs"] += 1
continue
req_info = row["request"] or {}
req = Request(url=row["url"], kind=req_info.get("kind", "default"), method=row["method"],
params=req_info.get("params"), json=req_info.get("json"), meta=req_info.get("meta") or {})
resp = Response(request=req, status=row["status"], url=row["url"],
headers={"content-type": row["content_type"] or ""}, body=body,
fetched_at=row["fetched_at"], sha256=row["sha256"], fetch_id=row["id"],
tdm_reserved=row["tdm_reserved"], from_archive=True)
ctx.stats["responses"] += 1
_parse_and_load(adapter, ctx, conn, loader, resp) # follow-ups are archived separately
if not ctx.stats.get("errors") and not ctx.stats.get("missing_blobs"):
_sweep_stale_claims(adapter, ctx, conn, started)
def _sweep_stale_claims(adapter: Adapter, ctx: RunContext, conn: Connection, started: datetime) -> None:
"""After a complete reparse, delete this adapter's claims that the current parser no longer produces
(e.g. a fixed parser emits a corrected fact with a new identity) - unless another source backs them.
Claims are reproducible from the archive, so nothing is lost; facets and evidence links cascade."""
deleted = conn.execute(
"""DELETE FROM claim c
WHERE c.created_by = %(adapter)s AND c.updated_at < %(started)s
AND NOT EXISTS (SELECT 1 FROM claim_evidence e JOIN document d ON d.id = e.document_id
JOIN source s ON s.id = d.source_id
WHERE e.claim_id = c.id AND s.slug <> %(source)s)""",
{"adapter": adapter.name, "started": started, "source": adapter.spec.slug},
).rowcount
conn.commit()
ctx.stats["stale_claims_deleted"] = deleted
def _run_enricher(enricher: Enricher, ctx: RunContext, conn: Connection) -> None:
loader = Loader(conn, ctx)
result = enricher.run(ctx)
pending = 0
for record in result or []:
try:
loader.load(record)
except Exception:
conn.rollback()
ctx.stats["errors"] += 1
ctx.log.exception("enricher %s failed to load a record", enricher.name)
loader = Loader(conn, ctx)
continue
pending += 1
if pending >= ENRICH_COMMIT_EVERY:
conn.commit()
pending = 0
conn.commit()
def parse_offline(adapter: Adapter, response: Response, params: dict[str, Any] | None = None) -> Iterable[Any]:
"""Run parse() without a database (golden tests, debugging)."""
ctx = RunContext(adapter=adapter.name, spec=adapter.spec, state=StateStore(None, adapter.name), mode="test",
params=params or {})
ctx.response = response
return list(adapter.parse(ctx, response))
def make_response(url: str, body: bytes, *, kind: str = "default", meta: dict | None = None,
content_type: str = "", method: str = "GET", json_body: Any = None,
params: dict | None = None) -> Response:
"""Build a Response for tests/fixtures."""
import hashlib
req = Request(url=url, kind=kind, meta=meta or {}, method=method, json=json_body, params=params)
return Response(request=req, status=200, url=url, headers={"content-type": content_type}, body=body,
fetched_at=datetime(2026, 1, 1, tzinfo=timezone.utc), sha256=hashlib.sha256(body).hexdigest())
+49
View File
@@ -0,0 +1,49 @@
"""SourceSpec: the single source of truth about a publisher we collect from.
Specs live next to the adapters that use them (backend/politrack/sources/*.py) and are synced
into the `source` table by `politrack migrate` / `politrack sources sync`.
"""
from dataclasses import dataclass
from typing import Literal
SourceKind = Literal["official", "ngo", "media", "social", "aggregator", "user", "derived"]
AccessMethod = Literal["api", "bulk", "rss", "scrape", "manual", "derived"]
@dataclass(frozen=True)
class SourceSpec:
slug: str
name: str
kind: SourceKind
homepage: str
license: str
access: AccessMethod
trust: int
#: may we keep body text (articles, posts, protocols)? False -> title, short excerpt and link only
store_fulltext: bool = True
#: may raw responses be archived? False for TDM-reserved publishers or restrictive terms
keep_raw: bool = True
#: minimum seconds between two requests to the same host (robots.txt crawl-delay can raise it)
min_interval: float = 2.0
#: URL prefixes of documented APIs whose operators allow programmatic use although robots.txt
#: lists them (justify in tos_notes). Everything else always obeys robots.txt.
robots_exempt: tuple[str, ...] = ()
tos_notes: str = ""
def __post_init__(self) -> None:
if not 0 <= self.trust <= 100:
raise ValueError(f"{self.slug}: trust must be 0-100")
# Sources used by the built-in enrichers for derived documents/claims.
DERIVED = SourceSpec(
slug="politrack",
name="Politrack (abgeleitet)",
kind="derived",
homepage="",
license="derived from cited sources",
access="derived",
trust=50,
tos_notes="Claims computed by Politrack from other sources; each cites its original evidence.",
)
+31
View File
@@ -0,0 +1,31 @@
import atexit
from collections.abc import Iterator
from contextlib import contextmanager
from functools import lru_cache
from psycopg import Connection
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool
from politrack.config import get_settings
@lru_cache
def get_pool() -> ConnectionPool:
pool = ConnectionPool(
get_settings().database_url,
min_size=1,
max_size=10,
kwargs={"row_factory": dict_row},
open=False,
)
pool.open()
atexit.register(pool.close)
return pool
@contextmanager
def connection() -> Iterator[Connection]:
"""A pooled connection inside a transaction (committed on success)."""
with get_pool().connection() as conn:
yield conn
Whitespace-only changes.
+244
View File
@@ -0,0 +1,244 @@
"""Corroboration: find claims from different sources that state the same fact and merge them.
Merging keeps the oldest claim, copies the duplicate's evidence onto it and marks
the duplicate with `merged_into`. Afterwards `refresh_claim_status` marks the
survivor "corroborated" when at least two *independent sources* support it.
Matchers:
* roles: same person, equivalent body/office, overlapping or near-identical periods
(e.g. Wikidata "Abgeordneter zum Nationalrat" == Parliament mandate)
* stories: news articles about the same event (similar headlines, shared people,
within 72 hours) are grouped; mentions of a person in one story are merged
* events: trips/meetings of the same person to the same place/counterpart within 3 days
"""
from collections import defaultdict
from datetime import timedelta
from psycopg import Connection
from rapidfuzz import fuzz
from politrack.core import Enricher, RunContext, register
from politrack.core.spec import DERIVED
from politrack.enrich.events import counterpart_anchor
from politrack.text import normalize
REFRESH_STATUS = """
UPDATE claim SET status = CASE
WHEN status = 'retracted' THEN status
WHEN EXISTS (SELECT 1 FROM claim_evidence e WHERE e.claim_id = claim.id AND e.stance = 'contradicts') THEN 'disputed'
WHEN (SELECT count(DISTINCT d.source_id) FROM claim_evidence e JOIN document d ON d.id = e.document_id
WHERE e.claim_id = claim.id AND e.stance IN ('supports', 'mentions')) >= 2 THEN 'corroborated'
ELSE 'reported' END
WHERE id = %s
"""
def merge_claims(conn: Connection, keep_id: int, dup_id: int) -> None:
if keep_id == dup_id:
return
conn.execute(
"""
INSERT INTO claim_evidence (claim_id, document_id, stance, quote, locator, extracted_by, confidence)
SELECT %s, document_id, stance, quote, locator, extracted_by, confidence
FROM claim_evidence WHERE claim_id = %s
ON CONFLICT (claim_id, document_id, locator) DO NOTHING
""",
(keep_id, dup_id),
)
conn.execute(
"""
INSERT INTO topic_assignment (topic_id, claim_id, confidence, method)
SELECT topic_id, %s, confidence, method FROM topic_assignment WHERE claim_id = %s
ON CONFLICT DO NOTHING
""",
(keep_id, dup_id),
)
conn.execute("UPDATE claim SET merged_into = %s, updated_at = now() WHERE id = %s", (keep_id, dup_id))
conn.execute("UPDATE claim SET merged_into = %s WHERE merged_into = %s", (keep_id, dup_id))
conn.execute(REFRESH_STATUS, (keep_id,))
# -- roles ---------------------------------------------------------------------
BODY_KEYWORDS = {
"nationalrat": ("nationalrat",),
"bundesrat": ("bundesrat", "bundesrates"),
}
def _same_office(a: dict, b: dict) -> bool:
"""a: authoritative role (Parliament), b: other source (Wikidata)."""
if a["role"] == "mp" and b["role"] == "mp":
return a["body_slug"] == b["body_slug"]
label = normalize(b["role"])
if a["role"] == "mp" and a["body_slug"] in BODY_KEYWORDS:
return any(k in label for k in BODY_KEYWORDS[a["body_slug"]]) and "prasident" not in label
return fuzz.token_set_ratio(normalize(a["role"]), label) >= 80
def _periods_match(a: dict, b: dict) -> bool:
a_start, b_start = a["start_date"], b["start_date"]
a_end, b_end = a["end_date"], b["end_date"]
if a_start and b_start and abs((a_start - b_start).days) <= 60:
return True
# Wikidata often only records one term for several consecutive mandates: accept containment.
if a_start and b_start and b_start <= a_start and (b_end is None or a_end is None or a_end <= b_end + timedelta(days=60)):
return True
return False
def merge_roles(conn: Connection) -> int:
rows = conn.execute(
"""
SELECT r.claim_id, r.person_id, r.role, r.start_date, r.end_date, c.created_by, o.slug AS body_slug
FROM role r JOIN claim c ON c.id = r.claim_id LEFT JOIN organization o ON o.id = r.organization_id
WHERE c.merged_into IS NULL AND r.level <> 'party'
AND (c.created_by LIKE 'parlament%%' OR c.created_by LIKE 'wikidata%%' OR c.created_by LIKE 'europarl%%')
ORDER BY r.person_id, r.start_date NULLS LAST
"""
).fetchall()
by_person: dict[int, dict[str, list[dict]]] = defaultdict(lambda: defaultdict(list))
for r in rows:
family = "wikidata" if r["created_by"].startswith("wikidata") else "parlament"
by_person[r["person_id"]][family].append(r)
merged = 0
for roles in by_person.values():
for other in roles.get("wikidata", []):
match = next((a for a in roles.get("parlament", []) if _same_office(a, other) and _periods_match(a, other)),
None)
if match:
merge_claims(conn, match["claim_id"], other["claim_id"])
merged += 1
conn.commit()
return merged
# -- news stories -------------------------------------------------------------
def cluster_stories(conn: Connection, days: int = 14) -> int:
"""Group recent news documents about the same thing into stories."""
docs = conn.execute(
"""
SELECT d.id, d.title, d.published_at, d.story_id, d.source_id, d.meta->'people' AS people
FROM document d JOIN source s ON s.id = d.source_id
WHERE (s.kind = 'media' OR s.slug = 'gdelt') AND d.published_at > now() - make_interval(days => %s)
AND d.title IS NOT NULL
ORDER BY d.published_at
""",
(days,),
).fetchall()
stories: list[dict] = [] # {id, docs: [...]}
for doc in docs:
title = normalize(doc["title"])
people = set(doc["people"] or [])
best, best_score = None, 0
for story in stories[-400:]:
lead = story["docs"][0]
if doc["published_at"] - lead["published_at"] > timedelta(hours=72):
continue
score = fuzz.token_set_ratio(title, lead["norm"])
shared = people & lead["people"]
if score >= 82 or (shared and score >= 60):
if score > best_score:
best, best_score = story, score
entry = {"id": doc["id"], "norm": title, "people": people, "published_at": doc["published_at"],
"source_id": doc["source_id"], "story_id": doc["story_id"], "title": doc["title"]}
if best:
best["docs"].append(entry)
else:
stories.append({"docs": [entry]})
assigned = 0
for story in stories:
if len(story["docs"]) < 2:
continue
existing = next((d["story_id"] for d in story["docs"] if d["story_id"]), None)
lead = story["docs"][0]
if existing is None:
existing = conn.execute(
"INSERT INTO story (title, first_seen) VALUES (%s, %s) RETURNING id", (lead["title"], lead["published_at"])
).fetchone()["id"]
ids = [d["id"] for d in story["docs"]]
conn.execute("UPDATE document SET story_id = %s WHERE id = ANY(%s)", (existing, ids))
conn.execute(
"""
UPDATE story SET last_seen = sub.last_seen, document_count = sub.n, source_count = sub.sources
FROM (SELECT max(published_at) AS last_seen, count(*) AS n, count(DISTINCT source_id) AS sources
FROM document WHERE story_id = %s) sub
WHERE story.id = %s
""",
(existing, existing),
)
assigned += len(ids)
conn.commit()
return assigned
def merge_story_mentions(conn: Connection) -> int:
"""One media_mention claim per (person or party, story); other outlets become additional evidence."""
rows = conn.execute(
"""
SELECT c.id, c.person_id, c.organization_id, d.story_id, c.occurred_at
FROM claim c
JOIN claim_evidence e ON e.claim_id = c.id
JOIN document d ON d.id = e.document_id
WHERE c.claim_type = 'media_mention' AND c.merged_into IS NULL AND d.story_id IS NOT NULL
ORDER BY c.occurred_at NULLS LAST, c.id
"""
).fetchall()
keep: dict[tuple, int] = {}
merged = 0
for r in rows:
key = (r["person_id"], r["organization_id"], r["story_id"])
if key in keep:
if keep[key] != r["id"]:
merge_claims(conn, keep[key], r["id"])
merged += 1
else:
keep[key] = r["id"]
conn.commit()
return merged
# -- events --------------------------------------------------------------------
def merge_events(conn: Connection) -> int:
rows = conn.execute(
"""
SELECT c.id, c.person_id, c.claim_type, c.occurred_at, c.data->>'country' AS country,
c.data->>'counterpart' AS counterpart
FROM claim c
WHERE c.claim_type IN ('travel', 'meeting') AND c.merged_into IS NULL
ORDER BY c.person_id, c.claim_type, c.occurred_at
"""
).fetchall()
merged = 0
previous: dict[tuple, dict] = {}
for r in rows:
anchor = r["country"] if r["claim_type"] == "travel" else counterpart_anchor(r["counterpart"] or "")
key = (r["person_id"], r["claim_type"], anchor)
prev = previous.get(key)
if prev and r["occurred_at"] and prev["occurred_at"] and r["occurred_at"] - prev["occurred_at"] <= timedelta(days=3):
merge_claims(conn, prev["id"], r["id"])
merged += 1
continue
previous[key] = r
conn.commit()
return merged
@register
class Corroborator(Enricher):
spec = DERIVED
name = "enrich.corroborate"
description = "Merge near-duplicate claims from different sources; cluster news stories"
schedule = "9-59/10 * * * *"
def run(self, ctx: RunContext) -> None:
conn = ctx.conn
ctx.stats["roles_merged"] = merge_roles(conn)
ctx.stats["story_documents"] = cluster_stories(conn)
ctx.stats["story_mentions_merged"] = merge_story_mentions(conn)
ctx.stats["events_merged"] = merge_events(conn)
return None
+74
View File
@@ -0,0 +1,74 @@
"""Curated lists maintained by the operator in config/ (accounts that no official register lists).
* config.social_accounts - config/social_accounts.yaml -> social_account rows for persons/organisations,
e.g. party Telegram channels or Bluesky accounts linked from party websites. The social adapters then
collect their public posts.
"""
from __future__ import annotations
from collections.abc import Iterator
from typing import Any
import yaml
from politrack.config import get_settings
from politrack.core import Enricher, RunContext, SourceSpec, register
from politrack.core.records import OrgRef, PersonRef, SocialAccount
CURATED = SourceSpec(
slug="curated",
name="Kuratierte Listen (config/)",
kind="derived",
homepage="",
license="operator-maintained",
access="manual",
trust=60,
tos_notes="Entries maintained by the operator; each should name the public page that links the account.",
)
def person_ref(ctx: RunContext, entry: dict[str, Any]) -> PersonRef | None:
"""`person: <politrack slug>`, `person: {wikidata: Q..}` / `{parlament: ..}` or `name: <full name>`."""
person = entry.get("person")
if isinstance(person, dict):
return PersonRef(ids={k: str(v) for k, v in person.items()})
if isinstance(person, str):
row = ctx.conn.execute("SELECT id FROM person WHERE slug = %s", (person,)).fetchone()
return PersonRef(ids={"politrack": str(row["id"])}) if row else None
if entry.get("name"):
return PersonRef(name=entry["name"])
return None
def org_ref(entry: dict[str, Any]) -> OrgRef | None:
"""`org: <slug>` for existing organisations, or `org: {slug, name, kind}` to create one."""
org = entry.get("org")
if isinstance(org, str):
return OrgRef(slug=org, create=False)
if isinstance(org, dict):
return OrgRef(slug=org.get("slug"), name=org.get("name"), short_name=org.get("short_name"),
kind=org.get("kind", "other"), create=bool(org.get("name")))
return None
@register
class CuratedSocialAccounts(Enricher):
spec = CURATED
name = "config.social_accounts"
description = "Import social accounts from config/social_accounts.yaml"
schedule = "5 3 * * *"
def run(self, ctx: RunContext) -> Iterator[SocialAccount]:
path = get_settings().config_dir / "social_accounts.yaml"
if not path.exists():
return
for entry in (yaml.safe_load(path.read_text(encoding="utf-8")) or {}).get("accounts") or []:
person = person_ref(ctx, entry)
org = None if person else org_ref(entry)
if person is None and org is None:
ctx.stats["entries_without_owner"] += 1
ctx.log.warning("social_accounts.yaml: no person/org for %s", entry.get("url"))
continue
yield SocialAccount(platform=entry["platform"], url=entry["url"], handle=entry.get("handle"),
person=person, org=org, verified=bool(entry.get("verified")))
+118
View File
@@ -0,0 +1,118 @@
"""Chunking + embeddings for retrieval (RAG).
Every claim gets one chunk (its summary plus evidence titles) so agents can
retrieve facts *with citations*; documents with licensed full text are split
into ~1,200 character chunks. Embeddings are computed by an Ollama server
(model bge-m3, multilingual, 1024 dims) if POLITRACK_OLLAMA_URL is set;
otherwise only German full-text search is available.
"""
import httpx
from psycopg import Connection
from politrack.config import get_settings
from politrack.core import Enricher, RunContext, register
from politrack.core.spec import DERIVED
CHUNK_SIZE = 1200
UPSERT_CHUNK = """
INSERT INTO chunk (claim_id, document_id, chunk_index, text, lang) VALUES (%s, %s, %s, %s, 'de')
ON CONFLICT (COALESCE(document_id, 0), COALESCE(claim_id, 0), chunk_index)
DO UPDATE SET text = EXCLUDED.text, embedding = CASE WHEN chunk.text = EXCLUDED.text THEN chunk.embedding END
"""
@register
class Chunker(Enricher):
spec = DERIVED
name = "enrich.embed"
description = "Build search chunks for claims/documents and embed them (Ollama, optional)"
schedule = "*/15 * * * *"
def run(self, ctx: RunContext) -> None:
while (n := build_chunks(ctx.conn)):
ctx.stats["chunks"] += n
ctx.stats["embedded"] = embed_pending(ctx.conn)
return None
def split_text(text: str, size: int = CHUNK_SIZE) -> list[str]:
paragraphs, chunks, current = [p.strip() for p in text.split("\n") if p.strip()], [], ""
for p in paragraphs:
if current and len(current) + len(p) > size:
chunks.append(current)
current = ""
current = f"{current}\n{p}".strip()
while len(current) > size * 1.5:
chunks.append(current[:size])
current = current[size:]
if current:
chunks.append(current)
return chunks
def build_chunks(conn: Connection, batch: int = 5000) -> int:
claims = conn.execute(
"""
SELECT c.id, c.claim_type, c.summary_de, c.occurred_at, p.full_name, o.name AS org_name,
string_agg(DISTINCT s.name || ': ' || COALESCE(d.title, d.url), ' | ') AS sources
FROM claim c
LEFT JOIN person p ON p.id = c.person_id
LEFT JOIN organization o ON o.id = c.organization_id
JOIN claim_evidence e ON e.claim_id = c.id
JOIN document d ON d.id = e.document_id
JOIN source s ON s.id = d.source_id
WHERE c.merged_into IS NULL AND c.review_status = 'published'
AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.claim_id = c.id)
GROUP BY c.id, p.full_name, o.name
LIMIT %s
""",
(batch,),
).fetchall()
for c in claims:
who = c["full_name"] or c["org_name"] or ""
when = c["occurred_at"].date().isoformat() if c["occurred_at"] else ""
text = f"[{c['claim_type']}] {when} {who}\n{c['summary_de']}\nQuellen: {c['sources']}"
conn.execute(UPSERT_CHUNK, (c["id"], None, 0, text.strip()))
docs = conn.execute(
"""
SELECT d.id, d.title, d.full_text FROM document d
WHERE d.full_text IS NOT NULL AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.document_id = d.id)
LIMIT %s
""",
(batch // 10,),
).fetchall()
for d in docs:
for i, part in enumerate(split_text(d["full_text"])):
conn.execute(UPSERT_CHUNK, (None, d["id"], i, f"{d['title'] or ''}\n{part}".strip()))
conn.commit()
return len(claims) + len(docs)
def embed_texts(texts: list[str]) -> list[list[float]]:
settings = get_settings()
resp = httpx.post(f"{settings.ollama_url}/api/embed", json={"model": settings.embedding_model, "input": texts},
timeout=300)
resp.raise_for_status()
return resp.json()["embeddings"]
def embed_pending(conn: Connection, batch: int = 64, max_batches: int = 50) -> int:
settings = get_settings()
if not settings.ollama_url:
return 0
total = 0
for _ in range(max_batches):
rows = conn.execute("SELECT id, text FROM chunk WHERE embedding IS NULL ORDER BY id LIMIT %s", (batch,)).fetchall()
if not rows:
break
vectors = embed_texts([r["text"] for r in rows])
with conn.cursor() as cur:
cur.executemany(
"UPDATE chunk SET embedding = %s::vector, embedding_model = %s WHERE id = %s",
[(str(v), settings.embedding_model, r["id"]) for r, v in zip(rows, vectors)],
)
conn.commit()
total += len(rows)
return total
+179
View File
@@ -0,0 +1,179 @@
"""Extract trips and meetings from official press releases and news headlines.
Rule-based and conservative. A sentence becomes an event only if it names a
known politician and contains a travel or meeting trigger:
travel: "reist nach Brüssel", "Arbeitsbesuch in Israel", "beim Gipfel in Montenegro", "in China"
meeting: "empfängt den tschechischen Außenminister", "Treffen mit ...", "traf ... zu einem Gespräch"
Sources:
* Parlamentskorrespondenz and government news: title + full text (official works / CC BY),
* news headlines: title only (we do not store article text).
The event date is the publication date of the source (`data.date_precision =
"publication"`). The claim key is (kind, person, place/counterpart, ISO week),
so reports from different sources about the same trip attach to the same claim
as additional evidence and corroborate it. corroborate.merge_events() also
merges near-duplicates whose dates differ by a few days.
"""
from __future__ import annotations
import re
from collections.abc import Iterator
from datetime import datetime
from typing import Any
from psycopg.types.json import Jsonb
from politrack.core import Enricher, RunContext, register
from politrack.core.records import Claim, Event, Evidence, Participation, PersonRef
from politrack.core.spec import DERIVED
from politrack.enrich.gazetteer import AUSTRIA, PLACES
from politrack.enrich.linker import MentionLinker
from politrack.text import excerpt, fingerprint, normalize
# (?<!-)/(?!-): skip names inside compounds such as "Berlin-Prozess" or "EU-Brüssel-Treffen".
PLACE_RE = re.compile(
r"(?<![-\w])(" + "|".join(sorted((re.escape(p) for p in PLACES), key=len, reverse=True)) + r")(?![-\w])"
)
DESTINATION_RE = re.compile(r"\b(?:in|nach|aus)\s+(?:de[mnr]\s+|die\s+)?$")
STRONG_TRAVEL = re.compile(r"\b(reist\w*|Reise\w*|\w*[Bb]esuch\w*|\w*Gipfel\w*|Delegation\w*|Konferenz\w*|Tagung\w*)\b")
MEETING_RE = re.compile(
r"\b(?:empfängt|empfing|empfangen|trifft|traf|getroffen|Treffen mit|Gespräch mit|Arbeitsgespräch mit|"
r"Unterredung mit|tauschte sich mit|im Austausch mit|zusammengetroffen mit|kam mit)\s+"
r"(?P<counterpart>(?:de[mnr]|die|das|seine[mnr]?|ihre[mnr]?)?\s*[^.,;:]{3,90})",
re.IGNORECASE,
)
COUNTERPART_ROLE = re.compile(
r"(minister|präsident|kanzler|botschafter|delegation|amtskolleg|vertreter|bürgermeister|abgeordnet|chef|"
r"premier|könig|papst|kommissar|generalsekretär|regierungschef|staatssekretär|landeshauptm|gouverneur|"
r"bischof|kardinal|vorsitzend|ceo|gewerkschaft|sozialpartner|außenminister)",
re.IGNORECASE,
)
FULL_NAME = re.compile(r"\b[A-ZÄÖÜ][a-zäöüß]+(?:-[A-ZÄÖÜ][a-zäöüß]+)?\s+[A-ZÄÖÜ][a-zäöüß]+")
SENTENCE_SPLIT = re.compile(r"(?<=[.!?])\s+(?=[A-ZÄÖÜ„\"])|\n+")
FILLER = re.compile(r"^(?:(?:heute|gestern|morgen|am|vorgestern|Montag|Dienstag|Mittwoch|Donnerstag|Freitag|"
r"Samstag|Sonntag|de[mnr]|die|das|seine[mnr]?|ihre[mnr]?|eine[mnr]?)\s+)+", re.IGNORECASE)
def clean_counterpart(text: str) -> str:
return FILLER.sub("", text.strip()).strip()
def counterpart_anchor(text: str) -> str:
"""Stable key for a counterpart: its last capitalised word (usually the surname or country)."""
words = re.findall(r"[A-ZÄÖÜ][\wäöüß-]+", clean_counterpart(text))
return normalize(words[-1]) if words else normalize(text)[:25]
def find_places(text: str) -> list[tuple[str, str]]:
"""Places in order of preference: those introduced by "in/nach" first (the destination)."""
matches = list(PLACE_RE.finditer(text))
preferred = [m for m in matches if DESTINATION_RE.search(text[: m.start()])]
ordered = preferred + [m for m in matches if m not in preferred]
return [(m.group(1), PLACES[m.group(1)]) for m in ordered]
def detect(text: str) -> list[dict]:
"""Return candidate events found in one sentence (no person resolution)."""
events = []
places = find_places(text)
foreign = [(p, iso) for p, iso in places if iso != "AT"]
if foreign and STRONG_TRAVEL.search(text):
place, iso = foreign[0]
events.append({"kind": "foreign_trip", "claim_type": "travel", "place": place, "country": iso})
meeting = MEETING_RE.search(text)
if meeting:
counterpart = re.split(r"\s+(?:in|im|am|zu|über|bei|und|sowie)\s+", meeting.group("counterpart").strip())[0]
place, iso = (places[0] if places else (None, None))
# "traf eine Entscheidung" is not a meeting: require a role word or a person-like name.
if COUNTERPART_ROLE.search(counterpart) or FULL_NAME.search(counterpart):
events.append({"kind": "meeting", "claim_type": "meeting", "counterpart": clean_counterpart(counterpart),
"place": place, "country": iso})
return events
@register
class EventExtractor(Enricher):
spec = DERIVED
name = "enrich.events"
description = "Trips and meetings from press releases, government news and headlines"
schedule = "3-59/10 * * * *"
def run(self, ctx: RunContext) -> Iterator[Any]:
conn = ctx.conn
linker = MentionLinker(conn)
while True:
docs = conn.execute(
"""SELECT d.id, d.url, d.title, d.full_text, d.published_at, d.meta, s.slug AS source, s.kind AS source_kind
FROM document d JOIN source s ON s.id = d.source_id
WHERE NOT (d.meta ? 'events_at') AND d.published_at IS NOT NULL
AND (d.doc_type = 'press_release' OR s.kind = 'media' OR s.slug = 'gdelt')
ORDER BY d.id LIMIT 300"""
).fetchall()
if not docs:
return
for doc in docs:
found = 0
for record in self.document_events(conn, linker, doc):
found += isinstance(record, Claim)
yield record
conn.execute("UPDATE document SET meta = meta || %s WHERE id = %s",
(Jsonb({"events_at": datetime.now().isoformat(), "events": found}), doc["id"]))
ctx.stats["documents"] += 1
def document_events(self, conn, linker: MentionLinker, doc: dict) -> Iterator[Any]:
official = doc["source_kind"] == "official"
title = doc["title"] or ""
body = (doc["full_text"] or "")[:4000] if official else ""
pads = (doc["meta"] or {}).get("person_pads") or []
doc_people: set[int] = set()
if pads:
doc_people = {r["entity_id"] for r in conn.execute(
"SELECT entity_id FROM identifier WHERE entity_type = 'person' AND scheme = 'parlament' AND value = ANY(%s)",
([str(p) for p in pads],))}
sentences = [title] + [s for s in SENTENCE_SPLIT.split(body) if s][:25]
for i, sentence in enumerate(sentences):
candidates = detect(sentence)
if not candidates:
continue
people = linker.all_people(sentence)
# In the headline of an official release the actors are usually the people it links/names.
if not people and i == 0:
people = doc_people or linker.all_people(f"{title} {body[:600]}")
for person_id in people:
for ev in candidates:
yield from event_records(conn, doc, person_id, ev, sentence, official)
def event_records(conn, doc: dict, person_id: int, ev: dict, sentence: str, official: bool) -> Iterator[Any]:
published = doc["published_at"]
week = published.strftime("%G-W%V")
anchor = ev.get("country") if ev["kind"] == "foreign_trip" else counterpart_anchor(ev.get("counterpart") or "")
row = conn.execute("SELECT full_name, family_name FROM person WHERE id = %s", (person_id,)).fetchone()
# A sentence like "X empfing Y" yields the meeting for X and Y; never record "Y met Y".
if ev["kind"] == "meeting" and row["family_name"] and normalize(row["family_name"]) in normalize(ev["counterpart"]):
return
if ev["kind"] == "foreign_trip":
summary_de, summary_en = f"Reise/Aufenthalt in {ev['place']}", f"Trip to {ev['place']}"
else:
where = f" ({ev['place']})" if ev.get("place") else ""
summary_de = f"Treffen mit {ev['counterpart']}{where}"
summary_en = f"Meeting with {ev['counterpart']}{where}"
event_key = fingerprint("event", ev["kind"], person_id, anchor, week)
yield Event(key=event_key, kind=ev["kind"], title=f"{row['full_name']}: {summary_de}", starts_at=published,
location=ev.get("place"), country=ev.get("country"), url=doc["url"], description=excerpt(sentence, 400))
foreign = ev.get("country") and ev["country"] not in AUSTRIA.values()
yield Claim(
claim_type=ev["claim_type"], person=PersonRef(ids={"politrack": str(person_id)}), occurred_at=published,
summary_de=summary_de, summary_en=summary_en,
data={"place": ev.get("place"), "country": ev.get("country"), "counterpart": ev.get("counterpart"),
"date_precision": "publication", "extracted": "rules"},
evidence=[Evidence(url=doc["url"], source=doc["source"], title=doc["title"], published_at=published,
quote=excerpt(sentence, 400), confidence=0.8 if official else 0.6)],
topics=("Außenpolitik",) if foreign else (),
facet=Participation(event_key=event_key),
)
+52
View File
@@ -0,0 +1,52 @@
"""German place names -> ISO 3166-1 alpha-2, for detecting trips in German texts.
Countries in nominative form (as used after "nach"/"in"); cities that are
frequent destinations of Austrian politicians. Austrian places map to "AT" so
domestic visits are recognised but not counted as trips abroad.
"""
COUNTRIES = {
"Ägypten": "EG", "Albanien": "AL", "Algerien": "DZ", "Argentinien": "AR", "Armenien": "AM", "Aserbaidschan": "AZ",
"Äthiopien": "ET", "Australien": "AU", "Belgien": "BE", "Bosnien und Herzegowina": "BA", "Bosnien": "BA",
"Brasilien": "BR", "Bulgarien": "BG", "Chile": "CL", "China": "CN", "Dänemark": "DK", "Deutschland": "DE",
"Estland": "EE", "Finnland": "FI", "Frankreich": "FR", "Georgien": "GE", "Ghana": "GH", "Griechenland": "GR",
"Großbritannien": "GB", "Vereinigten Königreich": "GB", "Vereinigtes Königreich": "GB", "Indien": "IN",
"Indonesien": "ID", "Irak": "IQ", "Iran": "IR", "Irland": "IE", "Island": "IS", "Israel": "IL", "Italien": "IT",
"Japan": "JP", "Jordanien": "JO", "Kanada": "CA", "Kasachstan": "KZ", "Katar": "QA", "Kenia": "KE",
"Kolumbien": "CO", "Kosovo": "XK", "Kroatien": "HR", "Kuba": "CU", "Lettland": "LV", "Libanon": "LB",
"Libyen": "LY", "Liechtenstein": "LI", "Litauen": "LT", "Luxemburg": "LU", "Malta": "MT", "Marokko": "MA",
"Mexiko": "MX", "Moldau": "MD", "Moldawien": "MD", "Monaco": "MC", "Mongolei": "MN", "Montenegro": "ME",
"Neuseeland": "NZ", "Niederlande": "NL", "Nigeria": "NG", "Nordmazedonien": "MK", "Norwegen": "NO",
"Oman": "OM", "Pakistan": "PK", "Palästina": "PS", "Polen": "PL", "Portugal": "PT", "Ruanda": "RW",
"Rumänien": "RO", "Russland": "RU", "Saudi-Arabien": "SA", "Schweden": "SE", "Schweiz": "CH", "Senegal": "SN",
"Serbien": "RS", "Singapur": "SG", "Slowakei": "SK", "Slowenien": "SI", "Spanien": "ES", "Südafrika": "ZA",
"Südkorea": "KR", "Syrien": "SY", "Taiwan": "TW", "Thailand": "TH", "Tschechien": "CZ", "Tunesien": "TN",
"Türkei": "TR", "Ukraine": "UA", "Ungarn": "HU", "USA": "US", "Vereinigten Staaten": "US",
"Vereinigte Staaten": "US", "Usbekistan": "UZ", "Vatikan": "VA", "Venezuela": "VE",
"Vereinigten Arabischen Emiraten": "AE", "Vereinigte Arabische Emirate": "AE", "Vietnam": "VN",
"Weißrussland": "BY", "Belarus": "BY", "Zypern": "CY", "Andorra": "AD", "San Marino": "SM",
}
CITIES = {
"Brüssel": "BE", "Straßburg": "FR", "Luxemburg": "LU", "Berlin": "DE", "München": "DE", "Frankfurt": "DE",
"Paris": "FR", "Rom": "IT", "Mailand": "IT", "Madrid": "ES", "Lissabon": "PT", "London": "GB", "Dublin": "IE",
"Amsterdam": "NL", "Den Haag": "NL", "Kopenhagen": "DK", "Stockholm": "SE", "Oslo": "NO", "Helsinki": "FI",
"Warschau": "PL", "Prag": "CZ", "Bratislava": "SK", "Pressburg": "SK", "Budapest": "HU", "Ljubljana": "SI",
"Laibach": "SI", "Zagreb": "HR", "Belgrad": "RS", "Sarajevo": "BA", "Podgorica": "ME", "Skopje": "MK",
"Tirana": "AL", "Pristina": "XK", "Sofia": "BG", "Bukarest": "RO", "Chișinău": "MD", "Kiew": "UA",
"Kyjiw": "UA", "Moskau": "RU", "Minsk": "BY", "Athen": "GR", "Nikosia": "CY", "Valletta": "MT",
"Ankara": "TR", "Istanbul": "TR", "Bern": "CH", "Genf": "CH", "Zürich": "CH", "Davos": "CH", "Vaduz": "LI",
"Washington": "US", "New York": "US", "Ottawa": "CA", "Peking": "CN", "Beijing": "CN", "Shanghai": "CN",
"Tokio": "JP", "Neu-Delhi": "IN", "Jerusalem": "IL", "Tel Aviv": "IL", "Kairo": "EG", "Amman": "JO",
"Riad": "SA", "Doha": "QA", "Abu Dhabi": "AE", "Dubai": "AE", "Teheran": "IR", "Baku": "AZ", "Tiflis": "GE",
"Eriwan": "AM", "Astana": "KZ", "Taschkent": "UZ", "Brdo": "SI", "Göttweig": "AT",
}
AUSTRIA = {
"Österreich": "AT", "Wien": "AT", "Graz": "AT", "Linz": "AT", "Salzburg": "AT", "Innsbruck": "AT",
"Klagenfurt": "AT", "Villach": "AT", "Wels": "AT", "St. Pölten": "AT", "Dornbirn": "AT", "Bregenz": "AT",
"Eisenstadt": "AT", "Niederösterreich": "AT", "Oberösterreich": "AT", "Steiermark": "AT", "Kärnten": "AT",
"Tirol": "AT", "Vorarlberg": "AT", "Burgenland": "AT", "Alpbach": "AT",
}
PLACES = {**AUSTRIA, **COUNTRIES, **CITIES}
+91
View File
@@ -0,0 +1,91 @@
"""Detect reports about legal proceedings in news headlines (and WKStA press releases).
Everything found here is a *proposal*: claims are created with review=True (hidden until a human approves
them via `politrack review`). Headline wording is ambiguous, the data is specially protected (Art. 10 GDPR)
and wrong attributions can be defamatory. Official parliament immunity data does not go through this path.
Several outlets reporting the same person + stage in the same month attach to one claim (corroboration).
"""
from __future__ import annotations
import re
from collections.abc import Iterator
from datetime import datetime
from psycopg.types.json import Jsonb
from politrack.core import Enricher, RunContext, register
from politrack.core.records import Claim, Evidence, Legal, PersonRef
from politrack.core.spec import DERIVED
# Most specific first.
STAGES = [
# "nicht rechtskräftig verurteilt" must never be read as a final conviction.
("convicted_final", re.compile(r"(?<!nicht )\brechtskräftig\s+(verurteilt|schuldig)", re.I)),
("convicted_not_final", re.compile(r"\b(nicht rechtskräftig\b.*\bverurteilt|verurteilt\b.*nicht rechtskräftig|"
r"schuldig gesprochen|Schuldspruch|\bverurteilt\b)", re.I)),
("acquitted", re.compile(r"\b(freigesprochen|Freispruch)\b", re.I)),
("diversion", re.compile(r"\bDiversion\b", re.I)),
("proceedings_dropped", re.compile(r"\b(Verfahren|Ermittlungen)\b.*\beingestellt\b|\beingestellt\b.*\bVerfahren\b", re.I)),
("trial", re.compile(r"\b(Prozess|Strafprozess|vor Gericht|Prozessauftakt|Hauptverhandlung)\b", re.I)),
("indicted", re.compile(r"\b(angeklagt|Anklage|Strafantrag)\b", re.I)),
("investigation", re.compile(r"\b(ermittelt|Ermittlungen|Ermittlungsverfahren|Beschuldigte[rn]?|Hausdurchsuchung|"
r"WKStA|Staatsanwaltschaft|Razzia)\b", re.I)),
]
STAGE_DE = {
"convicted_final": "rechtskräftige Verurteilung (laut Bericht)",
"convicted_not_final": "nicht rechtskräftige Verurteilung (laut Bericht)",
"acquitted": "Freispruch (laut Bericht)",
"diversion": "Diversion (laut Bericht)",
"proceedings_dropped": "Verfahren eingestellt (laut Bericht)",
"trial": "Gerichtsverfahren (laut Bericht)",
"indicted": "Anklage (laut Bericht)",
"investigation": "Ermittlungen (laut Bericht)",
}
def classify(headline: str) -> str | None:
for stage, pattern in STAGES:
if pattern.search(headline):
return stage
return None
@register
class LegalDetector(Enricher):
spec = DERIVED
name = "enrich.legal"
description = "Propose legal-proceeding claims from headlines (human review required)"
schedule = "6-59/10 * * * *"
def run(self, ctx: RunContext) -> Iterator[Claim]:
conn = ctx.conn
while True:
rows = conn.execute(
"""SELECT c.id, c.person_id, c.occurred_at, d.url, d.title, d.published_at, s.slug AS source
FROM claim c
JOIN claim_evidence e ON e.claim_id = c.id
JOIN document d ON d.id = e.document_id
JOIN source s ON s.id = d.source_id
WHERE c.claim_type = 'media_mention' AND c.person_id IS NOT NULL AND NOT (c.data ? 'legal_checked')
AND (s.kind IN ('media', 'aggregator') OR s.slug = 'wksta')
ORDER BY c.id LIMIT 1000"""
).fetchall()
if not rows:
return
for row in rows:
stage = classify(row["title"] or "")
if stage:
when = row["published_at"] or row["occurred_at"] or datetime.now()
yield Claim(
claim_type="legal_proceeding", person=PersonRef(ids={"politrack": str(row["person_id"])}),
key=("legal-news", row["person_id"], stage, when.strftime("%Y-%m")), occurred_at=when,
summary_de=STAGE_DE[stage], summary_en=f"Reported legal matter ({stage.replace('_', ' ')})",
data={"detected_from": "headline"}, sensitive=True, review=True,
evidence=[Evidence(url=row["url"], source=row["source"], title=row["title"], published_at=when,
quote=row["title"], confidence=0.5)],
topics=("Inneres und Recht",),
facet=Legal(stage=stage, stage_date=when.date()),
)
ctx.stats["proposed"] += 1
conn.execute("UPDATE claim SET data = data || %s WHERE id = %s", (Jsonb({"legal_checked": True}), row["id"]))
+153
View File
@@ -0,0 +1,153 @@
"""Link documents (headlines, press releases, government news) to the politicians and parties they mention.
Conservative by design: full names (>= 2 tokens) only, plus "office title + surname" of *current* office
holders ("Bundeskanzler Stocker"); an ambiguous name is only linked if exactly one candidate is active.
False links are worse than missed ones on a site like this.
"""
from __future__ import annotations
import re
from collections import defaultdict
from collections.abc import Iterator
from datetime import datetime, timezone
from psycopg import Connection
from psycopg.types.json import Jsonb
from politrack.core import Enricher, RunContext, register
from politrack.core.records import Claim, Evidence, OrgRef, PersonRef
from politrack.core.spec import DERIVED
from politrack.text import normalize, strip_titles
TOKEN = re.compile(r"[\w'-]+")
OFFICE_TITLE = re.compile(
r"\b(?:Bundeskanzler(?:in)?|Vizekanzler(?:in)?|[A-ZÄÖÜ][a-zäöüß]*minister(?:in)?|Minister(?:in)?|"
r"Staatssekretär(?:in)?|Nationalratspräsident(?:in)?|Bundesratspräsident(?:in)?|Bundespräsident(?:in)?|"
r"Landeshauptmann|Landeshauptfrau|Landesrät(?:in)?|Landesrat|Bürgermeister(?:in)?|Klubobmann|Klubobfrau|"
r"Klubchef(?:in)?|Parteichef(?:in)?|Generalsekretär(?:in)?|Abgeordnete[rn]?|EU-Abgeordnete[rn]?|"
r"EU-Kommissar(?:in)?|Präsident(?:in)?)\s+(?:(?:Dr|Mag|DI|Ing)\.\s+)?(?P<surname>[A-ZÄÖÜ][a-zäöüß]+(?:-[A-ZÄÖÜ][a-zäöüß]+)?)\b"
# "Nationalratspräsident Walter Rosenkranz": a following capitalised word means we matched a first name.
r"(?!\s+(?:von\s+)?[A-ZÄÖÜ][a-zäöüß]+)"
)
CONTEXT_PATTERNS = [
("interview", re.compile(r"\binterview\b", re.I)),
("tv", re.compile(r"\b(ZIB ?2|ZiB ?2|Pressestunde|Im Zentrum|Sommergespräch|Report|Puls ?24|ServusTV|Talk im Hangar|"
r"Milborn|Pro und Contra)\b", re.I)),
("radio", re.compile(r"\b(Ö1|Morgenjournal|Mittagsjournal|Journal zu Gast)\b", re.I)),
("op_ed", re.compile(r"\b(Gastkommentar|Kommentar der anderen)\b", re.I)),
]
def tokens(text: str) -> list[str]:
return TOKEN.findall(normalize(text))
def detect_context(text: str) -> str | None:
for name, pattern in CONTEXT_PATTERNS:
if pattern.search(text):
return name
return None
class MentionLinker:
def __init__(self, conn: Connection) -> None:
self.person_index: dict[tuple[str, ...], set[int]] = defaultdict(set)
for row in conn.execute("SELECT entity_id, alias FROM alias WHERE entity_type = 'person'"):
key = tuple(tokens(strip_titles(row["alias"])))
if len(key) >= 2:
self.person_index[key].add(row["entity_id"])
self.active = {r["person_id"] for r in conn.execute("SELECT person_id FROM person_status WHERE status = 'active'")}
self.active_surnames: dict[str, set[int]] = defaultdict(set)
for row in conn.execute(
"SELECT p.id, p.family_name FROM person p JOIN person_status s ON s.person_id = p.id "
"WHERE s.status = 'active' AND p.family_name IS NOT NULL"
):
self.active_surnames[normalize(row["family_name"])].add(row["id"])
self.org_index: dict[tuple[str, ...], int] = {}
for row in conn.execute(
"""SELECT o.id, a.alias FROM organization o JOIN alias a ON a.entity_type = 'organization' AND a.entity_id = o.id
WHERE o.kind = 'party'
UNION SELECT id, short_name FROM organization WHERE kind = 'party' AND short_name IS NOT NULL"""
):
key = tuple(tokens(row["alias"]))
if key and len("".join(key)) >= 3:
self.org_index[key] = row["id"]
self.max_len = max((len(k) for k in self.person_index), default=2)
def people(self, text: str) -> set[int]:
toks = tokens(text)
found: set[int] = set()
for n in range(2, self.max_len + 1):
for i in range(len(toks) - n + 1):
candidates = self.person_index.get(tuple(toks[i : i + n]))
if not candidates:
continue
if len(candidates) == 1:
found |= candidates
elif len(active := candidates & self.active) == 1:
found |= active
return found
def people_by_title(self, text: str) -> set[int]:
found: set[int] = set()
for m in OFFICE_TITLE.finditer(text):
candidates = self.active_surnames.get(normalize(m.group("surname")), set())
if len(candidates) == 1:
found |= candidates
return found
def all_people(self, text: str) -> set[int]:
return self.people(text) | self.people_by_title(text)
def parties(self, text: str) -> set[int]:
toks = tokens(text)
found = set()
for n in (1, 2, 3):
for i in range(len(toks) - n + 1):
if org := self.org_index.get(tuple(toks[i : i + n])):
found.add(org)
return found
@register
class Linker(Enricher):
spec = DERIVED
name = "enrich.linker"
description = "Link news headlines, press releases and government news to politicians and parties"
schedule = "*/10 * * * *"
def run(self, ctx: RunContext) -> Iterator[Claim]:
conn = ctx.conn
linker = MentionLinker(conn)
while True:
docs = conn.execute(
"""SELECT d.id, d.url, d.title, d.excerpt, left(d.full_text, 3000) AS full_text, d.published_at,
s.slug AS source, s.name AS source_name, s.kind AS source_kind,
COALESCE((d.meta->>'tdm_reserved')::boolean, false) AS tdm_reserved
FROM document d JOIN source s ON s.id = d.source_id
WHERE (s.kind IN ('media', 'user') OR s.slug IN ('gdelt', 'bundesregierung', 'wksta'))
AND NOT (d.meta ? 'linked_at') AND d.doc_type IN ('article', 'press_release')
ORDER BY d.id LIMIT 500"""
).fetchall()
if not docs:
return
for doc in docs:
title = doc["title"] or ""
text = title if doc["tdm_reserved"] else f"{title} {doc['full_text'] or doc['excerpt'] or ''}"
context = detect_context(title)
people, parties = linker.all_people(text), linker.parties(title)
evidence = Evidence(url=doc["url"], source=doc["source"], title=title, quote=title, stance="mentions",
confidence=0.8, published_at=doc["published_at"])
for person_id in people:
yield Claim(claim_type="media_mention", person=PersonRef(ids={"politrack": str(person_id)}),
occurred_at=doc["published_at"], summary_de=f"{doc['source_name']}: {title}",
data={"context": context, "source": doc["source"]}, evidence=[evidence])
for org_id in parties:
yield Claim(claim_type="media_mention", org=OrgRef(ids={"politrack_org": str(org_id)}, create=False),
occurred_at=doc["published_at"], summary_de=f"{doc['source_name']}: {title}",
data={"context": context, "source": doc["source"]}, evidence=[evidence])
conn.execute("UPDATE document SET meta = meta || %s WHERE id = %s",
(Jsonb({"linked_at": datetime.now(timezone.utc).isoformat(), "people": sorted(people),
"parties": sorted(parties)}), doc["id"]))
ctx.stats["documents_linked"] += 1
+81
View File
@@ -0,0 +1,81 @@
"""Housekeeping enrichers that work directly on canonical rows (no new facts, only derived structure).
* enrich.roles - give open-ended offices the party of the holder's party membership at that time
(Wikidata offices carry no party), close open roles at the holder's date of death
* enrich.rollcall - derive each faction's majority position from individual roll-call votes, so party
pages show roll calls too ("ÖVP mostly for: ... 70 yes, 2 no")
"""
from __future__ import annotations
from collections import Counter, defaultdict
from collections.abc import Iterator
from politrack.core import Enricher, RunContext, register
from politrack.core.records import Claim, Evidence, OrgRef, VotePosition
from politrack.core.spec import DERIVED
@register
class RoleMaintenance(Enricher):
spec = DERIVED
name = "enrich.roles"
description = "Assign parties to offices from party memberships; close roles at death"
schedule = "50 5 * * *"
def run(self, ctx: RunContext) -> None:
conn = ctx.conn
ctx.stats["roles_with_party"] = conn.execute(
"""UPDATE role r SET party_id = pm.party_id
FROM role pm JOIN claim c ON c.id = pm.claim_id
WHERE r.party_id IS NULL AND r.level <> 'party' AND pm.role = 'party_member' AND c.merged_into IS NULL
AND pm.person_id = r.person_id
AND (pm.start_date IS NULL OR r.end_date IS NULL OR pm.start_date <= r.end_date)
AND (pm.end_date IS NULL OR r.start_date IS NULL OR pm.end_date >= r.start_date)"""
).rowcount
ctx.stats["roles_closed_at_death"] = conn.execute(
"""UPDATE role r SET end_date = p.death_date FROM person p
WHERE p.id = r.person_id AND r.end_date IS NULL AND p.death_date IS NOT NULL"""
).rowcount
return None
@register
class RollCallFactions(Enricher):
spec = DERIVED
name = "enrich.rollcall"
description = "Faction majority positions derived from individual roll-call votes"
schedule = "55 4 * * *"
def run(self, ctx: RunContext) -> Iterator[Claim]:
rows = ctx.conn.execute(
"""SELECT ve.event_key, ve.item_title, ve.voted_on, ve.url, ch.slug AS chamber, vp.party_id, vp.position,
o.short_name, o.name
FROM vote_event ve JOIN vote_position vp ON vp.vote_event_id = ve.id
JOIN organization ch ON ch.id = ve.chamber_id
JOIN organization o ON o.id = vp.party_id
JOIN claim c ON c.id = vp.claim_id AND c.merged_into IS NULL
WHERE ve.roll_call AND vp.person_id IS NOT NULL"""
).fetchall()
tally: dict[tuple, Counter] = defaultdict(Counter)
info: dict[tuple, dict] = {}
for r in rows:
key = (r["event_key"], r["party_id"])
tally[key][r["position"]] += 1
info[key] = r
for key, counts in tally.items():
r = info[key]
position, _ = counts.most_common(1)[0]
party = r["short_name"] or r["name"]
yield Claim(
claim_type="vote", org=OrgRef(ids={"politrack_org": str(r["party_id"])}, create=False),
counterpart_org=OrgRef(slug=r["chamber"]), occurred_at=r["voted_on"],
summary_de=f"{party} stimmte mehrheitlich {'für' if position == 'for' else 'gegen'}: {r['item_title']} "
f"(namentliche Abstimmung, {counts['for']} Ja, {counts['against']} Nein)",
summary_en=f"{party} mostly voted {'for' if position == 'for' else 'against'}: {r['item_title']} "
f"(roll call, {counts['for']} yes / {counts['against']} no)",
data={"position": position, "roll_call": True, "yes": counts["for"], "no": counts["against"]},
evidence=[Evidence(url=r["url"], source="parlament", doc_type="api_record", title=r["item_title"],
published_at=r["voted_on"], locator=f"rollcall:{r['voted_on']}")],
facet=VotePosition(event_key=r["event_key"], position=position),
)
+61
View File
@@ -0,0 +1,61 @@
"""Keyword-based topic tagging for claims that did not get topics from their source.
Parliament items carry official topics; news mentions, donations and posts
are tagged here with the keyword lists of the `topic` table. An optional LLM
classifier can replace this later (see docs/roadmap.md).
"""
import re
from politrack.core import Enricher, RunContext, register
from politrack.core.spec import DERIVED
def keyword_pattern(keywords: list[str]) -> re.Pattern:
"""Keywords match as word prefixes ("Klima" -> "Klimaschutz"); short ones (<= 3 chars) only as whole words."""
parts = [re.escape(k) + (r"\b" if len(k) <= 3 else "") for k in keywords]
return re.compile(r"\b(" + "|".join(parts) + ")", re.IGNORECASE)
@register
class TopicTagger(Enricher):
spec = DERIVED
name = "enrich.topics"
description = "Keyword topics for claims without official topics"
schedule = "8-59/10 * * * *"
def run(self, ctx: RunContext) -> None:
while (n := tag_claims(ctx.conn)):
ctx.stats["claims_tagged"] += n
return None
def tag_claims(conn, batch: int = 2000) -> int:
topics = [
(row["id"], keyword_pattern(row["keywords"]))
for row in conn.execute("SELECT id, keywords FROM topic WHERE cardinality(keywords) > 0")
]
claims = conn.execute(
"""
SELECT c.id, c.summary_de || ' ' || COALESCE(string_agg(d.title, ' '), '') AS text
FROM claim c
LEFT JOIN claim_evidence e ON e.claim_id = c.id
LEFT JOIN document d ON d.id = e.document_id
WHERE c.claim_type IN ('media_mention', 'statement', 'donation', 'side_job', 'appearance', 'meeting', 'travel',
'money_flow', 'government_decision', 'wiki_edit', 'career')
AND NOT (c.data ? 'topics_tagged')
GROUP BY c.id ORDER BY c.id LIMIT %s
""",
(batch,),
).fetchall()
for claim in claims:
for topic_id, pattern in topics:
if pattern.search(claim["text"]):
conn.execute(
"INSERT INTO topic_assignment (topic_id, claim_id, confidence, method) "
"VALUES (%s, %s, 0.5, 'keyword') ON CONFLICT DO NOTHING",
(topic_id, claim["id"]),
)
conn.execute("UPDATE claim SET data = data || '{\"topics_tagged\": true}' WHERE id = %s", (claim["id"],))
conn.commit()
return len(claims)
+189
View File
@@ -0,0 +1,189 @@
"""MCP (Model Context Protocol) server so AI agents can query Politrack with citations.
Connect an MCP client (Claude Desktop/Code, etc.) to http://<host>:8001/mcp
(streamable HTTP transport). All tools return evidence URLs; agents should cite them.
Tools are thin wrappers around politrack.queries - the same functions the REST API uses.
"""
import json
from datetime import date
from mcp.server.fastmcp import FastMCP
from politrack import queries
from politrack.db import connection
from politrack.queries import SourceFilter
mcp = FastMCP(
"politrack",
instructions=(
"Evidence-backed data on Austrian politicians and parties. Every claim has an `evidence` list of URLs: "
"always cite them. status 'corroborated' = confirmed by at least two independent sources. "
"Claims of type legal_proceeding: unless legal_stage is 'convicted_final' (and the claim is not "
"'disputed'), the presumption of innocence applies - always state 'Es gilt die Unschuldsvermutung' / "
"'presumed innocent'. Parliamentary votes in Austria are mostly recorded per faction; roll_call=true "
"votes contain individual positions. Travel/meeting dates are publication dates of the source. "
"Money flows (donations, public contracts, government ads, party funding) come from official registers; "
"a flow to an organisation connected to a person is a connection, not an accusation. "
"Social accounts marked verified=false come from Wikidata and were not confirmed by an official source."
),
)
def _dump(rows) -> str:
return json.dumps(rows, default=str, ensure_ascii=False)
def _date(value: str | None) -> date | None:
return date.fromisoformat(value) if value else None
@mcp.tool()
def search(query: str, limit: int = 10, since: str | None = None, until: str | None = None,
exclude_sources: list[str] | None = None) -> str:
"""Hybrid (keyword + semantic) search over all claims and documents. Dates as YYYY-MM-DD."""
with connection() as conn:
return _dump(queries.search(conn, query, sources=SourceFilter(exclude_sources or []), since=_date(since),
until=_date(until), limit=limit))
@mcp.tool()
def find_person(name: str, status: str = "all") -> str:
"""Find politicians by (part of) their name. status: active | former | party_member | deceased | all."""
with connection() as conn:
return _dump(queries.list_persons(conn, q=name, status=None if status == "all" else status, limit=20))
@mcp.tool()
def get_person(slug: str) -> str:
"""Profile: activity status, roles/mandates over time (with sources), social accounts, claim counts."""
with connection() as conn:
return _dump(queries.get_person(conn, slug))
@mcp.tool()
def person_network(slug: str) -> str:
"""Organisations connected to a person (side jobs, career, party, meetings, lobby contacts) with the
money those organisations paid or received (donations, public contracts, government ads)."""
with connection() as conn:
return _dump(queries.person_network(conn, slug))
@mcp.tool()
def list_parties() -> str:
"""All parties with number of currently active officials."""
with connection() as conn:
return _dump(queries.list_parties(conn))
@mcp.tool()
def get_party(slug: str) -> str:
"""Party overview: current officials, former members, money per year, finance reports, votes, dissenters,
affiliated organisations and holdings."""
with connection() as conn:
return _dump(queries.get_party(conn, slug))
@mcp.tool()
def find_organization(name: str, kind: str | None = None) -> str:
"""Find organisations (companies, associations, public bodies, media, lobby groups) by name.
kind: party | company | association | public_body | ministry | media | ngo | eu_institution | other."""
with connection() as conn:
return _dump(queries.list_organizations(conn, q=name, kind=kind, limit=20))
@mcp.tool()
def get_organization(slug: str) -> str:
"""Organisation profile: identifiers (Firmenbuch, UID, EU transparency register), money paid/received by
kind, top counterparties and connected people."""
with connection() as conn:
return _dump(queries.get_organization(conn, slug))
@mcp.tool()
def list_claims(
person: str | None = None,
party: str | None = None,
organization: str | None = None,
claim_types: list[str] | None = None,
topics: list[str] | None = None,
since: str | None = None,
until: str | None = None,
limit: int = 30,
exclude_sources: list[str] | None = None,
min_trust: int = 0,
) -> str:
"""Claims about a person/party/organisation, newest first. claim_types: role, vote, donation, money_flow,
statement, appearance, meeting, travel, side_job, legal_proceeding, lobbying, media_mention, career,
parliamentary_question, government_decision, wiki_edit, declaration, ownership, affiliation, party_finance.
Dates as YYYY-MM-DD."""
with connection() as conn:
return _dump(queries.list_claims(
conn, sources=SourceFilter(exclude_sources or [], min_trust), person=person, party=party,
organization=organization, claim_types=claim_types, topics=topics, since=_date(since),
until=_date(until), limit=limit))
@mcp.tool()
def money_flows(kinds: list[str] | None = None, organization: str | None = None, person: str | None = None,
query: str | None = None, since: str | None = None, until: str | None = None,
min_amount: float | None = None, limit: int = 30) -> str:
"""Money flows from official registers. kinds: donation, contract, government_ad, political_ad, subsidy,
party_funding, party_ad, loan, side_income, sponsorship. `person` = flows of organisations connected to
that person. Dates as YYYY-MM-DD."""
with connection() as conn:
return _dump(queries.list_money(conn, sources=SourceFilter(), kinds=kinds, org=organization, person=person,
q=query, since=_date(since), until=_date(until), min_amount=min_amount,
limit=limit))
@mcp.tool()
def social_posts(person: str | None = None, party: str | None = None, platforms: list[str] | None = None,
limit: int = 30) -> str:
"""Public posts from politicians' accounts. platforms: bluesky, mastodon, telegram, youtube."""
with connection() as conn:
return _dump(queries.social_feed(conn, sources=SourceFilter(), person=person, party=party,
platforms=platforms, limit=limit))
@mcp.tool()
def wikipedia_edits(limit: int = 50) -> str:
"""Wikipedia edits made from IP ranges of parliament/government (anonymous edits only)."""
with connection() as conn:
return _dump(queries.wiki_edits(conn, limit=limit))
@mcp.tool()
def calendar(start: str, end: str, person: str | None = None, party: str | None = None) -> str:
"""Events (sessions, hearings, council of ministers, trips, meetings) and per-day activity between two dates."""
with connection() as conn:
return _dump(queries.calendar(conn, sources=SourceFilter(), start=date.fromisoformat(start),
end=date.fromisoformat(end), person=person, party=party))
@mcp.tool()
def person_votes(slug: str, topics: list[str] | None = None, limit: int = 50) -> str:
"""Votes: individual roll-call votes (with dissent flag) and faction votes during the person's mandates.
`spoke` links a speech by the person on the same item (say vs. vote)."""
with connection() as conn:
return _dump(queries.person_votes(conn, slug, topics=topics, limit=limit))
@mcp.tool()
def former_politicians(party: str | None = None, limit: int = 30) -> str:
"""Former politicians with their latest news coverage and career entries."""
with connection() as conn:
return _dump(queries.former_overview(conn, party=party, limit=limit))
@mcp.tool()
def list_topics() -> str:
"""Topic taxonomy (slugs usable as filters)."""
with connection() as conn:
return _dump(queries.list_topics(conn))
def serve(host: str, port: int) -> None:
mcp.settings.host = host
mcp.settings.port = port
mcp.run(transport="streamable-http")
+35
View File
@@ -0,0 +1,35 @@
"""Minimal forward-only SQL migration runner (files in backend/migrations)."""
import logging
from pathlib import Path
import psycopg
from psycopg import sql
from politrack.config import get_settings
log = logging.getLogger(__name__)
MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations"
def migrate() -> None:
settings = get_settings()
with psycopg.connect(settings.database_url, autocommit=False) as conn:
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations ("
" name TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT now())"
)
applied = {r[0] for r in conn.execute("SELECT name FROM schema_migrations")}
for path in sorted(MIGRATIONS_DIR.glob("*.sql")):
if path.name in applied:
continue
log.info("applying %s", path.name)
conn.execute(path.read_text(encoding="utf-8"))
conn.execute("INSERT INTO schema_migrations (name) VALUES (%s)", (path.name,))
conn.commit()
if settings.reader_password:
conn.execute(
sql.SQL("ALTER ROLE politrack_reader PASSWORD {}").format(sql.Literal(settings.reader_password))
)
conn.commit()
+79
View File
@@ -0,0 +1,79 @@
"""Generate TypeScript types from the OpenAPI schema (no npm needed).
Supports the JSON-Schema subset FastAPI/Pydantic emit: objects, arrays, enums/Literals, anyOf with null,
$ref, additionalProperties, primitive formats. Output: one `export type X = {...}` per component schema
plus `Paths` mapping GET endpoints to their response types.
"""
from __future__ import annotations
from typing import Any
HEADER = "// Generated by `politrack openapi --ts` from the FastAPI schema. Do not edit by hand.\n\n"
def ts_type(schema: dict[str, Any]) -> str:
if "$ref" in schema:
return schema["$ref"].rsplit("/", 1)[-1]
if "const" in schema:
return _literal(schema["const"])
if "enum" in schema:
return " | ".join(_literal(v) for v in schema["enum"])
for key in ("anyOf", "oneOf"):
if key in schema:
parts = []
for sub in schema[key]:
part = ts_type(sub)
if part not in parts:
parts.append(part)
return " | ".join(parts)
kind = schema.get("type")
if kind == "array":
inner = ts_type(schema.get("items", {}))
return f"({inner})[]" if "|" in inner else f"{inner}[]"
if kind == "object":
if "properties" in schema:
return _object(schema)
extra = schema.get("additionalProperties")
value = ts_type(extra) if isinstance(extra, dict) and extra else "unknown"
return f"Record<string, {value}>"
if kind in ("integer", "number"):
return "number"
if kind == "boolean":
return "boolean"
if kind == "string":
return "string"
if kind == "null":
return "null"
return "unknown"
def _literal(value: Any) -> str:
return f"'{value}'" if isinstance(value, str) else ("null" if value is None else str(value).lower())
def _object(schema: dict[str, Any], indent: str = "") -> str:
required = set(schema.get("required", []))
lines = ["{"]
for name, prop in schema.get("properties", {}).items():
optional = "" if name in required else "?"
lines.append(f"{indent} {name}{optional}: {ts_type(prop)};")
lines.append(f"{indent}}}")
return "\n".join(lines)
def to_typescript(openapi: dict[str, Any]) -> str:
out = [HEADER]
for name, schema in sorted(openapi.get("components", {}).get("schemas", {}).items()):
if name in ("HTTPValidationError", "ValidationError"):
continue
out.append(f"export type {name} = {ts_type(schema)};\n")
out.append("\nexport type Paths = {")
for path, ops in sorted(openapi.get("paths", {}).items()):
get = ops.get("get")
if not get:
continue
content = get.get("responses", {}).get("200", {}).get("content", {}).get("application/json", {})
out.append(f" '{path}': {ts_type(content.get('schema', {})) if content.get('schema') else 'unknown'};")
out.append("};\n")
return "\n".join(out)
+17
View File
@@ -0,0 +1,17 @@
"""Read queries shared by the REST API and the MCP server (one module per domain)."""
from politrack.queries.admin import list_sources, list_topics, pending_claims, recent_runs, set_review, stats
from politrack.queries.claims import calendar, list_claims, timeline_histogram
from politrack.queries.common import SourceFilter
from politrack.queries.money import list_money, money_summary, social_feed, wiki_edits
from politrack.queries.organizations import get_organization, get_party, list_organizations, list_parties
from politrack.queries.people import former_overview, get_person, list_persons, person_network
from politrack.queries.search import search
from politrack.queries.votes import list_votes, person_votes, vote_detail
__all__ = [
"SourceFilter", "calendar", "former_overview", "get_organization", "get_party", "get_person", "list_claims",
"list_money", "list_organizations", "list_parties", "list_persons", "list_sources", "list_topics", "list_votes",
"money_summary", "pending_claims", "recent_runs", "person_network", "person_votes", "search", "set_review", "social_feed",
"stats", "timeline_histogram", "vote_detail", "wiki_edits",
]
+69
View File
@@ -0,0 +1,69 @@
"""Sources, topics, statistics, moderation."""
from __future__ import annotations
from psycopg import Connection
def list_sources(conn: Connection) -> list[dict]:
return conn.execute(
"""
SELECT s.slug, s.name, s.kind, s.homepage, s.license, s.access_method, s.default_trust, s.tos_notes,
(SELECT count(*) FROM document d WHERE d.source_id = s.id) AS documents
FROM source s ORDER BY s.kind, s.default_trust DESC, s.name
"""
).fetchall()
def list_topics(conn: Connection) -> list[dict]:
return conn.execute("SELECT slug, name_de, name_en FROM topic ORDER BY name_de").fetchall()
def pending_claims(conn: Connection, limit: int = 50) -> list[dict]:
return conn.execute(
"""
SELECT c.id, c.claim_type, c.summary_de, c.occurred_at, p.full_name,
(SELECT jsonb_agg(jsonb_build_object('url', d.url, 'title', d.title))
FROM claim_evidence e JOIN document d ON d.id = e.document_id WHERE e.claim_id = c.id) AS evidence
FROM claim c LEFT JOIN person p ON p.id = c.person_id
WHERE c.review_status = 'pending' AND c.merged_into IS NULL
ORDER BY c.occurred_at DESC NULLS LAST LIMIT %s
""",
(limit,),
).fetchall()
def set_review(conn: Connection, claim_id: int, decision: str, reviewer: str) -> None:
if decision not in ("published", "rejected"):
raise ValueError("decision must be 'published' or 'rejected'")
conn.execute(
"UPDATE claim SET review_status = %s, reviewed_by = %s, reviewed_at = now() WHERE id = %s",
(decision, reviewer, claim_id),
)
def stats(conn: Connection) -> dict:
return conn.execute(
"""
SELECT (SELECT count(*) FROM person) AS persons,
(SELECT count(*) FROM person_status WHERE status = 'active') AS active_persons,
(SELECT count(*) FROM organization WHERE kind = 'party') AS parties,
(SELECT count(*) FROM claim WHERE merged_into IS NULL AND review_status = 'published') AS claims,
(SELECT count(*) FROM claim WHERE merged_into IS NULL AND status = 'corroborated') AS corroborated,
(SELECT count(*) FROM claim WHERE review_status = 'pending') AS pending_review,
(SELECT count(*) FROM document) AS documents,
(SELECT count(*) FROM vote_event) AS vote_events,
(SELECT count(*) FROM vote_event WHERE roll_call) AS roll_call_votes,
(SELECT count(*) FROM event) AS events,
(SELECT count(*) FROM money_flow) AS money_flows,
(SELECT max(finished_at) FROM run WHERE status IN ('ok', 'partial')) AS last_update
"""
).fetchone()
def recent_runs(conn: Connection, limit: int = 50) -> list[dict]:
return conn.execute(
"SELECT adapter, mode, status, started_at, finished_at, stats, left(error, 2000) AS error FROM run "
"ORDER BY id DESC LIMIT %s",
(limit,),
).fetchall()
+157
View File
@@ -0,0 +1,157 @@
"""Claims (the generic fact list), timeline histogram and calendar."""
from __future__ import annotations
from datetime import date
from psycopg import Connection
from politrack.queries.common import VISIBLE, SourceFilter
def list_claims(
conn: Connection,
*,
sources: SourceFilter,
person: str | None = None,
party: str | None = None,
organization: str | None = None,
claim_types: list[str] | None = None,
topics: list[str] | None = None,
since: date | None = None,
until: date | None = None,
q: str | None = None,
include_pending: bool = False,
limit: int = 50,
offset: int = 0,
) -> list[dict]:
source_sql, params = sources.sql("s")
params.update({
"person": person, "party": party, "organization": organization, "claim_types": claim_types, "topics": topics, "since": since,
"until": until, "q": q, "limit": limit, "offset": offset, "include_pending": include_pending,
})
return conn.execute(
f"""
SELECT c.id, c.claim_type, c.summary_de, c.summary_en, c.occurred_at, c.valid_from, c.valid_to, c.status,
c.sensitive, c.review_status, c.data,
p.slug AS person_slug, p.full_name AS person_name,
o.slug AS organization_slug, o.short_name AS organization_short, o.name AS organization_name,
lp.stage AS legal_stage, lp.offense AS legal_offense, lp.authority AS legal_authority,
cp.slug AS counterpart_slug, cp.name AS counterpart_name, cp.kind AS counterpart_kind,
cpp.slug AS counterpart_person_slug, cpp.full_name AS counterpart_person_name,
to_jsonb(mf) - 'claim_id' AS money, st.platform AS platform,
(SELECT jsonb_agg(jsonb_build_object('slug', t.slug, 'name_de', t.name_de, 'name_en', t.name_en))
FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id WHERE ta.claim_id = c.id) AS topics,
ev.evidence, ev.source_count
FROM claim c
LEFT JOIN person p ON p.id = c.person_id
LEFT JOIN organization o ON o.id = c.organization_id
LEFT JOIN legal_proceeding lp ON lp.claim_id = c.id
LEFT JOIN organization cp ON cp.id = c.counterpart_org_id
LEFT JOIN person cpp ON cpp.id = c.counterpart_person_id
LEFT JOIN money_flow mf ON mf.claim_id = c.id
LEFT JOIN statement st ON st.claim_id = c.id
CROSS JOIN LATERAL (
SELECT jsonb_agg(jsonb_build_object(
'url', d.url, 'title', d.title, 'source', s.slug, 'source_name', s.name, 'source_kind', s.kind,
'published_at', d.published_at, 'quote', e.quote, 'stance', e.stance, 'archive_url', d.archive_url)
ORDER BY s.default_trust DESC, d.published_at) AS evidence,
count(DISTINCT s.id) AS source_count
FROM claim_evidence e JOIN document d ON d.id = e.document_id JOIN source s ON s.id = d.source_id
WHERE e.claim_id = c.id AND {source_sql}
) ev
WHERE ev.evidence IS NOT NULL AND {VISIBLE}
AND (%(person)s::text IS NULL OR p.slug = %(person)s OR cpp.slug = %(person)s)
AND (%(party)s::text IS NULL OR o.slug = %(party)s
OR c.counterpart_org_id = (SELECT id FROM organization WHERE slug = %(party)s))
AND (%(organization)s::text IS NULL OR o.slug = %(organization)s
OR c.counterpart_org_id = (SELECT id FROM organization WHERE slug = %(organization)s))
AND (%(claim_types)s::text[] IS NULL OR c.claim_type = ANY(%(claim_types)s))
AND (%(topics)s::text[] IS NULL OR c.claim_type IN ('role', 'career') OR EXISTS (
SELECT 1 FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id
WHERE ta.claim_id = c.id AND t.slug = ANY(%(topics)s)))
AND (%(since)s::date IS NULL OR COALESCE(c.occurred_at::date, c.valid_to, c.valid_from, current_date) >= %(since)s)
AND (%(until)s::date IS NULL OR COALESCE(c.occurred_at::date, c.valid_from) <= %(until)s)
AND (%(q)s::text IS NULL OR c.summary_de ILIKE '%%' || %(q)s || '%%')
ORDER BY COALESCE(c.occurred_at, c.valid_from::timestamptz, c.created_at) DESC, c.id DESC
LIMIT %(limit)s OFFSET %(offset)s
""",
params,
).fetchall()
def timeline_histogram(conn: Connection, *, sources: SourceFilter, person: str | None = None,
party: str | None = None) -> list[dict]:
"""Claim counts per month and type - drives the timeline overview."""
source_sql, params = sources.sql("s")
params.update({"person": person, "party": party, "include_pending": False})
return conn.execute(
f"""
SELECT to_char(date_trunc('month', COALESCE(c.occurred_at, c.valid_from::timestamptz)), 'YYYY-MM') AS month,
c.claim_type, count(DISTINCT c.id) AS count
FROM claim c
LEFT JOIN person p ON p.id = c.person_id
LEFT JOIN person cpp ON cpp.id = c.counterpart_person_id
LEFT JOIN organization o ON o.id = c.organization_id
JOIN claim_evidence e ON e.claim_id = c.id JOIN document d ON d.id = e.document_id
JOIN source s ON s.id = d.source_id
WHERE {VISIBLE} AND {source_sql} AND COALESCE(c.occurred_at, c.valid_from::timestamptz) IS NOT NULL
AND (%(person)s::text IS NULL OR p.slug = %(person)s OR cpp.slug = %(person)s)
AND (%(party)s::text IS NULL OR o.slug = %(party)s)
AND c.claim_type <> 'role'
GROUP BY 1, 2 ORDER BY 1
""",
params,
).fetchall()
def calendar(conn: Connection, *, sources: SourceFilter, start: date, end: date, person: str | None = None,
party: str | None = None, kinds: list[str] | None = None) -> dict:
"""Events (sessions, trips, meetings, appearances) plus per-day claim counts in [start, end]."""
source_sql, params = sources.sql("s")
params.update({"start": start, "end": end, "person": person, "party": party, "kinds": kinds,
"include_pending": False})
events = conn.execute(
"""
SELECT ev.id, ev.kind, ev.title, ev.starts_at, ev.ends_at, ev.location_name, ev.country, ev.url,
ev.description, o.name AS body,
COALESCE(jsonb_agg(DISTINCT jsonb_build_object('slug', p.slug, 'name', p.full_name))
FILTER (WHERE p.id IS NOT NULL), '[]'::jsonb) AS participants
FROM event ev
LEFT JOIN organization o ON o.id = ev.organization_id
LEFT JOIN event_participant ep ON ep.event_id = ev.id
LEFT JOIN claim c ON c.id = ep.claim_id
LEFT JOIN person p ON p.id = ep.person_id AND c.merged_into IS NULL
WHERE ev.starts_at::date BETWEEN %(start)s AND %(end)s
AND (%(kinds)s::text[] IS NULL OR ev.kind = ANY(%(kinds)s))
AND (%(person)s::text IS NULL OR EXISTS (
SELECT 1 FROM event_participant ep2 JOIN person p2 ON p2.id = ep2.person_id
JOIN claim c2 ON c2.id = ep2.claim_id AND c2.merged_into IS NULL
WHERE ep2.event_id = ev.id AND p2.slug = %(person)s))
AND (%(party)s::text IS NULL OR EXISTS (
SELECT 1 FROM event_participant ep3 JOIN person_status ps3 ON ps3.person_id = ep3.person_id
JOIN organization o3 ON o3.id = ANY(ps3.current_party_ids)
WHERE ep3.event_id = ev.id AND o3.slug = %(party)s))
GROUP BY ev.id, o.name
ORDER BY ev.starts_at
LIMIT 2000
""",
params,
).fetchall()
days = conn.execute(
f"""
SELECT c.occurred_at::date AS day, c.claim_type, count(DISTINCT c.id) AS count
FROM claim c
LEFT JOIN person p ON p.id = c.person_id
LEFT JOIN person cpp ON cpp.id = c.counterpart_person_id
LEFT JOIN organization o ON o.id = c.organization_id
JOIN claim_evidence e ON e.claim_id = c.id JOIN document d ON d.id = e.document_id
JOIN source s ON s.id = d.source_id
WHERE {VISIBLE} AND {source_sql} AND c.occurred_at::date BETWEEN %(start)s AND %(end)s
AND (%(person)s::text IS NULL OR p.slug = %(person)s OR cpp.slug = %(person)s)
AND (%(party)s::text IS NULL OR o.slug = %(party)s)
GROUP BY 1, 2 ORDER BY 1
""",
params,
).fetchall()
return {"events": events, "days": days}
+41
View File
@@ -0,0 +1,41 @@
"""Rules shared by all read queries.
* Source filtering happens at the evidence level: a claim is returned only if at least one document comes
from an allowed source, and only allowed evidence is shown.
* Claims merged into another claim (corroboration) are hidden; the survivor carries all evidence.
* Claims pending human review are hidden unless include_pending is set (moderation).
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
VISIBLE = ("c.merged_into IS NULL AND (c.review_status = 'published' OR "
"(%(include_pending)s AND c.review_status = 'pending'))")
@dataclass
class SourceFilter:
exclude_sources: list[str] = field(default_factory=list)
min_trust: int = 0
def sql(self, alias: str = "s") -> tuple[str, dict[str, Any]]:
return (
f"(NOT ({alias}.slug = ANY(%(exclude_sources)s)) AND {alias}.default_trust >= %(min_trust)s)",
{"exclude_sources": self.exclude_sources, "min_trust": self.min_trust},
)
def evidence_json(source_sql: str) -> str:
"""Lateral subquery aggregating the allowed evidence of claim `c`."""
return f"""
CROSS JOIN LATERAL (
SELECT jsonb_agg(jsonb_build_object(
'url', d.url, 'title', d.title, 'source', s.slug, 'source_name', s.name, 'source_kind', s.kind,
'published_at', d.published_at, 'quote', e.quote, 'stance', e.stance, 'archive_url', d.archive_url)
ORDER BY s.default_trust DESC, d.published_at) AS evidence,
count(DISTINCT s.id) AS source_count
FROM claim_evidence e JOIN document d ON d.id = e.document_id JOIN source s ON s.id = d.source_id
WHERE e.claim_id = c.id AND {source_sql}
) ev"""
+114
View File
@@ -0,0 +1,114 @@
"""Money flows (donations, contracts, government ads, party funding, political ads), social posts, wiki edits."""
from __future__ import annotations
from datetime import date
from psycopg import Connection
from politrack.queries.common import VISIBLE, SourceFilter, evidence_json
def list_money(conn: Connection, *, sources: SourceFilter, kinds: list[str] | None = None, org: str | None = None,
person: str | None = None, q: str | None = None, since: date | None = None, until: date | None = None,
min_amount: float | None = None, limit: int = 50, offset: int = 0) -> list[dict]:
"""Money flows, filterable by kind, organisation (either side), connected person and period.
`person` = flows of organisations connected to that person (side jobs, careers, party) - the
"follow the money" view.
"""
source_sql, params = sources.sql("s")
params.update({"kinds": kinds, "org": org, "person": person, "q": q, "since": since, "until": until,
"min_amount": min_amount, "limit": limit, "offset": offset, "include_pending": False})
return conn.execute(
f"""
WITH person_orgs AS (
SELECT c.counterpart_org_id AS org_id FROM claim c JOIN person p ON p.id = c.person_id
WHERE p.slug = %(person)s AND c.counterpart_org_id IS NOT NULL AND c.merged_into IS NULL
UNION SELECT r.party_id FROM role r JOIN person p ON p.id = r.person_id WHERE p.slug = %(person)s
)
SELECT c.id, c.summary_de, c.summary_en, c.occurred_at, c.status, c.data, mf.kind, mf.amount, mf.amount_category,
mf.currency, mf.paid_on, mf.period, mf.reference, mf.category, mf.payer_name, mf.payee_name,
pa.slug AS payer_slug, pa.kind AS payer_kind, pe.slug AS payee_slug, pe.kind AS payee_kind,
ev.evidence, ev.source_count
FROM money_flow mf
JOIN claim c ON c.id = mf.claim_id
LEFT JOIN organization pa ON pa.id = mf.payer_org_id
LEFT JOIN organization pe ON pe.id = mf.payee_org_id
{evidence_json(source_sql)}
WHERE ev.evidence IS NOT NULL AND {VISIBLE}
AND (%(kinds)s::text[] IS NULL OR mf.kind = ANY(%(kinds)s))
AND (%(org)s::text IS NULL OR pa.slug = %(org)s OR pe.slug = %(org)s)
AND (%(person)s::text IS NULL OR mf.payer_org_id IN (SELECT org_id FROM person_orgs)
OR mf.payee_org_id IN (SELECT org_id FROM person_orgs))
AND (%(q)s::text IS NULL OR mf.payer_name ILIKE '%%' || %(q)s || '%%' OR mf.payee_name ILIKE '%%' || %(q)s || '%%'
OR c.summary_de ILIKE '%%' || %(q)s || '%%')
AND (%(since)s::date IS NULL OR COALESCE(mf.paid_on, c.occurred_at::date, c.valid_from) >= %(since)s)
AND (%(until)s::date IS NULL OR COALESCE(mf.paid_on, c.occurred_at::date, c.valid_from) <= %(until)s)
AND (%(min_amount)s::numeric IS NULL OR mf.amount >= %(min_amount)s)
ORDER BY COALESCE(mf.paid_on, c.occurred_at::date, c.valid_from) DESC NULLS LAST, mf.amount DESC NULLS LAST
LIMIT %(limit)s OFFSET %(offset)s
""",
params,
).fetchall()
def money_summary(conn: Connection, *, kinds: list[str] | None = None, since: date | None = None) -> list[dict]:
"""Top payers/payees per kind (for the money overview page)."""
return conn.execute(
"""
SELECT mf.kind, o.slug, o.name, o.kind AS org_kind,
CASE WHEN o.id = mf.payer_org_id THEN 'payer' ELSE 'payee' END AS side,
sum(mf.amount) AS total, count(*) AS count
FROM money_flow mf JOIN claim c ON c.id = mf.claim_id AND c.merged_into IS NULL
JOIN organization o ON o.id IN (mf.payer_org_id, mf.payee_org_id)
WHERE (%(kinds)s::text[] IS NULL OR mf.kind = ANY(%(kinds)s))
AND (%(since)s::date IS NULL OR COALESCE(mf.paid_on, c.valid_from) >= %(since)s)
GROUP BY 1, 2, 3, 4, 5
ORDER BY total DESC NULLS LAST LIMIT 100
""",
{"kinds": kinds, "since": since},
).fetchall()
def social_feed(conn: Connection, *, sources: SourceFilter, person: str | None = None, party: str | None = None,
platforms: list[str] | None = None, limit: int = 50, offset: int = 0) -> list[dict]:
"""Public posts of politicians and of party/institution channels (`party` = the party's own channels
plus posts of its current members)."""
source_sql, params = sources.sql("s")
params.update({"person": person, "party": party, "platforms": platforms, "limit": limit, "offset": offset,
"include_pending": False})
return conn.execute(
f"""
SELECT c.id, st.platform, st.said_at, st.excerpt, st.url, st.title, c.data, p.slug AS person_slug,
p.full_name AS person_name, o.slug AS org_slug, o.name AS org_name, ev.evidence
FROM statement st JOIN claim c ON c.id = st.claim_id
LEFT JOIN person p ON p.id = st.person_id
LEFT JOIN organization o ON o.id = c.organization_id AND st.person_id IS NULL
{evidence_json(source_sql)}
WHERE st.context = 'social_post' AND ev.evidence IS NOT NULL AND {VISIBLE}
AND (%(person)s::text IS NULL OR p.slug = %(person)s)
AND (%(platforms)s::text[] IS NULL OR st.platform = ANY(%(platforms)s))
AND (%(party)s::text IS NULL OR o.slug = %(party)s OR EXISTS (
SELECT 1 FROM person_status ps JOIN organization po ON po.id = ANY(ps.current_party_ids)
WHERE ps.person_id = p.id AND po.slug = %(party)s))
ORDER BY st.said_at DESC NULLS LAST
LIMIT %(limit)s OFFSET %(offset)s
""",
params,
).fetchall()
def wiki_edits(conn: Connection, *, limit: int = 100, offset: int = 0) -> list[dict]:
return conn.execute(
"""
SELECT c.id, c.occurred_at, c.summary_de, c.summary_en, c.data, o.slug AS org_slug, o.name AS org_name,
p.slug AS person_slug, p.full_name AS person_name,
(SELECT d.url FROM claim_evidence e JOIN document d ON d.id = e.document_id WHERE e.claim_id = c.id LIMIT 1) AS url
FROM claim c JOIN organization o ON o.id = c.organization_id
LEFT JOIN person p ON p.id = c.counterpart_person_id
WHERE c.claim_type = 'wiki_edit' AND c.merged_into IS NULL
ORDER BY c.occurred_at DESC NULLS LAST LIMIT %s OFFSET %s
""",
(limit, offset),
).fetchall()
+160
View File
@@ -0,0 +1,160 @@
"""Parties and organisations (companies, public bodies, media owners, lobby groups)."""
from __future__ import annotations
from psycopg import Connection
def list_parties(conn: Connection) -> list[dict]:
return conn.execute(
"""
SELECT o.slug, o.name, o.short_name, o.color, o.logo_url,
(SELECT count(*) FROM person_status ps WHERE ps.status = 'active' AND o.id = ANY(ps.current_party_ids))
AS active_officials,
count(DISTINCT r.person_id) AS total_people
FROM organization o LEFT JOIN role r ON r.party_id = o.id
WHERE o.kind = 'party'
GROUP BY o.id
ORDER BY active_officials DESC, total_people DESC, o.name
"""
).fetchall()
def get_party(conn: Connection, slug: str) -> dict | None:
party = conn.execute(
"SELECT id, slug, name, short_name, color, logo_url, kind FROM organization WHERE slug = %s", (slug,)
).fetchone()
if not party:
return None
pid = party["id"]
party["members"] = conn.execute(
"""
SELECT DISTINCT ON (p.id) p.slug, p.full_name, p.photo_url, r.role, r.level, r.region, r.start_date,
ch.name AS body
FROM role r
JOIN claim c ON c.id = r.claim_id AND c.merged_into IS NULL
JOIN person p ON p.id = r.person_id
JOIN person_status ps ON ps.person_id = p.id AND ps.status = 'active'
LEFT JOIN organization ch ON ch.id = r.organization_id
WHERE r.party_id = %s AND (r.end_date IS NULL OR r.end_date >= current_date) AND r.level <> 'party'
AND r.role = ANY(ps.current_roles)
ORDER BY p.id, CASE r.level WHEN 'eu' THEN 0 WHEN 'federal' THEN 1 WHEN 'state' THEN 2 ELSE 3 END
""",
(pid,),
).fetchall()
party["former_members"] = conn.execute(
"""
SELECT p.slug, p.full_name, ps.last_office_end,
(SELECT max(c.occurred_at) FROM claim c WHERE c.person_id = p.id AND c.claim_type = 'media_mention')
AS last_mention
FROM person p JOIN person_status ps ON ps.person_id = p.id
WHERE ps.status IN ('former', 'party_member') AND EXISTS (SELECT 1 FROM role r WHERE r.person_id = p.id AND r.party_id = %s)
ORDER BY last_mention DESC NULLS LAST, ps.last_office_end DESC NULLS LAST LIMIT 50
""",
(pid,),
).fetchall()
party["money_by_year"] = conn.execute(
"""
SELECT COALESCE(extract(year FROM mf.paid_on)::int, left(mf.period, 4)::int) AS year, mf.kind,
sum(mf.amount) AS total, count(*) AS count
FROM money_flow mf JOIN claim c ON c.id = mf.claim_id AND c.merged_into IS NULL
WHERE (mf.payee_org_id = %(id)s OR c.organization_id = %(id)s)
AND mf.kind IN ('donation', 'sponsorship', 'party_ad', 'party_funding', 'loan')
AND COALESCE(mf.paid_on::text, mf.period) ~ '^\\d{4}'
GROUP BY 1, 2 ORDER BY 1 DESC, 2
""",
{"id": pid},
).fetchall()
party["finance"] = conn.execute(
"""SELECT c.data, c.valid_from FROM claim c WHERE c.organization_id = %s AND c.claim_type = 'party_finance'
AND c.merged_into IS NULL ORDER BY c.valid_from DESC NULLS LAST LIMIT 5""",
(pid,),
).fetchall()
party["vote_stats"] = conn.execute(
"SELECT vp.position, count(*) AS count FROM vote_position vp WHERE vp.organization_id = %s GROUP BY 1", (pid,)
).fetchall()
party["dissent"] = conn.execute(
"""
SELECT p.slug, p.full_name, count(*) AS dissenting_votes
FROM vote_position vp
JOIN person p ON p.id = vp.person_id
JOIN vote_position fp ON fp.vote_event_id = vp.vote_event_id AND fp.organization_id = vp.party_id
WHERE vp.party_id = %s AND vp.position <> fp.position
GROUP BY p.id ORDER BY dissenting_votes DESC LIMIT 20
""",
(pid,),
).fetchall()
party["related"] = conn.execute(
"""SELECT c.claim_type, o.slug, o.name, o.kind FROM claim c JOIN organization o ON o.id = c.counterpart_org_id
WHERE c.organization_id = %s AND c.claim_type IN ('ownership', 'affiliation') AND c.merged_into IS NULL
ORDER BY c.claim_type, o.name""",
(pid,),
).fetchall()
return party
def list_organizations(conn: Connection, *, q: str | None = None, kind: str | None = None, limit: int = 50,
offset: int = 0) -> list[dict]:
return conn.execute(
"""
SELECT o.slug, o.name, o.short_name, o.kind,
(SELECT sum(amount) FROM money_flow WHERE payer_org_id = o.id) AS paid,
(SELECT sum(amount) FROM money_flow WHERE payee_org_id = o.id) AS received
FROM organization o
WHERE (%(q)s::text IS NULL OR o.name ILIKE '%%' || %(q)s || '%%'
OR EXISTS (SELECT 1 FROM alias a WHERE a.entity_type = 'organization' AND a.entity_id = o.id
AND a.alias_norm LIKE '%%' || lower(unaccent(%(q)s)) || '%%'))
AND (%(kind)s::text IS NULL OR o.kind = %(kind)s)
ORDER BY COALESCE((SELECT sum(amount) FROM money_flow WHERE payer_org_id = o.id OR payee_org_id = o.id), 0) DESC,
o.name
LIMIT %(limit)s OFFSET %(offset)s
""",
{"q": q, "kind": kind, "limit": limit, "offset": offset},
).fetchall()
def get_organization(conn: Connection, slug: str) -> dict | None:
org = conn.execute(
"""SELECT o.id, o.slug, o.name, o.short_name, o.kind, parent.slug AS parent_slug, parent.name AS parent_name,
(SELECT jsonb_object_agg(i.scheme, i.value) FROM identifier i
WHERE i.entity_type = 'organization' AND i.entity_id = o.id) AS identifiers
FROM organization o LEFT JOIN organization parent ON parent.id = o.parent_id WHERE o.slug = %s""",
(slug,),
).fetchone()
if not org:
return None
oid = org["id"]
org["money"] = conn.execute(
"""
SELECT mf.kind, CASE WHEN mf.payer_org_id = %(id)s THEN 'paid' ELSE 'received' END AS direction,
sum(mf.amount) AS total, count(*) AS count, min(COALESCE(mf.paid_on::text, mf.period)) AS first,
max(COALESCE(mf.paid_on::text, mf.period)) AS last
FROM money_flow mf JOIN claim c ON c.id = mf.claim_id AND c.merged_into IS NULL
WHERE mf.payer_org_id = %(id)s OR mf.payee_org_id = %(id)s
GROUP BY 1, 2 ORDER BY 3 DESC NULLS LAST
""",
{"id": oid},
).fetchall()
org["counterparties"] = conn.execute(
"""
SELECT o.slug, o.name, o.kind, mf.kind AS money_kind,
CASE WHEN mf.payer_org_id = %(id)s THEN 'paid_to' ELSE 'received_from' END AS direction,
sum(mf.amount) AS total, count(*) AS count
FROM money_flow mf JOIN claim c ON c.id = mf.claim_id AND c.merged_into IS NULL
JOIN organization o ON o.id = CASE WHEN mf.payer_org_id = %(id)s THEN mf.payee_org_id ELSE mf.payer_org_id END
WHERE mf.payer_org_id = %(id)s OR mf.payee_org_id = %(id)s
GROUP BY 1, 2, 3, 4, 5 ORDER BY total DESC NULLS LAST LIMIT 50
""",
{"id": oid},
).fetchall()
org["people"] = conn.execute(
"""
SELECT DISTINCT ON (p.id, c.claim_type) p.slug, p.full_name, c.claim_type AS via, c.summary_de
FROM claim c JOIN person p ON p.id = c.person_id
WHERE c.counterpart_org_id = %(id)s AND c.merged_into IS NULL AND c.review_status = 'published'
AND c.claim_type IN ('side_job', 'career', 'meeting', 'role', 'lobbying')
ORDER BY p.id, c.claim_type LIMIT 100
""",
{"id": oid},
).fetchall()
return org
+146
View File
@@ -0,0 +1,146 @@
"""Persons: lists, profiles, former politicians, connections (network)."""
from __future__ import annotations
from psycopg import Connection
def list_persons(conn: Connection, *, q: str | None = None, party: str | None = None, level: str | None = None,
status: str | None = "active", limit: int = 50, offset: int = 0) -> list[dict]:
"""status: active | former | party_member | deceased | None (all)."""
return conn.execute(
"""
SELECT p.slug, p.full_name, p.photo_url, ps.status, ps.verified AS status_verified, ps.last_office_end,
ps.current_roles,
(SELECT jsonb_agg(DISTINCT jsonb_build_object('slug', o.slug, 'short_name', o.short_name, 'color', o.color))
FROM organization o WHERE o.id = ANY(ps.current_party_ids)) AS parties
FROM person p
JOIN person_status ps ON ps.person_id = p.id
WHERE (%(status)s::text IS NULL OR ps.status = %(status)s)
AND (%(q)s::text IS NULL OR p.full_name ILIKE '%%' || %(q)s || '%%'
OR EXISTS (SELECT 1 FROM alias a WHERE a.entity_type = 'person' AND a.entity_id = p.id
AND a.alias_norm LIKE '%%' || lower(unaccent(%(q)s)) || '%%'))
AND (%(party)s::text IS NULL OR EXISTS (
SELECT 1 FROM role r JOIN organization o ON o.id = r.party_id
WHERE r.person_id = p.id AND o.slug = %(party)s
AND (ps.status <> 'active' OR o.id = ANY(ps.current_party_ids))))
AND (%(level)s::text IS NULL OR EXISTS (
SELECT 1 FROM role r WHERE r.person_id = p.id AND r.level = %(level)s
AND (ps.status <> 'active' OR r.role = ANY(ps.current_roles))))
AND EXISTS (SELECT 1 FROM role r WHERE r.person_id = p.id)
ORDER BY p.family_name NULLS LAST, p.full_name
LIMIT %(limit)s OFFSET %(offset)s
""",
{"q": q, "party": party, "level": level, "status": status, "limit": limit, "offset": offset},
).fetchall()
def get_person(conn: Connection, slug: str) -> dict | None:
person = conn.execute(
"""
SELECT p.id, p.slug, p.full_name, p.given_name, p.family_name, p.honorific_prefix, p.birth_date, p.death_date,
p.photo_url, pi.wikidata_qid, pi.parlament_pad, pi.ep_id, p.summary_de, p.summary_en, ps.status,
ps.verified AS status_verified, ps.last_office_end
FROM person p JOIN person_status ps ON ps.person_id = p.id JOIN person_identifier pi ON pi.person_id = p.id
WHERE p.slug = %s
""",
(slug,),
).fetchone()
if not person:
return None
person["roles"] = conn.execute(
"""
SELECT r.role, r.level, r.region, r.start_date, r.end_date, o.name AS body, o.slug AS body_slug,
pa.slug AS party_slug, pa.short_name AS party_short, pa.color AS party_color, c.status AS claim_status,
(SELECT jsonb_agg(DISTINCT jsonb_build_object('url', d.url, 'source', s.slug))
FROM claim_evidence e JOIN document d ON d.id = e.document_id JOIN source s ON s.id = d.source_id
WHERE e.claim_id = r.claim_id) AS evidence
FROM role r
JOIN claim c ON c.id = r.claim_id AND c.merged_into IS NULL
LEFT JOIN organization o ON o.id = r.organization_id
LEFT JOIN organization pa ON pa.id = r.party_id
WHERE r.person_id = %s
ORDER BY r.end_date DESC NULLS FIRST, r.start_date DESC NULLS LAST
""",
(person["id"],),
).fetchall()
person["social_accounts"] = conn.execute(
"SELECT platform, handle, url, verified FROM social_account WHERE person_id = %s ORDER BY verified DESC, platform",
(person["id"],),
).fetchall()
person["claim_counts"] = {
r["claim_type"]: r["count"]
for r in conn.execute(
"SELECT claim_type, count(*) AS count FROM claim WHERE person_id = %s AND merged_into IS NULL "
"AND review_status = 'published' GROUP BY claim_type",
(person["id"],),
)
}
return person
def former_overview(conn: Connection, *, party: str | None = None, limit: int = 50, offset: int = 0) -> list[dict]:
"""Former politicians, most recently in the news first ("Was macht eigentlich ...?")."""
return conn.execute(
"""
SELECT p.slug, p.full_name, p.photo_url, ps.status, ps.last_office_end,
(SELECT r.role FROM role r JOIN claim c ON c.id = r.claim_id AND c.merged_into IS NULL
WHERE r.person_id = p.id AND r.level <> 'party' ORDER BY r.end_date DESC NULLS LAST LIMIT 1) AS last_role,
latest.summary_de AS latest_news, latest.occurred_at AS latest_news_at, latest.url AS latest_news_url,
career.summary_de AS latest_career
FROM person p
JOIN person_status ps ON ps.person_id = p.id
LEFT JOIN LATERAL (
SELECT c.summary_de, c.occurred_at, d.url FROM claim c
JOIN claim_evidence e ON e.claim_id = c.id JOIN document d ON d.id = e.document_id
WHERE c.person_id = p.id AND c.claim_type IN ('media_mention', 'statement') AND c.merged_into IS NULL
ORDER BY c.occurred_at DESC NULLS LAST LIMIT 1) latest ON true
LEFT JOIN LATERAL (
SELECT c.summary_de FROM claim c WHERE c.person_id = p.id AND c.claim_type = 'career'
AND c.merged_into IS NULL ORDER BY c.valid_from DESC NULLS LAST LIMIT 1) career ON true
WHERE ps.status IN ('former', 'party_member')
AND (%(party)s::text IS NULL OR EXISTS (
SELECT 1 FROM role r JOIN organization o ON o.id = r.party_id
WHERE r.person_id = p.id AND o.slug = %(party)s))
AND (latest.occurred_at IS NOT NULL OR career.summary_de IS NOT NULL)
ORDER BY latest.occurred_at DESC NULLS LAST, ps.last_office_end DESC NULLS LAST
LIMIT %(limit)s OFFSET %(offset)s
""",
{"party": party, "limit": limit, "offset": offset},
).fetchall()
def person_network(conn: Connection, slug: str) -> dict:
"""Organisations connected to a person and what flows through them.
Connections: employers/boards from side-job disclosures and careers, parties, orgs met (meetings).
For each organisation: donations it made, contracts it won, ads/funding it received or paid.
"""
person = conn.execute("SELECT id FROM person WHERE slug = %s", (slug,)).fetchone()
if not person:
return {"organizations": []}
orgs = conn.execute(
"""
WITH links AS (
SELECT c.counterpart_org_id AS org_id, c.claim_type AS via, max(c.summary_de) AS label
FROM claim c
WHERE c.person_id = %(pid)s AND c.counterpart_org_id IS NOT NULL AND c.merged_into IS NULL
AND c.review_status = 'published'
AND c.claim_type IN ('side_job', 'career', 'meeting', 'role', 'lobbying')
GROUP BY 1, 2
UNION
SELECT r.party_id, 'party', NULL FROM role r WHERE r.person_id = %(pid)s AND r.party_id IS NOT NULL
)
SELECT o.id, o.slug, o.name, o.kind, array_agg(DISTINCT l.via) AS via, max(l.label) AS label,
(SELECT jsonb_object_agg(k, v) FROM (
SELECT mf.kind || CASE WHEN mf.payer_org_id = o.id THEN '_paid' ELSE '_received' END AS k,
jsonb_build_object('total', sum(mf.amount), 'count', count(*)) AS v
FROM money_flow mf JOIN claim mc ON mc.id = mf.claim_id AND mc.merged_into IS NULL
WHERE mf.payer_org_id = o.id OR mf.payee_org_id = o.id GROUP BY 1) x) AS money
FROM links l JOIN organization o ON o.id = l.org_id
GROUP BY o.id
ORDER BY (o.kind = 'party') DESC, o.name
""",
{"pid": person["id"]},
).fetchall()
return {"organizations": orgs}
+68
View File
@@ -0,0 +1,68 @@
"""Hybrid search (German full text + optional vector similarity)."""
from __future__ import annotations
from datetime import date
import httpx
from psycopg import Connection
from politrack.config import get_settings
from politrack.queries.common import VISIBLE, SourceFilter
def search(conn: Connection, q: str, *, sources: SourceFilter, since: date | None = None, until: date | None = None,
limit: int = 20) -> list[dict]:
"""Hybrid search: German full-text + (if configured) vector similarity, fused by reciprocal rank."""
settings = get_settings()
source_sql, params = sources.sql("s")
params.update({"q": q, "limit": limit * 3, "since": since, "until": until, "include_pending": False})
base = f"""
SELECT ch.id, ch.text, ch.claim_id, ch.document_id, c.summary_de, c.claim_type, c.occurred_at,
p.slug AS person_slug, p.full_name AS person_name, o.slug AS organization_slug,
dd.url AS document_url, dd.title AS document_title, dd.published_at AS document_date,
(SELECT jsonb_agg(DISTINCT jsonb_build_object('url', d.url, 'title', d.title, 'source', s.slug))
FROM claim_evidence e JOIN document d ON d.id = e.document_id JOIN source s ON s.id = d.source_id
WHERE e.claim_id = ch.claim_id AND {source_sql}) AS evidence
FROM chunk ch
LEFT JOIN claim c ON c.id = ch.claim_id
LEFT JOIN person p ON p.id = c.person_id
LEFT JOIN organization o ON o.id = c.organization_id
LEFT JOIN document dd ON dd.id = ch.document_id
LEFT JOIN source ds ON ds.id = dd.source_id
WHERE (ch.claim_id IS NULL OR ({VISIBLE}))
AND (ch.document_id IS NULL OR (NOT (ds.slug = ANY(%(exclude_sources)s)) AND ds.default_trust >= %(min_trust)s))
AND (%(since)s::date IS NULL OR COALESCE(c.occurred_at::date, c.valid_from, dd.published_at::date) >= %(since)s)
AND (%(until)s::date IS NULL OR COALESCE(c.occurred_at::date, c.valid_from, dd.published_at::date) <= %(until)s)
"""
fts = conn.execute(
base + " AND ch.tsv @@ websearch_to_tsquery('german', %(q)s)"
" ORDER BY ts_rank_cd(ch.tsv, websearch_to_tsquery('german', %(q)s)) DESC LIMIT %(limit)s",
params,
).fetchall()
ranked: dict[int, tuple[float, dict]] = {}
for rank, row in enumerate(fts):
ranked[row["id"]] = (1 / (60 + rank), row)
if settings.ollama_url:
try:
resp = httpx.post(f"{settings.ollama_url}/api/embed",
json={"model": settings.embedding_model, "input": [q]}, timeout=30)
resp.raise_for_status()
params["vec"] = str(resp.json()["embeddings"][0])
vec_rows = conn.execute(
base + " AND ch.embedding IS NOT NULL ORDER BY ch.embedding <=> %(vec)s::vector LIMIT %(limit)s",
params,
).fetchall()
for rank, row in enumerate(vec_rows):
score = ranked.get(row["id"], (0.0, row))[0] + 1 / (60 + rank)
ranked[row["id"]] = (score, row)
except httpx.HTTPError:
pass
results = [row | {"score": score} for score, row in sorted(ranked.values(), key=lambda x: -x[0])]
# Hide results whose only evidence comes from sources the user excluded.
return [r for r in results if r["evidence"] or r["claim_id"] is None][:limit]
# -- moderation -----------------------------------------------------------------
+127
View File
@@ -0,0 +1,127 @@
"""Votes: per person (individual roll calls + faction votes during mandates, with "say vs. vote"), lists, details."""
from __future__ import annotations
from datetime import date
from psycopg import Connection
# Did the person speak on this item on the day of the vote? Speech titles are the item titles of the
# parliament's speech list; vote_event titles come from the item page - similar, not identical.
SPOKE = """
(SELECT jsonb_build_object('url', st.url, 'title', st.title) FROM statement st
JOIN claim sc ON sc.id = st.claim_id AND sc.merged_into IS NULL
WHERE st.person_id = me.id AND st.context = 'parliament_speech' AND st.said_at::date = ve.voted_on
AND similarity(st.title, ve.item_title) > 0.35
ORDER BY similarity(st.title, ve.item_title) DESC LIMIT 1)
"""
def person_votes(conn: Connection, slug: str, *, topics: list[str] | None = None, limit: int = 100,
offset: int = 0) -> list[dict]:
"""Votes concerning a person.
* individual=true: roll-call votes with the person's own vote; dissent=true when it differs from the
majority of their faction (vote_position.party_id = their faction that day).
* individual=false: faction votes cast while the person held a mandate for that faction.
* spoke: the person's speech on that item on the same day ("say vs. vote"), if any.
"""
return conn.execute(
f"""
WITH me AS (SELECT id FROM person WHERE slug = %(slug)s)
SELECT ve.id, ve.item_title, ve.item_ref, ve.stage, ve.voted_on, ve.result, ve.url, ve.protocol_url,
ve.roll_call, vp.position, fo.short_name AS faction, ch.name AS chamber, true AS individual,
(fp.position IS NOT NULL AND fp.position <> vp.position) AS dissent, {SPOKE} AS spoke,
(SELECT jsonb_agg(DISTINCT t.slug) FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id
WHERE ta.claim_id = vp.claim_id) AS topics
FROM vote_position vp
JOIN me ON me.id = vp.person_id
JOIN claim c ON c.id = vp.claim_id AND c.merged_into IS NULL
JOIN vote_event ve ON ve.id = vp.vote_event_id
JOIN organization ch ON ch.id = ve.chamber_id
LEFT JOIN organization fo ON fo.id = vp.party_id
LEFT JOIN vote_position fp ON fp.vote_event_id = ve.id AND fp.organization_id = vp.party_id
WHERE (%(topics)s::text[] IS NULL OR EXISTS (
SELECT 1 FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id
WHERE ta.claim_id = vp.claim_id AND t.slug = ANY(%(topics)s)))
UNION ALL
SELECT ve.id, ve.item_title, ve.item_ref, ve.stage, ve.voted_on, ve.result, ve.url, ve.protocol_url,
ve.roll_call, vp.position, o.short_name, ch.name, false, false, {SPOKE},
(SELECT jsonb_agg(DISTINCT t.slug) FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id
WHERE ta.claim_id = vp.claim_id)
FROM me
JOIN role r ON r.person_id = me.id AND r.role = 'mp'
JOIN claim rc ON rc.id = r.claim_id AND rc.merged_into IS NULL
JOIN vote_event ve ON ve.chamber_id = r.organization_id AND NOT ve.roll_call
AND ve.voted_on BETWEEN r.start_date AND COALESCE(r.end_date, 'infinity'::date)
JOIN vote_position vp ON vp.vote_event_id = ve.id AND vp.organization_id = r.party_id
JOIN organization o ON o.id = vp.organization_id
JOIN organization ch ON ch.id = ve.chamber_id
WHERE (%(topics)s::text[] IS NULL OR EXISTS (
SELECT 1 FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id
WHERE ta.claim_id = vp.claim_id AND t.slug = ANY(%(topics)s)))
ORDER BY voted_on DESC NULLS LAST
LIMIT %(limit)s OFFSET %(offset)s
""",
{"slug": slug, "topics": topics, "limit": limit, "offset": offset},
).fetchall()
def list_votes(conn: Connection, *, party: str | None = None, topics: list[str] | None = None,
roll_call: bool | None = None, since: date | None = None, until: date | None = None,
limit: int = 50, offset: int = 0) -> list[dict]:
return conn.execute(
"""
SELECT ve.id, ve.item_title, ve.item_ref, ve.stage, ve.voted_on, ve.result, ve.url, ve.protocol_url,
ve.raw_text, ve.roll_call, ch.name AS chamber,
COALESCE(jsonb_agg(jsonb_build_object('party', o.short_name, 'slug', o.slug, 'color', o.color,
'position', vp.position) ORDER BY o.short_name)
FILTER (WHERE o.id IS NOT NULL), '[]'::jsonb) AS positions,
(SELECT count(*) FROM vote_position x WHERE x.vote_event_id = ve.id AND x.person_id IS NOT NULL)
AS individual_votes
FROM vote_event ve
JOIN organization ch ON ch.id = ve.chamber_id
LEFT JOIN vote_position vp ON vp.vote_event_id = ve.id AND vp.organization_id IS NOT NULL
LEFT JOIN organization o ON o.id = vp.organization_id
WHERE (%(party)s::text IS NULL OR EXISTS (
SELECT 1 FROM vote_position v2 JOIN organization o2 ON o2.id = v2.organization_id
WHERE v2.vote_event_id = ve.id AND o2.slug = %(party)s))
AND (%(topics)s::text[] IS NULL OR EXISTS (
SELECT 1 FROM vote_position v3 JOIN topic_assignment ta ON ta.claim_id = v3.claim_id
JOIN topic t ON t.id = ta.topic_id WHERE v3.vote_event_id = ve.id AND t.slug = ANY(%(topics)s)))
AND (%(roll_call)s::boolean IS NULL OR ve.roll_call = %(roll_call)s)
AND (%(since)s::date IS NULL OR ve.voted_on >= %(since)s)
AND (%(until)s::date IS NULL OR ve.voted_on <= %(until)s)
GROUP BY ve.id, ch.name
ORDER BY ve.voted_on DESC NULLS LAST, ve.id DESC
LIMIT %(limit)s OFFSET %(offset)s
""",
{"party": party, "topics": topics, "roll_call": roll_call, "since": since, "until": until,
"limit": limit, "offset": offset},
).fetchall()
def vote_detail(conn: Connection, vote_id: int) -> dict | None:
vote = conn.execute(
"SELECT ve.*, ch.name AS chamber FROM vote_event ve JOIN organization ch ON ch.id = ve.chamber_id WHERE ve.id = %s",
(vote_id,),
).fetchone()
if not vote:
return None
vote["positions"] = conn.execute(
"""SELECT o.short_name AS party, o.slug, o.color, vp.position FROM vote_position vp
JOIN organization o ON o.id = vp.organization_id WHERE vp.vote_event_id = %s ORDER BY o.short_name""",
(vote_id,),
).fetchall()
vote["individual"] = conn.execute(
"""
SELECT p.slug, p.full_name, vp.position, o.short_name AS faction,
(fp.position IS NOT NULL AND fp.position <> vp.position) AS dissent
FROM vote_position vp JOIN person p ON p.id = vp.person_id
LEFT JOIN organization o ON o.id = vp.party_id
LEFT JOIN vote_position fp ON fp.vote_event_id = vp.vote_event_id AND fp.organization_id = vp.party_id
WHERE vp.vote_event_id = %s ORDER BY o.short_name, p.family_name
""",
(vote_id,),
).fetchall()
return vote
+5
View File
@@ -0,0 +1,5 @@
"""Built-in source adapters. Every module in this package (and sub-packages) is imported by
politrack.core.registry.load_builtin(); modules register their adapters with @register.
To add a source: create one module here (see docs/contributing/adapters.md).
"""
@@ -0,0 +1,183 @@
"""Bundeskanzleramt: government news archive and Council of Ministers minutes (Ministerratsprotokolle).
Official works (§7 UrhG); robots.txt only blocks /401.
* News ("Nachrichten der Bundesregierung", one index per year): stored with full text; the event
extractor turns summits, trips abroad and bilateral meetings into travel/meeting claims.
* Council of Ministers minutes (Beschlussprotokolle): every agenda item names the reporting office(s)
("Bericht der Bundesministerin für Justiz, Zahl ..., betreffend ..."). Each becomes a
government_decision claim for the office holder on that date (resolved via the office hint).
"""
from __future__ import annotations
import re
from collections.abc import Iterator
from datetime import date
from typing import Any
from urllib.parse import urljoin
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Document, Event, Evidence, OrgRef, Participation, PersonRef
from politrack.text import excerpt, fingerprint
BASE = "https://www.bundeskanzleramt.gv.at"
NEWS_INDEX = BASE + "/bundeskanzleramt/nachrichten-der-bundesregierung/{year}.html"
MR_INDEX = BASE + "/bundeskanzleramt/die-bundesregierung/ministerratsprotokolle.html"
SPEC = SourceSpec(
slug="bundesregierung",
name="Bundeskanzleramt – Bundesregierung",
kind="official",
homepage=BASE,
license="Amtliches Werk (§7 UrhG)",
access="scrape",
trust=90,
min_interval=2.0,
tos_notes="Government news archive and Council of Ministers minutes. robots.txt only blocks /401.",
)
MONTHS = {m: i for i, m in enumerate(["jänner", "februar", "märz", "april", "mai", "juni", "juli", "august",
"september", "oktober", "november", "dezember"], start=1)}
MONTHS["januar"] = 1
GERMAN_DATE = re.compile(r"(\d{1,2})\.\s*([A-Za-zäÄ]+)\s+(\d{4})")
# Office names may contain commas ("für Wohnen, Kunst, Kultur, Medien und Sport"); file numbers sometimes use
# non-breaking hyphens or spaces ("2026 0.040.056"); subjects span lines (joined with " | ") and end at the
# decision ("Der Ministerrat beschließt ...") or at the next item.
ITEM = re.compile(r"(?:(?:Gemeinsamer\s+)?Bericht|Vortrag|Antrag)\s+(?P<offices>de[sr]\s+[^|]+?),?\s+"
r"Zahl\s+(?P<zahl>\d[\d./\-\u2010-\u2015 ]*?)\s*,\s*betreffend\s+(?P<subject>.+?)"
r"(?=\s*\|?\s*Der Ministerrat\b"
r"|\s*\|\s*(?:\d+(?:\.\d+)*\.?\s+)?(?:Gemeinsamer\s+)?(?:Bericht|Vortrag|Antrag)\s+de[sr]\s|$)")
OFFICE_SPLIT = re.compile(r",\s*(?=de[sr]\s)|\s+und\s+(?=de[sr]\s)")
GENITIVE = [(r"\bBundeskanzlers\b", "Bundeskanzler"), (r"\bVizekanzlers\b", "Vizekanzler"),
(r"\bBundesministers\b", "Bundesminister"), (r"\bStaatssekretärs\b", "Staatssekretär")]
def german_date(text: str) -> date | None:
m = GERMAN_DATE.search(text)
if not m or m.group(2).lower() not in MONTHS:
return None
return date(int(m.group(3)), MONTHS[m.group(2).lower()], int(m.group(1)))
def period_start(path: str) -> int:
"""'.../ministerratsprotokolle-der-xxvi-regierungsperiode-2017-2018.html' -> 2018 (latest year named)."""
return max((int(y) for y in re.findall(r"(20\d\d)", path)), default=0)
def offices(text: str) -> list[str]:
"""'des Bundeskanzlers, des Bundesministers für Wohnen, Kunst, Kultur, Medien und Sport und der
Bundesministerin für Justiz' -> ['Bundeskanzler', 'Bundesminister für Wohnen, ...', 'Bundesministerin für Justiz']."""
result = []
for part in OFFICE_SPLIT.split(text):
part = re.sub(r"^\s*de[sr]\s+", "", part).strip(" ,")
for pattern, repl in GENITIVE:
part = re.sub(pattern, repl, part)
if part:
result.append(part)
return result
def main_text(response: Response) -> tuple[str, str]:
soup = response.soup
main = soup.find("main") or soup
title_el = main.find(class_="title") or main.find("h1")
title = title_el.get_text(" ", strip=True) if title_el else (soup.title.string if soup.title else response.url)
for noise in main.find_all(["nav", "aside", "script", "style", "form", "footer"]):
noise.decompose()
parts = [p.get_text(" ", strip=True) for p in main.find_all(["p", "li", "h2", "h3"])]
return title, "\n".join(p for p in parts if p)
@register
class GovernmentNews(Adapter):
spec = SPEC
name = "bundesregierung.news"
description = "Government news archive (summits, trips, meetings of government members)"
schedule = "10 */3 * * *"
def start(self, ctx: RunContext) -> Iterator[Request]:
first = int(ctx.params.get("since_year", date.today().year - 1))
for year in range(first, date.today().year + 1):
yield Request(url=NEWS_INDEX.format(year=year), kind="index", meta={"year": year})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if response.request.kind == "index":
year = response.meta["year"]
for path in sorted(set(re.findall(
rf'href="(/bundeskanzleramt/nachrichten-der-bundesregierung/{year}/[^"]+\.html)"', response.text))):
yield Request(url=BASE + path, kind="article")
return
title, text = main_text(response)
time_el = response.soup.find("time")
yield Document(Evidence(url=response.request.url, doc_type="press_release", title=title,
published_at=time_el.get("datetime") if time_el else None,
excerpt=excerpt(text, 300), full_text=f"{title}\n{text}"))
@register
class CouncilOfMinisters(Adapter):
spec = SPEC
name = "bundesregierung.ministerrat"
description = "Council of Ministers minutes: decisions per reporting minister"
schedule = "20 6 * * 3,4"
parser_version = 2
def start(self, ctx: RunContext) -> Iterator[Request]:
yield Request(url=MR_INDEX, kind="index")
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
kind = response.request.kind
if kind == "index":
periods = sorted(set(re.findall(r'href="(/bundeskanzleramt/die-bundesregierung/ministerratsprotokolle/'
r'[^"?]+\.html)"', response.text)), key=period_start)
# Period pages are named after the year a government took office ("...-xxviii-2025") and list all
# its sessions, so the latest period that started before `since_year` may still contain newer ones.
since = int(ctx.params.get("since_year", date.today().year - 1))
started_before = [period_start(p) for p in periods if period_start(p) < since]
keep_from = max(started_before, default=0)
for path in reversed([p for p in periods if period_start(p) >= keep_from]): # newest first
yield Request(url=BASE + path, kind="period")
elif kind == "period":
base = response.request.url.split("?")[0]
for href in sorted(set(re.findall(r'href="([^"]+)"', response.text))):
url = urljoin(response.url, href)
if url.startswith(base.removesuffix(".html") + "/"):
yield Request(url=url, kind="protocol")
elif url.startswith(base + "?currentPage="):
yield Request(url=url, kind="period")
else:
yield from self.parse_protocol(response)
def parse_protocol(self, response: Response) -> Iterator[Any]:
title, text = main_text(response)
held_on = german_date(title) or german_date(text[:300])
url = response.request.url
yield Document(Evidence(url=url, doc_type="protocol", title=title, published_at=held_on,
excerpt=excerpt(text, 300), full_text=f"{title}\n{text}"))
event_key = fingerprint("ministerrat", url)
yield Event(key=event_key, kind="government_meeting", title=title, starts_at=held_on, location="Wien",
country="AT", url=url, body=OrgRef(slug="bundesregierung"))
flat = " | ".join(text.split("\n"))
for n, m in enumerate(ITEM.finditer(flat)):
subject = re.sub(r"\s*\|\s*", " ", m.group("subject")).strip(" .")
zahl = re.sub(r"[\s\u2010-\u2015]+", "-", m.group("zahl").strip())
for office in offices(m.group("offices")):
yield Claim(
claim_type="government_decision",
person=PersonRef(name=None, hints={"office": office, "on": held_on}),
key=("ministerrat", url, zahl, office), occurred_at=held_on,
summary_de=f"Bericht im Ministerrat ({office}): {excerpt(subject, 220)}",
summary_en=f"Report to the Council of Ministers ({office}): {excerpt(subject, 220)}",
data={"office": office, "zahl": zahl, "item": n + 1},
evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=held_on,
quote=excerpt(re.sub(r"\s*\|\s*", " ", m.group(0)), 500),
locator=f"item:{zahl}")],
)
# Participation of the reporting ministers in this session (calendar)
for office in offices(m.group("offices")):
yield Claim(
claim_type="appearance", person=PersonRef(hints={"office": office, "on": held_on}),
occurred_at=held_on, summary_de=f"Teilnahme am Ministerrat ({title})",
evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=held_on)],
facet=Participation(event_key=event_key, participation="reporting_minister"),
)
+193
View File
@@ -0,0 +1,193 @@
"""EU institutions: Commission lobby meetings and the European Parliament (Austrian MEPs, declarations).
* European Commission - meetings with interest representatives (open data XLSX per Commission term).
Every row names the cabinet ("Cabinet of Commissioner Magnus Brunner"). Only meetings of Austrian
Commissioners' cabinets are kept (names passed from start(): Commission roles in the DB + a fallback list).
* European Parliament Open Data API v2 (CC BY 4.0): current Austrian MEPs (authoritative mandate data),
and their declarations (financial interests, conflicts of interest, gifts, ...).
"""
from __future__ import annotations
import io
import re
from collections.abc import Iterator
from datetime import date
from typing import Any
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import (
Claim,
Event,
Evidence,
OrgRef,
Participation,
PersonRecord,
PersonRef,
Role,
)
from politrack.text import excerpt, fingerprint, normalize
EC_SPEC = SourceSpec(
slug="ec-meetings", name="Europäische Kommission – Treffen mit Interessenvertretern", kind="official",
homepage="https://commission.europa.eu/about/service-standards-and-principles/transparency_en",
license="EU open data (reuse permitted, attribution)", access="bulk", trust=90, min_interval=2.0,
tos_notes="Open data export of meetings of Commissioners and their cabinets with interest representatives.",
)
EP_SPEC = SourceSpec(
slug="europarl", name="Europäisches Parlament Open Data", kind="official",
homepage="https://data.europarl.europa.eu", license="CC BY 4.0", access="api", trust=95, min_interval=1.0,
tos_notes="EP Open Data API v2 (limit 500 requests / 5 min / endpoint).",
)
EC_EXPORT = "https://ec.europa.eu/transparency-initiative/meetings/data/meetings/dataxlsx"
EC_TERMS = ("meetingscommissionrepresentatives2429", "meetingscommissionrepresentatives1924")
EP_API = "https://data.europarl.europa.eu/api/v2"
# Austrian Commissioners in the terms covered by EC_TERMS (fallback if the DB has no Commission roles yet).
AUSTRIAN_COMMISSIONERS = ("Johannes Hahn", "Magnus Brunner")
CABINET = re.compile(r"Cabinet of (?:Executive Vice-President|Vice-President|Commissioner|President)\s+(?P<name>.+)$")
@register
class CommissionMeetings(Adapter):
spec = EC_SPEC
name = "eu.commission_meetings"
description = "Meetings of (Austrian) EU Commissioners and their cabinets with interest representatives"
schedule = "0 5 * * 1"
queue = "bulk"
def start(self, ctx: RunContext) -> Iterator[Request]:
names = set(AUSTRIAN_COMMISSIONERS)
if ctx.conn is not None:
names |= {row["full_name"] for row in ctx.conn.execute(
"""SELECT DISTINCT p.full_name FROM role r JOIN person p ON p.id = r.person_id
JOIN organization o ON o.id = r.organization_id WHERE o.slug = 'europaeische-kommission'""")}
for term in EC_TERMS:
yield Request(url=EC_EXPORT, params={"name": term}, kind="xlsx", meta={"commissioners": sorted(names)})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
import openpyxl
wb = openpyxl.load_workbook(io.BytesIO(response.body), read_only=True, data_only=True)
rows = list(wb.worksheets[0].iter_rows(values_only=True))
header_index = next((i for i, r in enumerate(rows) if r and r[0] == "Name of cabinet"), None)
if header_index is None:
ctx.stats["unexpected_layout"] += 1
return
header = [str(h) for h in rows[header_index]]
url = f"{EC_EXPORT}?name={response.request.params['name']}"
wanted = {normalize(n) for n in response.meta.get("commissioners") or AUSTRIAN_COMMISSIONERS}
for row in rows[header_index + 1:]:
rec = dict(zip(header, row))
cabinet = rec.get("Name of cabinet") or ""
m = CABINET.search(cabinet.strip())
if not m or not rec.get("Date of meeting"):
continue
commissioner = m.group("name").strip()
if normalize(commissioner) not in wanted:
ctx.stats["other_commissioners"] += 1
continue
org_name = (rec.get("Name of interest representative") or "").strip()
held = str(rec["Date of meeting"])[:10]
subject = (rec.get("Subject of the meeting") or "").strip()
reps = rec.get("Name of EC representative") or ""
titles = rec.get("Title of EC representative") or ""
register_id = (rec.get("Transparency register ID") or "").strip()
event_key = fingerprint("ec-meeting", cabinet, held, org_name, subject)
yield Event(key=event_key, kind="meeting", title=f"{cabinet}: {org_name}", starts_at=held,
location=rec.get("Location"), url=url, description=subject,
body=OrgRef(slug="europaeische-kommission"))
yield Claim(
claim_type="meeting", person=PersonRef(name=commissioner), occurred_at=held,
counterpart_org=OrgRef(name=org_name, kind="lobby",
ids={"eu_transparency": register_id} if register_id else {}) if org_name else None,
summary_de=f"Treffen ({'Kommissar persönlich' if 'Commissioner' in titles else 'Kabinett'}) mit "
f"{org_name}: {excerpt(subject, 160)}",
summary_en=f"Meeting ({'Commissioner' if 'Commissioner' in titles else 'cabinet'}) with {org_name}: "
f"{excerpt(subject, 160)}",
data={"cabinet": cabinet, "representatives": reps, "titles": titles, "location": rec.get("Location"),
"register_id": register_id, "date_precision": "day"},
evidence=[Evidence(url=url, doc_type="dataset_row", title=f"EU-Kommission: Treffen {cabinet}",
published_at=held, quote=f"{held} | {org_name} | {subject}",
locator=f"{held}:{org_name}:{subject[:60]}")],
topics=("Europäische Union",),
facet=Participation(event_key=event_key, participation="host"),
)
@register
class EuropeanParliament(Adapter):
spec = EP_SPEC
name = "europarl.meps"
description = "Current Austrian MEPs (mandates) and their declarations"
schedule = "30 4 * * 2"
creates_persons = True
def start(self, ctx: RunContext) -> Iterator[Request]:
yield Request(url=f"{EP_API}/meps/show-current", kind="meps",
params={"country-of-representation": "AT", "format": "application/ld+json", "limit": 100})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
data = response.json().get("data") or []
if response.request.kind == "meps":
for mep in data:
mep_id = str(mep.get("identifier"))
given, family = mep.get("givenName"), mep.get("familyName")
name = f"{given} {family}" if given and family else mep.get("label")
ref = PersonRef(ids={"ep": mep_id}, name=name)
yield PersonRecord(ref=ref, full_name=name, given_name=given, family_name=family,
aliases=(mep.get("label") or "",), match_by_name=True)
yield Request(url=f"{EP_API}/meps/{mep_id}", params={"format": "application/ld+json"}, kind="mep",
meta={"mep_id": mep_id, "name": name, "group": mep.get("api:political-group")})
yield Request(url=f"{EP_API}/meps-declarations", kind="declarations",
params={"person-id": mep_id, "format": "application/ld+json", "limit": 100},
meta={"mep_id": mep_id, "name": name})
elif response.request.kind == "mep":
yield from self.parse_mep(response, data[0] if data else {})
else:
yield from self.parse_declarations(response, data)
def parse_mep(self, response: Response, mep: dict) -> Iterator[Any]:
meta = response.meta
ref = PersonRef(ids={"ep": meta["mep_id"]})
if mep.get("img"):
yield PersonRecord(ref=ref, full_name=meta["name"], photo_url=mep["img"],
birth_date=date.fromisoformat(mep["bday"][:10]) if mep.get("bday") else None)
starts = [m.get("memberDuring", {}).get("startDate") for m in mep.get("hasMembership") or []
if "EU_PARLIAMENT" in str(m.get("membershipClassification", ""))]
starts = sorted(s for s in starts if s)
body = OrgRef(slug="europaeisches-parlament")
start = date.fromisoformat(starts[-1]) if starts else None
url = f"https://www.europarl.europa.eu/meps/de/{meta['mep_id']}"
yield Claim(
claim_type="role", person=ref, org=body, valid_from=start,
summary_de=f"Mitglied des Europäischen Parlaments (Fraktion {meta.get('group') or '?'})",
summary_en=f"Member of the European Parliament ({meta.get('group') or '?'})",
data={"ep_group": meta.get("group")},
evidence=[Evidence(url=url, doc_type="api_record", title=f"Europäisches Parlament: {meta['name']}")],
facet=Role(role="mp", level="eu", body=body, start=start),
)
def parse_declarations(self, response: Response, docs: list[dict]) -> Iterator[Claim]:
meta = response.meta
for doc in docs:
titles = []
pdf = None
for expression in doc.get("is_realized_by") or []:
title = (expression.get("title") or {})
titles.extend(v for v in title.values() if v)
for manifestation in expression.get("is_embodied_by") or []:
if "pdf" in str(manifestation.get("media_type", "")).lower() and not pdf:
pdf = manifestation.get("is_exemplified_by")
title = next((t for t in titles if t), doc.get("id"))
url = f"https://www.europarl.europa.eu/{pdf}" if pdf and not pdf.startswith("http") else (
pdf or f"https://data.europarl.europa.eu/{doc.get('id')}")
kind = str(doc.get("work_type", "")).rsplit("/", 1)[-1]
yield Claim(
claim_type="declaration", person=PersonRef(ids={"ep": meta["mep_id"]}),
key=("ep-declaration", doc.get("id")), occurred_at=doc.get("document_date"),
summary_de=f"Erklärung im Europäischen Parlament: {excerpt(title, 200)}",
summary_en=f"Declaration filed with the European Parliament: {excerpt(title, 200)}",
data={"work_type": kind},
evidence=[Evidence(url=url, doc_type="pdf", title=title, published_at=doc.get("document_date"))],
topics=("Transparenz und Korruption",),
)
+70
View File
@@ -0,0 +1,70 @@
"""GDELT DOC 2.0 API: news coverage of politicians across many more outlets (metadata only).
Free, open, max. one request every 5 s (we use 6 s and back off on the plain-text throttle notice).
Coverage of German-language Austrian media is thin, so this is a supplement. Queries: active
federal/EU politicians daily, former politicians in a weekly rotation ("what are they up to?").
"""
from __future__ import annotations
import json
from collections.abc import Iterator
from datetime import date
from typing import Any
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Document, Evidence
API = "https://api.gdeltproject.org/api/v2/doc/doc"
SPEC = SourceSpec(
slug="gdelt", name="Weitere Medien (via GDELT)", kind="aggregator", homepage="https://www.gdeltproject.org",
license="GDELT open data (citation required)", access="api", trust=55, store_fulltext=False, keep_raw=False,
min_interval=6.0, tos_notes="GDELT DOC 2.0 API: article metadata (URL, title, date). Max 1 request / 5 s.",
)
DOMAIN_SOURCES = {
"orf.at": "orf", "derstandard.at": "derstandard", "diepresse.com": "diepresse", "kurier.at": "kurier",
"profil.at": "profil", "falter.at": "falter", "krone.at": "krone", "sn.at": "sn", "kleinezeitung.at": "kleinezeitung",
"ots.at": "apa-ots", "vol.at": "vol", "vienna.at": "vol",
}
@register
class Gdelt(Adapter):
spec = SPEC
name = "gdelt.coverage"
description = "Supplementary news coverage via GDELT (active and former politicians)"
schedule = "0 1 * * *"
queue = "bulk"
def start(self, ctx: RunContext) -> Iterator[Request]:
if ctx.conn is None:
return
rows = ctx.conn.execute(
"""SELECT p.id, p.given_name, p.family_name, s.status FROM person p
JOIN person_status s ON s.person_id = p.id
WHERE p.given_name IS NOT NULL AND p.family_name IS NOT NULL AND (
(s.status = 'active' AND EXISTS (SELECT 1 FROM role r WHERE r.person_id = p.id
AND r.level IN ('federal', 'eu') AND r.end_date IS NULL))
OR (s.status IN ('former', 'party_member') AND s.last_office_end > current_date - interval '25 years'
AND p.id %% 7 = %s))
ORDER BY s.status, p.id LIMIT %s""",
(date.today().weekday(), int(ctx.params.get("max_queries", 400))),
).fetchall()
for row in rows:
name = f"{row['given_name']} {row['family_name']}"
yield Request(url=API, kind="artlist", meta={"person_id": row["id"]}, conditional=False,
params={"query": f'"{name}" sourcecountry:AU', "mode": "artlist", "format": "json",
"maxrecords": 50, "timespan": "7d" if row["status"] == "active" else "1m",
"sort": "datedesc"})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if not response.text.strip().startswith("{"):
ctx.stats["throttled"] += 1 # plain-text rate-limit notice
return
for article in json.loads(response.text, strict=False).get("articles", []):
domain = (article.get("domain") or "").removeprefix("www.")
seen = article.get("seendate") or ""
published = f"{seen[:4]}-{seen[4:6]}-{seen[6:8]}T{seen[9:11]}:{seen[11:13]}:00Z" if len(seen) >= 13 else None
yield Document(Evidence(url=article["url"], source=DOMAIN_SOURCES.get(domain, "gdelt"),
title=article.get("title"), doc_type="article", published_at=published,
meta={"via": "gdelt", "domain": domain}))
+63
View File
@@ -0,0 +1,63 @@
"""Press releases of the Wirtschafts- und Korruptionsstaatsanwaltschaft (WKStA, anti-corruption prosecutors).
Official statements on high-profile proceedings. They usually anonymise suspects ("ein ehemaliger
Bundesminister"), so we store them as official documents (full text, searchable, RAG) and let the
legal-matter detector propose claims - which, as always for legal matters, need human review.
"""
from __future__ import annotations
import re
from collections.abc import Iterator
from datetime import date
from typing import Any
from urllib.parse import urljoin
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Document, Evidence
from politrack.sources.bundesregierung import german_date
from politrack.text import excerpt
BASE = "https://www.justiz.gv.at"
START = BASE + "/wksta/wirtschafts-und-korruptionsstaatsanwaltschaft.312.de.html"
SPEC = SourceSpec(
slug="wksta", name="WKStA – Pressemitteilungen", kind="official", homepage=START,
license="Amtliches Werk (§7 UrhG)", access="scrape", trust=90, min_interval=3.0,
tos_notes="Press releases of the anti-corruption prosecutors. robots.txt allows all.",
)
RELEASE = re.compile(r"/medienstelle/pressemitteilungen/pressemitteilungen-(\d{4})/")
LISTING = re.compile(r"/medienstelle(?:/pressemitteilungen(?:/pressemitteilungen-\d{4})?)?\.[0-9a-f]+\.de\.html")
def clean(url: str) -> str:
return re.sub(r";jsessionid=[^?#]*", "", url)
@register
class WkstaPress(Adapter):
spec = SPEC
name = "justiz.wksta"
description = "WKStA press releases (official documents for search and reviewed legal matters)"
schedule = "40 8 * * *"
def start(self, ctx: RunContext) -> Iterator[Request]:
yield Request(url=START, kind="listing", meta={"depth": 0})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if response.request.kind == "listing":
since = int(ctx.params.get("since_year", date.today().year - 2))
for a in response.soup.find_all("a", href=True):
url = clean(urljoin(response.url, a["href"]))
year = RELEASE.search(url)
if year and int(year.group(1)) >= since:
yield Request(url=url, kind="release")
elif LISTING.search(url) and response.meta["depth"] < 2:
yield Request(url=url, kind="listing", meta={"depth": response.meta["depth"] + 1})
return
main = response.soup.find("main") or response.soup.find(id="content") or response.soup
title_el = main.find("h1")
title = title_el.get_text(" ", strip=True) if title_el else response.url
text = "\n".join(p.get_text(" ", strip=True) for p in main.find_all(["p", "li"]) if p.get_text(strip=True))
yield Document(Evidence(url=clean(response.request.url), doc_type="press_release", title=title,
published_at=german_date(text[:400]) or german_date(title), excerpt=excerpt(text, 300),
full_text=f"{title}\n{text}"))
+86
View File
@@ -0,0 +1,86 @@
"""Headline monitoring via publishers' RSS feeds and Google-News sitemaps (listed in robots.txt).
Policy (docs/sources.md): only feeds/sitemaps publishers offer for automated discovery; store headline,
teaser/keywords and link - never article text (store_fulltext=False, keep_raw=False). If a site declares
a TDM reservation, the Loader drops even the teaser and linking uses the headline only.
The feed list lives in config/feeds.yaml (user-editable); every `source` there needs a SourceSpec below.
"""
from __future__ import annotations
import xml.etree.ElementTree as ET
from collections.abc import Iterator
from datetime import datetime, timezone
from time import mktime
from typing import Any
import feedparser
import yaml
from politrack.config import get_settings
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register, register_spec
from politrack.core.records import Document, Evidence
from politrack.text import html_to_text
def media(slug: str, name: str, homepage: str, trust: int, notes: str = "RSS/news sitemap only; headline + link.",
license: str = "All rights reserved") -> SourceSpec:
return register_spec(SourceSpec(slug=slug, name=name, kind="media", homepage=homepage, license=license,
access="rss", trust=trust, store_fulltext=False, keep_raw=False,
min_interval=2.0, tos_notes=notes))
ORF = media("orf", "ORF.at", "https://orf.at", 75, "Public RSS feeds (rss.orf.at). Headline, teaser, link.")
media("derstandard", "DER STANDARD", "https://www.derstandard.at", 75,
"RSS + news sitemap. robots.txt blocks AI crawlers by name; we only use feed/sitemap and link out.")
media("diepresse", "Die Presse", "https://www.diepresse.com", 75)
media("kurier", "KURIER", "https://kurier.at", 70)
media("profil", "profil", "https://www.profil.at", 75)
media("falter", "FALTER", "https://www.falter.at", 75)
media("krone", "Kronen Zeitung", "https://www.krone.at", 55)
media("sn", "Salzburger Nachrichten", "https://www.sn.at", 75)
media("kleinezeitung", "Kleine Zeitung", "https://www.kleinezeitung.at", 75)
media("vol", "Vorarlberg Online", "https://www.vol.at", 65)
media("apa-ots", "APA-OTS Presseaussendungen", "https://www.ots.at", 60,
"tdmrep.json declares tdm-reservation=1 for the whole site: headline + link only.")
NS = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9", "news": "http://www.google.com/schemas/sitemap-news/0.9"}
@register
class NewsFeeds(Adapter):
spec = ORF
name = "news.feeds"
description = "Headlines from outlets' RSS feeds and Google-News sitemaps (config/feeds.yaml)"
schedule = "*/20 * * * *"
def start(self, ctx: RunContext) -> Iterator[Request]:
config = yaml.safe_load((get_settings().config_dir / "feeds.yaml").read_text(encoding="utf-8"))
for feed in config.get("feeds") or []:
yield Request(url=feed["url"], kind=feed.get("type", "rss"), meta={"source": feed["source"]},
skip_unchanged=True)
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
source = response.meta["source"]
doc_type = "press_release" if source == "apa-ots" else "article"
if response.request.kind == "sitemap":
root = ET.fromstring(response.body)
for node in root.findall("sm:url", NS):
link, title = node.findtext("sm:loc", namespaces=NS), node.findtext("news:news/news:title", namespaces=NS)
if not link or not title:
continue
keywords = node.findtext("news:news/news:keywords", namespaces=NS)
yield Document(Evidence(url=link.strip(), source=source, title=html_to_text(title), doc_type=doc_type,
published_at=node.findtext("news:news/news:publication_date", namespaces=NS),
excerpt=f"Schlagworte: {keywords}" if keywords else None,
meta={"sitemap": response.request.url}))
return
for entry in feedparser.parse(response.body).entries:
if not entry.get("link"):
continue
published = (datetime.fromtimestamp(mktime(entry.published_parsed), tz=timezone.utc)
if entry.get("published_parsed") else None)
yield Document(Evidence(url=entry["link"], source=source, title=html_to_text(entry.get("title")),
doc_type=doc_type, published_at=published, excerpt=html_to_text(entry.get("summary")),
meta={"feed": response.request.url}))
@@ -0,0 +1 @@
"""Austrian Parliament adapters: persons, items (votes, immunity, questions, hearings), press, calendar."""
@@ -0,0 +1,199 @@
"""Shared spec and pure parsing helpers for the Austrian Parliament adapters (CC BY 4.0).
API (https://www.parlament.gv.at/recherchieren/open-data/):
* list exports: POST /Filter/api/filter/data/<listeId>?js=eval&showAll=true (JSON body = filter)
409 persons since 1918 · 101 parliamentary items · 251 speeches of a person
110 Parlamentskorrespondenz (press releases) · 600 calendar (Termine)
* detail pages: <page URL>?json=TRUE
"""
from __future__ import annotations
import re
from datetime import date, datetime
from typing import Any
from politrack.core import Request, SourceSpec, register_spec
from politrack.text import html_to_text
BASE = "https://www.parlament.gv.at"
CHAMBER_SLUG = {"NR": "nationalrat", "BR": "bundesrat"}
CHAMBER_NAME_DE = {"NR": "Nationalrat", "BR": "Bundesrat"}
GOVERNMENT_RE = re.compile(r"Bundeskanzler|Vizekanzler|Bundesminister|Staatssekretär", re.IGNORECASE)
SPEC = register_spec(SourceSpec(
slug="parlament",
name="Parlament Österreich (Open Data)",
kind="official",
homepage="https://www.parlament.gv.at/recherchieren/open-data/",
license="CC BY 4.0",
access="api",
trust=95,
store_fulltext=True,
min_interval=1.0,
tos_notes="Official JSON API (?json=TRUE, /Filter/api/filter/data/<list>). Attribution required. The "
"Parlamentsdirektion may prohibit uses incompatible with the dignity of parliament.",
))
def list_request(list_id: int, body: dict, kind: str, meta: dict | None = None) -> Request:
return Request(url=f"{BASE}/Filter/api/filter/data/{list_id}?js=eval&showAll=true", method="POST", json=body,
kind=kind, meta=meta or {})
def detail_request(path: str, kind: str, meta: dict | None = None) -> Request:
return Request(url=f"{BASE}{path}", params={"json": "TRUE"}, kind=kind, meta=meta or {})
def list_rows(data: dict) -> list[dict[str, Any]]:
"""Turn a filter-API export into dicts keyed by column label (duplicates get a _<index> suffix)."""
labels: list[str] = []
for i, header in enumerate(data.get("header", [])):
label = header.get("label") or header.get("feld_name") or f"col{i}"
labels.append(label if label not in labels else f"{label}_{i}")
return [dict(zip(labels, row)) for row in data.get("rows", [])]
def parse_date(value: str | None) -> date | None:
if not value:
return None
for fmt in ("%d.%m.%Y", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d"):
try:
return datetime.strptime(value[:19], fmt).date()
except ValueError:
continue
return None
def walk_dicts(obj: Any, required_key: str):
"""Yield every nested dict that contains `required_key`."""
if isinstance(obj, dict):
if required_key in obj:
yield obj
for value in obj.values():
yield from walk_dicts(value, required_key)
elif isinstance(obj, list):
for value in obj:
yield from walk_dicts(value, required_key)
# -- faction votes ------------------------------------------------------------------------
PARTY_TOKEN = re.compile(r"^[A-ZÄÖÜ][A-Za-zÄÖÜäöüß.\- ]{0,30}$")
# Split votes ("teilweise", "bzw.", per-part results in parentheses) can't be attributed to factions.
SPLIT_VOTE = re.compile(r"\(|\bbzw\.|\bteilweise\b|\btlw\.|\bmehrstimmig\b|\bgetrennt", re.IGNORECASE)
def parse_party_list(value: str | None) -> list[str]:
if not value:
return []
parts = [p.strip(" .;") for p in re.split(r",|\bund\b", value)]
return [p for p in parts if PARTY_TOKEN.match(p)]
def parse_vote_text(text: str) -> dict[str, Any] | None:
"""'137. Sitzung …: Gesetzesvorschlag in dritter Lesung angenommen Dafür: ÖVP, SPÖ, dagegen: FPÖ'"""
m = re.search(r"\b(angenommen|abgelehnt)\b", text, re.IGNORECASE)
if not m:
return None
rest = text[m.end():]
roll_call = "namentliche abstimmung" in text.lower()
split = bool(SPLIT_VOTE.search(rest)) and not roll_call
pro = re.search(r"Dafür:\s*(.*?)(?:,?\s*dagegen:|$)", rest, re.IGNORECASE)
contra = re.search(r"dagegen:\s*(.*)$", rest, re.IGNORECASE)
return {
"stage": text[: m.start()].strip(" :,"),
"result": "accepted" if m.group(1).lower() == "angenommen" else "rejected",
"for": [] if split or roll_call else parse_party_list(pro.group(1) if pro else None),
"against": [] if split or roll_call else parse_party_list(contra.group(1) if contra else None),
"unanimous": "einstimmig" in text.lower(),
"split": split,
"roll_call": roll_call,
}
# -- roll-call votes ----------------------------------------------------------------------
ROLL_CALL_HEADER = re.compile(r"Mit\s*[„\"“]?\s*(Ja|Nein)\s*[“\"”]?\s*stimmten[^:]*:", re.IGNORECASE)
NEW_VOTE = re.compile(r"\b(gelangen|kommen)\s+(?:\w+\s+){0,2}zur\s+Abstimmung\b", re.IGNORECASE)
PERSON_LINK = re.compile(r"<a[^>]*href=\"[^\"]*/person/(\d+)\"[^>]*>.*?</a>", re.IGNORECASE | re.DOTALL)
def parse_roll_call(html_text: str, anchor: str | None) -> dict[str, list[tuple[str, int | None]]]:
"""Extract the "Mit „Ja“ stimmten …" / "Mit „Nein“ stimmten …" lists that follow `anchor`.
Returns {"for": [(label, pad|None)], "against": [...]}. Names are mostly surnames; where a surname is
ambiguous the protocol links the person, which gives us the parliament id directly.
"""
start = 0
if anchor:
m = re.search(r'id="%s"' % re.escape(anchor), html_text)
if not m:
return {}
start = m.start()
first = ROLL_CALL_HEADER.search(html_text, start)
if not first:
return {}
# The lists must belong to this anchor: they follow within a few numbered paragraphs and no new vote
# may be called in between.
between = html_text[start + 10 : first.start()]
if anchor and (len(re.findall(r'class="[^"]*randnummer', between)) > 3 or NEW_VOTE.search(html_to_text(between))):
return {}
end_match = re.compile(r"<hr|class=\"[^\"]*randnummer").search(html_text, first.end())
segment = html_text[first.start() : end_match.start() if end_match else len(html_text)]
segment = PERSON_LINK.sub(lambda m: f" @@{m.group(1)}@@ ", segment)
text = html_to_text(segment)
result: dict[str, list[tuple[str, int | None]]] = {"for": [], "against": []}
headers = list(ROLL_CALL_HEADER.finditer(text))
for i, header in enumerate(headers):
block = text[header.end() : headers[i + 1].start() if i + 1 < len(headers) else len(text)]
key = "for" if header.group(1).lower() == "ja" else "against"
for token in re.split(r"[,;]|\.\s*$", block):
token = token.strip(" .")
if not token:
continue
link = re.fullmatch(r"@@(\d+)@@", token)
result[key].append((token, int(link.group(1))) if link else (token, None))
return result
# -- immunity -----------------------------------------------------------------------------
STGB_OFFENSES = {
"111": "Üble Nachrede", "115": "Beleidigung", "146": "Betrug", "153": "Untreue", "165": "Geldwäscherei",
"201": "Vergewaltigung", "283": "Verhetzung", "288": "Falsche Beweisaussage", "293": "Fälschung eines Beweismittels",
"295": "Unterdrückung eines Beweismittels", "297": "Verleumdung", "302": "Missbrauch der Amtsgewalt",
"304": "Bestechlichkeit", "305": "Vorteilsannahme", "306": "Vorteilsannahme zur Beeinflussung",
"307": "Bestechung", "308": "Verbotene Intervention", "310": "Verletzung des Amtsgeheimnisses",
}
def parse_immunity_report(text: str) -> dict[str, Any]:
authority = re.search(r"Ersuchen (?:der|des) (.+?),\s*(?:do\.|GZ|Zl)", text)
case_ref = re.search(r"\bGZ\.?\s*([^,]+?),", text)
# "wegen des Verdachtes einer strafbaren Handlung nach § 283 Abs. 1 Z 1 und Z 2, Abs. 2 StGB"
offense = re.search(r"wegen[^§]{0,90}?(§§? ?\d+.{0,120}?\b(?:StGB|VerbotsG|FinStrG|[A-Z][A-Za-z]*G)\b)", text)
offense_text = offense.group(1).strip() if offense else None
if offense_text:
section = re.search(r"§ ?(\d+)", offense_text)
if section and section.group(1) in STGB_OFFENSES and "StGB" in offense_text:
offense_text = f"{offense_text} ({STGB_OFFENSES[section.group(1)]})"
decision = re.search(r"(?:wird|werde)\s+(nicht\s+)?zugestimmt|(nicht\s+)?zuzustimmen", text)
recommendation = None
if decision:
recommendation = "refuse" if (decision.group(1) or decision.group(2)) else "consent"
connection = None
if re.search(r"\bkein Zusammenhang\b", text):
connection = False
elif re.search(r"\bein Zusammenhang\b", text):
connection = True
quote = re.search(r"Als Ergebnis seiner Beratungen.*?(?:zugestimmt|nicht zugestimmt)\s*\.", text)
return {
"authority": authority.group(1).strip() if authority else None,
"case_ref": case_ref.group(1).strip() if case_ref else None,
"offense": offense_text,
"recommendation": recommendation,
"political_connection": connection,
"quote": quote.group(0) if quote else None,
}
@@ -0,0 +1,285 @@
"""Parliament items (Gegenstände) of the configured legislative periods.
From one list export per period (list 101) we derive:
* votes -> item detail -> VoteEvent + faction VotePositions
roll calls -> stenographic protocol -> individual VotePositions
* immunity requests ("Ersuchen um Zustimmung zur behördlichen Verfolgung") -> committee report -> Legal claim
* written questions and answers (ITYP J / AB) -> parliamentary_question claims (directly from the list)
* investigative-committee hearings ("öffentliche Befragung der Auskunftsperson ...") -> hearing Event +
appearance claim for politicians who testified (other witnesses are not stored)
"""
from __future__ import annotations
import json
import re
from collections.abc import Iterator
from typing import Any
from politrack.config import get_settings
from politrack.core import Adapter, Request, Response, RunContext, register
from politrack.core.records import (
Claim,
Event,
Evidence,
Legal,
OrgRef,
Participation,
PersonRef,
VoteEvent,
VotePosition,
)
from politrack.sources.parlament.common import (
BASE,
CHAMBER_SLUG,
SPEC,
detail_request,
list_request,
list_rows,
parse_date,
parse_immunity_report,
parse_roll_call,
parse_vote_text,
walk_dicts,
)
from politrack.text import fingerprint, html_to_text, strip_titles
HEARING = re.compile(r"Befragung der Auskunftsperson\s+(?P<name>.+)$")
WITNESS_TITLES = re.compile(r"^(?:(?:Oberstaatsanwalt|Staatsanwalt|Sektionschef(?:in)?|MinR|Generalsekretär(?:in)?|"
r"Bundesminister(?:in)?|Abgeordnete[rn]?|Klubobmann|Klubobfrau|Bundeskanzler|"
r"Vizekanzler|Präsident(?:in)?)\s+)+")
@register
class ParlamentItems(Adapter):
spec = SPEC
name = "parlament.items"
description = "Votes (faction + roll call), immunity requests, written questions, U-committee hearings"
schedule = "45 3 * * *"
parser_version = 2
def start(self, ctx: RunContext) -> Iterator[Request]:
for gp in ctx.params.get("periods") or get_settings().parlament_periods:
yield list_request(101, {"GP_CODE": [gp], "NRBR": ["NR"]}, kind="items", meta={"gp": gp})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
handler = {
"items": self.parse_items,
"votes": self.parse_votes,
"rollcall": self.parse_rollcall,
"immunity": self.parse_immunity_item,
"immunity_report": self.parse_immunity_report,
"hearing": self.parse_hearing_item,
}[response.request.kind]
yield from handler(ctx, response)
# -- list ---------------------------------------------------------------------------
def parse_items(self, ctx: RunContext, response: Response) -> Iterator[Any]:
for row in list_rows(response.json()):
path, subject, kind = row.get("HIS_URL"), row.get("Betreff") or "", row.get("ITYP")
if not path:
continue
if row.get("Dafür") or row.get("Dagegen") or row.get("Abstimmungstext"):
# The list row changes when new votes happen, so the detail is re-fetched only then
# (conditional request + unchanged-body detection do the rest).
yield detail_request(path, kind="votes", meta={"title": subject, "ref": row.get("Nummer")})
if subject.startswith("Ersuchen") and "behördlichen Verfolgung" in subject:
yield detail_request(path, kind="immunity",
meta={"title": subject, "date": row.get("Datum"), "status": row.get("Status")})
if kind in ("J", "AB"):
yield from self.question_claims(row, path)
if kind == "KOMM" and HEARING.search(subject):
yield detail_request(path, kind="hearing", meta={"title": subject, "date": row.get("Datum")})
def question_claims(self, row: dict, path: str) -> Iterator[Claim]:
pads = row.get("Personen") or "[]"
pads = json.loads(pads) if isinstance(pads, str) else pads
subject = row.get("Betreff") or ""
is_question = row.get("ITYP") == "J"
asked = parse_date(row.get("Datum"))
topics = tuple(json.loads(row.get("THEMEN") or "[]")) if isinstance(row.get("THEMEN"), str) else ()
for pad in pads:
if not str(pad).strip():
continue
verb_de = "stellte die schriftliche Anfrage" if is_question else "beantwortete die Anfrage"
verb_en = "submitted the written question" if is_question else "answered the question"
yield Claim(
claim_type="parliamentary_question", person=PersonRef(ids={"parlament": str(pad)}),
key=("question", path, str(pad)), occurred_at=asked,
summary_de=f"{verb_de[0].upper()}{verb_de[1:]} „{subject}“ ({row.get('Nummer')})",
summary_en=f"{verb_en[0].upper()}{verb_en[1:]} \"{subject}\" ({row.get('Nummer')})",
data={"kind": "question" if is_question else "answer", "number": row.get("Nummer")},
evidence=[Evidence(url=f"{BASE}{path}", doc_type="api_record", title=subject, published_at=asked)],
topics=topics,
)
# -- votes ----------------------------------------------------------------------------
def parse_votes(self, ctx: RunContext, response: Response) -> Iterator[Any]:
content = response.json().get("content") or {}
path = response.request.url.removeprefix(BASE)
title = html_to_text(content.get("title")) or response.meta.get("title") or path
item_ref = html_to_text(content.get("zitation")) or response.meta.get("ref")
item_url = f"{BASE}{path}"
topics = tuple(b.get("label") for b in ((content.get("topics") or {}).get("data") or {}).get("bubbles") or [])
eurovoc = [b.get("label") for b in ((content.get("eurovoc") or {}).get("data") or {}).get("bubbles") or []]
for stage in walk_dicts(content.get("phase") or content, "text"):
text = html_to_text(stage.get("text"))
if "angenommen" not in text and "abgelehnt" not in text:
continue
parsed = parse_vote_text(text)
if not parsed or not (parsed["for"] or parsed["against"] or parsed["unanimous"] or parsed["split"]
or parsed["roll_call"]):
continue
voted_on = parse_date(stage.get("date"))
refs = stage.get("fsth") or []
protocol_url = f"{BASE}{refs[0]['url']}" if refs and refs[0].get("url") else None
chamber = "BR" if (refs and refs[0].get("gp_code") == "BR") or "Bundesrat" in text else "NR"
chamber_ref = OrgRef(slug=CHAMBER_SLUG[chamber])
event_key = fingerprint("vote", path, voted_on, parsed["stage"])
yield VoteEvent(key=event_key, chamber=chamber_ref, title=title, voted_on=voted_on, result=parsed["result"],
stage=parsed["stage"], item_ref=item_ref, roll_call=parsed["roll_call"], url=item_url,
protocol_url=protocol_url, raw_text=text)
evidence = [Evidence(url=item_url, doc_type="api_record", title=title, published_at=voted_on, quote=text,
locator=f"stage:{voted_on}:{parsed['stage'][:80]}")]
if protocol_url:
evidence.append(Evidence(url=protocol_url, doc_type="protocol", published_at=voted_on,
title=refs[0].get("title") or "Stenographisches Protokoll"))
outcome = "angenommen" if parsed["result"] == "accepted" else "abgelehnt"
for party, position in [(p, "for") for p in parsed["for"]] + [(p, "against") for p in parsed["against"]]:
verb_de, verb_en = ("stimmte für", "voted for") if position == "for" else ("stimmte gegen", "voted against")
yield Claim(
claim_type="vote", org=OrgRef(name=party, short_name=party, kind="parliamentary_group"),
counterpart_org=chamber_ref, occurred_at=voted_on,
summary_de=f"{party} {verb_de}: {title} ({parsed['stage']}) – {outcome}",
summary_en=f"{party} {verb_en}: {title} ({parsed['stage']})",
data={"position": position, "result": parsed["result"], "item_ref": item_ref, "chamber": chamber,
"eurovoc": eurovoc},
evidence=evidence, topics=topics, facet=VotePosition(event_key=event_key, position=position),
)
if parsed["roll_call"]:
# Individual votes are listed in the protocol after one of the referenced anchors; the
# result is usually behind the last one, sometimes in another fragment (session break).
refs_list = [r["url"].split("#", 1) for r in reversed(refs) if r.get("url") and "#" in r["url"]]
if refs_list:
yield Request(url=f"{BASE}{refs_list[0][0]}", kind="rollcall",
meta={"event_key": event_key, "refs": refs_list, "tried": [refs_list[0][0]],
"title": title, "chamber": chamber, "date": str(voted_on),
"item_url": item_url, "topics": list(topics)})
def parse_rollcall(self, ctx: RunContext, response: Response) -> Iterator[Any]:
meta = response.meta
doc = response.request.url.removeprefix(BASE)
names: dict[str, list] = {}
anchor_used = None
for ref_doc, anchor in meta["refs"]:
if ref_doc != doc:
continue
names = parse_roll_call(response.text, anchor)
if names.get("for") or names.get("against"):
anchor_used = anchor
break
if anchor_used is None:
remaining = [d for d, _ in meta["refs"] if d not in meta["tried"]]
if remaining:
ctx.stats["rollcall_followup"] += 1
yield Request(url=f"{BASE}{remaining[0]}", kind="rollcall",
meta={**meta, "tried": [*meta["tried"], remaining[0]]})
else:
ctx.stats["rollcall_unparsed"] += 1
return
voted_on = parse_date(meta["date"])
body = CHAMBER_SLUG[meta["chamber"]]
source_url = f"{response.request.url}#{anchor_used}"
evidence = [
Evidence(url=source_url, doc_type="protocol", published_at=voted_on,
title="Stenographisches Protokoll – Ergebnis der namentlichen Abstimmung", locator=anchor_used),
Evidence(url=meta["item_url"], doc_type="api_record", title=meta["title"], published_at=voted_on,
locator=f"rollcall:{voted_on}"),
]
for position, entries in names.items():
answer = "Ja" if position == "for" else "Nein"
for label, pad in entries:
ref = (PersonRef(ids={"parlament": str(pad)}) if pad is not None else
PersonRef(name=strip_titles(label), hints={"surname_in_body": body, "on": voted_on}))
yield Claim(
claim_type="vote", person=ref, counterpart_org=OrgRef(slug=body), occurred_at=voted_on,
summary_de=f"Stimmte in namentlicher Abstimmung mit „{answer}“: {meta['title']}",
summary_en=f"Voted \"{'yes' if position == 'for' else 'no'}\" in a roll-call vote: {meta['title']}",
data={"position": position, "roll_call": True, "chamber": meta["chamber"], "label": label},
evidence=evidence, topics=tuple(meta.get("topics") or ()),
facet=VotePosition(event_key=meta["event_key"], position=position),
)
# -- immunity -----------------------------------------------------------------------
def parse_immunity_item(self, ctx: RunContext, response: Response) -> Iterator[Request]:
content = response.json().get("content") or {}
report = next((d["link"] for group in content.get("documents") or [] for d in group.get("documents") or []
if d.get("type") == "HTML"), None)
if report:
yield Request(url=f"{BASE}{report}", kind="immunity_report",
meta={**response.meta, "item": response.request.url.removeprefix(BASE),
"title": html_to_text(content.get("title")) or response.meta.get("title")})
def parse_immunity_report(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
meta = response.meta
text = html_to_text(response.text)
details = parse_immunity_report(text)
title = meta["title"]
name = re.sub(r"^.*?(Abgeordneten zum Nationalrat|Mitglied(es)? des Bundesrates)\s*(Klubobmann|Klubobfrau)?",
"", title).strip()
decided = str(meta.get("status")) == "5"
stage = {"consent": "immunity_waived" if decided else "immunity_waiver_requested",
"refuse": "immunity_refused" if decided else "immunity_waiver_requested"}.get(
details["recommendation"] or "", "immunity_waiver_requested")
stage_de = {"immunity_waived": "Nationalrat stimmte der behördlichen Verfolgung zu (Immunität aufgehoben)",
"immunity_refused": "Nationalrat verweigerte die Zustimmung zur behördlichen Verfolgung",
"immunity_waiver_requested": "Ersuchen um Aufhebung der Immunität"}[stage]
offense = details["offense"]
requested = parse_date(meta.get("date"))
yield Claim(
claim_type="legal_proceeding", person=PersonRef(name=strip_titles(name)),
key=("immunity", meta["item"]), occurred_at=requested, sensitive=True,
summary_de=stage_de + (f" – Verdacht nach {offense}" if offense else "")
+ (f" (Ersuchen: {details['authority']})" if details["authority"] else ""),
data={"political_connection": details["political_connection"],
"recommendation": details["recommendation"], "item": meta["item"]},
evidence=[
Evidence(url=response.request.url, doc_type="register_entry", published_at=requested,
title=f"Bericht des Immunitätsausschusses: {title}", quote=details["quote"], full_text=text),
Evidence(url=f"{BASE}{meta['item']}", doc_type="api_record", title=title, published_at=requested),
],
topics=("Inneres und Recht",),
facet=Legal(stage=stage, offense=offense, authority=details["authority"], case_ref=details["case_ref"],
stage_date=requested),
)
# -- investigative-committee hearings ----------------------------------------------
def parse_hearing_item(self, ctx: RunContext, response: Response) -> Iterator[Any]:
content = response.json().get("content") or {}
title = html_to_text(content.get("title")) or response.meta["title"]
m = HEARING.search(title)
if not m:
return
witness = WITNESS_TITLES.sub("", strip_titles(m.group("name"))).strip()
published = parse_date(response.meta.get("date"))
protocol = next((d["link"] for group in content.get("documents") or [] for d in group.get("documents") or []
if d.get("type") == "HTML"), None)
url = f"{BASE}{protocol}" if protocol else response.request.url.split("?")[0]
event_key = fingerprint("hearing", response.request.url)
yield Event(key=event_key, kind="hearing", title=f"Befragung im Untersuchungsausschuss: {witness}",
starts_at=published, location="Parlament, Wien", country="AT", url=url,
description=title, body=OrgRef(slug="nationalrat"))
yield Claim(
claim_type="appearance", person=PersonRef(name=witness), occurred_at=published,
summary_de=f"Befragung als Auskunftsperson im Untersuchungsausschuss ({title})",
summary_en="Questioned as a witness by a parliamentary investigative committee",
data={"context": "investigative_committee"},
evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=published)],
topics=("Transparenz und Korruption",),
facet=Participation(event_key=event_key, participation="witness"),
)
+124
View File
@@ -0,0 +1,124 @@
"""Parliament press releases (Parlamentskorrespondenz, list 110) and calendar (Termine, list 600). CC BY 4.0.
Press releases are stored with full text; the people they link (/person/<id>) get a mention claim.
The event extractor later turns visits, receptions and trips described in them into meeting/travel
claims. The calendar fills the calendar view with sessions, committee meetings, hearings and events.
"""
from __future__ import annotations
import json
import re
from collections.abc import Iterator
from datetime import date, timedelta
from typing import Any
from politrack.core import Adapter, Request, Response, RunContext, register
from politrack.core.records import Claim, Document, Event, Evidence, OrgRef, PersonRef
from politrack.sources.parlament.common import BASE, CHAMBER_SLUG, SPEC, list_request, list_rows, parse_date
from politrack.text import excerpt, fingerprint, html_to_text
ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}")
SKIP_TERMIN = re.compile(r"Guided Tour|Führung|Galeriebesuch|Besuch einer Plenarsitzung|Demokratiebildung", re.I)
TERMIN_KIND = [
(re.compile(r"Plenarsitzung|Bundesversammlung|Fest-|Gedenksitzung", re.I), "parliament_session"),
(re.compile(r"Untersuchungsausschuss", re.I), "hearing"),
(re.compile(r"Ausschuss|Enquete", re.I), "committee"),
(re.compile(r"Pressekonferenz", re.I), "press_conference"),
(re.compile(r"Internationales|Konferenz", re.I), "meeting"),
]
def release_body(content: list) -> str:
"""The detail JSON is a list of render blocks; the article is the one wrapped in <div class=pk>."""
for block in content:
text = (block.get("data") or {}).get("text") if isinstance(block, dict) else None
if isinstance(text, str) and re.search(r"class=\"?pk\b", text):
return text
return ""
@register
class ParlamentPress(Adapter):
spec = SPEC
name = "parlament.press"
description = "Parlamentskorrespondenz: official press releases with linked people"
schedule = "5 */2 * * *"
def start(self, ctx: RunContext) -> Iterator[Request]:
last = ctx.state.get("last_date")
backfill = int(ctx.params.get("backfill_days", 365))
since = (date.fromisoformat(last) - timedelta(days=2)) if last else date.today() - timedelta(days=backfill)
yield list_request(110, {"DATERANGE": [since.isoformat(), date.today().isoformat()]}, kind="list",
meta={"since": since.isoformat()})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if response.request.kind == "list":
newest = response.meta["since"]
for row in list_rows(response.json()):
if not row.get("Link"):
continue
published = parse_date(row.get("Datum")) # "23.10.2026" (German) in list 110
if published:
newest = max(newest, published.isoformat())
yield Request(url=f"{BASE}{row['Link']}", params={"json": "TRUE"}, kind="release",
meta={k: row.get(k) for k in ("Titel", "Untertitel", "Datum", "Nr.", "Abothemen",
"Stichworte")})
if ctx.mode != "test":
ctx.state.set("last_date", newest)
return
yield from self.parse_release(response)
def parse_release(self, response: Response) -> Iterator[Any]:
meta = response.meta
url = response.request.url
body_html = release_body(response.json().get("content") or [])
pads = sorted({p for p in re.findall(r"/person/(\d+)", body_html)})
text = html_to_text(body_html)
title, subtitle = meta.get("Titel") or "", meta.get("Untertitel") or ""
topics_raw = meta.get("Abothemen")
topics = tuple(json.loads(topics_raw)) if isinstance(topics_raw, str) and topics_raw.startswith("[") else ()
published = parse_date(meta.get("Datum"))
yield Document(Evidence(url=url, doc_type="press_release", title=title, published_at=published,
excerpt=subtitle or excerpt(text, 300), full_text=f"{title}\n{subtitle}\n{text}",
meta={"pk_number": meta.get("Nr."), "person_pads": pads, "topics": list(topics)}))
for pad in pads:
yield Claim(
claim_type="media_mention", person=PersonRef(ids={"parlament": pad}), occurred_at=published,
summary_de=f"Parlamentskorrespondenz: {title}",
data={"context": "parlamentskorrespondenz"},
evidence=[Evidence(url=url, doc_type="press_release", title=title, published_at=published,
quote=subtitle or None, stance="mentions")],
topics=topics,
)
@register
class ParlamentCalendar(Adapter):
spec = SPEC
name = "parlament.calendar"
description = "Parliament calendar: plenary sessions, committees, hearings, events (tours skipped)"
schedule = "30 2 * * *"
def start(self, ctx: RunContext) -> Iterator[Request]:
yield list_request(600, {}, kind="list")
def parse(self, ctx: RunContext, response: Response) -> Iterator[Event]:
since = str(ctx.params.get("since_year", date.today().year - 2))
for row in list_rows(response.json()):
kind_label = (row.get("Terminart") or "").replace("<BR>", " ")
starts = next((v for k, v in row.items() if k.startswith("Datum") and k != "Datum Bis"
and isinstance(v, str) and ISO_DATE.match(v)), None)
if not starts or starts[:4] < since or SKIP_TERMIN.search(f"{kind_label} {row.get('Bezeichnung') or ''}"):
continue
title = row.get("Bezeichnung") or kind_label
kind = next((k for pattern, k in TERMIN_KIND if pattern.search(f"{kind_label} {title}")), "public_event")
chamber = {"Nationalrat": "NR", "Bundesrat": "BR"}.get(row.get("Gremium") or "")
ends = next((v for k, v in row.items() if k.startswith("Datum Bis") and isinstance(v, str)
and ISO_DATE.match(v)), None)
yield Event(
key=fingerprint("termin", row.get("WENTRY_ID") or row.get("Link")), kind=kind, title=title,
starts_at=starts, ends_at=ends, location=(row.get("Ort") or "").replace(" | ", ", ") or "Parlament, Wien",
country="AT", url=f"{BASE}{row['Link']}" if row.get("Link") else None, description=kind_label,
body=OrgRef(slug=CHAMBER_SLUG[chamber]) if chamber else None,
)
@@ -0,0 +1,239 @@
"""Parliament: people, mandates, government functions, side-job disclosures, social links, speeches.
Flow: list 409 (all MPs since 1918) -> PersonRecord + mandate roles
-> detail /person/<id> for members of the configured periods
detail -> photo, social accounts, functions, side jobs (§6 Unv-Transparenz-G)
-> list 251 (speeches of that person)
"""
from __future__ import annotations
import json
import re
from collections.abc import Iterator
from typing import Any
from politrack.config import get_settings
from politrack.core import Adapter, Request, Response, RunContext, register
from politrack.core.records import Claim, Evidence, OrgRef, PersonRecord, PersonRef, Role, SocialAccount, Statement
from politrack.sources.parlament.common import (
BASE,
CHAMBER_NAME_DE,
CHAMBER_SLUG,
GOVERNMENT_RE,
SPEC,
detail_request,
list_request,
list_rows,
parse_date,
walk_dicts,
)
from politrack.text import firmenbuch_number, html_to_text, strip_titles
SOCIAL_PLATFORMS = {"TWITTER": "twitter", "FACEBOOK": "facebook", "INSTAGRAM": "instagram", "TIKTOK": "tiktok",
"YOUTUBE": "youtube", "LINKEDIN": "linkedin", "WEBSITE": "website", "BLUESKY": "bluesky",
"MASTODON": "mastodon", "TELEGRAM": "telegram", "THREADS": "threads"}
FN_IN_TEXT = re.compile(r"\bFN\s*\d{1,6}\s?[a-zA-Z]\b")
POSTCODE_CITY = re.compile(r"(?:(?<=\d)|\b)\d{4}\s+[A-ZÄÖÜ][\wäöüß .\-]*$")
LEGAL_FORM_END = re.compile(r"(?i:gesmbh|gmbh|ges\.\s?m\.\s?b\.\s?h\.?|m\.\s?b\.\s?h\.?|aktiengesellschaft|\bag|\bkg|\bog|"
r"e\.\s?u\.|\bse|privatstiftung|genossenschaft|sparkasse)(?=[A-ZÄÖÜ\s,]|$)")
STREET_WORD = r"\S*(?:straße|strasse|str\.|gasse|platz|weg|kai|ring|allee|zeile|lände|markt|steig)"
def strip_street(text: str) -> str:
"""'Verein Musterhilfe Hauptstraße 12' -> 'Verein Musterhilfe' (greedy name, shortest street suffix)."""
m = (re.match(rf"^(.*\S)\s+{STREET_WORD}\s*\d+[a-z]?(?:/\d+)*\s*$", text, re.IGNORECASE)
or re.match(r"^(.*\S)\s+\S+\s*\d+[a-z]?(?:/\d+)*\s*$", text))
return m.group(1) if m else text
def employer(cell: str) -> tuple[str | None, str | None]:
"""Employer name without business address, plus its company register number.
'Hypo Tirol Bank AG, Meraner Straße 8, 6020 Innsbruck' -> ('Hypo Tirol Bank AG', None)
'25 PersonaldienstleistungsgesmbHFranz Josef Kai 95020 Salzburg FN306527a' (parliament's JSON glues
the lines together) -> ('25 PersonaldienstleistungsgesmbH', '306527a')
"""
cell = re.sub(r"^lit\.\s*\w\s*:\s*", "", cell).strip()
fn_match = FN_IN_TEXT.search(cell)
fn = firmenbuch_number(fn_match.group(0)) if fn_match else None
name = FN_IN_TEXT.sub("", cell).split(",")[0].strip(" *")
address = POSTCODE_CITY.search(name)
if address:
head = name[:address.start()]
ends = [m.end() for m in LEGAL_FORM_END.finditer(head)]
# cut after the last legal form; otherwise drop "<street> <number>" before the postcode
name = head[:ends[-1]] if ends else strip_street(head)
name = name.strip(" ,-*")
return (name if len(name) >= 3 and not re.fullmatch(r"[\d\s.€-]+", name) else None), fn
@register
class ParlamentPersons(Adapter):
spec = SPEC
name = "parlament.persons"
description = "Members of Nationalrat and Bundesrat since 1918, mandates, functions, side jobs, speeches"
schedule = "15 3 * * *"
creates_persons = True
parser_version = 3
def start(self, ctx: RunContext) -> Iterator[Request]:
yield list_request(409, {"NRBR": ["NR", "BR"]}, kind="persons")
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
kind = response.request.kind
if kind == "persons":
yield from self.parse_persons(ctx, response)
elif kind == "person":
yield from self.parse_person(ctx, response)
elif kind == "speeches":
yield from self.parse_speeches(ctx, response)
# -- persons & mandates -------------------------------------------------------------
def parse_persons(self, ctx: RunContext, response: Response) -> Iterator[Any]:
periods = set(ctx.params.get("periods") or get_settings().parlament_periods)
for row in list_rows(response.json()):
attr = row.get("Attribute") or {}
if isinstance(attr, str):
attr = json.loads(attr)
m = re.search(r"/person/(\d+)", attr.get("uri") or "")
if not m:
continue
pad = m.group(1)
ref = PersonRef(ids={"parlament": pad})
display = attr.get("zit") or strip_titles(row.get("Name") or "")
yield PersonRecord(ref=ref, full_name=display, given_name=row.get("vorname"), family_name=row.get("nachname"),
gender={"M": "male", "W": "female"}.get(row.get("GESCHL_CODE") or attr.get("geschlecht")))
mandates = attr.get("mandate_detail") or []
for mandate in mandates:
claim = self.mandate_claim(pad, display, mandate)
if claim:
yield claim
if any(md.get("gp_code") in periods for md in mandates):
yield detail_request(f"/person/{pad}", kind="person", meta={"pad": pad, "name": display})
def mandate_claim(self, pad: str, name: str, m: dict) -> Claim | None:
code = m.get("mand_code") or "NR"
if code not in CHAMBER_SLUG:
return None
party_code = m.get("wahlpartei_code")
party_name = m.get("wahlpartei_full_txt") or m.get("wahlpartei_txt") or party_code
party = OrgRef(name=party_name, short_name=party_code, kind="party") if party_name else None
start, end = parse_date(m.get("mandat_von")), parse_date(m.get("mandat_bis"))
gp = m.get("gp_code") or ""
region = m.get("wahlkreis_bundesland")
label = party_code or party_name or "?"
body = OrgRef(slug=CHAMBER_SLUG[code])
return Claim(
claim_type="role",
person=PersonRef(ids={"parlament": pad}),
org=body,
counterpart_org=party,
valid_from=start,
valid_to=end,
summary_de=f"{name}: Mandat im {CHAMBER_NAME_DE[code]} ({gp}. GP) für {label}"
+ (f", Wahlkreis {region}" if region else ""),
summary_en=f"{name}: member of the {CHAMBER_NAME_DE[code]} ({gp} legislative period) for {label}",
data={"gp": gp, "party_code": party_code, "faction_code": m.get("fraktionscode"), "region": region},
evidence=[Evidence(url=f"{BASE}/person/{pad}", doc_type="api_record", title=f"Parlament: {name}",
locator=f"mandat:{code}:{gp}:{start}")],
facet=Role(role="mp", level="federal", body=body, party=party, region=region, start=start, end=end),
)
# -- person detail ----------------------------------------------------------------
def parse_person(self, ctx: RunContext, response: Response) -> Iterator[Any]:
data = response.json()
content = data.get("content") or {}
pad, name = response.meta["pad"], response.meta["name"]
ref = PersonRef(ids={"parlament": pad})
url = f"{BASE}/person/{pad}"
photo = ((data.get("meta") or {}).get("openGraph") or {}).get("image")
if photo:
yield PersonRecord(ref=ref, full_name=name, photo_url=photo)
for account in (content.get("kontakt") or {}).get("socialMedia") or []:
platform = SOCIAL_PLATFORMS.get((account.get("type") or "").upper(), (account.get("type") or "website").lower())
yield SocialAccount(platform=platform, url=account["url"], person=ref, verified=True)
for fn in walk_dicts(content.get("biografie") or {}, "funktion_text"):
if fn.get("gremium") in CHAMBER_SLUG:
continue
text = fn.get("funktion_text") or fn.get("bez")
if not text:
continue
start, end = parse_date(fn.get("funktion_von")), parse_date(fn.get("funktion_bis"))
is_gov = bool(GOVERNMENT_RE.search(text))
body = OrgRef(slug="bundesregierung") if is_gov else None
yield Claim(
claim_type="role", person=ref, org=body, valid_from=start, valid_to=end,
summary_de=f"{name}: {text}",
evidence=[Evidence(url=url, doc_type="api_record", title=f"Parlament: {name}",
locator=f"funktion:{text}:{start}")],
facet=Role(role=text, level="federal" if is_gov else "other", body=body, start=start, end=end),
)
yield from self.side_jobs(ref, pad, name, content.get("unvtrans") or {})
yield list_request(251, {"PAD_INTERN": [int(pad)]}, kind="speeches", meta={"pad": pad, "name": name})
def side_jobs(self, ref: PersonRef, pad: str, name: str, unv: dict) -> Iterator[Claim]:
"""Disclosures under §6 Unv-Transparenz-G: leading positions, paid side jobs, honorary positions."""
tables = unv.get("unvtransData") or {}
pdf = next((d["link"] for group in unv.get("linkliste") or [] for d in group.get("documents") or []
if d.get("link", "").endswith(f"/person/{pad}/BezBegrBVGPar9.pdf")), None)
for key, table in tables.items():
m = re.fullmatch(r"(z\d+)Table", key)
if not m or not isinstance(table, dict):
continue
header = html_to_text(tables.get(f"{m.group(1)}Header")) or key
for row in (table.get("data") or {}).get("rows") or []:
cells = [c for c in (html_to_text(c) for c in row if isinstance(c, str)) if c]
if not cells:
continue
name_, fn = employer(cells[0]) if m.group(1) in ("z1", "z2", "z3") else (None, None)
if name_: # never keep the (business) address that parliament publishes with the employer
cells = [name_ + (f" (FN {fn})" if fn else ""), *cells[1:]]
text = " – ".join(cells)
yield Claim(
claim_type="side_job", person=ref, key=("unvtrans", pad, m.group(1), text),
counterpart_org=OrgRef(name=name_, kind="company" if LEGAL_FORM_END.search(name_) else "other",
ids={"firmenbuch": fn} if fn else {}) if name_ else None,
summary_de=f"{name} – {header.split(' - ')[0]}: {text}",
data={"category": header, "cells": cells, "unpaid": "*" in "".join(str(c) for c in row),
"employer": name_, "firmenbuch": fn},
evidence=[Evidence(url=f"{BASE}{pdf}" if pdf else f"{BASE}/person/{pad}", doc_type="register_entry",
title=f"Unvereinbarkeits-/Transparenzliste {name}", quote=text,
locator=f"unvtrans:{m.group(1)}")],
topics=("Transparenz und Korruption",),
)
# -- speeches -----------------------------------------------------------------------
def parse_speeches(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
periods = set(ctx.params.get("periods") or get_settings().parlament_periods)
pad, name = response.meta["pad"], response.meta["name"]
for row in list_rows(response.json()):
if row.get("GP") not in periods and row.get("GREMIUM") != "B":
continue
html_cells = " ".join(str(v) for v in row.values() if isinstance(v, str))
protocol = re.search(r'href="(/dokument/[^"]+)"', html_cells)
session = html_to_text(row.get("Sitzung"))
topic = html_to_text(row.get("Rede"))
said_at = row.get("fromDate")
url = f"{BASE}{protocol.group(1)}" if protocol else f"{BASE}/person/{pad}"
chamber = "Bundesrat" if row.get("GREMIUM") == "B" else "Nationalrat"
yield Claim(
claim_type="statement", person=PersonRef(ids={"parlament": pad}),
key=("parl-speech", pad, row.get("Sortier"), topic), occurred_at=said_at,
summary_de=f"{name} sprach im {chamber} zu: {topic}",
summary_en=f"{name} spoke in the {chamber} on: {topic}",
data={"session": session, "gp": row.get("GP"), "item_title": topic},
evidence=[Evidence(url=url, doc_type="protocol", title=session, published_at=parse_date(said_at),
locator=url.split("#", 1)[-1] if "#" in url else "")],
facet=Statement(context="parliament_speech", url=url, said_at=said_at, medium=chamber, title=topic),
)
+145
View File
@@ -0,0 +1,145 @@
"""Political advertising on online platforms.
* Google Political Ads Transparency Report - public download bundle (zip with CSVs, ~300 MB). We keep
ads served in Austria (Regions contains "AT"): advertiser, period, impressions bracket, spend range.
Note: Google stopped serving political ads in the EU in October 2025 (EU TTPA regulation), so this
is historical data (2019-2025).
* Meta Ad Library API (political and issue ads reaching Austria) - requires an identity-verified developer
token (POLITRACK_META_AD_LIBRARY_TOKEN) and page ids in config/meta_pages.yaml; skipped otherwise.
Advertisers are resolved to parties/organisations by name where possible (e.g. "FPÖ", "SPÖ Wien").
"""
from __future__ import annotations
import csv
import io
import zipfile
from collections.abc import Iterator
from datetime import date
from typing import Any
import yaml
from politrack.config import get_settings
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Evidence, Money, OrgRef
from politrack.text import format_eur_de
GOOGLE_BUNDLE = "https://storage.googleapis.com/political-csv/google-political-ads-transparency-bundle.zip"
GOOGLE_SPEC = SourceSpec(
slug="google-political-ads", name="Google Political Ads Transparency Report", kind="aggregator",
homepage="https://adstransparency.google.com/political", license="Google transparency data (public download)",
access="bulk", trust=85, keep_raw=False, min_interval=5.0,
tos_notes="Public CSV bundle of political ads. ~300 MB; fetched monthly with conditional requests.",
)
META_SPEC = SourceSpec(
slug="meta-adlibrary", name="Meta Ad Library", kind="social", homepage="https://www.facebook.com/ads/library/api",
license="Meta Platform Terms", access="api", trust=80, min_interval=2.0,
tos_notes="Official Ad Library API for political and issue ads (identity-verified developer token required).",
)
META_API = "https://graph.facebook.com/v21.0/ads_archive"
def col(row: dict[str, str], *names: str) -> str | None:
lowered = {k.lower(): v for k, v in row.items()}
for name in names:
if lowered.get(name.lower()) not in (None, ""):
return lowered[name.lower()]
return None
@register
class GooglePoliticalAds(Adapter):
spec = GOOGLE_SPEC
name = "ads.google"
description = "Political ads served in Austria on Google/YouTube (historical, until 10/2025)"
schedule = "0 3 1 * *"
queue = "bulk"
def start(self, ctx: RunContext) -> Iterator[Request]:
yield Request(url=GOOGLE_BUNDLE, kind="bundle")
def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
since = ctx.params.get("since") or get_settings().ad_archive_since
with zipfile.ZipFile(io.BytesIO(response.body)) as bundle:
name = next((n for n in bundle.namelist() if n.endswith("creative-stats.csv")), None)
if name is None:
ctx.stats["creative_stats_missing"] += 1
return
with bundle.open(name) as fh:
for row in csv.DictReader(io.TextIOWrapper(fh, encoding="utf-8", errors="replace")):
regions = (col(row, "Regions") or "").replace(" ", "").split(",")
if "AT" not in regions:
continue
start = col(row, "Date_Range_Start") or ""
if start < since:
continue
advertiser = col(row, "Advertiser_Name") or "?"
low, high = col(row, "Spend_Range_Min_EUR"), col(row, "Spend_Range_Max_EUR")
ad_url = col(row, "Ad_URL") or GOOGLE_BUNDLE
payer = OrgRef(name=advertiser, kind="other", ids={"google_advertiser": col(row, "Advertiser_ID") or ""})
spend = f"{format_eur_de(low)}–{format_eur_de(high)} €" if low and high else None
yield Claim(
claim_type="money_flow", org=payer, occurred_at=start,
summary_de=f"Politische Google-Werbung von {advertiser} ({start} – {col(row, 'Date_Range_End')})"
+ (f", Ausgaben {spend}" if spend else ""),
summary_en=f"Political Google ad by {advertiser} ({start} – {col(row, 'Date_Range_End')})",
data={"impressions": col(row, "Impressions"), "ad_type": col(row, "Ad_Type"),
"age_targeting": col(row, "Age_Targeting"), "gender_targeting": col(row, "Gender_Targeting"),
"geo_included": col(row, "Geo_Targeting_Included")},
evidence=[Evidence(url=ad_url, doc_type="dataset_row", title=f"Google Ads Transparency: {advertiser}",
published_at=start)],
facet=Money(kind="political_ad", payer=payer, payer_name=advertiser,
payee=OrgRef(name="Google", kind="company"), payee_name="Google",
paid_on=date.fromisoformat(start[:10]) if start else None,
amount_category=spend, reference=col(row, "Ad_ID"), category=col(row, "Ad_Type")),
)
@register
class MetaPoliticalAds(Adapter):
spec = META_SPEC
name = "ads.meta"
description = "Political/issue ads reaching Austria from Meta's Ad Library (token + page list required)"
schedule = "0 4 * * 4"
queue = "bulk"
FIELDS = ("id,page_id,page_name,bylines,ad_delivery_start_time,ad_delivery_stop_time,spend,impressions,"
"currency,ad_snapshot_url,ad_creative_bodies")
def start(self, ctx: RunContext) -> Iterator[Request]:
token = get_settings().meta_ad_library_token
path = get_settings().config_dir / "meta_pages.yaml"
if not token or not path.exists():
ctx.stats["skipped_not_configured"] = 1
return
pages = [str(p["page_id"]) for p in (yaml.safe_load(path.read_text(encoding="utf-8")) or {}).get("pages", [])]
for i in range(0, len(pages), 10):
yield Request(url=META_API, kind="ads", conditional=False, params={
"access_token": token, "ad_reached_countries": '["AT"]', "ad_type": "POLITICAL_AND_ISSUE_ADS",
"search_page_ids": ",".join(pages[i:i + 10]), "fields": self.FIELDS, "limit": 250,
"ad_delivery_date_min": ctx.params.get("since") or get_settings().ad_archive_since})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
data = response.json()
next_url = (data.get("paging") or {}).get("next")
if next_url:
yield Request(url=next_url, kind="ads", conditional=False)
for ad in data.get("data") or []:
spend = ad.get("spend") or {}
payer_name = ad.get("bylines") or ad.get("page_name") or "?"
payer = OrgRef(name=ad.get("page_name") or payer_name, kind="other", ids={"facebook_page": ad.get("page_id")})
rng = f"{spend.get('lower_bound')}–{spend.get('upper_bound')} {ad.get('currency') or 'EUR'}" if spend else None
start = (ad.get("ad_delivery_start_time") or "")[:10]
yield Claim(
claim_type="money_flow", org=payer, occurred_at=start or None,
summary_de=f"Politische Meta-Werbung von {payer_name} ab {start}" + (f", Ausgaben {rng}" if rng else ""),
data={"impressions": ad.get("impressions"), "bodies": (ad.get("ad_creative_bodies") or [])[:2]},
evidence=[Evidence(url=ad.get("ad_snapshot_url") or f"https://www.facebook.com/ads/library/?id={ad['id']}",
doc_type="dataset_row", title=f"Meta Ad Library: {payer_name}", published_at=start or None)],
facet=Money(kind="political_ad", payer=payer, payer_name=payer_name,
payee=OrgRef(name="Meta Platforms", kind="company"), payee_name="Meta",
paid_on=date.fromisoformat(start) if start else None, amount_category=rng,
reference=ad.get("id")),
)
+254
View File
@@ -0,0 +1,254 @@
"""Public procurement: contract award notices published as open "core data" (Kerndaten, §62/64 BVergG 2018).
Every contracting authority (or the platform it uses) publishes a "Kerndatenquelle" (KDQ): an XML index
<kdq><item id lastmod><url>…</url></item>…</kdq> listed as a dataset on data.gv.at. Each item is one
notice; since 2023/24 notices use the EU eForms UBL schema. We keep ContractAwardNotice documents
(who bought what from whom, for how much) and skip calls for tenders.
Correlations this enables: suppliers that are also party donors, sponsors or side-job employers of
politicians; contracts by ministries led by a given minister (organisation pages).
Volume: tens of thousands of notices; first run backfills from POLITRACK_PROCUREMENT_SINCE, later runs only
fetch items whose lastmod changed. Runs on the `bulk` queue.
"""
from __future__ import annotations
import xml.etree.ElementTree as ET
from collections.abc import Iterator
from datetime import date
from typing import Any
from politrack.config import get_settings
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Evidence, Money, OrgRef
from politrack.text import firmenbuch_number, format_eur_de
HUB = "https://www.data.gv.at/api/hub/search/search"
SPEC = SourceSpec(
slug="vergabe", name="Öffentliche Aufträge (Kerndaten, data.gv.at)", kind="official",
homepage="https://www.data.gv.at", license="Open data (free licence required by §64 BVergG 2018)", access="bulk",
trust=90, min_interval=1.0,
tos_notes="Contracting authorities must publish award core data as open, machine-readable data under a free "
"licence (BVergG 2018). We read each publisher's Kerndatenquelle (KDQ) XML and its eForms notices.",
)
NS = {
"cbc": "urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2",
"cac": "urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2",
"efac": "http://data.europa.eu/p27/eforms-ubl-extension-aggregate-components/1",
"efbc": "http://data.europa.eu/p27/eforms-ubl-extension-basic-components/1",
"efext": "http://data.europa.eu/p27/eforms-ubl-extensions/1",
"ext": "urn:oasis:names:specification:ubl:schema:xsd:CommonExtensionComponents-2",
}
CATALOG_QUERIES = ("Kerndaten", "Kerndatenquelle", "KDQ BVergG")
def local(tag: str) -> str:
return tag.rsplit("}", 1)[-1]
def text(node: ET.Element | None, path: str) -> str | None:
if node is None:
return None
found = node.find(path, NS)
return found.text.strip() if found is not None and found.text else None
def parse_kerndaten(root: ET.Element) -> dict[str, Any] | None:
"""Austrian core-data format (BVergG 2018, schema brz.gv.at/eproc/kd): KD_8_2_* = awarded contracts.
Still used by many publishers (e-shops, below-threshold awards). Contractor NATIONALID is usually the
company register number ("183286p"), occasionally the VAT id.
"""
if not local(root.tag).startswith("KD_8_2"):
return None
def t(node: ET.Element | None, path: str) -> str | None:
found = node.find(path) if node is not None else None
return found.text.strip() if found is not None and found.text else None
buyer = root.find("{*}CONTRACTING_BODY/{*}ADDRESS_CONTRACTING_BODY")
title = " ".join(p.text.strip() for p in root.iterfind("{*}OBJECT_CONTRACT/{*}TITLE/{*}P") if p.text)
cpv = root.find("{*}OBJECT_CONTRACT/{*}CPV_MAIN/{*}CPV_CODE")
contracts = []
for awarded in root.iterfind(".//{*}AWARD_CONTRACT/{*}AWARDED_CONTRACT"):
contracts.append({
"id": None,
"title": None,
"awarded": t(awarded, "{*}DATE_CONCLUSION_CONTRACT"),
"amount": t(awarded, "{*}VAL_TOTAL"),
"winners": [{"name": t(c, "{*}ADDRESS_CONTRACTOR/{*}OFFICIALNAME"),
"company_id": t(c, "{*}ADDRESS_CONTRACTOR/{*}NATIONALID")}
for c in awarded.iterfind("{*}CONTRACTOR")],
})
return {
"notice_id": t(root, "{*}EXTENSION[@KEY='Referenz']") or t(root, "{*}OBJECT_CONTRACT/{*}REFERENCE_NUMBER"),
"issued": None,
"title": title or None,
"cpv": cpv.get("CODE") if cpv is not None else None,
"total": None,
"buyers": [{"name": t(buyer, "{*}OFFICIALNAME"), "company_id": None}] if buyer is not None else [],
"contracts": contracts,
}
def parse_award_notice(xml: bytes) -> dict[str, Any] | None:
"""Extract buyer, winners, amounts, title, CPV and dates from an award notice: eForms
ContractAwardNotice, or the Austrian core-data format (parse_kerndaten)."""
root = ET.fromstring(xml)
if local(root.tag).startswith("KD_"):
return parse_kerndaten(root)
if local(root.tag) != "ContractAwardNotice":
return None
orgs: dict[str, dict[str, Any]] = {}
for org in root.iterfind(".//efac:Organizations/efac:Organization/efac:Company", NS):
org_id = text(org, "cac:PartyIdentification/cbc:ID")
if org_id:
orgs[org_id] = {"name": text(org, "cac:PartyName/cbc:Name"),
"company_id": text(org, "cac:PartyLegalEntity/cbc:CompanyID"),
"city": text(org, "cac:PostalAddress/cbc:CityName")}
buyer_ids = [e.text for e in root.iterfind("cac:ContractingParty/cac:Party/cac:PartyIdentification/cbc:ID", NS)
if e.text]
result = root.find(".//efac:NoticeResult", NS)
tender_amounts = {text(t, "cbc:ID"): text(t, "cac:LegalMonetaryTotal/cbc:PayableAmount")
for t in (result.iterfind("efac:LotTender", NS) if result is not None else [])}
tendering_parties = {text(tp, "cbc:ID"): [e.text for e in tp.iterfind("efac:Tenderer/cbc:ID", NS) if e.text]
for tp in (result.iterfind("efac:TenderingParty", NS) if result is not None else [])}
lot_tender_party = {text(t, "cbc:ID"): text(t, "efac:TenderingParty/cbc:ID")
for t in (result.iterfind("efac:LotTender", NS) if result is not None else [])}
contracts = []
for contract in (result.iterfind("efac:SettledContract", NS) if result is not None else []):
tender_id = text(contract, "efac:LotTender/cbc:ID")
winners = tendering_parties.get(lot_tender_party.get(tender_id), [])
contracts.append({
"id": text(contract, "cbc:ID"),
"title": text(contract, "cbc:Title"),
"awarded": text(contract, "cbc:AwardDate") or text(contract, "cbc:IssueDate"),
"amount": tender_amounts.get(tender_id),
"winners": [orgs.get(w, {"name": None}) | {"ref": w} for w in winners],
})
project_title = text(root, "cac:ProcurementProject/cbc:Name")
return {
"notice_id": text(root, "cbc:ID"),
"issued": text(root, "cbc:IssueDate"),
"title": project_title,
"cpv": text(root, "cac:ProcurementProject/cac:MainCommodityClassification/cbc:ItemClassificationCode"),
"total": text(root, ".//efac:NoticeResult/cbc:TotalAmount"),
"buyers": [orgs.get(b, {"name": None}) | {"ref": b} for b in buyer_ids],
"contracts": contracts,
}
def org_ref(org: dict[str, Any], kind: str) -> OrgRef | None:
if not org.get("name"):
return None
company_id = (org.get("company_id") or "").replace(" ", "")
ids = {}
if company_id.upper().startswith("ATU"):
ids["uid"] = company_id.upper()
elif fn := firmenbuch_number(company_id):
ids["firmenbuch"] = fn
return OrgRef(name=org["name"], kind=kind, ids=ids)
def as_date(value: str | None) -> date | None:
if not value:
return None
try:
return date.fromisoformat(value[:10])
except ValueError:
return None
@register
class PublicContracts(Adapter):
spec = SPEC
name = "vergabe.awards"
description = "Contract awards of Austrian public bodies (eForms and BVergG core data via data.gv.at)"
schedule = "0 2 * * *"
queue = "bulk"
parser_version = 2
def start(self, ctx: RunContext) -> Iterator[Request]:
for query in CATALOG_QUERIES:
yield Request(url=HUB, params={"q": query, "limit": 100, "page": 0}, kind="catalog", conditional=False)
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
kind = response.request.kind
if kind == "catalog":
result = response.json().get("result") or {}
for dataset in result.get("results") or []:
for dist in dataset.get("distributions") or []:
url = (dist.get("access_url") or dist.get("download_url") or [None])
url = url[0] if isinstance(url, list) else url
if url and url.startswith("http") and "data.gv.at/katalog" not in url:
if ctx.params.get("only_url") and ctx.params["only_url"] not in url:
continue # targeted runs: -p only_url=bbg.gv.at
title = (dataset.get("title") or {}).get("de") or ""
yield Request(url=url, kind="kdq", meta={"publisher": title.split("von", 1)[-1].strip(" '")})
if result.get("count", 0) > (response.request.params["page"] + 1) * 100:
yield Request(url=HUB, params={**response.request.params, "page": response.request.params["page"] + 1},
kind="catalog", conditional=False)
elif kind == "kdq":
yield from self.parse_kdq(ctx, response)
elif kind == "notice":
yield from self.parse_notice(ctx, response)
def parse_kdq(self, ctx: RunContext, response: Response) -> Iterator[Request]:
try:
root = ET.fromstring(response.body)
except ET.ParseError:
ctx.stats["kdq_not_xml"] += 1
return
state_key = f"kdq:{response.request.url}"
since = ctx.params.get("since") or ctx.state.get(state_key) or get_settings().procurement_since
limit = int(ctx.params.get("max_notices_per_source", 3000))
items = [(item.get("lastmod") or "", item.findtext("{*}url") or item.findtext("url"))
for item in root.iter() if local(item.tag) == "item"]
# Oldest first so an interrupted backfill resumes where it stopped (cursor = newest lastmod handled).
items = sorted(i for i in items if i[1] and i[0] > since)[:limit]
for lastmod, url in items:
# lastmod in meta (not in the URL) + conditional requests: unchanged notices cost a 304 at most.
# The cursor advances only when a notice has been parsed (see advance_cursor), so an interrupted
# run resumes with the first notice it did not get to.
yield Request(url=url.strip(), kind="notice", meta={"publisher": response.meta.get("publisher"),
"lastmod": lastmod, "state_key": state_key})
@staticmethod
def advance_cursor(ctx: RunContext, response: Response) -> None:
key, lastmod = response.meta.get("state_key"), response.meta.get("lastmod")
if key and lastmod and lastmod > (ctx.state.get(key) or ""):
ctx.state.set(key, lastmod)
def parse_notice(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
self.advance_cursor(ctx, response) # notices arrive oldest-first per publisher
try:
notice = parse_award_notice(response.body)
except ET.ParseError:
ctx.stats["notice_not_xml"] += 1
return
if notice is None:
ctx.stats["not_an_award"] += 1
return
buyer = next((org_ref(b, "public_body") for b in notice["buyers"] if b.get("name")), None)
if buyer is None:
return
for contract in notice["contracts"]:
for winner in contract["winners"]:
supplier = org_ref(winner, "company")
if supplier is None:
continue
amount = contract["amount"] or (notice["total"] if len(notice["contracts"]) == 1 else None)
awarded = as_date(contract["awarded"]) or as_date(notice["issued"])
title = contract["title"] or notice["title"] or "Auftrag"
yield Claim(
claim_type="money_flow", org=buyer, counterpart_org=supplier, occurred_at=awarded,
summary_de=f"Auftrag „{title}“ an {supplier.name}" + (f" – {format_eur_de(amount)} €" if amount else ""),
summary_en=f"Contract \"{title}\" awarded to {supplier.name}" + (f" – EUR {amount}" if amount else ""),
data={"notice_id": notice["notice_id"], "cpv": notice["cpv"], "publisher": response.meta.get("publisher")},
evidence=[Evidence(url=response.request.url, doc_type="register_entry", title=f"Vergabe: {title}",
published_at=awarded, quote=f"{buyer.name} → {supplier.name}: {title}")],
facet=Money(kind="contract", amount=amount, payer=buyer, payer_name=buyer.name, payee=supplier,
payee_name=supplier.name, paid_on=awarded,
reference=f"{notice['notice_id']}/{contract['id']}", category=notice["cpv"]),
)
+551
View File
@@ -0,0 +1,551 @@
"""Rechnungshof (Court of Audit): published party donations and parties' annual financial reports.
Official works (§7 UrhG); no robots restrictions.
Donations: HTML pages + one CSV per year/election. The CSVs come in UTF-8, Windows-1252 and Mac Roman,
with or without title rows, and one older file without header (date; amount; donor; address; party).
Annual reports (Rechenschaftsberichte, 2023+ as XLSX): income statement per organisation level
(public funding "Fördermittel", membership fees, donations, sponsoring, ad income...) and named lists:
donations > EUR 500, sponsors, advertisers in party media, party-owned companies, loans, affiliated
organisations.
Data minimisation: donor addresses and postcodes are never mapped; private donors are stored as names
only (no person entity is created); company donors become organisation entities (contract/donor correlation).
"""
from __future__ import annotations
import csv
import io
import re
from collections.abc import Iterator
from datetime import date, datetime
from decimal import Decimal, InvalidOperation
from typing import Any
from urllib.parse import urljoin
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Document, Evidence, Money, OrgRef
from politrack.text import firmenbuch_number, format_eur_de, normalize, normalize_org_name
BASE = "https://www.rechnungshof.gv.at"
SPEC = SourceSpec(
slug="rechnungshof",
name="Rechnungshof Österreich",
kind="official",
homepage=BASE,
license="Amtliches Werk (§7 UrhG)",
access="scrape",
trust=95,
min_interval=2.0,
tos_notes="Party donation lists and audited annual party reports. Official works; no robots restrictions.",
)
DONATION_INDEX = [
"/rh/home/was-wir-tun/was-wir-tun_5/Kontrolle_der_Parteien.html",
"/rh/home/was-wir-tun/was-wir-tun_5/was-wir-tun_5/Parteispenden/Parteispenden_2024.html",
]
REPORTS_INDEX = "/rh/home/was-wir-tun/was-wir-tun_5/was-wir-tun_6/Kontrolle_der_Parteien.html"
SECTION_PREFIX = "/rh/home/was-wir-tun/"
DONATION_LINK = re.compile(r"spende", re.IGNORECASE)
PARTY_PREFIXES = {"ovp": "oevp", "osterreichische volkspartei": "oevp", "die neue volkspartei": "oevp",
"spo": "spoe", "sozialdemokratische": "spoe", "fpo": "fpoe", "freiheitliche": "fpoe",
"die freiheitliche": "fpoe", "neos": "neos", "die grunen": "gruene", "grune": "gruene",
"kpo": "kpoe", "kommunistische": "kpoe", "bier": "bier", "die bierpartei": "bier", "mfg": "mfg"}
COMPANY = re.compile(r"\b(GmbH|AG|KG|OG|e\.U\.|GesmbH|Ges\.m\.b\.H|Gesellschaft|Verein|Stiftung|Bank|Holding|"
r"Genossenschaft|Verband|Kammer|Sparkasse|Versicherung|Privatstiftung)\b", re.IGNORECASE)
# -- shared helpers -----------------------------------------------------------------------
def decode(raw: bytes) -> str:
"""UTF-8, Windows-1252 or Mac Roman: pick the decoding that yields real umlauts."""
try:
return raw.decode("utf-8-sig")
except UnicodeDecodeError:
pass
candidates = [raw.decode(enc, errors="replace") for enc in ("cp1252", "mac_roman")]
return max(candidates, key=lambda t: sum(t.count(c) for c in "äöüÄÖÜß"))
def parse_amount(value: Any) -> Decimal | None:
if isinstance(value, (int, float, Decimal)):
return Decimal(str(value)).quantize(Decimal("0.01"))
value = (value or "").strip().replace(".", "").replace(",", ".").replace("€", "").strip()
try:
return Decimal(value)
except InvalidOperation:
return None
def parse_de_date(value: str) -> date | None:
for fmt in ("%d.%m.%Y", "%d.%m.%y"):
try:
return datetime.strptime((value or "").strip(), fmt).date()
except ValueError:
continue
return None
def format_de(amount: Decimal) -> str:
return format_eur_de(amount)
POSTAL_ADDRESS = re.compile(r"^\d{4}\s")
def clean_name(value: str) -> str:
"""Guard against malformed cells ('"2522 Ort Straße 4;Partei"') - an address never ends up in a name."""
value = (value or "").split(";")[-1].strip()
return "" if POSTAL_ADDRESS.match(value) else value
def party_ref(label: str) -> OrgRef | None:
if not label:
return None
norm = normalize(label)
for prefix, slug in PARTY_PREFIXES.items():
if norm.startswith(prefix):
return OrgRef(slug=slug)
return OrgRef(name=label.strip(), kind="party")
def donor_ref(name: str) -> OrgRef | None:
"""Organisations become entities (for correlations); private persons stay a plain name."""
return OrgRef(name=name, kind="company") if COMPANY.search(name) else None
def column(header: list[str], *needles: str) -> int | None:
for i, h in enumerate(header):
h = normalize(h)
if any(n in h for n in needles):
return i
return None
DATE_CELL = re.compile(r"^\d{1,2}\.\d{1,2}\.\d{2,4}$")
AMOUNT_CELL = re.compile(r"^[\d.]+(,\d{1,2})?$")
def find_columns(rows: list[list[str]]) -> tuple[int, dict[str, int | None]]:
"""Locate the header row (older files have a title row above it) and map the columns we use.
Address/postcode columns are never mapped."""
for i, row in enumerate(rows[:5]):
norm = [normalize(c) for c in row]
if any("betrag" in c for c in norm) and any("name" in c or "spender" in c for c in norm):
return i + 1, {
"party": column(row, "partei"),
"date": column(row, "eingang") if column(row, "eingang") is not None else column(row, "datum"),
"donor": column(row, "name", "spender"),
"amount": column(row, "betrag"),
"currency": column(row, "wahrung", "waehrung"),
"recipient": column(row, "empfang", "empfaeng"),
"note": column(row, "anmerk"),
}
first = next((r for r in rows if any(c.strip() for c in r)), [])
date_i = next((i for i, c in enumerate(first) if DATE_CELL.match(c.strip())), None)
amount_i = next((i for i, c in enumerate(first) if AMOUNT_CELL.match(c.strip()) and i != date_i), None)
if date_i is None or amount_i is None:
return 0, {}
return 0, {"party": len(first) - 1, "date": date_i, "donor": amount_i + 1, "amount": amount_i,
"currency": None, "recipient": len(first) - 1, "note": None}
def read_rows(text: str) -> tuple[dict[str, int | None], list[list[str]]]:
"""Parse the semicolon CSV; repairs rows broken by stray line breaks."""
reader = list(csv.reader(io.StringIO(text), delimiter=";"))
if not reader:
return {}, []
start, idx = find_columns(reader)
if not idx:
return {}, []
width = max(i for i in idx.values() if i is not None) + 1
rows, pending = [], []
for row in reader[start:]:
if pending:
pending[-1] += " " + (row[0] if row else "")
pending.extend(row[1:])
else:
pending = list(row)
if len(pending) >= width - 1:
rows.append(pending)
pending = []
return idx, [r for r in rows if any(c.strip() for c in r)]
# -- donations ----------------------------------------------------------------------------
@register
class RechnungshofDonations(Adapter):
spec = SPEC
name = "rechnungshof.donations"
description = "Published party donations (CSV per year / election)"
schedule = "30 5 * * *"
def start(self, ctx: RunContext) -> Iterator[Request]:
for path in DONATION_INDEX:
yield Request(url=urljoin(BASE, path), kind="page", meta={"depth": 0})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if response.request.kind == "page":
depth = response.meta.get("depth", 0)
for a in response.soup.find_all("a", href=True):
href = urljoin(response.url, a["href"])
path = href.removeprefix(BASE)
if href.lower().endswith(".csv") and DONATION_LINK.search(href):
yield Request(url=href, kind="csv", meta={"page": response.url})
elif (depth < 2 and path.startswith(SECTION_PREFIX) and path.endswith(".html")
and DONATION_LINK.search(path)):
yield Request(url=href, kind="page", meta={"depth": depth + 1})
return
yield from self.parse_csv(ctx, response)
def parse_csv(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
idx, rows = read_rows(decode(response.body))
if idx.get("donor") is None or idx.get("amount") is None:
ctx.stats["unparsed_files"] += 1
return
def cell(row: list[str], key: str) -> str:
i = idx[key]
value = row[i].strip() if i is not None and i < len(row) else ""
return clean_name(value) if key in ("party", "donor", "recipient") else value
for n, row in enumerate(rows):
party_label, donor, recipient = cell(row, "party"), cell(row, "donor"), cell(row, "recipient")
amount, received = parse_amount(cell(row, "amount")), parse_de_date(cell(row, "date"))
if not donor or amount is None:
continue
currency = cell(row, "currency") or "EUR"
currency = "EUR" if currency.lower().startswith("eur") else currency
party = party_ref(party_label) or party_ref(recipient)
line = " | ".join(c for c in (cell(row, "date"), party_label, donor, cell(row, "amount"), currency,
recipient) if c)
yield Claim(
claim_type="donation", org=party, occurred_at=received,
summary_de=f"{recipient or party_label} erhielt {format_de(amount)} {currency} von {donor}",
summary_en=f"{recipient or party_label} received {currency} {amount:,.2f} from {donor}",
data={"party_label": party_label, "recipient": recipient, "note": cell(row, "note")},
evidence=[
Evidence(url=response.meta["page"], doc_type="register_entry",
title="Rechnungshof: veröffentlichte Parteispenden", quote=line, locator=f"row:{line}"),
Evidence(url=response.request.url, doc_type="register_entry",
title="Rechnungshof: Parteispenden (CSV)", locator=f"row:{n + 1}"),
],
topics=("Transparenz und Korruption",),
facet=Money(kind="donation", amount=amount, currency=currency, payer=donor_ref(donor), payer_name=donor,
payee=party, payee_name=recipient or party_label, paid_on=received),
)
# -- annual party reports -------------------------------------------------------------------
# Lines of the income statement (§ 5 Abs. 4/5 PartG), matched on the normalised label. Order matters:
# the first matching pattern wins ("Inserate und Werbeeinschaltungen" is an expense, "Inserate" income).
FINANCE_LINES = [
(re.compile(r"^(gesamt)?summe (der )?ertrage"), "total_income"),
(re.compile(r"^(gesamt)?summe (der )?aufwendungen"), "total_expenses"),
(re.compile(r"^fordermittel"), "public_funding"),
(re.compile(r"^mitgliedsbeitrage"), "membership_fees"),
(re.compile(r"^beitrage der .*angehorenden mandatare"), "mandate_holder_levies"),
(re.compile(r"^geldspenden"), "donations"),
(re.compile(r"^sponsoring"), "sponsoring"),
(re.compile(r"^inserate und werbeeinschaltungen"), "ad_spending"),
(re.compile(r"^inserate\b"), "ad_income"),
(re.compile(r"^ertrage aus nahestehenden organisationen"), "from_affiliates"),
(re.compile(r"^personalaufwand"), "staff_costs"),
(re.compile(r"^au(ss|ß)enwerbung"), "outdoor_advertising"),
(re.compile(r"^direktwerbung"), "direct_marketing"),
]
def finance_lines(rows: list[list[Any]], year: str) -> dict[str, Decimal]:
"""Income/expense lines of one sheet, summed over all blocks in it.
Some parties stack one block per organisation in a sheet (e.g. every state below the other); each block
has its own header row with the years. The value is taken from the reporting-year column of the latest
header (columns are "2024, 2023" in some files and "2023, 2024" in others); without a header, the first
number after the label.
"""
total: dict[str, Decimal] = {}
block: dict[str, Decimal] = {}
year_col: int | None = None
for row in rows:
years = [i for i, c in enumerate(row) if str(c).strip() in (year, f"{year}.0")]
if years and sum(1 for c in row if re.fullmatch(r"(19|20)\d\d(\.0)?", str(c).strip() if c is not None else "")) >= 1:
year_col = years[0]
for i, cell in enumerate(row):
if not isinstance(cell, str) or len(cell.strip()) < 4:
continue
label = re.sub(r"\s+", " ", normalize(cell))
key = next((k for pattern, k in FINANCE_LINES if pattern.search(label)), None)
if key:
value = number(row[year_col]) if year_col is not None and year_col > i and year_col < len(row) else None
if value is None:
value = next((number(c) for c in row[i + 1:] if number(c) is not None), None)
if value is not None:
if key in block: # the same line again: a new block (next organisation) starts
for k, v in block.items():
total[k] = total.get(k, Decimal(0)) + v
block = {}
block[key] = value
break # only the first text cell of a row is its label
for k, v in block.items():
total[k] = total.get(k, Decimal(0)) + v
return total
def sum_levels(levels: dict[str, dict[str, Decimal]]) -> tuple[dict[str, Decimal], dict[str, dict[str, Decimal]]]:
"""Sum all organisation levels (federal, states, capitals, districts ...) -> (totals, levels used).
Overview sheets that add up other levels are left out: titles listing several levels ("BO&LOs&LH-St.&NTG")
and sheets whose public funding equals the sum of all the others. Totals (income, expenses) are only
reported when every level states them - a partial sum would be misleading.
"""
levels = {t: v for t, v in levels.items() if "&" not in t}
funding = {title: v.get("public_funding", Decimal(0)) for title, v in levels.items()}
for title, value in funding.items():
others = sum(v for t, v in funding.items() if t != title)
if value > 0 and len(funding) > 2 and abs(value - others) <= Decimal(1):
levels = {t: v for t, v in levels.items() if t != title}
break
totals: dict[str, Decimal] = {}
for values in levels.values():
for key, value in values.items():
totals[key] = totals.get(key, Decimal(0)) + value
for key in ("total_income", "total_expenses"):
if key in totals and not all(key in v for v in levels.values()):
del totals[key]
return totals, levels
AFFILIATE_SKIP = re.compile(
r"^(anlage|liste|ertrag|gem\.|gemass|\(?§|name$|zvr|reg\.? ?nr|organisationseinheit|nahestehende|"
r"bundesorganisation|landesorganisation|zweigvereine:?$|[a-z]\)|[a-z]\. anlage|keine|leermeldung|rechtsform|zweck)")
ZVR = re.compile(r"\(?ZVR(?:-Zahl)?:?\s*(\d{6,10})\)?", re.IGNORECASE)
def affiliate_from_row(row: list[Any]) -> tuple[str, str | None] | None:
"""Name (and association-register number) of an affiliated organisation from one row of the list."""
cells = [str(c).strip() for c in row if c is not None and str(c).strip()]
number_cell = next((c for c in cells if re.fullmatch(r"\d{6,10}", c)), None)
for cell in cells:
if re.fullmatch(r"[\d\s./-]+", cell) or cell.lower() in ("partei", "verein", "gmbh", "stiftung"):
continue
if AFFILIATE_SKIP.search(normalize(cell)) or len(cell) <= 3:
return None
zvr = ZVR.search(cell)
name = ZVR.sub("", cell)
name = re.sub(r'"?,?\s*Kurzbezeichnung:?\s*"?([^"]+)"?\s*$', r" (\1)", name).strip(' "„“,;-')
return (name, (zvr.group(1) if zvr else number_cell)) if len(name) > 3 else None
return None
def transliterate(text: str) -> str:
"""'Kaernten Koefer Gruenen' -> 'Kärnten Köfer Grünen' (file names; not after vowels: 'Neue', 'Steuer')."""
for plain, umlaut in (("ae", "ä"), ("oe", "ö"), ("ue", "ü")):
text = re.sub(rf"(?<![aeiouqAEIOUQ]){plain}", umlaut, text)
text = re.sub(rf"(?<![aeiouqAEIOUQ]){plain.capitalize()}", umlaut.upper(), text)
return text
def report_party_name(sheets: dict[str, list[list[Any]]], url: str) -> str:
"""Cover sheet, else the 'Territoriale Gliederung ... : <party>' line, else the file name."""
cover = [str(c).strip(" -–") for row in sheets.get("Deckblatt", []) for c in row if c and str(c).strip(" -–")]
names = [c for c in cover if len(c) > 2 and not re.search(
r"(?i)rechenschaftsbericht|bericht|kalenderjahr|jahr\b|partg|gemäß|name der|politische partei$|^\d", c)]
if names:
return names[0]
for rows in sheets.values():
for row in rows:
for cell in row:
if isinstance(cell, str) and (m := re.search(r"Territoriale Gliederung[^:]*:\s*(.+)$", cell.strip())):
return m.group(1).strip()
stem = url.rsplit("/", 1)[-1].rsplit(".", 1)[0]
stem = re.sub(r"(?i)^Rechenschaftsbericht[_-]?\d{4}[_-]?|[_-](gepr(ue|ü)ft|Anlagen|Verm(oe|ö)gensaufstellung)$", "", stem)
return transliterate(re.sub(r"[_-]+", " ", stem)).strip()
def sheets_titled(sheets: dict[str, list[list[Any]]], *keywords: str) -> list[list[Any]]:
"""All rows of every sheet whose (normalised) title contains one of the keywords."""
return [row for title, rows in sheets.items() if any(k in normalize(title) for k in keywords) for row in rows]
def number(value: Any) -> Decimal | None:
if value is None or isinstance(value, bool):
return None
if isinstance(value, (int, float, Decimal)):
return Decimal(str(value)).quantize(Decimal("0.01"))
text = str(value).strip()
if re.fullmatch(r"-?\d+\.\d{1,2}", text): # '301561.19' - plain decimal point (some exports)
return Decimal(text).quantize(Decimal("0.01"))
return parse_amount(text) if re.fullmatch(r"[\d.,\s€-]+", text or "x") else None
def sheet_lists(rows: list[list[Any]], header_needles: tuple[str, ...]) -> Iterator[dict[str, Any]]:
"""Yield dict rows of every table in a sheet whose header contains all needles (case-insensitive)."""
header: list[str] | None = None
for row in rows:
cells = ["" if c is None else str(c).strip() for c in row]
lowered = [normalize(c) for c in cells]
if all(any(n in c for c in lowered) for n in header_needles):
header = lowered
continue
if header is None or not any(cells):
continue
if any("leermeldung" in c or "keine" in c for c in lowered[:2]):
header = None
continue
record = {header[i]: row[i] for i in range(min(len(header), len(row))) if header[i]}
amount = next((number(row[i]) for i in range(len(row) - 1, -1, -1) if number(row[i]) is not None), None)
if amount is not None and any(isinstance(v, str) and v.strip() for v in record.values()):
record["__amount"] = amount
yield record
def pick(record: dict[str, Any], *needles: str) -> str:
for key, value in record.items():
if any(n in key for n in needles) and isinstance(value, str) and value.strip():
return value.strip()
return ""
@register
class RechnungshofPartyReports(Adapter):
spec = SPEC
name = "rechnungshof.party_reports"
description = "Parties' annual financial reports: funding, income, sponsors, advertisers, companies, loans"
schedule = "0 6 * * 1"
parser_version = 3
def start(self, ctx: RunContext) -> Iterator[Request]:
yield Request(url=f"{BASE}{REPORTS_INDEX}", kind="index")
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if response.request.kind == "index":
for a in response.soup.find_all("a", href=True):
href = urljoin(response.url, a["href"])
if re.search(r"Rechenschaftsbericht[_-]?\d{4}.*\.xlsx$", href, re.I):
yield Request(url=href, kind="xlsx")
elif re.search(r"(Ergebnis[_-]der[_-]Pruefung|Rechenschaftsbericht).*\.pdf$", href, re.I):
yield Document(Evidence(url=href, doc_type="pdf", title=a.get_text(" ", strip=True) or href.rsplit("/", 1)[-1],
meta={"kind": "party_report_pdf"}))
return
yield from self.parse_report(ctx, response)
def parse_report(self, ctx: RunContext, response: Response) -> Iterator[Any]:
import openpyxl
wb = openpyxl.load_workbook(io.BytesIO(response.body), data_only=True, read_only=True)
sheets = {ws.title.strip(): [list(r) for r in ws.iter_rows(values_only=True)] for ws in wb.worksheets}
url = response.request.url
year_match = re.search(r"Rechenschaftsbericht[_-]?(\d{4})", url) or re.search(r"(\d{4})", url)
year = year_match.group(1) if year_match else "?"
audited = bool(re.search(r"gepr(ue|ü)ft", url, re.IGNORECASE))
party_name = report_party_name(sheets, url)
party = party_ref(party_name)
version_de = "vom Rechnungshof geprüft" if audited else "eingereicht"
evidence = [Evidence(url=url, doc_type="register_entry",
title=f"Rechenschaftsbericht {year} ({version_de}): {party_name}")]
# Income statement per organisation level (federal, states, capitals, districts...), then summed.
levels = {title: lines for title, rows in sheets.items()
if (lines := finance_lines(rows, year)) and ("public_funding" in lines or "total_income" in lines)}
totals, levels = sum_levels(levels)
if totals:
income = totals.get("total_income")
yield Claim(
claim_type="party_finance", org=party,
key=("rh-report", normalize_org_name(party_name), year, "audited" if audited else "submitted"),
valid_from=date(int(year), 1, 1) if year.isdigit() else None,
valid_to=date(int(year), 12, 31) if year.isdigit() else None,
summary_de=f"Rechenschaftsbericht {year} ({version_de}): "
+ (f"Erträge {format_de(income)} €, " if income is not None else "")
+ f"davon Fördermittel {format_de(totals.get('public_funding', Decimal(0)))} €"
+ " (Summe aller ausgewiesenen Gliederungen)",
summary_en=f"Annual report {year} ({'audited' if audited else 'submitted'}): "
+ (f"income EUR {income:,.2f}, " if income is not None else "")
+ f"public funding EUR {totals.get('public_funding', 0):,.2f}",
data={"year": year, "audited": audited, **{k: str(v) for k, v in totals.items()},
"levels": {t: {k: str(v) for k, v in lines.items()} for t, lines in levels.items()}},
evidence=evidence, topics=("Transparenz und Korruption",),
)
if totals.get("public_funding"):
yield Claim(
claim_type="money_flow", org=party, valid_from=date(int(year), 1, 1) if year.isdigit() else None,
summary_de=f"{party_name} erhielt {year} öffentliche Fördermittel von {format_de(totals['public_funding'])} €",
summary_en=f"{party_name} received EUR {totals['public_funding']:,.2f} in public funding in {year}",
evidence=evidence, topics=("Transparenz und Korruption",),
facet=Money(kind="party_funding", amount=totals["public_funding"], payee=party,
payee_name=party_name, payer_name="Öffentliche Parteienförderung", period=year),
)
else:
ctx.stats["reports_without_income_statement"] += 1
def money_claims(rows: list[list[Any]], needles: tuple[str, ...], kind: str, who_keys: tuple[str, ...],
label_de: str) -> Iterator[Claim]:
for rec in sheet_lists(rows, needles):
who = clean_name(pick(rec, *who_keys))
if not who:
continue
recipient = pick(rec, "empfanger", "empfager")
medium = pick(rec, "medium")
amount = rec["__amount"]
yield Claim(
claim_type="donation" if kind == "donation" else "money_flow", org=party,
valid_from=date(int(year), 1, 1) if year.isdigit() else None,
summary_de=f"{label_de}: {who} → {recipient or party_name}"
+ (f" ({medium})" if medium else "") + f", {format_de(amount)} € ({year})",
data={"year": year, "recipient": recipient, "medium": medium},
evidence=[Evidence(url=url, doc_type="register_entry", title=evidence[0].title,
quote=f"{who} | {recipient} | {medium} | {amount}", locator=f"{kind}:{who}:{amount}")],
topics=("Transparenz und Korruption",),
facet=Money(kind=kind, amount=amount, payer=donor_ref(who), payer_name=who, payee=party,
payee_name=recipient or party_name, period=year, category=medium or None),
)
# List sheets are named differently by every party ("Spenden", "Spendenliste (§ 6 PartG)", "Spenden,
# Sponsoring, Inserate" ...): select by title keyword, find the tables by their header.
yield from money_claims(sheets_titled(sheets, "spende"), ("empfanger", "name", "plz"), "donation", ("name",),
"Spende laut Rechenschaftsbericht")
yield from money_claims(sheets_titled(sheets, "sponsor"), ("sponsor",), "sponsorship", ("sponsor",), "Sponsoring")
yield from money_claims(sheets_titled(sheets, "inserat"), ("inserent",), "party_ad", ("inserent",),
"Inserat in Parteimedium")
for rec in sheet_lists(sheets_titled(sheets, "kredit", "darlehen", "verbindlichkeit"), ("kreditgeber",)):
lender = clean_name(pick(rec, "kreditgeber"))
if lender:
yield Claim(
claim_type="money_flow", org=party,
summary_de=f"Kredit/Darlehen von {lender} an {party_name}: {format_de(rec['__amount'])} € ({year})",
evidence=evidence, facet=Money(kind="loan", amount=rec["__amount"], payer=OrgRef(name=lender, kind="company"),
payer_name=lender, payee=party, payee_name=party_name, period=year),
)
for row in sheets_titled(sheets, "beteiligung"):
cells = [str(c).strip() for c in row if c is not None and str(c).strip()]
fn = next((f for f in (firmenbuch_number(c) for c in cells[1:3]) if f), None)
if fn:
company = re.sub(r"\s*\d\)$", "", cells[0]) # footnote markers: "Werbeagentur 1)"
yield Claim(
claim_type="ownership", org=party,
counterpart_org=OrgRef(name=company, kind="company", ids={"firmenbuch": fn}),
key=("rh-ownership", normalize_org_name(party_name), year, fn),
summary_de=f"{party_name} hält eine Beteiligung an {company} (FN {fn}) – laut Rechenschaftsbericht {year}",
evidence=evidence, topics=("Transparenz und Korruption",),
)
for title, rows in sheets.items():
# list of affiliated organisations (not the sheet of income received from them)
if "nahestehend" not in normalize(title) or "ertr" in normalize(title):
continue
for row in rows:
affiliate = affiliate_from_row(row)
if affiliate:
name, zvr = affiliate
yield Claim(
claim_type="affiliation", org=party,
counterpart_org=OrgRef(name=name, kind="ngo", ids={"zvr": zvr} if zvr else {}),
key=("rh-affiliate", normalize_org_name(party_name), year, normalize_org_name(name)),
summary_de=f"Nahestehende Organisation von {party_name}: {name} ({year})",
evidence=evidence,
)
ctx.stats["reports"] += 1
+106
View File
@@ -0,0 +1,106 @@
"""RTR / KommAustria media transparency (MedKF-TG): advertising and media subsidies paid by public bodies.
Open data API (no usage restrictions, attribution): https://data.rtr.at/api/v1/tables/<table>.json
* mt-bekanntgabe2024 (half-yearly since 2024): payer, medium, media owner, campaign, amount, ad sujet link
* mt-bekanntgabe (quarterly until 2023): payer, medium/owner, amount
Each row -> money_flow (kind government_ad for advertising, subsidy for media funding) from the public body
(ministry, state government, municipality, public company...) to the media owner.
"""
from __future__ import annotations
from collections.abc import Iterator
from datetime import date
from typing import Any
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Evidence, Money, OrgRef
from politrack.text import format_eur_de
API = "https://data.rtr.at/api/v1/tables"
PAGE_SIZE = 5000
SPEC = SourceSpec(
slug="rtr-medientransparenz", name="RTR/KommAustria Medientransparenz", kind="official",
homepage="https://www.rtr.at/medien/was_wir_tun/medientransparenz/",
license="Amtliches Werk (§7 UrhG); attribution \"KommAustria – data.rtr.at\"", access="api",
trust=95, min_interval=2.0,
# data.rtr.at/robots.txt disallows everything for crawlers, but RTR documents this API as "der Endpunkt zur
# automatischen Abfrage der Daten" (rtr.at/rtr/service/opendata/OD_Schnittstellenbeschreibung.de.html).
robots_exempt=(API + "/",),
tos_notes="Government advertising and media funding reported under MedKF-TG. The REST API is documented by "
"RTR for automated queries (OD_Schnittstellenbeschreibung); robots.txt on data.rtr.at blocks "
"crawlers only. Terms (OD_Nutzungsbedingungen): free works under §7 UrhG, attribution requested, "
"no use for defamatory/discriminatory purposes.",
)
KIND = {2: "government_ad", 4: "subsidy"}
def half_years(since_year: int) -> list[str]:
today = date.today()
return [f"{y}{h}" for y in range(since_year, today.year + 1) for h in (1, 2)
if (y, h) <= (today.year, 1 if today.month <= 6 else 2)]
def quarters(since_year: int, until_year: int = 2023) -> list[str]:
return [f"{y}{q}" for y in range(since_year, until_year + 1) for q in (1, 2, 3, 4)]
@register
class MediaTransparency(Adapter):
spec = SPEC
name = "rtr.medientransparenz"
description = "Government advertising and media subsidies per public body and media owner"
schedule = "0 4 * * 2"
queue = "bulk"
def start(self, ctx: RunContext) -> Iterator[Request]:
for period in half_years(int(ctx.params.get("since_year", 2024))):
yield Request(url=f"{API}/mt-bekanntgabe2024.json", params={"halbjahr": period, "size": PAGE_SIZE, "page": 0},
kind="halbjahr", meta={"period": period})
for period in quarters(int(ctx.params.get("legacy_since_year", 2021))):
yield Request(url=f"{API}/mt-bekanntgabe.json", params={"quartal": period, "size": PAGE_SIZE, "page": 0},
kind="quartal", meta={"period": period})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
rows = response.json().get("data") or []
params = response.request.params or {}
if len(rows) >= PAGE_SIZE:
yield Request(url=response.request.url, params={**params, "page": params.get("page", 0) + 1},
kind=response.request.kind, meta=response.meta)
period = response.meta["period"]
label = f"{period[:4]}-H{period[4:]}" if response.request.kind == "halbjahr" else f"{period[:4]}-Q{period[4:]}"
min_euro = float(ctx.params.get("min_euro", 0))
for row in rows:
amount = row.get("euro")
if row.get("leermeldung") or amount is None or float(amount) < min_euro:
continue
kind = KIND.get(row.get("bekanntgabe"), "other")
payer_name = (row.get("rechtstraeger") or "").strip().strip('"')
owner = (row.get("medieninhaber") or row.get("mediumMedieninhaber") or "").strip()
medium = (row.get("medium") or row.get("mediumMedieninhaber") or "").strip()
if not payer_name or not owner:
continue
payer = OrgRef(name=payer_name, kind="public_body",
ids={"rtr": str(row["rechtstraeger_id"])} if row.get("rechtstraeger_id") else {})
payee = OrgRef(name=owner, kind="media")
campaign = row.get("kampagne")
what = "Werbung" if kind == "government_ad" else "Förderung"
evidence_url = (f"{API}/{'mt-bekanntgabe2024' if response.request.kind == 'halbjahr' else 'mt-bekanntgabe'}"
f".json?{'halbjahr' if response.request.kind == 'halbjahr' else 'quartal'}={period}"
+ (f"&rechtstraeger_id={row['rechtstraeger_id']}" if row.get("rechtstraeger_id") else ""))
yield Claim(
claim_type="money_flow", org=payer, counterpart_org=payee,
summary_de=f"{payer_name}: {what} in {medium} ({owner}), {format_eur_de(amount)} € ({label})"
+ (f" – Kampagne „{campaign}“" if campaign else ""),
summary_en=f"{payer_name}: {'advertising' if kind == 'government_ad' else 'funding'} in {medium} "
f"({owner}), EUR {amount:,.2f} ({label})",
data={"medium": medium, "category": row.get("kategorie"), "subcategory": row.get("subkategorie"),
"campaign": campaign, "sujet": row.get("sujet"), "period": label},
evidence=[Evidence(url=evidence_url, doc_type="dataset_row",
title=f"Medientransparenz {label}: {payer_name}",
quote=f"{payer_name} → {medium} ({owner}): {amount} €")],
topics=("Information und Medien",),
facet=Money(kind=kind, amount=amount, payer=payer, payer_name=payer_name, payee=payee, payee_name=owner,
period=label, reference=f"{campaign or ''}|{medium}|{row.get('sujet_dateiname') or ''}",
category=row.get("kategorie")),
)
+279
View File
@@ -0,0 +1,279 @@
"""Public posts of politicians on open platforms: Bluesky, Mastodon, Telegram (public channels), YouTube,
and X via the official (paid) API when a token is configured.
Accounts come from the `social_account` table: parliament profiles (verified), Wikidata
P2002/P12361/P4033/P3789/P2397 (unverified until confirmed) and the curated config/social_accounts.yaml
(enricher `config.social_accounts`). Accounts belong to a person or to an organisation (party channels).
Not collected (see docs/sources.md): X without the official API (ToS forbid scraping and proxies such as
Nitter), Facebook/Instagram (ToS), WhatsApp channels (no public read access, ToS forbid automated
collection), Signal (private by design, no public channels), private Telegram groups.
"""
from __future__ import annotations
import re
from collections.abc import Iterator
from typing import Any
from politrack.config import get_settings
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Evidence, OrgRef, PersonRef, Statement
from politrack.text import excerpt, html_to_text
def social_spec(slug: str, name: str, homepage: str, notes: str, robots_exempt: tuple[str, ...] = ()) -> SourceSpec:
return SourceSpec(slug=slug, name=name, kind="social", homepage=homepage, license="Per-post copyright of the author",
access="api", trust=40, store_fulltext=True, keep_raw=True, min_interval=2.0,
robots_exempt=robots_exempt, tos_notes=notes)
BLUESKY = social_spec("bluesky", "Bluesky", "https://bsky.app",
"Public AT Protocol AppView API, no auth for public reads. Posts by the politicians themselves.")
MASTODON = social_spec("mastodon", "Mastodon / Fediverse", "https://joinmastodon.org",
"Public REST API of the account's instance (no auth for public statuses). Respect instance rules.")
TELEGRAM = social_spec("telegram", "Telegram (öffentliche Kanäle)", "https://telegram.org",
"Public web preview of *public broadcast channels* (t.me/s/<channel>); no robots.txt restrictions. "
"Private groups/chats are never accessed. Low frequency, latest posts only.")
YOUTUBE = social_spec("youtube", "YouTube", "https://www.youtube.com",
"YouTube Data API v3 (free quota, API key required via POLITRACK_YOUTUBE_API_KEY).")
X_API = "https://api.x.com/2/"
X = SourceSpec(slug="x", name="X (Twitter)", kind="social", homepage="https://x.com",
license="Per-post copyright of the author; X Developer Agreement", access="api", trust=40,
store_fulltext=False, keep_raw=False, min_interval=2.0, robots_exempt=(X_API,),
tos_notes="Official X API v2 only (paid tier, POLITRACK_X_BEARER_TOKEN). Scraping x.com or using "
"mirrors/proxies (Nitter etc.) violates the ToS and is not done. The Developer Agreement requires "
"removing posts that were deleted on X; no raw archive or full text is kept, but the "
"deletion sync is still open (docs/roadmap.md).")
PLATFORM_LABELS = {"bluesky": "Bluesky", "mastodon": "Mastodon", "telegram": "Telegram", "youtube": "YouTube",
"x": "X"}
def accounts(ctx: RunContext, platform: str, url_pattern: str | None = None) -> list[dict[str, Any]]:
"""Accounts of persons or organisations on one platform; each row carries an `owner` dict for Request.meta.
`url_pattern` also picks up accounts stored as plain websites, so it must match only that platform's
own host (never a generic shape like "https://host/@user", which TikTok and others share).
"""
if ctx.conn is None:
return []
rows = ctx.conn.execute(
"""SELECT person_id, organization_id, url, handle, verified FROM social_account
WHERE (platform = %s OR (%s::text IS NOT NULL AND url ~* %s))
AND (person_id IS NOT NULL OR organization_id IS NOT NULL)
ORDER BY verified DESC, id""",
(platform, url_pattern, url_pattern),
).fetchall()
for row in rows:
row["owner"] = {"person_id": str(row["person_id"]) if row["person_id"] else None,
"org_id": None if row["person_id"] else str(row["organization_id"]),
"verified": row["verified"]}
return rows
def post_claim(*, owner: dict[str, Any], platform: str, url: str, text: str, created: Any,
title: str | None = None, data: dict | None = None) -> Claim:
"""A public post as a `statement` claim of the account owner (person, or organisation for party channels)."""
label = PLATFORM_LABELS[platform]
return Claim(
claim_type="statement", occurred_at=created,
person=PersonRef(ids={"politrack": owner["person_id"]}) if owner.get("person_id") else None,
org=OrgRef(ids={"politrack_org": owner["org_id"]}, create=False) if owner.get("org_id") else None,
summary_de=f"Auf {label}: „{excerpt(title or text, 200)}“",
summary_en=f"On {label}: \"{excerpt(title or text, 200)}\"",
data={"platform": platform, "account_verified": owner.get("verified", False), **(data or {})},
evidence=[Evidence(url=url, source=platform, doc_type="social_post", title=excerpt(title or text, 120),
published_at=created, full_text=text, excerpt=text, quote=excerpt(text, 500))],
facet=Statement(context="social_post", url=url, said_at=created, platform=platform, title=title,
excerpt=text),
)
@register
class BlueskyPosts(Adapter):
spec = BLUESKY
name = "social.bluesky"
description = "Public Bluesky posts of politicians"
schedule = "7 * * * *"
queue = "social"
def start(self, ctx: RunContext) -> Iterator[Request]:
for acc in accounts(ctx, "bluesky", r"bsky\.app/profile/"):
m = re.search(r"bsky\.app/profile/([^/?#]+)", acc["url"])
if m:
yield Request(url="https://public.api.bsky.app/xrpc/app.bsky.feed.getAuthorFeed",
params={"actor": m.group(1), "limit": 50, "filter": "posts_no_replies"}, kind="feed",
meta={**acc["owner"], "handle": m.group(1)})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
handle = response.meta["handle"]
for item in response.json().get("feed", []):
post = item.get("post") or {}
if item.get("reason"): # repost of someone else's post
continue
record = post.get("record") or {}
text = record.get("text") or ""
author = (post.get("author") or {}).get("handle") or handle
rkey = post.get("uri", "").rsplit("/", 1)[-1]
yield post_claim(owner=response.meta, platform="bluesky",
url=f"https://bsky.app/profile/{author}/post/{rkey}", text=text,
created=record.get("createdAt"),
data={"likes": post.get("likeCount"), "reposts": post.get("repostCount")})
@register
class MastodonPosts(Adapter):
spec = MASTODON
name = "social.mastodon"
description = "Public Mastodon/Fediverse posts of politicians"
schedule = "17 * * * *"
queue = "social"
def start(self, ctx: RunContext) -> Iterator[Request]:
for acc in accounts(ctx, "mastodon"): # instances have arbitrary hosts: platform only
m = re.match(r"^https?://([^/]+)/@([^/@]+)", acc["url"])
if m:
host, user = m.groups()
yield Request(url=f"https://{host}/api/v1/accounts/lookup", params={"acct": user}, kind="lookup",
meta={**acc["owner"], "host": host})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
meta = response.meta
if response.request.kind == "lookup":
account_id = response.json().get("id")
if account_id:
yield Request(url=f"https://{meta['host']}/api/v1/accounts/{account_id}/statuses",
params={"limit": 40, "exclude_replies": "true", "exclude_reblogs": "true"},
kind="statuses", meta=meta)
return
for status in response.json():
if status.get("visibility") not in (None, "public"):
continue
yield post_claim(owner=meta, platform="mastodon", url=status.get("url") or status.get("uri"),
text=html_to_text(status.get("content")), created=status.get("created_at"),
data={"favourites": status.get("favourites_count"), "reblogs": status.get("reblogs_count")})
@register
class TelegramChannels(Adapter):
spec = TELEGRAM
name = "social.telegram"
description = "Public Telegram broadcast channels of politicians (web preview, latest posts)"
schedule = "27 */2 * * *"
queue = "social"
def start(self, ctx: RunContext) -> Iterator[Request]:
for acc in accounts(ctx, "telegram", r"(t\.me|telegram\.me)/"):
m = re.search(r"(?:t\.me|telegram\.me)/(?:s/)?([A-Za-z0-9_]{4,})", acc["url"])
if m:
yield Request(url=f"https://t.me/s/{m.group(1)}", kind="channel",
meta={**acc["owner"], "channel": m.group(1)})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
soup = response.soup
subscribers = soup.select_one(".tgme_channel_info_counter .counter_value")
for msg in soup.select("div.tgme_widget_message[data-post]"):
text_el = msg.select_one(".tgme_widget_message_text")
time_el = msg.select_one(".tgme_widget_message_date time")
text = text_el.get_text("\n", strip=True) if text_el else ""
if not text:
continue # media-only posts
forwarded = msg.select_one(".tgme_widget_message_forwarded_from")
views = msg.select_one(".tgme_widget_message_views")
yield post_claim(
owner=response.meta, platform="telegram", url=f"https://t.me/{msg['data-post']}",
text=text, created=time_el.get("datetime") if time_el else None,
data={"views": views.get_text(strip=True) if views else None,
"forwarded_from": forwarded.get_text(" ", strip=True) if forwarded else None,
"channel_subscribers": subscribers.get_text(strip=True) if subscribers else None},
)
@register
class YouTubeVideos(Adapter):
spec = YOUTUBE
name = "social.youtube"
description = "Latest YouTube uploads of politicians' channels (requires POLITRACK_YOUTUBE_API_KEY)"
schedule = "37 */6 * * *"
queue = "social"
def start(self, ctx: RunContext) -> Iterator[Request]:
key = get_settings().youtube_api_key
if not key:
ctx.stats["skipped_no_api_key"] = 1
return
api = "https://www.googleapis.com/youtube/v3/channels"
for acc in accounts(ctx, "youtube", r"youtube\.com/"):
meta = acc["owner"]
if m := re.search(r"youtube\.com/channel/(UC[\w-]+)", acc["url"]):
params = {"id": m.group(1)}
elif m := re.search(r"youtube\.com/@([\w.-]+)", acc["url"]):
params = {"forHandle": f"@{m.group(1)}"}
elif m := re.search(r"youtube\.com/(?:user/|c/)?([\w.-]+)/?$", acc["url"]):
params = {"forUsername": m.group(1)}
else:
continue
yield Request(url=api, params={**params, "part": "contentDetails", "key": key}, kind="channel", meta=meta)
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
data = response.json()
meta = response.meta
if response.request.kind == "channel":
for item in data.get("items", []):
uploads = item["contentDetails"]["relatedPlaylists"]["uploads"]
yield Request(url="https://www.googleapis.com/youtube/v3/playlistItems", kind="uploads", meta=meta,
params={"playlistId": uploads, "part": "snippet", "maxResults": 25,
"key": get_settings().youtube_api_key})
return
for item in data.get("items", []):
snippet = item.get("snippet") or {}
video = (snippet.get("resourceId") or {}).get("videoId")
if not video:
continue
yield post_claim(owner=meta, platform="youtube",
url=f"https://www.youtube.com/watch?v={video}", title=snippet.get("title"),
text=f"{snippet.get('title', '')}\n{snippet.get('description', '')}".strip(),
created=snippet.get("publishedAt"))
@register
class XPosts(Adapter):
spec = X
name = "social.x"
description = "Posts of politicians on X via the official API v2 (requires POLITRACK_X_BEARER_TOKEN)"
schedule = "47 */6 * * *"
queue = "social"
def start(self, ctx: RunContext) -> Iterator[Request]:
token = get_settings().x_bearer_token
if not token:
ctx.stats["skipped_no_api_key"] = 1
return
for acc in accounts(ctx, "twitter", r"//(www\.)?(twitter|x)\.com/"):
m = re.search(r"(?:twitter|x)\.com/(?:#!/)?@?(\w{1,15})/?$", acc["url"])
if m:
yield Request(url=f"{X_API}users/by/username/{m.group(1)}", kind="user", headers=self.auth(),
meta=acc["owner"])
@staticmethod
def auth() -> dict[str, str]:
return {"Authorization": f"Bearer {get_settings().x_bearer_token}"}
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
data = response.json()
meta = response.meta
if response.request.kind == "user":
user = data.get("data") or {}
if user.get("id"):
yield Request(url=f"{X_API}users/{user['id']}/tweets", kind="tweets", headers=self.auth(),
params={"max_results": 50, "exclude": "replies,retweets",
"tweet.fields": "created_at,public_metrics"},
meta={**meta, "username": user.get("username")})
return
for tweet in data.get("data") or []:
metrics = tweet.get("public_metrics") or {}
yield post_claim(owner=meta, platform="x",
url=f"https://x.com/{meta['username']}/status/{tweet['id']}", text=tweet.get("text", ""),
created=tweet.get("created_at"),
data={"likes": metrics.get("like_count"), "reposts": metrics.get("retweet_count")})
+305
View File
@@ -0,0 +1,305 @@
"""Wikidata (CC0) via the public SPARQL endpoint.
Fills what the federal parliament API does not cover: state parliaments (Landtage), state
governments, mayors and other offices, party membership, careers (employers, positions abroad,
EU roles) and social-media handles. People are merged with parliament records through P2280
(Austrian Parliament ID = PAD_INTERN).
Seats in the Nationalrat/Bundesrat/European Parliament are mapped to the canonical role "mp" in
that chamber, so a Wikidata mandate and a Parliament mandate with the same start date are the
same claim (corroborated by construction).
Flow: list Austrian positions -> holders in batches of 40 positions -> for each batch of persons:
party memberships, careers, social handles. Batching keeps queries within the endpoint's 60 s limit.
"""
from __future__ import annotations
import json
import re
from collections.abc import Iterator
from datetime import date
from typing import Any
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Evidence, OrgRef, PersonRecord, PersonRef, Role, SocialAccount
SPARQL = "https://query.wikidata.org/sparql"
SPEC = SourceSpec(
slug="wikidata",
name="Wikidata",
kind="aggregator",
homepage="https://www.wikidata.org",
license="CC0",
access="api",
trust=65,
min_interval=2.0,
robots_exempt=(SPARQL,),
tos_notes="SPARQL endpoint. robots.txt disallows /sparql for crawlers, but the Wikimedia User-Agent policy "
"allows bots with a descriptive User-Agent to use the query service. Community-edited.",
)
POSITION_LIST_QUERY = """
SELECT DISTINCT ?position ?positionLabel WHERE {
?position wdt:P17 wd:Q40 .
[] ps:P39 ?position .
SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". }
}
"""
POSITIONS_QUERY = """
SELECT ?person ?personLabel ?givenLabel ?familyLabel ?birth ?death ?pad ?ep ?position ?positionLabel ?start ?end
?group ?groupLabel WHERE {
VALUES ?position { %(positions)s }
?person p:P39 ?st .
?st ps:P39 ?position .
OPTIONAL { ?st pq:P580 ?start }
OPTIONAL { ?st pq:P582 ?end }
FILTER(!BOUND(?end) || ?end >= "%(since)s-01-01"^^xsd:dateTime)
OPTIONAL { ?st pq:P4100 ?group }
OPTIONAL { ?person wdt:P2280 ?pad }
# Generic positions with country=Austria are also held by foreigners (e.g. "Bundessprecher" of the German
# Green Youth): require an Austrian connection - parliament id, citizenship or an Austrian party.
FILTER(BOUND(?pad) || EXISTS { ?person wdt:P27 wd:Q40 } || EXISTS { ?person wdt:P102 ?ap . ?ap wdt:P17 wd:Q40 })
OPTIONAL { ?person wdt:P1186 ?ep }
OPTIONAL { ?person wdt:P569 ?birth }
OPTIONAL { ?person wdt:P570 ?death }
OPTIONAL { ?person wdt:P735 ?given }
OPTIONAL { ?person wdt:P734 ?family }
SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". }
}
"""
PARTY_QUERY = """
SELECT ?person ?party ?partyLabel ?partyShort ?start ?end WHERE {
VALUES ?person { %(persons)s }
?person p:P102 ?st .
?st ps:P102 ?party .
?party wdt:P17 wd:Q40 .
OPTIONAL { ?st pq:P580 ?start }
OPTIONAL { ?st pq:P582 ?end }
OPTIONAL { ?party wdt:P1813 ?partyShort . FILTER(LANG(?partyShort) = "de") }
SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". }
}
"""
CAREER_QUERY = """
SELECT ?person ?org ?orgLabel ?start ?end ?kind WHERE {
VALUES ?person { %(persons)s }
{ ?person p:P108 ?st . ?st ps:P108 ?org . BIND("employer" AS ?kind) }
UNION
{ ?person p:P39 ?st . ?st ps:P39 ?org . FILTER NOT EXISTS { ?org wdt:P17 wd:Q40 } BIND("position" AS ?kind) }
OPTIONAL { ?st pq:P580 ?start }
OPTIONAL { ?st pq:P582 ?end }
SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". }
}
"""
# Social-media properties: P2002 X/Twitter, P12361 Bluesky, P4033 Mastodon, P3789 Telegram,
# P2397 YouTube channel id, P2003 Instagram, P2013 Facebook, P7085 TikTok
SOCIAL_QUERY = """
SELECT ?person ?prop ?value WHERE {
VALUES ?person { %(persons)s }
VALUES ?prop { wdt:P2002 wdt:P12361 wdt:P4033 wdt:P3789 wdt:P2397 wdt:P2003 wdt:P2013 wdt:P7085 }
?person ?prop ?value .
}
"""
SOCIAL_PROPS = {
"P2002": ("twitter", "https://x.com/{}"),
"P12361": ("bluesky", "https://bsky.app/profile/{}"),
"P4033": ("mastodon", None),
"P3789": ("telegram", "https://t.me/{}"),
"P2397": ("youtube", "https://www.youtube.com/channel/{}"),
"P2003": ("instagram", "https://www.instagram.com/{}"),
"P2013": ("facebook", "https://www.facebook.com/{}"),
"P7085": ("tiktok", "https://www.tiktok.com/@{}"),
}
EU_POLITICAL = re.compile(r"Europäischen Parlament|Europaparlament|Europäischen Kommission|EU-Kommissar|"
r"Member of the European Parliament", re.IGNORECASE)
EXCLUDE = re.compile(
r"professor|kommerzialrat|kommerzienrat|hofrat|ministerialrat|regierungsrat|gerichtshof|richter|"
r"botschafter|bischof|rektor|ökonomierat|medizinalrat|schulrat|studienrat|baurat|veterinärrat|"
r"bergrat|amtsrat|kammerrat|konsul",
re.IGNORECASE,
)
CHAMBER_LABELS = [
(re.compile(r"Abgeordnete[rn]? zum Nationalrat|Mitglied des Nationalrates", re.I), "nationalrat", "federal"),
(re.compile(r"Mitglied des (österreichischen )?Bundesrate?s", re.I), "bundesrat", "federal"),
(re.compile(r"Mitglied des Europäischen Parlaments|Member of the European Parliament", re.I),
"europaeisches-parlament", "eu"),
]
def classify_level(label: str) -> str:
if re.search(r"europäischen parlament|europaabgeordnet|eu-kommiss", label, re.I):
return "eu"
if re.search(r"bürgermeister|gemeinderat|stadtrat|vizebürgermeister|gemeinde", label, re.I):
return "municipal" if "Wiener Landtag" not in label else "state"
if re.search(r"landtag|landesrat|landeshauptmann|landesregierung|landesamtsdirektor", label, re.I):
return "state"
if re.search(r"bezirk", label, re.I):
return "district"
return "federal"
def wd_date(binding: dict, key: str) -> date | None:
value = binding.get(key, {}).get("value")
if not value or value.startswith("-"):
return None
try:
return date.fromisoformat(value[:10])
except ValueError:
return None
def qid(binding: dict, key: str) -> str | None:
value = binding.get(key, {}).get("value")
return value.rsplit("/", 1)[-1] if value else None
def label(binding: dict, key: str) -> str | None:
value = binding.get(key, {}).get("value")
return None if not value or re.fullmatch(r"Q\d+", value) else value
def sparql(query: str, kind: str, meta: dict | None = None) -> Request:
return Request(url=SPARQL, params={"query": query, "format": "json"}, kind=kind, meta=meta or {},
headers={"Accept": "application/sparql-results+json"})
def role_for(position_label: str) -> tuple[str, OrgRef | None, str]:
for pattern, slug, level in CHAMBER_LABELS:
if pattern.search(position_label):
return "mp", OrgRef(slug=slug), level
return position_label, None, classify_level(position_label)
@register
class WikidataPoliticians(Adapter):
spec = SPEC
name = "wikidata.politicians"
description = "Austrian office holders (all levels), party membership, careers, social handles"
schedule = "45 4 * * 0"
creates_persons = True
parser_version = 2
def start(self, ctx: RunContext) -> Iterator[Request]:
yield sparql(POSITION_LIST_QUERY, kind="positions")
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
bindings = json.loads(response.text, strict=False)["results"]["bindings"]
kind = response.request.kind
if kind == "positions":
positions = [qid(b, "position") for b in bindings
if label(b, "positionLabel") and not EXCLUDE.search(label(b, "positionLabel") or "")]
since = ctx.params.get("since_year", 2000)
for i in range(0, len(positions), 40):
values = " ".join(f"wd:{q}" for q in positions[i:i + 40])
yield sparql(POSITIONS_QUERY % {"positions": values, "since": since}, kind="holders")
elif kind == "holders":
yield from self.parse_holders(bindings)
elif kind == "parties":
yield from self.parse_parties(bindings)
elif kind == "careers":
yield from self.parse_careers(bindings)
elif kind == "social":
yield from self.parse_social(bindings)
def parse_holders(self, bindings: list[dict]) -> Iterator[Any]:
seen: set[str] = set()
for b in bindings:
person_qid, name, position = qid(b, "person"), label(b, "personLabel"), label(b, "positionLabel")
if not name or not position or EXCLUDE.search(position):
continue
pad = b.get("pad", {}).get("value")
ids = {"wikidata": person_qid}
if pad and pad.isdigit():
ids["parlament"] = str(int(pad))
if b.get("ep", {}).get("value"):
ids["ep"] = b["ep"]["value"]
ref = PersonRef(ids=ids)
if person_qid not in seen:
seen.add(person_qid)
yield PersonRecord(ref=ref, full_name=name, given_name=label(b, "givenLabel"),
family_name=label(b, "familyLabel"), birth_date=wd_date(b, "birth"),
death_date=wd_date(b, "death"))
start, end = wd_date(b, "start"), wd_date(b, "end") or wd_date(b, "death")
group_label = label(b, "groupLabel")
group = OrgRef(name=group_label, kind="parliamentary_group", ids={"wikidata": qid(b, "group")}) \
if group_label else None
role, body, level = role_for(position)
yield Claim(
claim_type="role", person=ref, org=body, counterpart_org=group, valid_from=start, valid_to=end,
summary_de=f"{name}: {position}" + (f" ({group_label})" if group_label else ""),
data={"position_qid": qid(b, "position"), "position_label": position},
evidence=[Evidence(url=f"https://www.wikidata.org/wiki/{person_qid}", doc_type="api_record",
title=f"Wikidata: {name}", locator=f"P39:{qid(b, 'position')}:{start}")],
facet=Role(role=role, level=level, body=body, start=start, end=end),
)
persons = sorted(seen)
for i in range(0, len(persons), 300):
values = " ".join(f"wd:{q}" for q in persons[i:i + 300])
yield sparql(PARTY_QUERY % {"persons": values}, kind="parties")
yield sparql(CAREER_QUERY % {"persons": values}, kind="careers")
yield sparql(SOCIAL_QUERY % {"persons": values}, kind="social")
def parse_parties(self, bindings: list[dict]) -> Iterator[Claim]:
for b in bindings:
party_label = label(b, "partyLabel")
if not party_label:
continue
person_qid = qid(b, "person")
party = OrgRef(name=party_label, short_name=b.get("partyShort", {}).get("value"), kind="party",
ids={"wikidata": qid(b, "party")})
start, end = wd_date(b, "start"), wd_date(b, "end")
yield Claim(
claim_type="role", person=PersonRef(ids={"wikidata": person_qid}), org=party,
valid_from=start, valid_to=end,
summary_de=f"Mitglied der Partei {party_label}", summary_en=f"Member of {party_label}",
evidence=[Evidence(url=f"https://www.wikidata.org/wiki/{person_qid}", doc_type="api_record",
title="Wikidata", locator=f"P102:{qid(b, 'party')}:{start}")],
facet=Role(role="party_member", level="party", body=party, party=party, start=start, end=end),
)
def parse_careers(self, bindings: list[dict]) -> Iterator[Claim]:
for b in bindings:
org_label = label(b, "orgLabel")
if not org_label or EXCLUDE.search(org_label):
continue
person_qid = qid(b, "person")
ref = PersonRef(ids={"wikidata": person_qid})
start, end = wd_date(b, "start"), wd_date(b, "end")
kind = b["kind"]["value"]
evidence = [Evidence(url=f"https://www.wikidata.org/wiki/{person_qid}", doc_type="api_record",
title="Wikidata", locator=f"{kind}:{qid(b, 'org')}:{start}")]
if kind == "position" and EU_POLITICAL.search(org_label):
role, body, level = role_for(org_label)
yield Claim(claim_type="role", person=ref, org=body, valid_from=start, valid_to=end,
summary_de=org_label, evidence=evidence,
facet=Role(role=role, level="eu", body=body, start=start, end=end))
continue
employer = OrgRef(name=org_label, ids={"wikidata": qid(b, "org")}) if kind == "employer" else None
yield Claim(
claim_type="career", person=ref, counterpart_org=employer, valid_from=start, valid_to=end,
occurred_at=start, key=("wikidata-career", person_qid, kind, qid(b, "org"), start),
summary_de=f"{'Arbeitet(e) bei' if kind == 'employer' else 'Funktion:'} {org_label}",
summary_en=f"{'Employed by' if kind == 'employer' else 'Position:'} {org_label}",
data={"kind": kind, "org_qid": qid(b, "org")}, evidence=evidence,
)
def parse_social(self, bindings: list[dict]) -> Iterator[SocialAccount]:
for b in bindings:
prop = b["prop"]["value"].rsplit("/", 1)[-1]
value = b["value"]["value"].strip()
platform, template = SOCIAL_PROPS[prop]
if platform == "mastodon":
user, _, host = value.lstrip("@").partition("@")
if not host:
continue
url = f"https://{host}/@{user}"
else:
url = template.format(value)
yield SocialAccount(platform=platform, url=url, handle=value,
person=PersonRef(ids={"wikidata": qid(b, "person")}), verified=False)
+83
View File
@@ -0,0 +1,83 @@
"""Wikipedia edits made from the networks of Parliament and federal government (classic transparency bot).
Anonymous Wikipedia edits are publicly attributed to the editor's IP address. We query the MediaWiki API
(list=usercontribs, uciprange) for the IP ranges registered to public institutions in the RIPE database
(config/ip_ranges.yaml) and record each edit as a `wiki_edit` claim of the institution, linked to the
person whose article was edited when the title matches a known politician. Individual IP addresses are
not displayed - only the institution owning the network.
The API is used per the MediaWiki API etiquette (descriptive User-Agent, sequential requests); robots.txt
disallows /w/ for crawlers, which does not apply to API clients (robots_exempt below).
"""
from __future__ import annotations
from collections.abc import Iterator
from datetime import date, timedelta
from typing import Any
import yaml
from politrack.config import get_settings
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Evidence, OrgRef, PersonRef
from politrack.text import excerpt
WIKIS = ("de.wikipedia.org", "en.wikipedia.org")
SPEC = SourceSpec(
slug="wikipedia", name="Wikipedia (Bearbeitungen aus Behördennetzen)", kind="aggregator",
homepage="https://de.wikipedia.org", license="CC BY-SA 4.0", access="api", trust=80, min_interval=1.0,
robots_exempt=tuple(f"https://{w}/w/api.php" for w in WIKIS),
tos_notes="MediaWiki API (API etiquette: descriptive User-Agent, no parallel requests). Edits from IP ranges "
"registered to Parliament / Bundeskanzleramt in the RIPE database.",
)
@register
class InstitutionalWikiEdits(Adapter):
spec = SPEC
name = "wikipedia.institution_edits"
description = "Wikipedia edits from the IP ranges of Parliament and federal government"
schedule = "15 7 * * *"
def start(self, ctx: RunContext) -> Iterator[Request]:
config = yaml.safe_load((get_settings().config_dir / "ip_ranges.yaml").read_text(encoding="utf-8"))
days = int(ctx.params.get("days", 3650 if not ctx.state.get("initialised") else 30))
end = (date.today() - timedelta(days=days)).isoformat() + "T00:00:00Z"
for network in config.get("networks") or []:
for cidr in network["ranges"]:
for wiki in WIKIS:
yield Request(
url=f"https://{wiki}/w/api.php", kind="contribs",
params={"action": "query", "list": "usercontribs", "uciprange": cidr, "uclimit": 500,
"ucend": end, "ucprop": "ids|title|timestamp|comment|sizediff", "format": "json",
"formatversion": 2},
meta={"wiki": wiki, "org": network["org"], "label": network["label"]}, conditional=False)
ctx.state.set("initialised", True)
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
data = response.json()
meta = response.meta
if "continue" in data:
yield Request(url=response.request.url, kind="contribs", meta=meta, conditional=False,
params={**response.request.params, **data["continue"]})
for edit in (data.get("query") or {}).get("usercontribs") or []:
title, revid = edit.get("title"), edit.get("revid")
if not title or not revid or ":" in title.split(" ")[0] and not title.startswith("Liste"):
continue # skip talk/user/meta pages ("Diskussion:", "Benutzer:")
size = edit.get("sizediff")
comment = edit.get("comment") or ""
diff = f"https://{meta['wiki']}/w/index.php?diff={revid}"
yield Claim(
claim_type="wiki_edit", org=OrgRef(slug=meta["org"], create=False),
counterpart_person=PersonRef(name=title, hints={}), key=("wiki", meta["wiki"], revid),
occurred_at=edit.get("timestamp"),
summary_de=f"Wikipedia-Bearbeitung aus dem Netz von {meta['label']}: „{title}“"
+ (f" ({size:+d} Zeichen)" if isinstance(size, int) else "")
+ (f" – {excerpt(comment, 120)}" if comment else ""),
summary_en=f"Wikipedia edit from the network of {meta['label']}: \"{title}\""
+ (f" ({size:+d} characters)" if isinstance(size, int) else ""),
data={"wiki": meta["wiki"], "title": title, "sizediff": size, "comment": comment},
evidence=[Evidence(url=diff, doc_type="web_page", title=f"Wikipedia: {title}",
published_at=edit.get("timestamp"), quote=comment or None)],
)
+82
View File
@@ -0,0 +1,82 @@
import hashlib
import html
import re
import unicodedata
from bs4 import BeautifulSoup
from slugify import slugify
ACADEMIC_TITLES = re.compile(
r"\b(Mag\.?(\s?\(FH\))?|Dr\.?|DI|Dipl\.-Ing\.?|Ing\.?|MMag\.?|Prof\.?|Univ\.-Prof\.?|BA|MA|BSc|MSc|MBA|LL\.?M\.?|"
r"PhD|MAS|MPA|Bakk\.?|DDr\.?|KR|Abg\.?|Mmag\.?|MEd|BEd|Dipl\.-Päd\.?)(?=\s|,|$)",
re.IGNORECASE,
)
def normalize(text: str) -> str:
"""Lowercase, strip accents and collapse whitespace (matches SQL lower(unaccent()))."""
text = unicodedata.normalize("NFKD", text)
text = "".join(c for c in text if not unicodedata.combining(c))
return re.sub(r"\s+", " ", text).strip().lower()
def strip_titles(name: str) -> str:
name = ACADEMIC_TITLES.sub("", name)
name = re.sub(r"\s*,\s*", ", ", name)
return re.sub(r"\s+", " ", name).strip(" ,")
def make_slug(*parts: str) -> str:
return slugify(" ".join(p for p in parts if p), lowercase=True, replacements=[["ß", "ss"]])
def html_to_text(fragment: str | None) -> str:
if not fragment:
return ""
return re.sub(r"\s+", " ", BeautifulSoup(html.unescape(fragment), "html.parser").get_text(" ")).strip()
def fingerprint(*parts: object) -> str:
raw = "|".join(str(p) for p in parts)
return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:40]
LEGAL_FORMS = re.compile(
r"\b(gesellschaft\s+m\.?\s?b\.?\s?h\.?|ges\.?\s?m\.?\s?b\.?\s?h\.?|gmbh|g\.m\.b\.h\.?|aktiengesellschaft|ag|"
r"kommanditgesellschaft|kg|og|e\.?\s?u\.?|eg|e\.?\s?gen\.?|reg\.?\s?gen\.?\s?m\.?\s?b\.?\s?h\.?|se|ltd\.?|inc\.?|"
r"llc|s\.?a\.?|b\.?v\.?|n\.?v\.?|& co\.?|co\.?|und co\.?|gesmbh|mbh|kgaa|plc)\b",
re.IGNORECASE,
)
def normalize_org_name(name: str) -> str:
"""'HYPO TIROL BANK AG' == 'Hypo Tirol Bank Aktiengesellschaft' -> 'hypo tirol bank'.
Lowercase, strip accents, quotes, punctuation and legal-form suffixes, so the same company is
recognised across sources (side-job lists, procurement notices, donation lists, ad reports).
"""
text = normalize(name)
text = re.sub(r"[\"'„“”»«]", "", text)
text = LEGAL_FORMS.sub(" ", text)
text = re.sub(r"[^\w\s-]", " ", text)
return re.sub(r"\s+", " ", text).strip(" -")
def firmenbuch_number(value: str | None) -> str | None:
"""'FN 306527 a', 'FN306527a', '306527a' -> '306527a' (Austrian company register number, normalised)."""
m = re.fullmatch(r"\s*(?:FN\s*)?(\d{1,6})\s?([a-zA-Z])\s*", value or "", re.IGNORECASE)
return f"{m.group(1)}{m.group(2).lower()}" if m else None
def format_eur_de(amount: object) -> str:
"""1234567.8 -> '1.234.567,80' (German number format; formats only the number, never the sentence)."""
try:
value = float(amount) # type: ignore[arg-type]
except (TypeError, ValueError):
return str(amount)
return f"{value:,.2f}".replace(",", "X").replace(".", ",").replace("X", ".")
def excerpt(text: str, limit: int = 400) -> str:
text = re.sub(r"\s+", " ", text).strip()
return text if len(text) <= limit else text[: limit - 1].rsplit(" ", 1)[0] + "…"
+41
View File
@@ -0,0 +1,41 @@
"""Background jobs on Procrastinate (Postgres-backed queue; no Redis).
One periodic task is generated per registered adapter/enricher from its `schedule` (cron, UTC) and
`queue`. Run workers per queue so heavy jobs don't block light ones:
procrastinate --app=politrack.worker.app worker --queues=scrape,social
procrastinate --app=politrack.worker.app worker --queues=bulk
procrastinate --app=politrack.worker.app worker --queues=enrich
Periodic tasks are deferred by every running worker and de-duplicated by Procrastinate; queueing locks
prevent two runs of the same adapter at once. Workers can run on other machines that reach the DB.
"""
from __future__ import annotations
import logging
from procrastinate import App, PsycopgConnector
from politrack.config import get_settings
from politrack.core import registry
from politrack.core.runner import run
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
app = App(connector=PsycopgConnector(conninfo=get_settings().database_url))
def _make_task(name: str, queue: str, schedule: str | None) -> None:
def job(timestamp: int | None = None, **params) -> dict:
return run(name, params=params or None)
job.__name__ = f"run_{name.replace('.', '_')}"
task = app.task(name=f"politrack.run.{name}", queue=queue, queueing_lock=name, pass_context=False)(job)
if schedule:
app.periodic(cron=schedule, periodic_id=name)(task)
registry.load_builtin()
for _name, _cls in {**registry.ADAPTERS, **registry.ENRICHERS}.items():
_make_task(_name, _cls.queue, _cls.schedule)