Politrack: track Austrian politicians from public sources
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
583ccba31c
187 files changed
+39890
No files matched your search
Whitespace-only changes.
Whitespace-only changes.
@@ -0,0 +1,253 @@
|
||||
"""Public read API (JSON). OpenAPI docs at /api/docs; schema at /api/openapi.json.
|
||||
|
||||
Conventions: every list endpoint takes limit/offset; claim-like endpoints take the source filter
|
||||
(exclude_sources, min_trust) and topic/date filters; slugs identify persons, parties and organisations.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends, FastAPI, Header, HTTPException, Query
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from psycopg import Connection
|
||||
|
||||
from politrack import queries
|
||||
from politrack.api import schemas as s
|
||||
from politrack.config import get_settings
|
||||
from politrack.db import connection
|
||||
from politrack.queries import SourceFilter
|
||||
|
||||
app = FastAPI(
|
||||
title="Politrack API",
|
||||
version="0.2.0",
|
||||
description="Evidence-backed information on Austrian politicians and parties. Every claim links to its sources.",
|
||||
docs_url="/api/docs",
|
||||
openapi_url="/api/openapi.json",
|
||||
)
|
||||
app.add_middleware(CORSMiddleware, allow_origins=get_settings().api_cors_origins, allow_methods=["GET", "POST"],
|
||||
allow_headers=["*"])
|
||||
|
||||
|
||||
def db() -> Iterator[Connection]:
|
||||
with connection() as conn:
|
||||
yield conn
|
||||
|
||||
|
||||
def csv_list(value: str | None) -> list[str] | None:
|
||||
items = [v.strip() for v in (value or "").split(",") if v.strip()]
|
||||
return items or None
|
||||
|
||||
|
||||
def source_filter(
|
||||
exclude_sources: Annotated[str | None, Query(description="Comma-separated source slugs to ignore")] = None,
|
||||
min_trust: Annotated[int, Query(ge=0, le=100, description="Ignore sources below this trust score")] = 0,
|
||||
) -> SourceFilter:
|
||||
return SourceFilter(exclude_sources=csv_list(exclude_sources) or [], min_trust=min_trust)
|
||||
|
||||
|
||||
DB = Annotated[Connection, Depends(db)]
|
||||
Sources = Annotated[SourceFilter, Depends(source_filter)]
|
||||
Limit = Annotated[int, Query(ge=1, le=500)]
|
||||
Offset = Annotated[int, Query(ge=0)]
|
||||
|
||||
|
||||
def not_found(value, what: str):
|
||||
if not value:
|
||||
raise HTTPException(404, f"{what} not found")
|
||||
return value
|
||||
|
||||
|
||||
@app.get("/healthz", include_in_schema=False)
|
||||
def healthz(conn: DB) -> dict:
|
||||
conn.execute("SELECT 1")
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
# -- reference data -----------------------------------------------------------------------
|
||||
|
||||
@app.get("/api/v1/stats", response_model=s.Stats)
|
||||
def get_stats(conn: DB):
|
||||
return queries.stats(conn)
|
||||
|
||||
|
||||
@app.get("/api/v1/sources", response_model=list[s.Source])
|
||||
def get_sources(conn: DB):
|
||||
return queries.list_sources(conn)
|
||||
|
||||
|
||||
@app.get("/api/v1/topics", response_model=list[s.Topic])
|
||||
def get_topics(conn: DB):
|
||||
return queries.list_topics(conn)
|
||||
|
||||
|
||||
# -- parties & organisations ----------------------------------------------------------------
|
||||
|
||||
@app.get("/api/v1/parties", response_model=list[s.Party])
|
||||
def get_parties(conn: DB):
|
||||
return queries.list_parties(conn)
|
||||
|
||||
|
||||
@app.get("/api/v1/parties/{slug}", response_model=s.PartyDetail)
|
||||
def get_party(slug: str, conn: DB):
|
||||
return not_found(queries.get_party(conn, slug), "party")
|
||||
|
||||
|
||||
@app.get("/api/v1/organizations", response_model=list[s.OrganizationListItem])
|
||||
def get_organizations(conn: DB, q: str | None = None, kind: str | None = None, limit: Limit = 50, offset: Offset = 0):
|
||||
return queries.list_organizations(conn, q=q, kind=kind, limit=limit, offset=offset)
|
||||
|
||||
|
||||
@app.get("/api/v1/organizations/{slug}", response_model=s.OrganizationDetail)
|
||||
def get_organization(slug: str, conn: DB):
|
||||
return not_found(queries.get_organization(conn, slug), "organization")
|
||||
|
||||
|
||||
# -- persons --------------------------------------------------------------------------------
|
||||
|
||||
@app.get("/api/v1/persons", response_model=list[s.PersonListItem])
|
||||
def get_persons(
|
||||
conn: DB,
|
||||
q: str | None = None,
|
||||
party: str | None = None,
|
||||
level: Annotated[str | None, Query(pattern="^(eu|federal|state|district|municipal|party|other)$")] = None,
|
||||
status: Annotated[str, Query(pattern="^(active|former|party_member|deceased|all)$")] = "active",
|
||||
limit: Limit = 50,
|
||||
offset: Offset = 0,
|
||||
):
|
||||
return queries.list_persons(conn, q=q, party=party, level=level, status=None if status == "all" else status,
|
||||
limit=limit, offset=offset)
|
||||
|
||||
|
||||
@app.get("/api/v1/former", response_model=list[s.FormerPerson])
|
||||
def get_former(conn: DB, party: str | None = None, limit: Limit = 50, offset: Offset = 0):
|
||||
"""Former politicians, most recently in the news first."""
|
||||
return queries.former_overview(conn, party=party, limit=limit, offset=offset)
|
||||
|
||||
|
||||
@app.get("/api/v1/persons/{slug}", response_model=s.PersonDetail)
|
||||
def get_person(slug: str, conn: DB):
|
||||
return not_found(queries.get_person(conn, slug), "person")
|
||||
|
||||
|
||||
@app.get("/api/v1/persons/{slug}/votes", response_model=list[s.PersonVote])
|
||||
def get_person_votes(slug: str, conn: DB, topics: str | None = None, limit: Limit = 100, offset: Offset = 0):
|
||||
return queries.person_votes(conn, slug, topics=csv_list(topics), limit=limit, offset=offset)
|
||||
|
||||
|
||||
@app.get("/api/v1/persons/{slug}/network", response_model=s.Network)
|
||||
def get_person_network(slug: str, conn: DB):
|
||||
"""Organisations connected to the person (side jobs, careers, party, meetings) and their money flows."""
|
||||
return queries.person_network(conn, slug)
|
||||
|
||||
|
||||
# -- claims, timeline, calendar ----------------------------------------------------------------
|
||||
|
||||
@app.get("/api/v1/claims", response_model=list[s.Claim])
|
||||
def get_claims(
|
||||
conn: DB,
|
||||
sources: Sources,
|
||||
person: str | None = None,
|
||||
party: str | None = None,
|
||||
organization: str | None = None,
|
||||
types: Annotated[str | None, Query(description="Comma-separated claim types")] = None,
|
||||
topics: Annotated[str | None, Query(description="Comma-separated topic slugs")] = None,
|
||||
since: date | None = None,
|
||||
until: date | None = None,
|
||||
q: str | None = None,
|
||||
limit: Limit = 50,
|
||||
offset: Offset = 0,
|
||||
):
|
||||
return queries.list_claims(conn, sources=sources, person=person, party=party, organization=organization,
|
||||
claim_types=csv_list(types), topics=csv_list(topics), since=since, until=until, q=q,
|
||||
limit=limit, offset=offset)
|
||||
|
||||
|
||||
@app.get("/api/v1/timeline", response_model=list[s.TimelineBucket])
|
||||
def get_timeline(conn: DB, sources: Sources, person: str | None = None, party: str | None = None):
|
||||
return queries.timeline_histogram(conn, sources=sources, person=person, party=party)
|
||||
|
||||
|
||||
@app.get("/api/v1/calendar", response_model=s.CalendarData)
|
||||
def get_calendar(conn: DB, sources: Sources, start: date, end: date, person: str | None = None,
|
||||
party: str | None = None, kinds: str | None = None):
|
||||
if (end - start).days > 400:
|
||||
raise HTTPException(400, "range too large (max 400 days)")
|
||||
return queries.calendar(conn, sources=sources, start=start, end=end, person=person, party=party,
|
||||
kinds=csv_list(kinds))
|
||||
|
||||
|
||||
# -- money, social, wikipedia ---------------------------------------------------------------------
|
||||
|
||||
@app.get("/api/v1/money", response_model=list[s.MoneyFlow])
|
||||
def get_money(conn: DB, sources: Sources, kinds: str | None = None, org: str | None = None, person: str | None = None,
|
||||
q: str | None = None, since: date | None = None, until: date | None = None,
|
||||
min_amount: float | None = None, limit: Limit = 50, offset: Offset = 0):
|
||||
"""Donations, contracts, government ads, party funding, sponsoring, political ads."""
|
||||
return queries.list_money(conn, sources=sources, kinds=csv_list(kinds), org=org, person=person, q=q, since=since,
|
||||
until=until, min_amount=min_amount, limit=limit, offset=offset)
|
||||
|
||||
|
||||
@app.get("/api/v1/money/summary", response_model=list[s.MoneySummaryRow])
|
||||
def get_money_summary(conn: DB, kinds: str | None = None, since: date | None = None):
|
||||
return queries.money_summary(conn, kinds=csv_list(kinds), since=since)
|
||||
|
||||
|
||||
@app.get("/api/v1/social", response_model=list[s.SocialPost])
|
||||
def get_social(conn: DB, sources: Sources, person: str | None = None, party: str | None = None,
|
||||
platforms: str | None = None, limit: Limit = 50, offset: Offset = 0):
|
||||
return queries.social_feed(conn, sources=sources, person=person, party=party, platforms=csv_list(platforms),
|
||||
limit=limit, offset=offset)
|
||||
|
||||
|
||||
@app.get("/api/v1/wiki-edits", response_model=list[s.WikiEdit])
|
||||
def get_wiki_edits(conn: DB, limit: Limit = 100, offset: Offset = 0):
|
||||
return queries.wiki_edits(conn, limit=limit, offset=offset)
|
||||
|
||||
|
||||
# -- votes, search ------------------------------------------------------------------------------
|
||||
|
||||
@app.get("/api/v1/votes", response_model=list[s.VoteItem])
|
||||
def get_votes(conn: DB, party: str | None = None, topics: str | None = None, roll_call: bool | None = None,
|
||||
since: date | None = None, until: date | None = None, limit: Limit = 50, offset: Offset = 0):
|
||||
return queries.list_votes(conn, party=party, topics=csv_list(topics), roll_call=roll_call, since=since,
|
||||
until=until, limit=limit, offset=offset)
|
||||
|
||||
|
||||
@app.get("/api/v1/votes/{vote_id}", response_model=s.VoteDetail)
|
||||
def get_vote(vote_id: int, conn: DB):
|
||||
return not_found(queries.vote_detail(conn, vote_id), "vote")
|
||||
|
||||
|
||||
@app.get("/api/v1/search", response_model=list[s.SearchHit])
|
||||
def get_search(conn: DB, sources: Sources, q: Annotated[str, Query(min_length=2)], since: date | None = None,
|
||||
until: date | None = None, limit: Limit = 20):
|
||||
return queries.search(conn, q, sources=sources, since=since, until=until, limit=limit)
|
||||
|
||||
|
||||
# -- admin (POLITRACK_ADMIN_TOKEN) ------------------------------------------------------------------
|
||||
|
||||
def require_admin(authorization: Annotated[str | None, Header()] = None) -> str:
|
||||
token = get_settings().admin_token
|
||||
if not token or authorization != f"Bearer {token}":
|
||||
raise HTTPException(401, "admin token required")
|
||||
return "admin-api"
|
||||
|
||||
|
||||
@app.get("/api/v1/admin/runs", response_model=list[s.Run], include_in_schema=False)
|
||||
def get_runs(conn: DB, _: Annotated[str, Depends(require_admin)], limit: Limit = 50):
|
||||
return queries.recent_runs(conn, limit=limit)
|
||||
|
||||
|
||||
@app.get("/api/v1/review", response_model=list[s.PendingClaim], include_in_schema=False)
|
||||
def get_review_queue(conn: DB, _: Annotated[str, Depends(require_admin)], limit: Limit = 50):
|
||||
return queries.pending_claims(conn, limit=limit)
|
||||
|
||||
|
||||
@app.post("/api/v1/review/{claim_id}", include_in_schema=False)
|
||||
def post_review(claim_id: int, decision: Annotated[str, Query(pattern="^(published|rejected)$")], conn: DB,
|
||||
reviewer: Annotated[str, Depends(require_admin)]) -> dict:
|
||||
queries.set_review(conn, claim_id, decision, reviewer)
|
||||
return {"ok": True}
|
||||
@@ -0,0 +1,535 @@
|
||||
"""Response models: the API contract. `politrack openapi --ts ../web/src/lib/api-types.ts` turns them into
|
||||
TypeScript types for the web app, so frontend and backend cannot silently drift apart."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
ClaimType = Literal["role", "vote", "donation", "money_flow", "statement", "appearance", "meeting", "travel",
|
||||
"side_job", "legal_proceeding", "lobbying", "media_mention", "career", "parliamentary_question",
|
||||
"government_decision", "wiki_edit", "declaration", "ownership", "affiliation", "party_finance",
|
||||
"other"]
|
||||
PersonStatus = Literal["active", "former", "party_member", "deceased"]
|
||||
|
||||
|
||||
class Model(BaseModel):
|
||||
# Fields with defaults are always present in responses, so mark them required in the (serialization)
|
||||
# schema - the generated TypeScript types then have no needless optional fields.
|
||||
model_config = ConfigDict(from_attributes=True, json_schema_serialization_defaults_required=True)
|
||||
|
||||
|
||||
class Stats(Model):
|
||||
persons: int
|
||||
active_persons: int
|
||||
parties: int
|
||||
claims: int
|
||||
corroborated: int
|
||||
pending_review: int
|
||||
documents: int
|
||||
vote_events: int
|
||||
roll_call_votes: int
|
||||
events: int
|
||||
money_flows: int
|
||||
last_update: datetime | None = None
|
||||
|
||||
|
||||
class Source(Model):
|
||||
slug: str
|
||||
name: str
|
||||
kind: str
|
||||
homepage: str | None = None
|
||||
license: str | None = None
|
||||
access_method: str
|
||||
default_trust: int
|
||||
tos_notes: str | None = None
|
||||
documents: int
|
||||
|
||||
|
||||
class Topic(Model):
|
||||
slug: str
|
||||
name_de: str
|
||||
name_en: str
|
||||
|
||||
|
||||
class Evidence(Model):
|
||||
url: str
|
||||
title: str | None = None
|
||||
source: str
|
||||
source_name: str | None = None
|
||||
source_kind: str | None = None
|
||||
published_at: datetime | None = None
|
||||
quote: str | None = None
|
||||
stance: Literal["supports", "contradicts", "mentions"] = "supports"
|
||||
archive_url: str | None = None
|
||||
|
||||
|
||||
class MoneyFacet(Model):
|
||||
kind: str
|
||||
amount: float | None = None
|
||||
amount_category: str | None = None
|
||||
currency: str | None = None
|
||||
paid_on: date | None = None
|
||||
period: str | None = None
|
||||
reference: str | None = None
|
||||
category: str | None = None
|
||||
payer_name: str | None = None
|
||||
payee_name: str | None = None
|
||||
|
||||
|
||||
class Claim(Model):
|
||||
id: int
|
||||
claim_type: ClaimType
|
||||
summary_de: str
|
||||
summary_en: str | None = None
|
||||
occurred_at: datetime | None = None
|
||||
valid_from: date | None = None
|
||||
valid_to: date | None = None
|
||||
status: Literal["reported", "corroborated", "disputed", "retracted"]
|
||||
sensitive: bool
|
||||
review_status: Literal["pending", "published", "rejected"]
|
||||
data: dict[str, Any] = {}
|
||||
person_slug: str | None = None
|
||||
person_name: str | None = None
|
||||
organization_slug: str | None = None
|
||||
organization_short: str | None = None
|
||||
organization_name: str | None = None
|
||||
counterpart_slug: str | None = None
|
||||
counterpart_name: str | None = None
|
||||
counterpart_kind: str | None = None
|
||||
counterpart_person_slug: str | None = None
|
||||
counterpart_person_name: str | None = None
|
||||
legal_stage: str | None = None
|
||||
legal_offense: str | None = None
|
||||
legal_authority: str | None = None
|
||||
money: MoneyFacet | None = None
|
||||
platform: str | None = None
|
||||
topics: list[Topic] | None = None
|
||||
evidence: list[Evidence]
|
||||
source_count: int
|
||||
|
||||
|
||||
class PartyRef(Model):
|
||||
slug: str
|
||||
short_name: str | None = None
|
||||
color: str | None = None
|
||||
|
||||
|
||||
class PersonListItem(Model):
|
||||
slug: str
|
||||
full_name: str
|
||||
photo_url: str | None = None
|
||||
status: PersonStatus
|
||||
status_verified: bool
|
||||
last_office_end: date | None = None
|
||||
current_roles: list[str] | None = None
|
||||
parties: list[PartyRef] | None = None
|
||||
|
||||
|
||||
class RoleEvidence(Model):
|
||||
url: str
|
||||
source: str
|
||||
|
||||
|
||||
class Role(Model):
|
||||
role: str
|
||||
level: str
|
||||
region: str | None = None
|
||||
start_date: date | None = None
|
||||
end_date: date | None = None
|
||||
body: str | None = None
|
||||
body_slug: str | None = None
|
||||
party_slug: str | None = None
|
||||
party_short: str | None = None
|
||||
party_color: str | None = None
|
||||
claim_status: str
|
||||
evidence: list[RoleEvidence] | None = None
|
||||
|
||||
|
||||
class SocialAccountOut(Model):
|
||||
platform: str
|
||||
handle: str | None = None
|
||||
url: str
|
||||
verified: bool
|
||||
|
||||
|
||||
class PersonDetail(Model):
|
||||
slug: str
|
||||
full_name: str
|
||||
given_name: str | None = None
|
||||
family_name: str | None = None
|
||||
birth_date: date | None = None
|
||||
death_date: date | None = None
|
||||
photo_url: str | None = None
|
||||
wikidata_qid: str | None = None
|
||||
parlament_pad: str | None = None
|
||||
ep_id: str | None = None
|
||||
status: PersonStatus
|
||||
status_verified: bool
|
||||
last_office_end: date | None = None
|
||||
roles: list[Role]
|
||||
social_accounts: list[SocialAccountOut]
|
||||
claim_counts: dict[str, int]
|
||||
|
||||
|
||||
class FormerPerson(Model):
|
||||
slug: str
|
||||
full_name: str
|
||||
photo_url: str | None = None
|
||||
status: PersonStatus
|
||||
last_office_end: date | None = None
|
||||
last_role: str | None = None
|
||||
latest_news: str | None = None
|
||||
latest_news_at: datetime | None = None
|
||||
latest_news_url: str | None = None
|
||||
latest_career: str | None = None
|
||||
|
||||
|
||||
class NetworkOrg(Model):
|
||||
id: int
|
||||
slug: str
|
||||
name: str
|
||||
kind: str
|
||||
via: list[str]
|
||||
label: str | None = None
|
||||
money: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class Network(Model):
|
||||
organizations: list[NetworkOrg]
|
||||
|
||||
|
||||
class Party(Model):
|
||||
slug: str
|
||||
name: str
|
||||
short_name: str | None = None
|
||||
color: str | None = None
|
||||
logo_url: str | None = None
|
||||
active_officials: int = 0
|
||||
total_people: int = 0
|
||||
|
||||
|
||||
class PartyMember(Model):
|
||||
slug: str
|
||||
full_name: str
|
||||
photo_url: str | None = None
|
||||
role: str
|
||||
level: str
|
||||
region: str | None = None
|
||||
body: str | None = None
|
||||
|
||||
|
||||
class FormerMember(Model):
|
||||
slug: str
|
||||
full_name: str
|
||||
last_office_end: date | None = None
|
||||
last_mention: datetime | None = None
|
||||
|
||||
|
||||
class MoneyByYear(Model):
|
||||
year: int
|
||||
kind: str
|
||||
total: float | None = None
|
||||
count: int
|
||||
|
||||
|
||||
class Finance(Model):
|
||||
data: dict[str, Any]
|
||||
valid_from: date | None = None
|
||||
|
||||
|
||||
class CountByPosition(Model):
|
||||
position: str
|
||||
count: int
|
||||
|
||||
|
||||
class Dissenter(Model):
|
||||
slug: str
|
||||
full_name: str
|
||||
dissenting_votes: int
|
||||
|
||||
|
||||
class RelatedOrg(Model):
|
||||
claim_type: str
|
||||
slug: str
|
||||
name: str
|
||||
kind: str
|
||||
|
||||
|
||||
class PartyDetail(Model):
|
||||
slug: str
|
||||
name: str
|
||||
short_name: str | None = None
|
||||
color: str | None = None
|
||||
logo_url: str | None = None
|
||||
kind: str
|
||||
members: list[PartyMember]
|
||||
former_members: list[FormerMember]
|
||||
money_by_year: list[MoneyByYear]
|
||||
finance: list[Finance]
|
||||
vote_stats: list[CountByPosition]
|
||||
dissent: list[Dissenter]
|
||||
related: list[RelatedOrg]
|
||||
|
||||
|
||||
class OrganizationListItem(Model):
|
||||
slug: str
|
||||
name: str
|
||||
short_name: str | None = None
|
||||
kind: str
|
||||
paid: float | None = None
|
||||
received: float | None = None
|
||||
|
||||
|
||||
class OrgMoney(Model):
|
||||
kind: str
|
||||
direction: Literal["paid", "received"]
|
||||
total: float | None = None
|
||||
count: int
|
||||
first: str | None = None
|
||||
last: str | None = None
|
||||
|
||||
|
||||
class Counterparty(Model):
|
||||
slug: str
|
||||
name: str
|
||||
kind: str
|
||||
money_kind: str
|
||||
direction: Literal["paid_to", "received_from"]
|
||||
total: float | None = None
|
||||
count: int
|
||||
|
||||
|
||||
class OrgPerson(Model):
|
||||
slug: str
|
||||
full_name: str
|
||||
via: str
|
||||
summary_de: str
|
||||
|
||||
|
||||
class OrganizationDetail(Model):
|
||||
slug: str
|
||||
name: str
|
||||
short_name: str | None = None
|
||||
kind: str
|
||||
parent_slug: str | None = None
|
||||
parent_name: str | None = None
|
||||
identifiers: dict[str, str] | None = None
|
||||
money: list[OrgMoney]
|
||||
counterparties: list[Counterparty]
|
||||
people: list[OrgPerson]
|
||||
|
||||
|
||||
class MoneyFlow(Model):
|
||||
id: int
|
||||
summary_de: str
|
||||
summary_en: str | None = None
|
||||
occurred_at: datetime | None = None
|
||||
status: str
|
||||
data: dict[str, Any] = {}
|
||||
kind: str
|
||||
amount: float | None = None
|
||||
amount_category: str | None = None
|
||||
currency: str | None = None
|
||||
paid_on: date | None = None
|
||||
period: str | None = None
|
||||
reference: str | None = None
|
||||
category: str | None = None
|
||||
payer_name: str | None = None
|
||||
payee_name: str | None = None
|
||||
payer_slug: str | None = None
|
||||
payer_kind: str | None = None
|
||||
payee_slug: str | None = None
|
||||
payee_kind: str | None = None
|
||||
evidence: list[Evidence]
|
||||
source_count: int
|
||||
|
||||
|
||||
class MoneySummaryRow(Model):
|
||||
kind: str
|
||||
slug: str
|
||||
name: str
|
||||
org_kind: str
|
||||
side: Literal["payer", "payee"]
|
||||
total: float | None = None
|
||||
count: int
|
||||
|
||||
|
||||
class SocialPost(Model):
|
||||
id: int
|
||||
platform: str | None = None
|
||||
said_at: datetime | None = None
|
||||
excerpt: str | None = None
|
||||
url: str
|
||||
title: str | None = None
|
||||
data: dict[str, Any] = {}
|
||||
person_slug: str | None = None
|
||||
person_name: str | None = None
|
||||
org_slug: str | None = None
|
||||
org_name: str | None = None
|
||||
evidence: list[Evidence]
|
||||
|
||||
|
||||
class WikiEdit(Model):
|
||||
id: int
|
||||
occurred_at: datetime | None = None
|
||||
summary_de: str
|
||||
summary_en: str | None = None
|
||||
data: dict[str, Any] = {}
|
||||
org_slug: str
|
||||
org_name: str
|
||||
person_slug: str | None = None
|
||||
person_name: str | None = None
|
||||
url: str | None = None
|
||||
|
||||
|
||||
class TimelineBucket(Model):
|
||||
month: str
|
||||
claim_type: str
|
||||
count: int
|
||||
|
||||
|
||||
class Participant(Model):
|
||||
slug: str
|
||||
name: str
|
||||
|
||||
|
||||
class CalendarEvent(Model):
|
||||
id: int
|
||||
kind: str
|
||||
title: str
|
||||
starts_at: datetime | None = None
|
||||
ends_at: datetime | None = None
|
||||
location_name: str | None = None
|
||||
country: str | None = None
|
||||
url: str | None = None
|
||||
description: str | None = None
|
||||
body: str | None = None
|
||||
participants: list[Participant]
|
||||
|
||||
|
||||
class DayCount(Model):
|
||||
day: date
|
||||
claim_type: str
|
||||
count: int
|
||||
|
||||
|
||||
class CalendarData(Model):
|
||||
events: list[CalendarEvent]
|
||||
days: list[DayCount]
|
||||
|
||||
|
||||
class PartyPosition(Model):
|
||||
party: str | None = None
|
||||
slug: str
|
||||
color: str | None = None
|
||||
position: str
|
||||
|
||||
|
||||
class Speech(Model):
|
||||
url: str
|
||||
title: str | None = None
|
||||
|
||||
|
||||
class VoteItem(Model):
|
||||
id: int
|
||||
item_title: str
|
||||
item_ref: str | None = None
|
||||
stage: str | None = None
|
||||
voted_on: date | None = None
|
||||
result: str | None = None
|
||||
url: str | None = None
|
||||
protocol_url: str | None = None
|
||||
roll_call: bool
|
||||
raw_text: str | None = None
|
||||
chamber: str
|
||||
positions: list[PartyPosition] = []
|
||||
individual_votes: int = 0
|
||||
|
||||
|
||||
class PersonVote(Model):
|
||||
id: int
|
||||
item_title: str
|
||||
item_ref: str | None = None
|
||||
stage: str | None = None
|
||||
voted_on: date | None = None
|
||||
result: str | None = None
|
||||
url: str | None = None
|
||||
protocol_url: str | None = None
|
||||
roll_call: bool
|
||||
position: str
|
||||
faction: str | None = None
|
||||
chamber: str
|
||||
individual: bool
|
||||
dissent: bool
|
||||
spoke: Speech | None = None
|
||||
topics: list[str] | None = None
|
||||
|
||||
|
||||
class IndividualVote(Model):
|
||||
slug: str
|
||||
full_name: str
|
||||
position: str
|
||||
faction: str | None = None
|
||||
dissent: bool
|
||||
|
||||
|
||||
class VoteDetail(Model):
|
||||
id: int
|
||||
item_title: str
|
||||
item_ref: str | None = None
|
||||
stage: str | None = None
|
||||
voted_on: date | None = None
|
||||
result: str | None = None
|
||||
url: str | None = None
|
||||
protocol_url: str | None = None
|
||||
roll_call: bool
|
||||
raw_text: str | None = None
|
||||
chamber: str
|
||||
positions: list[PartyPosition]
|
||||
individual: list[IndividualVote]
|
||||
|
||||
|
||||
class SearchEvidence(Model):
|
||||
url: str
|
||||
title: str | None = None
|
||||
source: str
|
||||
|
||||
|
||||
class SearchHit(Model):
|
||||
id: int
|
||||
text: str
|
||||
claim_id: int | None = None
|
||||
document_id: int | None = None
|
||||
summary_de: str | None = None
|
||||
claim_type: str | None = None
|
||||
occurred_at: datetime | None = None
|
||||
person_slug: str | None = None
|
||||
person_name: str | None = None
|
||||
organization_slug: str | None = None
|
||||
document_url: str | None = None
|
||||
document_title: str | None = None
|
||||
document_date: datetime | None = None
|
||||
evidence: list[SearchEvidence] | None = None
|
||||
score: float
|
||||
|
||||
|
||||
class Run(Model):
|
||||
adapter: str
|
||||
mode: str
|
||||
status: str
|
||||
started_at: datetime
|
||||
finished_at: datetime | None = None
|
||||
stats: dict[str, Any] = {}
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class PendingClaim(Model):
|
||||
id: int
|
||||
claim_type: str
|
||||
summary_de: str
|
||||
occurred_at: datetime | None = None
|
||||
full_name: str | None = None
|
||||
evidence: list[dict[str, Any]] | None = None
|
||||
@@ -0,0 +1,191 @@
|
||||
"""`politrack` command line. Run `politrack --help` for all commands."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import typer
|
||||
|
||||
app = typer.Typer(help="Politrack command line", no_args_is_help=True)
|
||||
sources_app = typer.Typer(help="Source specs and adapters")
|
||||
app.add_typer(sources_app, name="sources")
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||||
|
||||
|
||||
def _params(values: list[str]) -> dict:
|
||||
"""--param key=value (value parsed as JSON when possible)."""
|
||||
out = {}
|
||||
for item in values:
|
||||
key, _, raw = item.partition("=")
|
||||
try:
|
||||
out[key] = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
out[key] = raw
|
||||
return out
|
||||
|
||||
|
||||
@app.command()
|
||||
def migrate() -> None:
|
||||
"""Apply SQL migrations, sync source specs and install the job-queue schema."""
|
||||
from politrack.core.registry import sync_sources
|
||||
from politrack.db import connection
|
||||
from politrack.migrate import migrate as run_migrations
|
||||
|
||||
run_migrations()
|
||||
with connection() as conn:
|
||||
typer.echo(f"synced {sync_sources(conn)} sources")
|
||||
installed = conn.execute("SELECT to_regclass('procrastinate_jobs') AS t").fetchone()["t"]
|
||||
if not installed:
|
||||
from politrack.worker import app as jobs
|
||||
|
||||
with jobs.open():
|
||||
jobs.schema_manager.apply_schema()
|
||||
|
||||
|
||||
@sources_app.command("list")
|
||||
def sources_list() -> None:
|
||||
"""List adapters, enrichers and their schedules."""
|
||||
from politrack.core import registry
|
||||
|
||||
registry.load_builtin()
|
||||
for name, cls in sorted({**registry.ADAPTERS, **registry.ENRICHERS}.items()):
|
||||
typer.echo(f"{name:34} {cls.queue:7} {cls.schedule or 'manual':16} {cls.spec.slug:22} {cls.description}")
|
||||
|
||||
|
||||
@sources_app.command("sync")
|
||||
def sources_sync() -> None:
|
||||
"""Upsert SourceSpecs into the database."""
|
||||
from politrack.core.registry import sync_sources
|
||||
from politrack.db import connection
|
||||
|
||||
with connection() as conn:
|
||||
typer.echo(f"synced {sync_sources(conn)} sources")
|
||||
|
||||
|
||||
@app.command("run")
|
||||
def run_cmd(
|
||||
name: str,
|
||||
param: list[str] = typer.Option([], "--param", "-p", help="key=value, e.g. -p periods='[\"XXVIII\"]'"),
|
||||
since: str = typer.Option(None, help="ISO date for incremental adapters"),
|
||||
max_requests: int = typer.Option(None, help="Stop after N requests (testing)"),
|
||||
) -> None:
|
||||
"""Run one adapter or enricher now, e.g. `politrack run parlament.persons`."""
|
||||
from politrack.core.runner import run
|
||||
|
||||
result = run(name, params=_params(param), since=datetime.fromisoformat(since) if since else None,
|
||||
max_requests=max_requests)
|
||||
typer.echo(json.dumps(result, indent=2, default=str))
|
||||
|
||||
|
||||
@app.command()
|
||||
def reparse(name: str, param: list[str] = typer.Option([], "--param", "-p")) -> None:
|
||||
"""Re-run parse() on the archived responses of an adapter (after a parser fix, no network)."""
|
||||
from politrack.core.runner import run
|
||||
|
||||
typer.echo(json.dumps(run(name, mode="reparse", params=_params(param)), indent=2, default=str))
|
||||
|
||||
|
||||
@app.command()
|
||||
def enrich() -> None:
|
||||
"""Run all enrichers once (linking, events, legal, topics, roll calls, corroboration, embeddings)."""
|
||||
from politrack.core import registry
|
||||
from politrack.core.runner import run
|
||||
|
||||
registry.load_builtin()
|
||||
for name in ("enrich.linker", "enrich.events", "enrich.legal", "enrich.topics", "enrich.roles",
|
||||
"enrich.rollcall", "enrich.corroborate", "enrich.embed"):
|
||||
typer.echo(f"{name}: {run(name)}")
|
||||
|
||||
|
||||
@app.command()
|
||||
def status(limit: int = 30) -> None:
|
||||
"""Show recent runs."""
|
||||
from politrack.db import connection
|
||||
|
||||
with connection() as conn:
|
||||
for r in conn.execute("SELECT adapter, mode, status, started_at, finished_at, stats FROM run "
|
||||
"ORDER BY id DESC LIMIT %s", (limit,)):
|
||||
took = (r["finished_at"] - r["started_at"]).seconds if r["finished_at"] else "…"
|
||||
typer.echo(f"{r['started_at']:%Y-%m-%d %H:%M} {r['adapter']:32} {r['status']:8} {took}s "
|
||||
f"{json.dumps(r['stats'])[:120]}")
|
||||
|
||||
|
||||
@app.command()
|
||||
def review(
|
||||
approve: list[int] = typer.Option([], help="Claim ids to publish"),
|
||||
reject: list[int] = typer.Option([], help="Claim ids to reject"),
|
||||
reviewer: str = typer.Option("cli", help="Your name, stored with the decision"),
|
||||
) -> None:
|
||||
"""List claims waiting for review (legal matters detected in headlines) or approve/reject them."""
|
||||
from politrack import queries
|
||||
from politrack.db import connection
|
||||
|
||||
with connection() as conn:
|
||||
for claim_id in approve:
|
||||
queries.set_review(conn, claim_id, "published", reviewer)
|
||||
for claim_id in reject:
|
||||
queries.set_review(conn, claim_id, "rejected", reviewer)
|
||||
if approve or reject:
|
||||
typer.echo(f"published {len(approve)}, rejected {len(reject)}")
|
||||
return
|
||||
for c in queries.pending_claims(conn, limit=100):
|
||||
typer.echo(f"[{c['id']}] {c['occurred_at']:%Y-%m-%d} {c['full_name']}: {c['summary_de']}")
|
||||
for ev in c["evidence"] or []:
|
||||
typer.echo(f" {ev['title']} {ev['url']}")
|
||||
|
||||
|
||||
@app.command()
|
||||
def fixture(name: str, url: str, case: str, kind: str = "default", meta: str = "{}", method: str = "GET",
|
||||
body: str = typer.Option(None, help="JSON request body for POST")) -> None:
|
||||
"""Capture a live response as a golden-test fixture (tests/fixtures/<adapter>/<case>.*)."""
|
||||
import httpx
|
||||
|
||||
from politrack.config import get_settings
|
||||
|
||||
out = Path(__file__).resolve().parent.parent / "tests" / "fixtures" / name
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
resp = httpx.request(method, url, json=json.loads(body) if body else None, timeout=120, follow_redirects=True,
|
||||
headers={"User-Agent": get_settings().user_agent})
|
||||
resp.raise_for_status()
|
||||
suffix = {"application/json": "json", "text/html": "html", "text/xml": "xml", "application/xml": "xml"}.get(
|
||||
resp.headers.get("content-type", "").split(";")[0], "bin")
|
||||
(out / f"{case}.{suffix}").write_bytes(resp.content)
|
||||
(out / f"{case}.request.json").write_text(json.dumps(
|
||||
{"url": url, "kind": kind, "meta": json.loads(meta), "method": method, "json": json.loads(body) if body else None,
|
||||
"content_type": resp.headers.get("content-type", ""), "body_file": f"{case}.{suffix}"}, indent=2), encoding="utf-8")
|
||||
typer.echo(f"saved {out / case}.* - now run: UPDATE_GOLDEN=1 pytest tests/test_golden.py -k {case}")
|
||||
|
||||
|
||||
@app.command()
|
||||
def openapi(json_out: Path = Path("openapi.json"), ts_out: Path = typer.Option(None, "--ts")) -> None:
|
||||
"""Export the OpenAPI schema and (optionally) TypeScript types for the web app."""
|
||||
from politrack.api.main import app as api_app
|
||||
from politrack.openapi_ts import to_typescript
|
||||
|
||||
schema = api_app.openapi()
|
||||
json_out.write_text(json.dumps(schema, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
if ts_out:
|
||||
ts_out.write_text(to_typescript(schema), encoding="utf-8")
|
||||
typer.echo(f"wrote {json_out}" + (f" and {ts_out}" if ts_out else ""))
|
||||
|
||||
|
||||
@app.command()
|
||||
def api(host: str = "0.0.0.0", port: int = 8000) -> None:
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run("politrack.api.main:app", host=host, port=port, proxy_headers=True)
|
||||
|
||||
|
||||
@app.command()
|
||||
def mcp(host: str = "0.0.0.0", port: int = 8001) -> None:
|
||||
"""Serve the MCP endpoint for AI agents (streamable HTTP at /mcp)."""
|
||||
from politrack.mcp_server import serve
|
||||
|
||||
serve(host, port)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app()
|
||||
@@ -0,0 +1,48 @@
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""All settings come from environment variables prefixed POLITRACK_ (see .env.example)."""
|
||||
|
||||
model_config = SettingsConfigDict(env_prefix="POLITRACK_", env_file=".env", extra="ignore")
|
||||
|
||||
database_url: str = "postgresql://politrack:politrack@localhost:5432/politrack"
|
||||
reader_password: str | None = None
|
||||
|
||||
# Crawling identity & politeness
|
||||
user_agent: str = "PolitrackBot/0.2 (+https://github.com/your-user/politics-tracking; contact: admin@example.org)"
|
||||
min_request_interval: float = 2.0
|
||||
http_timeout: float = 60.0
|
||||
|
||||
# Raw archive (fetched responses, gzip, content-addressed)
|
||||
raw_dir: Path = Path("/data/raw")
|
||||
|
||||
config_dir: Path = Path(__file__).resolve().parent.parent / "config"
|
||||
|
||||
# Parliament: legislative periods (GP) ingested in detail
|
||||
parlament_periods: list[str] = ["XXVII", "XXVIII"]
|
||||
|
||||
# Embeddings via Ollama (optional)
|
||||
ollama_url: str | None = None
|
||||
embedding_model: str = "bge-m3"
|
||||
|
||||
# Optional credentials for sources that require them (adapters skip themselves when unset)
|
||||
youtube_api_key: str | None = None
|
||||
meta_ad_library_token: str | None = None
|
||||
x_bearer_token: str | None = None
|
||||
|
||||
# Bulk sources: how far back to backfill on the first run
|
||||
procurement_since: str = "2024-01-01"
|
||||
ad_archive_since: str = "2023-01-01"
|
||||
|
||||
api_cors_origins: list[str] = ["http://localhost:3000"]
|
||||
# Bearer token for moderation/admin endpoints. Empty = disabled (use the CLI).
|
||||
admin_token: str | None = None
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_settings() -> Settings:
|
||||
return Settings()
|
||||
@@ -0,0 +1,11 @@
|
||||
"""The adapter framework. Most adapter modules only need:
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Evidence, PersonRef, OrgRef, ...
|
||||
"""
|
||||
|
||||
from politrack.core.adapter import Adapter, Enricher, Request, Response, RunContext
|
||||
from politrack.core.registry import register, register_spec
|
||||
from politrack.core.spec import SourceSpec
|
||||
|
||||
__all__ = ["Adapter", "Enricher", "Request", "Response", "RunContext", "SourceSpec", "register", "register_spec"]
|
||||
@@ -0,0 +1,177 @@
|
||||
"""Adapter / Enricher base classes and the request/response/context types they work with.
|
||||
|
||||
An Adapter turns one publisher's data into Records in two steps:
|
||||
|
||||
start(ctx) -> Request... entry points (lists, feeds, index pages)
|
||||
parse(ctx, resp) -> Record | Request... PURE: no DB access, no network
|
||||
|
||||
The runner fetches requests (policy-checked, archived), calls parse() and hands records to the
|
||||
Loader. See docs/contributing/adapters.md for a walkthrough.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable, Iterator
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from functools import cached_property
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from politrack.core.spec import SourceSpec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from psycopg import Connection
|
||||
|
||||
from politrack.core.records import Record
|
||||
|
||||
|
||||
SECRET_PARAMS = {"key", "api_key", "apikey", "token", "access_token"}
|
||||
|
||||
|
||||
def redact_url(url: str) -> str:
|
||||
"""Remove credentials from a URL before it is logged or stored."""
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
|
||||
parts = urlsplit(url)
|
||||
query = [(k, "<redacted>" if k.lower() in SECRET_PARAMS else v) for k, v in parse_qsl(parts.query, keep_blank_values=True)]
|
||||
return urlunsplit(parts._replace(query=urlencode(query)))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Request:
|
||||
url: str
|
||||
kind: str = "default" # parse() dispatch key
|
||||
method: str = "GET"
|
||||
params: dict[str, Any] | None = None
|
||||
json: Any = None
|
||||
headers: dict[str, str] | None = None
|
||||
meta: dict[str, Any] = field(default_factory=dict, compare=False, hash=False)
|
||||
conditional: bool = True # send If-None-Match / If-Modified-Since
|
||||
skip_unchanged: bool = True # body + parser_version unchanged: follow-ups only
|
||||
|
||||
@property
|
||||
def fingerprint(self) -> str:
|
||||
raw = json.dumps([self.method, self.url, self.params or {}, self.json], sort_keys=True, default=str)
|
||||
return hashlib.sha256(raw.encode()).hexdigest()[:32]
|
||||
|
||||
def describe(self) -> dict[str, Any]:
|
||||
"""What gets stored in the archive index (secrets removed)."""
|
||||
params = {k: ("<redacted>" if k.lower() in SECRET_PARAMS else v) for k, v in (self.params or {}).items()}
|
||||
return {"kind": self.kind, "params": params or None, "json": self.json, "meta": self.meta}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Response:
|
||||
request: Request
|
||||
status: int
|
||||
url: str
|
||||
headers: dict[str, str]
|
||||
body: bytes
|
||||
fetched_at: datetime
|
||||
sha256: str
|
||||
fetch_id: int | None = None
|
||||
tdm_reserved: bool = False
|
||||
from_archive: bool = False
|
||||
|
||||
@cached_property
|
||||
def text(self) -> str:
|
||||
encoding = "utf-8"
|
||||
content_type = self.headers.get("content-type", "")
|
||||
if "charset=" in content_type:
|
||||
encoding = content_type.split("charset=", 1)[1].split(";")[0].strip() or "utf-8"
|
||||
try:
|
||||
return self.body.decode(encoding)
|
||||
except (LookupError, UnicodeDecodeError):
|
||||
return self.body.decode("utf-8", errors="replace")
|
||||
|
||||
def json(self) -> Any:
|
||||
return json.loads(self.text, strict=False)
|
||||
|
||||
@cached_property
|
||||
def soup(self) -> BeautifulSoup:
|
||||
return BeautifulSoup(self.text, "html.parser")
|
||||
|
||||
@property
|
||||
def meta(self) -> dict[str, Any]:
|
||||
return self.request.meta
|
||||
|
||||
|
||||
class StateStore:
|
||||
"""Cursors of incremental adapters (persisted in the `state` table)."""
|
||||
|
||||
def __init__(self, conn: Connection | None, adapter: str) -> None:
|
||||
self.conn = conn
|
||||
self.adapter = adapter
|
||||
self._local: dict[str, Any] = {}
|
||||
|
||||
def get(self, key: str, default: Any = None) -> Any:
|
||||
if self.conn is None:
|
||||
return self._local.get(key, default)
|
||||
row = self.conn.execute("SELECT value FROM state WHERE adapter = %s AND key = %s",
|
||||
(self.adapter, key)).fetchone()
|
||||
return row["value"] if row else default
|
||||
|
||||
def set(self, key: str, value: Any) -> None:
|
||||
if self.conn is None:
|
||||
self._local[key] = value
|
||||
return
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
self.conn.execute(
|
||||
"""INSERT INTO state (adapter, key, value) VALUES (%s, %s, %s)
|
||||
ON CONFLICT (adapter, key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()""",
|
||||
(self.adapter, key, Jsonb(value)),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunContext:
|
||||
adapter: str
|
||||
spec: SourceSpec
|
||||
state: StateStore
|
||||
stats: Counter = field(default_factory=Counter)
|
||||
since: datetime | None = None
|
||||
params: dict[str, Any] = field(default_factory=dict) # CLI/worker parameters, e.g. {"gp": "XXVIII"}
|
||||
mode: str = "fetch" # fetch | reparse | test
|
||||
conn: Connection | None = None # only for enrichers / the loader, never in parse()
|
||||
response: Response | None = None # set by the runner while parsing
|
||||
creates_persons: bool = False
|
||||
log: logging.Logger = field(default_factory=lambda: logging.getLogger("politrack"))
|
||||
|
||||
|
||||
class Adapter:
|
||||
"""Base class for source adapters. Subclass, set the class attributes, implement start()/parse()."""
|
||||
|
||||
spec: ClassVar[SourceSpec]
|
||||
name: ClassVar[str]
|
||||
description: ClassVar[str] = ""
|
||||
schedule: ClassVar[str | None] = None # cron, UTC; None = manual only
|
||||
queue: ClassVar[str] = "scrape" # scrape | bulk | social | enrich
|
||||
parser_version: ClassVar[int] = 1 # bump when parse() output changes -> `politrack reparse`
|
||||
creates_persons: ClassVar[bool] = False # may this adapter create new person entities?
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterable[Request]:
|
||||
raise NotImplementedError
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Record | Request]:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class Enricher:
|
||||
"""Derives records from data already in the database (linking, events, corroboration...)."""
|
||||
|
||||
spec: ClassVar[SourceSpec]
|
||||
name: ClassVar[str]
|
||||
description: ClassVar[str] = ""
|
||||
schedule: ClassVar[str | None] = None
|
||||
queue: ClassVar[str] = "enrich"
|
||||
creates_persons: ClassVar[bool] = False
|
||||
|
||||
def run(self, ctx: RunContext) -> Iterable[Record] | None:
|
||||
raise NotImplementedError
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Raw archive: every fetch is logged in the `fetch_log` table; bodies are stored gzip-compressed and
|
||||
content-addressed (sha256) below POLITRACK_RAW_DIR when the source allows it.
|
||||
|
||||
Used for: conditional requests (ETag/Last-Modified), change detection (skip re-parsing identical
|
||||
bodies), `politrack reparse` after parser fixes, and as an audit trail.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from psycopg import Connection
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from politrack.config import get_settings
|
||||
|
||||
|
||||
class Archive:
|
||||
def __init__(self, raw_dir: Path | None = None) -> None:
|
||||
self.raw_dir = Path(raw_dir or get_settings().raw_dir)
|
||||
|
||||
# -- blobs ----------------------------------------------------------------------------
|
||||
|
||||
def _path(self, sha256: str) -> Path:
|
||||
return self.raw_dir / sha256[:2] / f"{sha256}.gz"
|
||||
|
||||
def write_blob(self, body: bytes) -> str:
|
||||
sha = hashlib.sha256(body).hexdigest()
|
||||
path = self._path(sha)
|
||||
if not path.exists():
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(f".tmp{os.getpid()}")
|
||||
with gzip.open(tmp, "wb") as fh:
|
||||
fh.write(body)
|
||||
tmp.replace(path)
|
||||
return sha
|
||||
|
||||
def read_blob(self, sha256: str) -> bytes | None:
|
||||
path = self._path(sha256)
|
||||
if not path.exists():
|
||||
return None
|
||||
with gzip.open(path, "rb") as fh:
|
||||
return fh.read()
|
||||
|
||||
# -- index ----------------------------------------------------------------------------
|
||||
|
||||
def last(self, conn: Connection, request_fp: str) -> dict[str, Any] | None:
|
||||
return conn.execute(
|
||||
"""SELECT id, sha256, etag, last_modified, stored, parsed_sha256, parser_version, status, url,
|
||||
content_type, tdm_reserved, fetched_at
|
||||
FROM fetch_log WHERE request_fp = %s AND status IS NOT NULL AND status < 400
|
||||
ORDER BY fetched_at DESC LIMIT 1""",
|
||||
(request_fp,),
|
||||
).fetchone()
|
||||
|
||||
def log(self, conn: Connection, *, adapter: str, source_id: int | None, request_fp: str, url: str, method: str,
|
||||
request: dict, status: int | None, body: bytes | None, headers: dict[str, str] | None,
|
||||
keep: bool, tdm_reserved: bool, error: str | None = None) -> tuple[int, str | None]:
|
||||
sha = hashlib.sha256(body).hexdigest() if body is not None else None
|
||||
stored = False
|
||||
if body is not None and keep and not tdm_reserved:
|
||||
self.write_blob(body)
|
||||
stored = True
|
||||
headers = headers or {}
|
||||
row = conn.execute(
|
||||
"""INSERT INTO fetch_log (adapter, source_id, request_fp, url, method, request, status, sha256, size,
|
||||
content_type, etag, last_modified, stored, tdm_reserved, error)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id""",
|
||||
(adapter, source_id, request_fp, url, method, Jsonb(request), status, sha,
|
||||
len(body) if body is not None else None, headers.get("content-type"), headers.get("etag"),
|
||||
headers.get("last-modified"), stored, tdm_reserved, error),
|
||||
).fetchone()
|
||||
return row["id"], sha
|
||||
|
||||
def mark_parsed(self, conn: Connection, fetch_id: int, sha256: str, parser_version: int) -> None:
|
||||
conn.execute("UPDATE fetch_log SET parsed_sha256 = %s, parser_version = %s WHERE id = %s",
|
||||
(sha256, parser_version, fetch_id))
|
||||
|
||||
def mark_error(self, conn: Connection, fetch_id: int, error: str) -> None:
|
||||
conn.execute("UPDATE fetch_log SET error = %s WHERE id = %s", (error[:2000], fetch_id))
|
||||
|
||||
def latest_stored(self, conn: Connection, adapter: str) -> list[dict[str, Any]]:
|
||||
"""Latest stored fetch per request of an adapter (input for `reparse`)."""
|
||||
return conn.execute(
|
||||
"""SELECT DISTINCT ON (request_fp) id, request_fp, url, method, request, status, sha256, fetched_at,
|
||||
content_type, tdm_reserved
|
||||
FROM fetch_log WHERE adapter = %s AND stored AND status < 400
|
||||
ORDER BY request_fp, fetched_at DESC""",
|
||||
(adapter,),
|
||||
).fetchall()
|
||||
@@ -0,0 +1,158 @@
|
||||
"""The Fetcher: the only component that talks to the network. It enforces the crawling policy:
|
||||
|
||||
* honest User-Agent with contact address,
|
||||
* robots.txt (cached per origin); exemptions only for documented APIs listed in SourceSpec.robots_exempt,
|
||||
* per-host throttling (max of SourceSpec.min_interval and robots.txt crawl-delay),
|
||||
* conditional requests (ETag / Last-Modified from the raw archive),
|
||||
* retries with exponential back-off for 429/5xx, honouring Retry-After,
|
||||
* TDM reservations (W3C TDMRep: `tdm-reservation` header, /.well-known/tdmrep.json, /tdmrep.json),
|
||||
* archiving of raw bodies (unless SourceSpec.keep_raw is False or TDM is reserved).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import httpx
|
||||
from protego import Protego
|
||||
from psycopg import Connection
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core.adapter import Request, Response, redact_url
|
||||
from politrack.core.archive import Archive
|
||||
from politrack.core.spec import SourceSpec
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
RETRY_STATUS = {429, 500, 502, 503, 504}
|
||||
|
||||
|
||||
class Disallowed(Exception):
|
||||
"""robots.txt forbids fetching this URL."""
|
||||
|
||||
|
||||
class Fetcher:
|
||||
def __init__(self, archive: Archive | None = None, client: httpx.Client | None = None) -> None:
|
||||
settings = get_settings()
|
||||
self.user_agent = settings.user_agent
|
||||
self.default_interval = settings.min_request_interval
|
||||
self.archive = archive or Archive()
|
||||
self.client = client or httpx.Client(
|
||||
headers={"User-Agent": self.user_agent, "Accept-Language": "de-AT,de;q=0.9,en;q=0.5"},
|
||||
timeout=settings.http_timeout,
|
||||
follow_redirects=True,
|
||||
)
|
||||
self._robots: dict[str, Protego | None] = {}
|
||||
self._tdm: dict[str, bool] = {}
|
||||
self._last: dict[str, float] = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# -- policy -------------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def origin(url: str) -> str:
|
||||
parts = urlsplit(url)
|
||||
return f"{parts.scheme}://{parts.netloc}"
|
||||
|
||||
def robots(self, url: str) -> Protego | None:
|
||||
origin = self.origin(url)
|
||||
if origin not in self._robots:
|
||||
try:
|
||||
resp = self.client.get(f"{origin}/robots.txt")
|
||||
self._robots[origin] = Protego.parse(resp.text) if resp.status_code == 200 else None
|
||||
except httpx.HTTPError:
|
||||
self._robots[origin] = None
|
||||
return self._robots[origin]
|
||||
|
||||
def allowed(self, url: str, spec: SourceSpec) -> bool:
|
||||
if any(url.startswith(prefix) for prefix in spec.robots_exempt):
|
||||
return True
|
||||
rules = self.robots(url)
|
||||
return rules is None or rules.can_fetch(url, self.user_agent)
|
||||
|
||||
def tdm_reserved(self, url: str) -> bool:
|
||||
origin = self.origin(url)
|
||||
if origin not in self._tdm:
|
||||
reserved = False
|
||||
for path in ("/.well-known/tdmrep.json", "/tdmrep.json"):
|
||||
try:
|
||||
resp = self.client.get(origin + path)
|
||||
except httpx.HTTPError:
|
||||
continue
|
||||
if resp.status_code == 200:
|
||||
try:
|
||||
rules = resp.json()
|
||||
except ValueError:
|
||||
continue
|
||||
reserved = any(str(r.get("tdm-reservation")) == "1" for r in rules if isinstance(r, dict))
|
||||
break
|
||||
self._tdm[origin] = reserved
|
||||
return self._tdm[origin]
|
||||
|
||||
def _throttle(self, url: str, spec: SourceSpec) -> None:
|
||||
rules = self._robots.get(self.origin(url))
|
||||
delay = rules.crawl_delay(self.user_agent) if rules else None
|
||||
interval = max(spec.min_interval or self.default_interval, float(delay or 0))
|
||||
host = urlsplit(url).netloc
|
||||
with self._lock:
|
||||
wait = self._last.get(host, 0) + interval - time.monotonic()
|
||||
if wait > 0:
|
||||
time.sleep(wait)
|
||||
self._last[host] = time.monotonic()
|
||||
|
||||
# -- fetching ---------------------------------------------------------------------
|
||||
|
||||
def fetch(self, conn: Connection, req: Request, *, adapter: str, spec: SourceSpec, source_id: int | None,
|
||||
attempts: int = 5) -> Response:
|
||||
if not self.allowed(req.url, spec):
|
||||
raise Disallowed(req.url)
|
||||
last = self.archive.last(conn, req.fingerprint) if req.conditional else None
|
||||
headers = dict(req.headers or {})
|
||||
if last and last["stored"]:
|
||||
if last["etag"]:
|
||||
headers["If-None-Match"] = last["etag"]
|
||||
if last["last_modified"]:
|
||||
headers["If-Modified-Since"] = last["last_modified"]
|
||||
|
||||
resp: httpx.Response | None = None
|
||||
for attempt in range(attempts):
|
||||
self._throttle(req.url, spec)
|
||||
try:
|
||||
resp = self.client.request(req.method, req.url, params=req.params, json=req.json, headers=headers)
|
||||
except httpx.TransportError as exc:
|
||||
if attempt == attempts - 1:
|
||||
raise
|
||||
log.warning("transport error %s (%s), retrying", req.url, exc)
|
||||
time.sleep(min(5 * 2 ** attempt, 120))
|
||||
continue
|
||||
if resp.status_code in RETRY_STATUS and attempt < attempts - 1:
|
||||
retry_after = resp.headers.get("Retry-After", "")
|
||||
time.sleep(min(float(retry_after) if retry_after.isdigit() else 5 * 2 ** attempt, 300))
|
||||
continue
|
||||
break
|
||||
assert resp is not None
|
||||
tdm = resp.headers.get("tdm-reservation") == "1" or self.tdm_reserved(req.url)
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
if resp.status_code == 304 and last:
|
||||
body = self.archive.read_blob(last["sha256"]) or b""
|
||||
return Response(request=req, status=200, url=last["url"], headers={"content-type": last["content_type"] or ""},
|
||||
body=body, fetched_at=now, sha256=last["sha256"], fetch_id=last["id"],
|
||||
tdm_reserved=last["tdm_reserved"], from_archive=True)
|
||||
|
||||
lowered = {k.lower(): v for k, v in resp.headers.items()}
|
||||
fetch_id, sha = self.archive.log(
|
||||
conn, adapter=adapter, source_id=source_id, request_fp=req.fingerprint, url=redact_url(str(resp.url)),
|
||||
method=req.method, request=req.describe(), status=resp.status_code, body=resp.content, headers=lowered,
|
||||
keep=spec.keep_raw, tdm_reserved=tdm,
|
||||
)
|
||||
conn.commit()
|
||||
resp.raise_for_status()
|
||||
return Response(request=req, status=resp.status_code, url=redact_url(str(resp.url)), headers=lowered, body=resp.content,
|
||||
fetched_at=now, sha256=sha or "", fetch_id=fetch_id, tdm_reserved=tdm)
|
||||
|
||||
def close(self) -> None:
|
||||
self.client.close()
|
||||
@@ -0,0 +1,448 @@
|
||||
"""The Loader: the only code that writes canonical rows.
|
||||
|
||||
Invariants enforced here (not in adapters):
|
||||
* no claim without evidence; claims whose person cannot be resolved are skipped (and counted),
|
||||
* full text is kept only for sources with store_fulltext and responses without TDM reservation,
|
||||
* claim identity is computed from *resolved* entity ids + type-specific fields, so the same fact
|
||||
from different sources becomes one claim with several evidence rows ("corroborated"),
|
||||
* claims marked `review=True` start as review_status='pending' (hidden until a human approves),
|
||||
* persons are only created by adapters with creates_persons=True.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Any
|
||||
|
||||
from psycopg import Connection
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from politrack.core.adapter import RunContext
|
||||
from politrack.core.records import (
|
||||
Claim,
|
||||
Document,
|
||||
Event,
|
||||
Evidence,
|
||||
Legal,
|
||||
Money,
|
||||
OrgRecord,
|
||||
OrgRef,
|
||||
Participation,
|
||||
PersonRecord,
|
||||
PersonRef,
|
||||
Role,
|
||||
SocialAccount,
|
||||
Statement,
|
||||
VoteEvent,
|
||||
VotePosition,
|
||||
)
|
||||
from politrack.core.resolve import Resolver
|
||||
from politrack.text import excerpt, fingerprint, make_slug, normalize, normalize_org_name, strip_titles
|
||||
|
||||
# Source policy is read from the DB once per process (synced from SourceSpecs).
|
||||
_SOURCES: dict[str, dict[str, Any]] = {}
|
||||
|
||||
|
||||
def source_row(conn: Connection, slug: str) -> dict[str, Any]:
|
||||
if slug not in _SOURCES:
|
||||
row = conn.execute("SELECT id, store_fulltext, kind FROM source WHERE slug = %s", (slug,)).fetchone()
|
||||
if row is None:
|
||||
raise KeyError(f"unknown source {slug!r}: run `politrack sources sync`")
|
||||
_SOURCES[slug] = row
|
||||
return _SOURCES[slug]
|
||||
|
||||
|
||||
class Loader:
|
||||
def __init__(self, conn: Connection, ctx: RunContext) -> None:
|
||||
self.conn = conn
|
||||
self.ctx = ctx
|
||||
self.resolver = Resolver(conn)
|
||||
|
||||
# -- dispatch ---------------------------------------------------------------------
|
||||
|
||||
def load(self, record: Any) -> int | None:
|
||||
if isinstance(record, Claim):
|
||||
return self.claim(record)
|
||||
if isinstance(record, Document):
|
||||
return self.document(record.evidence)
|
||||
if isinstance(record, PersonRecord):
|
||||
return self.person(record)
|
||||
if isinstance(record, OrgRecord):
|
||||
return self.organization(record)
|
||||
if isinstance(record, SocialAccount):
|
||||
return self.social_account(record)
|
||||
if isinstance(record, VoteEvent):
|
||||
return self.vote_event(record)
|
||||
if isinstance(record, Event):
|
||||
return self.event(record)
|
||||
raise TypeError(f"unknown record type {type(record).__name__}")
|
||||
|
||||
# -- documents --------------------------------------------------------------------
|
||||
|
||||
def document(self, ev: Evidence) -> int:
|
||||
slug = ev.source or self.ctx.spec.slug
|
||||
src = source_row(self.conn, slug)
|
||||
response = self.ctx.response
|
||||
tdm = bool(response and response.tdm_reserved)
|
||||
full_text = ev.full_text if (src["store_fulltext"] and not tdm) else None
|
||||
short = None if tdm else (excerpt(ev.excerpt, 400) if ev.excerpt else None)
|
||||
# Link the document to the raw response only if it *is* that response (not e.g. a protocol it cites).
|
||||
fetch_id = response.fetch_id if response and ev.url in (response.url, response.request.url) else None
|
||||
row = self.conn.execute(
|
||||
"""
|
||||
INSERT INTO document (source_id, url, title, doc_type, published_at, excerpt, full_text, fetch_id, meta)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (url) DO UPDATE SET
|
||||
title = COALESCE(EXCLUDED.title, document.title),
|
||||
published_at = COALESCE(EXCLUDED.published_at, document.published_at),
|
||||
excerpt = COALESCE(EXCLUDED.excerpt, document.excerpt),
|
||||
full_text = COALESCE(EXCLUDED.full_text, document.full_text),
|
||||
fetch_id = COALESCE(EXCLUDED.fetch_id, document.fetch_id),
|
||||
meta = document.meta || EXCLUDED.meta,
|
||||
retrieved_at = now()
|
||||
RETURNING id
|
||||
""",
|
||||
(src["id"], ev.url, ev.title, ev.doc_type, ev.published_at, short, full_text, fetch_id,
|
||||
Jsonb({**ev.meta, **({"tdm_reserved": True} if tdm else {})})),
|
||||
).fetchone()
|
||||
self.ctx.stats["documents"] += 1
|
||||
return row["id"]
|
||||
|
||||
# -- entities ---------------------------------------------------------------------
|
||||
|
||||
def person(self, rec: PersonRecord) -> int | None:
|
||||
person_id = self.resolver.person_by_ids(rec.ref.ids)
|
||||
if person_id is None and rec.match_by_name:
|
||||
person_id = self.resolver.person_by_name(rec.full_name)
|
||||
if person_id is None:
|
||||
if not self.ctx.creates_persons:
|
||||
self.ctx.stats["persons_not_created"] += 1
|
||||
return None
|
||||
base = make_slug(rec.given_name or "", rec.family_name or strip_titles(rec.full_name)) or "person"
|
||||
slug, n = base, 1
|
||||
while self.conn.execute("SELECT 1 FROM person WHERE slug = %s", (slug,)).fetchone():
|
||||
n += 1
|
||||
slug = f"{base}-{n}"
|
||||
person_id = self.conn.execute(
|
||||
"""INSERT INTO person (slug, full_name, given_name, family_name, honorific_prefix, honorific_suffix,
|
||||
birth_date, death_date, gender, photo_url)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id""",
|
||||
(slug, rec.full_name, rec.given_name, rec.family_name, rec.honorific_prefix, rec.honorific_suffix,
|
||||
rec.birth_date, rec.death_date, rec.gender, rec.photo_url),
|
||||
).fetchone()["id"]
|
||||
self.ctx.stats["persons_created"] += 1
|
||||
else:
|
||||
self.conn.execute(
|
||||
"""UPDATE person SET full_name = %s,
|
||||
given_name = COALESCE(%s, given_name), family_name = COALESCE(%s, family_name),
|
||||
honorific_prefix = COALESCE(%s, honorific_prefix), honorific_suffix = COALESCE(%s, honorific_suffix),
|
||||
birth_date = COALESCE(%s, birth_date), death_date = COALESCE(%s, death_date),
|
||||
gender = COALESCE(%s, gender), photo_url = COALESCE(%s, photo_url), updated_at = now()
|
||||
WHERE id = %s""",
|
||||
(rec.full_name, rec.given_name, rec.family_name, rec.honorific_prefix, rec.honorific_suffix,
|
||||
rec.birth_date, rec.death_date, rec.gender, rec.photo_url, person_id),
|
||||
)
|
||||
for scheme, value in rec.ref.ids.items():
|
||||
if value:
|
||||
self.conn.execute(
|
||||
"""INSERT INTO identifier (entity_type, entity_id, scheme, value, source_slug)
|
||||
VALUES ('person', %s, %s, %s, %s) ON CONFLICT DO NOTHING""",
|
||||
(person_id, scheme, str(value), self.ctx.spec.slug),
|
||||
)
|
||||
names = {rec.full_name, strip_titles(rec.full_name), *rec.aliases}
|
||||
if rec.given_name and rec.family_name:
|
||||
names.add(f"{rec.given_name} {rec.family_name}")
|
||||
for name in names:
|
||||
if name:
|
||||
self.conn.execute(
|
||||
"""INSERT INTO alias (entity_type, entity_id, alias, alias_norm) VALUES ('person', %s, %s, %s)
|
||||
ON CONFLICT DO NOTHING""",
|
||||
(person_id, name, normalize(strip_titles(name))),
|
||||
)
|
||||
self.ctx.stats["persons"] += 1
|
||||
return person_id
|
||||
|
||||
def organization(self, rec: OrgRecord) -> int | None:
|
||||
org_id = self.resolver.org(rec.ref)
|
||||
if org_id is None:
|
||||
return None
|
||||
if rec.color:
|
||||
self.conn.execute("UPDATE organization SET color = COALESCE(color, %s) WHERE id = %s", (rec.color, org_id))
|
||||
if rec.parent:
|
||||
parent = self.resolver.org(rec.parent)
|
||||
if parent and parent != org_id:
|
||||
self.conn.execute("UPDATE organization SET parent_id = %s WHERE id = %s", (parent, org_id))
|
||||
self.resolver.add_org_aliases(org_id, list(rec.aliases))
|
||||
return org_id
|
||||
|
||||
def social_account(self, rec: SocialAccount) -> int | None:
|
||||
person_id = self.resolver.person(rec.person)
|
||||
org_id = self.resolver.org(rec.org) if rec.org else None # OrgRef.create decides
|
||||
if person_id is None and org_id is None:
|
||||
self.ctx.stats["social_unresolved"] += 1
|
||||
return None
|
||||
row = self.conn.execute(
|
||||
"""INSERT INTO social_account (person_id, organization_id, platform, handle, url, verified, source_slug)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (platform, url) DO UPDATE SET
|
||||
handle = COALESCE(EXCLUDED.handle, social_account.handle),
|
||||
verified = social_account.verified OR EXCLUDED.verified,
|
||||
person_id = COALESCE(social_account.person_id, EXCLUDED.person_id),
|
||||
organization_id = COALESCE(social_account.organization_id, EXCLUDED.organization_id)
|
||||
RETURNING id""",
|
||||
(person_id, org_id, rec.platform.lower(), rec.handle, rec.url, rec.verified, self.ctx.spec.slug),
|
||||
).fetchone()
|
||||
self.ctx.stats["social_accounts"] += 1
|
||||
return row["id"]
|
||||
|
||||
# -- standalone facets ------------------------------------------------------------
|
||||
|
||||
def vote_event(self, rec: VoteEvent) -> int:
|
||||
chamber = self.resolver.org(rec.chamber, create=False) if rec.chamber else None
|
||||
row = self.conn.execute(
|
||||
"""INSERT INTO vote_event (event_key, chamber_id, item_ref, item_title, stage, voted_on, result, roll_call,
|
||||
url, protocol_url, raw_text)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (event_key) DO UPDATE SET result = EXCLUDED.result, raw_text = EXCLUDED.raw_text,
|
||||
roll_call = EXCLUDED.roll_call, item_title = EXCLUDED.item_title,
|
||||
protocol_url = COALESCE(EXCLUDED.protocol_url, vote_event.protocol_url)
|
||||
RETURNING id""",
|
||||
(rec.key, chamber, rec.item_ref, rec.title, rec.stage, rec.voted_on, rec.result, rec.roll_call, rec.url,
|
||||
rec.protocol_url, rec.raw_text),
|
||||
).fetchone()
|
||||
self.ctx.stats["vote_events"] += 1
|
||||
return row["id"]
|
||||
|
||||
def event(self, rec: Event) -> int:
|
||||
body = self.resolver.org(rec.body, create=False) if rec.body else None
|
||||
source_id = source_row(self.conn, self.ctx.spec.slug)["id"]
|
||||
row = self.conn.execute(
|
||||
"""INSERT INTO event (event_key, kind, title, starts_at, ends_at, location_name, country, url, description,
|
||||
organization_id, source_id)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (event_key) DO UPDATE SET title = EXCLUDED.title,
|
||||
starts_at = COALESCE(EXCLUDED.starts_at, event.starts_at), ends_at = EXCLUDED.ends_at,
|
||||
location_name = COALESCE(EXCLUDED.location_name, event.location_name),
|
||||
description = COALESCE(EXCLUDED.description, event.description)
|
||||
RETURNING id""",
|
||||
(rec.key, rec.kind, rec.title, rec.starts_at, rec.ends_at, rec.location, rec.country, rec.url,
|
||||
rec.description, body, source_id),
|
||||
).fetchone()
|
||||
self.ctx.stats["events"] += 1
|
||||
return row["id"]
|
||||
|
||||
# -- claims -----------------------------------------------------------------------
|
||||
|
||||
def claim(self, c: Claim) -> int | None:
|
||||
if not c.evidence:
|
||||
raise ValueError("a claim needs at least one piece of evidence")
|
||||
person_id = self.resolver.person(c.person)
|
||||
if c.person is not None and person_id is None:
|
||||
self.ctx.stats["unresolved_person"] += 1
|
||||
self.ctx.log.debug("unresolved person %s for %s", c.person, c.summary_de[:80])
|
||||
return None
|
||||
org_id = self.resolver.org(c.org) if c.org else None
|
||||
cp_person = self.resolver.person(c.counterpart_person)
|
||||
cp_org = self.resolver.org(c.counterpart_org) if c.counterpart_org else None
|
||||
if person_id is None and org_id is None and cp_org is None:
|
||||
self.ctx.stats["claims_without_subject"] += 1
|
||||
return None
|
||||
|
||||
fp = fingerprint(*self.identity(c, person_id, org_id, cp_org))
|
||||
row = self.conn.execute(
|
||||
"""
|
||||
INSERT INTO claim (claim_type, fingerprint, person_id, organization_id, counterpart_person_id,
|
||||
counterpart_org_id, occurred_at, valid_from, valid_to, summary_de, summary_en,
|
||||
data, sensitive, review_status, created_by)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (fingerprint) DO UPDATE SET
|
||||
summary_de = EXCLUDED.summary_de,
|
||||
summary_en = COALESCE(EXCLUDED.summary_en, claim.summary_en),
|
||||
valid_to = COALESCE(EXCLUDED.valid_to, claim.valid_to),
|
||||
occurred_at = COALESCE(claim.occurred_at, EXCLUDED.occurred_at),
|
||||
data = claim.data || EXCLUDED.data,
|
||||
sensitive = claim.sensitive OR EXCLUDED.sensitive,
|
||||
updated_at = now()
|
||||
RETURNING id, (xmax = 0) AS inserted
|
||||
""",
|
||||
(c.claim_type, fp, person_id, org_id, cp_person, cp_org, c.occurred_at, c.valid_from, c.valid_to,
|
||||
c.summary_de, c.summary_en, Jsonb(c.data), c.sensitive or c.claim_type == "legal_proceeding",
|
||||
"pending" if c.review else "published", self.ctx.adapter),
|
||||
).fetchone()
|
||||
claim_id = row["id"]
|
||||
self.ctx.stats["claims_new" if row["inserted"] else "claims_updated"] += 1
|
||||
|
||||
for ev in c.evidence:
|
||||
doc_id = self.document(ev)
|
||||
self.conn.execute(
|
||||
"""INSERT INTO claim_evidence (claim_id, document_id, stance, quote, locator, extracted_by, confidence)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (claim_id, document_id, locator) DO UPDATE SET
|
||||
quote = COALESCE(EXCLUDED.quote, claim_evidence.quote),
|
||||
confidence = GREATEST(claim_evidence.confidence, EXCLUDED.confidence)""",
|
||||
(claim_id, doc_id, ev.stance, ev.quote and excerpt(ev.quote, 600), ev.locator, self.ctx.adapter,
|
||||
ev.confidence),
|
||||
)
|
||||
for topic in c.topics:
|
||||
self.assign_topic(claim_id, topic)
|
||||
if c.facet is not None:
|
||||
self.facet(claim_id, c, person_id, org_id)
|
||||
self.refresh_status(claim_id)
|
||||
return claim_id
|
||||
|
||||
def identity(self, c: Claim, person_id: int | None, org_id: int | None, cp_org: int | None) -> tuple:
|
||||
"""Semantic identity of a claim (see docs/architecture/01-design-from-scratch.md §3.3)."""
|
||||
if c.key is not None:
|
||||
return (c.claim_type, "key", *c.key)
|
||||
f = c.facet
|
||||
if isinstance(f, Role):
|
||||
body = self.resolver.org(f.body) if f.body else cp_org
|
||||
return ("role", person_id, body, normalize(f.role), f.start)
|
||||
if isinstance(f, VotePosition):
|
||||
return ("vote", f.event_key, person_id, org_id)
|
||||
if isinstance(f, Money):
|
||||
def side(ref: Any, name: str | None) -> Any:
|
||||
if isinstance(ref, PersonRef):
|
||||
return ("p", self.resolver.person(ref))
|
||||
if isinstance(ref, OrgRef):
|
||||
return ("o", self.resolver.org(ref))
|
||||
return ("n", normalize_org_name(name or ""))
|
||||
return ("money", f.kind, side(f.payer, f.payer_name), side(f.payee, f.payee_name), _amount(f.amount),
|
||||
f.paid_on, f.period, f.reference)
|
||||
if isinstance(f, Statement):
|
||||
return ("statement", person_id, org_id, f.url)
|
||||
if isinstance(f, Participation):
|
||||
return ("participation", f.event_key, person_id, org_id)
|
||||
if c.claim_type == "media_mention":
|
||||
return ("mention", c.evidence[0].url, person_id, org_id)
|
||||
raise ValueError(f"claim of type {c.claim_type} needs an explicit key")
|
||||
|
||||
def facet(self, claim_id: int, c: Claim, person_id: int | None, org_id: int | None) -> None:
|
||||
f = c.facet
|
||||
if isinstance(f, Role):
|
||||
body = self.resolver.org(f.body) if f.body else None
|
||||
party = self.resolver.org(f.party) if f.party else None
|
||||
self.conn.execute(
|
||||
"""INSERT INTO role (claim_id, person_id, organization_id, party_id, role, level, region, start_date, end_date)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (claim_id) DO UPDATE SET end_date = COALESCE(EXCLUDED.end_date, role.end_date),
|
||||
party_id = COALESCE(EXCLUDED.party_id, role.party_id), region = COALESCE(EXCLUDED.region, role.region),
|
||||
organization_id = COALESCE(EXCLUDED.organization_id, role.organization_id)""",
|
||||
(claim_id, person_id, body, party, f.role, f.level, f.region, f.start, f.end),
|
||||
)
|
||||
elif isinstance(f, VotePosition):
|
||||
event = self.conn.execute(
|
||||
"""SELECT ve.id, ve.voted_on, o.slug AS chamber FROM vote_event ve
|
||||
LEFT JOIN organization o ON o.id = ve.chamber_id WHERE ve.event_key = %s""",
|
||||
(f.event_key,),
|
||||
).fetchone()
|
||||
if event is None:
|
||||
raise ValueError(f"vote event {f.event_key} must be loaded before its positions")
|
||||
party = self.resolver.org(f.party) if f.party else None
|
||||
if person_id is not None and party is None and event["chamber"]:
|
||||
# individual (roll-call) vote: remember the voter's faction on that day (dissent detection)
|
||||
party = self.resolver.faction_of(person_id, event["chamber"], event["voted_on"])
|
||||
self.conn.execute(
|
||||
"""INSERT INTO vote_position (claim_id, vote_event_id, organization_id, person_id, party_id, position)
|
||||
VALUES (%s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (claim_id) DO UPDATE SET position = EXCLUDED.position,
|
||||
party_id = COALESCE(EXCLUDED.party_id, vote_position.party_id)""",
|
||||
(claim_id, event["id"], org_id if person_id is None else None, person_id, party, f.position),
|
||||
)
|
||||
elif isinstance(f, Money):
|
||||
payer_org = self.resolver.org(f.payer) if isinstance(f.payer, OrgRef) else None
|
||||
payee_org = self.resolver.org(f.payee) if isinstance(f.payee, OrgRef) else None
|
||||
payer_person = self.resolver.person(f.payer) if isinstance(f.payer, PersonRef) else None
|
||||
payee_person = self.resolver.person(f.payee) if isinstance(f.payee, PersonRef) else None
|
||||
self.conn.execute(
|
||||
"""INSERT INTO money_flow (claim_id, kind, payer_name, payer_org_id, payer_person_id, payee_name,
|
||||
payee_org_id, payee_person_id, amount, amount_category, currency, paid_on,
|
||||
period, reference, category)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (claim_id) DO UPDATE SET amount = EXCLUDED.amount,
|
||||
payer_org_id = COALESCE(EXCLUDED.payer_org_id, money_flow.payer_org_id),
|
||||
payee_org_id = COALESCE(EXCLUDED.payee_org_id, money_flow.payee_org_id)""",
|
||||
(claim_id, f.kind, f.payer_name, payer_org, payer_person, f.payee_name, payee_org, payee_person,
|
||||
_amount(f.amount), f.amount_category, f.currency, f.paid_on, f.period, f.reference, f.category),
|
||||
)
|
||||
elif isinstance(f, Participation):
|
||||
event = self.conn.execute("SELECT id FROM event WHERE event_key = %s", (f.event_key,)).fetchone()
|
||||
if event is None:
|
||||
raise ValueError(f"event {f.event_key} must be loaded before participations")
|
||||
self.conn.execute(
|
||||
"""INSERT INTO event_participant (claim_id, event_id, person_id, organization_id, participation)
|
||||
VALUES (%s, %s, %s, %s, %s) ON CONFLICT (claim_id) DO NOTHING""",
|
||||
(claim_id, event["id"], person_id, org_id, f.participation),
|
||||
)
|
||||
elif isinstance(f, Statement):
|
||||
self.conn.execute(
|
||||
"""INSERT INTO statement (claim_id, person_id, context, platform, medium, title, said_at, excerpt, url)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) ON CONFLICT (claim_id) DO UPDATE SET
|
||||
excerpt = COALESCE(EXCLUDED.excerpt, statement.excerpt)""",
|
||||
(claim_id, person_id, f.context, f.platform, f.medium, f.title, f.said_at,
|
||||
f.excerpt and excerpt(f.excerpt, 600), f.url),
|
||||
)
|
||||
elif isinstance(f, Legal):
|
||||
self.conn.execute(
|
||||
"""INSERT INTO legal_proceeding (claim_id, person_id, organization_id, stage, offense, authority, case_ref,
|
||||
stage_date)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (claim_id) DO UPDATE SET stage = EXCLUDED.stage,
|
||||
offense = COALESCE(EXCLUDED.offense, legal_proceeding.offense),
|
||||
authority = COALESCE(EXCLUDED.authority, legal_proceeding.authority),
|
||||
case_ref = COALESCE(EXCLUDED.case_ref, legal_proceeding.case_ref)""",
|
||||
(claim_id, person_id, org_id, f.stage, f.offense, f.authority, f.case_ref, f.stage_date),
|
||||
)
|
||||
|
||||
# -- helpers ----------------------------------------------------------------------
|
||||
|
||||
def assign_topic(self, claim_id: int, topic_name: str, method: str | None = None, confidence: float = 1.0) -> None:
|
||||
norm = normalize(topic_name)
|
||||
row = self.conn.execute(
|
||||
"SELECT id FROM topic WHERE lower(unaccent(name_de)) = %s OR slug = %s", (norm, topic_name)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
row = self.conn.execute(
|
||||
"INSERT INTO topic (slug, name_de, name_en) VALUES (%s, %s, %s) ON CONFLICT (slug) DO UPDATE "
|
||||
"SET name_de = topic.name_de RETURNING id",
|
||||
(make_slug(topic_name), topic_name, topic_name),
|
||||
).fetchone()
|
||||
self.conn.execute(
|
||||
"INSERT INTO topic_assignment (topic_id, claim_id, confidence, method) VALUES (%s, %s, %s, %s) "
|
||||
"ON CONFLICT DO NOTHING",
|
||||
(row["id"], claim_id, confidence, method or self.ctx.adapter),
|
||||
)
|
||||
|
||||
def refresh_status(self, claim_id: int) -> None:
|
||||
"""corroborated = backed by >= 2 different sources; disputed = any contradicting evidence."""
|
||||
self.conn.execute(
|
||||
"""
|
||||
UPDATE claim SET status = CASE
|
||||
WHEN status = 'retracted' THEN status
|
||||
WHEN EXISTS (SELECT 1 FROM claim_evidence e WHERE e.claim_id = claim.id AND e.stance = 'contradicts')
|
||||
THEN 'disputed'
|
||||
WHEN (SELECT count(DISTINCT d.source_id) FROM claim_evidence e JOIN document d ON d.id = e.document_id
|
||||
WHERE e.claim_id = claim.id AND e.stance IN ('supports', 'mentions')) >= 2 THEN 'corroborated'
|
||||
ELSE 'reported' END
|
||||
WHERE id = %s
|
||||
""",
|
||||
(claim_id,),
|
||||
)
|
||||
|
||||
|
||||
def _amount(value: Any) -> Decimal | None:
|
||||
if value is None or value == "":
|
||||
return None
|
||||
try:
|
||||
return Decimal(str(value)).quantize(Decimal("0.01"))
|
||||
except InvalidOperation:
|
||||
return None
|
||||
|
||||
|
||||
def as_date(value: Any) -> date | None:
|
||||
if isinstance(value, datetime):
|
||||
return value.date()
|
||||
if isinstance(value, date):
|
||||
return value
|
||||
return None
|
||||
@@ -0,0 +1,265 @@
|
||||
"""Typed records: the only way adapters and enrichers hand data to the Loader.
|
||||
|
||||
Records are plain dataclasses so parsers stay pure and testable (see tests/golden). They refer to
|
||||
entities through PersonRef/OrgRef, which the Loader resolves (identifiers first, then hints, then
|
||||
names). Data minimisation is enforced by schema: there is, for example, no address field anywhere,
|
||||
so a parser cannot store a donor's street address even by accident.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field, fields, is_dataclass
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any, Literal, Union
|
||||
|
||||
# --------------------------------------------------------------------------------------------
|
||||
# References to entities
|
||||
# --------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class PersonRef:
|
||||
"""How a record points at a person.
|
||||
|
||||
ids: strong identifiers, e.g. {"parlament": "2339"} or {"wikidata": "Q123"}
|
||||
name: display/full name; used for alias matching if no id matches
|
||||
hints: resolution strategies, e.g.
|
||||
{"surname_in_body": "nationalrat", "on": date(...)} -> roll-call surname lists
|
||||
{"office": "Bundesministerin für Justiz", "on": date(...)} -> office holder on a date
|
||||
"""
|
||||
|
||||
ids: dict[str, str] = field(default_factory=dict)
|
||||
name: str | None = None
|
||||
hints: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class OrgRef:
|
||||
"""How a record points at an organization. `slug` addresses seeded orgs (e.g. "nationalrat")."""
|
||||
|
||||
ids: dict[str, str] = field(default_factory=dict)
|
||||
name: str | None = None
|
||||
short_name: str | None = None
|
||||
kind: str = "other"
|
||||
slug: str | None = None
|
||||
create: bool = True
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------------
|
||||
# Evidence & documents
|
||||
# --------------------------------------------------------------------------------------------
|
||||
|
||||
DocType = Literal["article", "press_release", "api_record", "protocol", "social_post", "register_entry",
|
||||
"web_page", "pdf", "dataset_row"]
|
||||
|
||||
|
||||
@dataclass
|
||||
class Evidence:
|
||||
url: str
|
||||
title: str | None = None
|
||||
doc_type: str = "web_page"
|
||||
published_at: datetime | date | str | None = None
|
||||
excerpt: str | None = None
|
||||
full_text: str | None = None
|
||||
quote: str | None = None
|
||||
locator: str = ""
|
||||
stance: Literal["supports", "contradicts", "mentions"] = "supports"
|
||||
confidence: float = 1.0
|
||||
source: str | None = None # defaults to the adapter's source
|
||||
meta: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Document:
|
||||
"""A document without a claim yet (e.g. a headline); enrichers derive claims from it later."""
|
||||
|
||||
evidence: Evidence
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------------
|
||||
# Entities
|
||||
# --------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class PersonRecord:
|
||||
ref: PersonRef
|
||||
full_name: str
|
||||
given_name: str | None = None
|
||||
family_name: str | None = None
|
||||
honorific_prefix: str | None = None
|
||||
honorific_suffix: str | None = None
|
||||
birth_date: date | None = None
|
||||
death_date: date | None = None
|
||||
gender: str | None = None
|
||||
photo_url: str | None = None
|
||||
aliases: tuple[str, ...] = ()
|
||||
#: if no identifier matches, merge with an existing person of exactly this (unambiguous) name before
|
||||
#: creating a new one - for authoritative sources that share no id with others (e.g. EP)
|
||||
match_by_name: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class OrgRecord:
|
||||
ref: OrgRef
|
||||
aliases: tuple[str, ...] = ()
|
||||
color: str | None = None
|
||||
parent: OrgRef | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SocialAccount:
|
||||
platform: str # bluesky | mastodon | telegram | youtube | twitter | facebook | ...
|
||||
url: str
|
||||
handle: str | None = None
|
||||
person: PersonRef | None = None
|
||||
org: OrgRef | None = None
|
||||
verified: bool = False # True only for official sources (parliament profile, official website)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------------
|
||||
# Facets (structured payload of a claim)
|
||||
# --------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Role:
|
||||
role: str # "mp" for chamber members, otherwise office title
|
||||
level: Literal["eu", "federal", "state", "district", "municipal", "party", "other"]
|
||||
body: OrgRef | None = None # chamber/government/party the role is in
|
||||
party: OrgRef | None = None # party/faction of the holder in this role
|
||||
region: str | None = None
|
||||
start: date | None = None
|
||||
end: date | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class VoteEvent:
|
||||
"""A vote (not a claim itself). Positions reference it by `key`."""
|
||||
|
||||
key: str
|
||||
chamber: OrgRef | None
|
||||
title: str
|
||||
voted_on: date | None
|
||||
result: Literal["accepted", "rejected", "unknown"] = "unknown"
|
||||
stage: str | None = None
|
||||
item_ref: str | None = None
|
||||
roll_call: bool = False
|
||||
url: str | None = None
|
||||
protocol_url: str | None = None
|
||||
raw_text: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class VotePosition:
|
||||
event_key: str
|
||||
position: Literal["for", "against", "abstain", "absent"]
|
||||
party: OrgRef | None = None # faction of an individual voter at that time
|
||||
|
||||
|
||||
@dataclass
|
||||
class Money:
|
||||
kind: Literal["donation", "contract", "government_ad", "political_ad", "subsidy", "party_funding", "party_ad",
|
||||
"loan", "side_income", "sponsorship", "other"]
|
||||
amount: Decimal | float | None = None
|
||||
currency: str = "EUR"
|
||||
payer: OrgRef | PersonRef | None = None
|
||||
payee: OrgRef | PersonRef | None = None
|
||||
payer_name: str | None = None
|
||||
payee_name: str | None = None
|
||||
paid_on: date | None = None
|
||||
period: str | None = None
|
||||
reference: str | None = None # notice id, campaign, ...
|
||||
category: str | None = None # CPV code, ad category, ...
|
||||
amount_category: str | None = None # when only a bracket is published
|
||||
|
||||
|
||||
@dataclass
|
||||
class Event:
|
||||
"""Something with time and place (not a claim). Participation claims reference it by `key`."""
|
||||
|
||||
key: str
|
||||
kind: str
|
||||
title: str
|
||||
starts_at: datetime | date | str | None
|
||||
ends_at: datetime | date | str | None = None
|
||||
location: str | None = None
|
||||
country: str | None = None
|
||||
url: str | None = None
|
||||
description: str | None = None
|
||||
body: OrgRef | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Participation:
|
||||
event_key: str
|
||||
participation: str = "participant"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Statement:
|
||||
context: Literal["parliament_speech", "interview", "tv", "radio", "social_post", "press_release",
|
||||
"event_speech", "op_ed", "testimony", "other"]
|
||||
url: str
|
||||
said_at: datetime | date | str | None = None
|
||||
platform: str | None = None
|
||||
medium: str | None = None
|
||||
title: str | None = None
|
||||
excerpt: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Legal:
|
||||
stage: str
|
||||
offense: str | None = None
|
||||
authority: str | None = None
|
||||
case_ref: str | None = None
|
||||
stage_date: date | None = None
|
||||
|
||||
|
||||
Facet = Union[Role, VotePosition, Money, Participation, Statement, Legal]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------------
|
||||
# Claims
|
||||
# --------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Claim:
|
||||
claim_type: str
|
||||
summary_de: str
|
||||
evidence: list[Evidence]
|
||||
person: PersonRef | None = None
|
||||
org: OrgRef | None = None
|
||||
counterpart_person: PersonRef | None = None
|
||||
counterpart_org: OrgRef | None = None
|
||||
occurred_at: datetime | date | str | None = None
|
||||
valid_from: date | None = None
|
||||
valid_to: date | None = None
|
||||
summary_en: str | None = None
|
||||
data: dict[str, Any] = field(default_factory=dict)
|
||||
topics: tuple[str, ...] = ()
|
||||
sensitive: bool = False
|
||||
review: bool = False # True -> published only after human review
|
||||
key: tuple | None = None # explicit identity (otherwise computed by the Loader)
|
||||
facet: Facet | None = None
|
||||
|
||||
|
||||
Record = Union[PersonRecord, OrgRecord, SocialAccount, VoteEvent, Event, Claim, Document]
|
||||
|
||||
|
||||
def to_jsonable(obj: Any) -> Any:
|
||||
"""Serialise records for golden tests and debugging (stable, sorted keys)."""
|
||||
if is_dataclass(obj) and not isinstance(obj, type):
|
||||
return {"__type__": type(obj).__name__, **{f.name: to_jsonable(getattr(obj, f.name)) for f in fields(obj)}}
|
||||
if isinstance(obj, dict):
|
||||
return {str(k): to_jsonable(v) for k, v in sorted(obj.items(), key=lambda kv: str(kv[0]))}
|
||||
if isinstance(obj, (list, tuple)):
|
||||
return [to_jsonable(v) for v in obj]
|
||||
if isinstance(obj, (datetime, date)):
|
||||
return obj.isoformat()
|
||||
if isinstance(obj, Decimal):
|
||||
return str(obj)
|
||||
return obj
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Registry of adapters, enrichers and source specs.
|
||||
|
||||
Built-in modules register themselves with @register when imported (see load_builtin()).
|
||||
External packages can add adapters via the entry-point group `politrack.adapters`
|
||||
(value: a module path whose import registers adapters), e.g. in their pyproject.toml:
|
||||
|
||||
[project.entry-points."politrack.adapters"]
|
||||
my_sources = "my_package.sources"
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import logging
|
||||
import pkgutil
|
||||
from importlib.metadata import entry_points
|
||||
|
||||
from politrack.core.adapter import Adapter, Enricher
|
||||
from politrack.core.spec import DERIVED, SourceSpec
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
ADAPTERS: dict[str, type[Adapter]] = {}
|
||||
ENRICHERS: dict[str, type[Enricher]] = {}
|
||||
SPECS: dict[str, SourceSpec] = {DERIVED.slug: DERIVED}
|
||||
_loaded = False
|
||||
|
||||
|
||||
def register(cls):
|
||||
"""Class decorator for Adapter and Enricher subclasses."""
|
||||
if issubclass(cls, Adapter):
|
||||
target = ADAPTERS
|
||||
elif issubclass(cls, Enricher):
|
||||
target = ENRICHERS
|
||||
else:
|
||||
raise TypeError(f"{cls} is neither an Adapter nor an Enricher")
|
||||
if cls.name in ADAPTERS or cls.name in ENRICHERS:
|
||||
raise ValueError(f"duplicate adapter name {cls.name!r}")
|
||||
target[cls.name] = cls
|
||||
register_spec(cls.spec)
|
||||
return cls
|
||||
|
||||
|
||||
def register_spec(spec: SourceSpec) -> SourceSpec:
|
||||
existing = SPECS.get(spec.slug)
|
||||
if existing is not None and existing != spec:
|
||||
raise ValueError(f"conflicting SourceSpec definitions for {spec.slug!r}")
|
||||
SPECS[spec.slug] = spec
|
||||
return spec
|
||||
|
||||
|
||||
def load_builtin() -> None:
|
||||
"""Import all built-in source and enricher modules (and plugins) exactly once."""
|
||||
global _loaded
|
||||
if _loaded:
|
||||
return
|
||||
_loaded = True
|
||||
for package_name in ("politrack.sources", "politrack.enrich"):
|
||||
package = importlib.import_module(package_name)
|
||||
for module in pkgutil.walk_packages(package.__path__, prefix=f"{package_name}."):
|
||||
importlib.import_module(module.name)
|
||||
for ep in entry_points(group="politrack.adapters"):
|
||||
try:
|
||||
importlib.import_module(ep.value)
|
||||
except Exception: # a broken plugin must not take the platform down
|
||||
log.exception("failed to load adapter plugin %s", ep.value)
|
||||
|
||||
|
||||
def get(name: str) -> type[Adapter] | type[Enricher]:
|
||||
load_builtin()
|
||||
if name in ADAPTERS:
|
||||
return ADAPTERS[name]
|
||||
if name in ENRICHERS:
|
||||
return ENRICHERS[name]
|
||||
raise KeyError(f"unknown adapter/enricher {name!r}; known: {', '.join(sorted(ADAPTERS) + sorted(ENRICHERS))}")
|
||||
|
||||
|
||||
def sync_sources(conn) -> int:
|
||||
"""Upsert all registered SourceSpecs into the `source` table."""
|
||||
load_builtin()
|
||||
for spec in SPECS.values():
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO source (slug, name, kind, homepage, license, access_method, store_fulltext, keep_raw,
|
||||
default_trust, tos_notes)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (slug) DO UPDATE SET name = EXCLUDED.name, kind = EXCLUDED.kind, homepage = EXCLUDED.homepage,
|
||||
license = EXCLUDED.license, access_method = EXCLUDED.access_method,
|
||||
store_fulltext = EXCLUDED.store_fulltext, keep_raw = EXCLUDED.keep_raw,
|
||||
default_trust = EXCLUDED.default_trust, tos_notes = EXCLUDED.tos_notes, updated_at = now()
|
||||
""",
|
||||
(spec.slug, spec.name, spec.kind, spec.homepage, spec.license, spec.access, spec.store_fulltext,
|
||||
spec.keep_raw, spec.trust, spec.tos_notes),
|
||||
)
|
||||
return len(SPECS)
|
||||
@@ -0,0 +1,269 @@
|
||||
"""Entity resolution: PersonRef/OrgRef -> database ids.
|
||||
|
||||
Order (first hit wins, ambiguity never guesses):
|
||||
persons: identifiers -> hint strategies -> exact alias (titles stripped; prefer active)
|
||||
organizations: identifiers -> slug -> exact alias -> normalised legal name -> create (if allowed)
|
||||
|
||||
Hint strategies for persons (documented in docs/data-model.md):
|
||||
surname_in_body + on members of a chamber on a date with that surname (roll-call lists)
|
||||
office + on holder of an office title on a date (Council of Ministers minutes)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from psycopg import Connection
|
||||
from rapidfuzz import fuzz
|
||||
|
||||
from politrack.core.records import OrgRef, PersonRef
|
||||
from politrack.text import make_slug, normalize, normalize_org_name, strip_titles
|
||||
|
||||
|
||||
OFFICE_FILLER = re.compile(r"\b(der republik osterreich|der bundesrepublik osterreich|im bundeskanzleramt)\b|"
|
||||
r"\(osterreich\)")
|
||||
|
||||
|
||||
def office_key(title: str) -> str:
|
||||
"""'Bundesministerin für Justiz' / 'Bundeskanzler der Republik Österreich' -> 'bundesminister fur justiz' /
|
||||
'bundeskanzler' (gender-neutral, without filler, punctuation-free) for comparing office titles."""
|
||||
text = normalize(title)
|
||||
text = re.sub(r"\b(\w*(?:minister|kanzler|sekretar|prasident|rat))in\b", r"\1", text)
|
||||
text = OFFICE_FILLER.sub(" ", text)
|
||||
return re.sub(r"\s+", " ", re.sub(r"[^\w\s]", " ", text)).strip()
|
||||
|
||||
|
||||
# Fallback for colloquial or English office labels (Wikidata: "Justizminister Österreichs", "Federal Minister for
|
||||
# Arts, Culture, the Civil Service and Sport"): same kind of office + at least one common portfolio.
|
||||
OFFICE_KINDS = [("vizekanzler", r"vizekanzler|vice.chancellor"), ("kanzler", r"kanzler|chancellor"),
|
||||
("staatssekretar", r"staatssekretar|state secretary"), ("minister", r"minist")]
|
||||
PORTFOLIOS = {
|
||||
"justiz": r"justiz|justice", "finanzen": r"finanz|financ", "inneres": r"\binnere|\binnenmin|interior",
|
||||
"aussen": r"au(ss|ß)en|auswartig|europaische und internationale|foreign|european and international",
|
||||
"verteidigung": r"verteidigung|defen[cs]e", "soziales": r"sozial|\barbeit|labou?r|social",
|
||||
"gesundheit": r"gesundheit|health", "bildung": r"bildung|education", "wissenschaft": r"wissenschaft|science",
|
||||
"frauen": r"frauen|women", "landwirtschaft": r"landwirtschaft|forstwirtschaft|agricultur",
|
||||
"umwelt": r"umwelt|klima|environment|climate", "kunst": r"kunst|kultur|\barts\b|culture|\bsport",
|
||||
"mobilitat": r"mobilitat|verkehr|infrastruktur|mobility|transport|infrastructure",
|
||||
"wirtschaft": r"\bwirtschaft|econom", "europa": r"\beuropa\b|integration|familie|famil",
|
||||
"digitalisierung": r"digital", "energie": r"energie|energy", "tourismus": r"touris",
|
||||
}
|
||||
|
||||
|
||||
def office_kind(key: str) -> str | None:
|
||||
return next((kind for kind, pattern in OFFICE_KINDS if re.search(pattern, key)), None)
|
||||
|
||||
|
||||
def portfolios(key: str) -> set[str]:
|
||||
return {name for name, pattern in PORTFOLIOS.items() if re.search(pattern, key)}
|
||||
|
||||
|
||||
def lead_portfolio(key: str) -> str | None:
|
||||
"""The portfolio named first ("Wirtschaft, Energie und Tourismus" -> wirtschaft): a role must cover it."""
|
||||
found = [(m.start(), name) for name, pattern in PORTFOLIOS.items() if (m := re.search(pattern, key))]
|
||||
return min(found)[1] if found else None
|
||||
|
||||
|
||||
class Resolver:
|
||||
def __init__(self, conn: Connection) -> None:
|
||||
self.conn = conn
|
||||
self._person_cache: dict[tuple, int | None] = {}
|
||||
self._org_cache: dict[tuple, int | None] = {}
|
||||
self._members_cache: dict[tuple, list[dict]] = {}
|
||||
self._office_cache: dict[tuple, int | None] = {}
|
||||
|
||||
# -- persons ------------------------------------------------------------------------
|
||||
|
||||
def person_by_ids(self, ids: dict[str, str]) -> int | None:
|
||||
for scheme, value in ids.items():
|
||||
if value is None:
|
||||
continue
|
||||
if scheme == "politrack": # internal id, used by adapters that plan requests from the database
|
||||
row = self.conn.execute("SELECT id FROM person WHERE id = %s", (int(value),)).fetchone()
|
||||
if row:
|
||||
return row["id"]
|
||||
continue
|
||||
row = self.conn.execute(
|
||||
"SELECT entity_id FROM identifier WHERE entity_type = 'person' AND scheme = %s AND value = %s",
|
||||
(scheme, str(value)),
|
||||
).fetchone()
|
||||
if row:
|
||||
return row["entity_id"]
|
||||
return None
|
||||
|
||||
def person(self, ref: PersonRef | None) -> int | None:
|
||||
if ref is None:
|
||||
return None
|
||||
cache_key = (tuple(sorted(ref.ids.items())), ref.name, tuple(sorted((k, str(v)) for k, v in ref.hints.items())))
|
||||
if cache_key in self._person_cache:
|
||||
return self._person_cache[cache_key]
|
||||
found = self.person_by_ids(ref.ids)
|
||||
if found is None and ref.hints:
|
||||
found = self._person_by_hints(ref)
|
||||
if found is None and ref.name and not ref.hints.get("no_name_match"):
|
||||
found = self.person_by_name(ref.name)
|
||||
self._person_cache[cache_key] = found
|
||||
return found
|
||||
|
||||
def person_by_name(self, name: str) -> int | None:
|
||||
norm = normalize(strip_titles(name))
|
||||
rows = self.conn.execute(
|
||||
"""SELECT DISTINCT a.entity_id AS id, ps.status
|
||||
FROM alias a LEFT JOIN person_status ps ON ps.person_id = a.entity_id
|
||||
WHERE a.entity_type = 'person' AND a.alias_norm = %s""",
|
||||
(norm,),
|
||||
).fetchall()
|
||||
if len(rows) == 1:
|
||||
return rows[0]["id"]
|
||||
active = [r for r in rows if r["status"] == "active"]
|
||||
return active[0]["id"] if len(active) == 1 else None
|
||||
|
||||
def _person_by_hints(self, ref: PersonRef) -> int | None:
|
||||
hints = ref.hints
|
||||
on: date | None = hints.get("on")
|
||||
if "surname_in_body" in hints and ref.name:
|
||||
members = self.members(hints["surname_in_body"], on)
|
||||
norm = normalize(strip_titles(ref.name))
|
||||
exact = [m for m in members if m["family_name"] and normalize(m["family_name"]) == norm]
|
||||
if len(exact) == 1:
|
||||
return exact[0]["id"]
|
||||
suffix = [m for m in members if normalize(strip_titles(m["full_name"])).endswith(norm)]
|
||||
return suffix[0]["id"] if len(suffix) == 1 else None
|
||||
if "office" in hints:
|
||||
return self.office_holder(hints["office"], on)
|
||||
return None
|
||||
|
||||
def office_holder(self, office: str, on: date | None) -> int | None:
|
||||
"""Holder of a (single-holder) office on a date. Titles are compared whole, gender-neutral and without
|
||||
filler ("Bundesministerin für Justiz" == "Bundesminister für Justiz"), so a generic "Bundesminister"
|
||||
never matches a specific ministry. Several matches (open-ended roles from Wikidata) -> the most recent
|
||||
appointment wins; no dated match -> None."""
|
||||
key = (office, on)
|
||||
if key not in self._office_cache:
|
||||
rows = self.conn.execute(
|
||||
"""SELECT DISTINCT r.person_id, r.role, r.start_date FROM role r JOIN claim c ON c.id = r.claim_id
|
||||
WHERE c.merged_into IS NULL AND r.level IN ('federal', 'state', 'eu')
|
||||
AND (%(on)s::date IS NULL OR ((r.start_date IS NULL OR r.start_date <= %(on)s)
|
||||
AND (r.end_date IS NULL OR r.end_date >= %(on)s)))""",
|
||||
{"on": on},
|
||||
).fetchall()
|
||||
target = office_key(office)
|
||||
candidates = [(r["start_date"] or date.min, r["person_id"]) for r in rows
|
||||
if fuzz.ratio(office_key(r["role"]), target) >= 92]
|
||||
kind, lead = office_kind(target), lead_portfolio(target)
|
||||
if not candidates and kind and (lead or kind in ("kanzler", "vizekanzler")):
|
||||
# Fuzzy labels only for appointments since the current government took office (the latest
|
||||
# chancellor appointment): stale open-ended roles of former ministers must not win.
|
||||
formed = max((r["start_date"] for r in rows if r["start_date"]
|
||||
and office_kind(office_key(r["role"])) == "kanzler"), default=date.min)
|
||||
candidates = [(r["start_date"], r["person_id"]) for r in rows
|
||||
if r["start_date"] and r["start_date"] >= formed
|
||||
and office_kind(office_key(r["role"])) == kind
|
||||
and (kind in ("kanzler", "vizekanzler") or lead in portfolios(office_key(r["role"])))]
|
||||
persons = {p for _, p in candidates}
|
||||
if len(persons) <= 1:
|
||||
self._office_cache[key] = persons.pop() if persons else None
|
||||
else:
|
||||
latest = max(start for start, _ in candidates)
|
||||
newest = {p for start, p in candidates if start == latest}
|
||||
self._office_cache[key] = newest.pop() if len(newest) == 1 and latest != date.min else None
|
||||
return self._office_cache[key]
|
||||
|
||||
def members(self, body_slug: str, on: date | None) -> list[dict[str, Any]]:
|
||||
key = (body_slug, on)
|
||||
if key not in self._members_cache:
|
||||
self._members_cache[key] = self.conn.execute(
|
||||
"""SELECT DISTINCT ON (p.id) p.id, p.full_name, p.family_name, r.party_id
|
||||
FROM role r JOIN person p ON p.id = r.person_id JOIN organization o ON o.id = r.organization_id
|
||||
JOIN claim c ON c.id = r.claim_id
|
||||
WHERE o.slug = %(body)s AND r.role = 'mp' AND c.merged_into IS NULL
|
||||
AND (%(on)s::date IS NULL OR (r.start_date <= %(on)s AND COALESCE(r.end_date, 'infinity') >= %(on)s))
|
||||
ORDER BY p.id, r.start_date DESC""",
|
||||
{"body": body_slug, "on": on},
|
||||
).fetchall()
|
||||
return self._members_cache[key]
|
||||
|
||||
def faction_of(self, person_id: int, body_slug: str, on: date | None) -> int | None:
|
||||
return next((m["party_id"] for m in self.members(body_slug, on) if m["id"] == person_id), None)
|
||||
|
||||
# -- organizations ------------------------------------------------------------------
|
||||
|
||||
def org(self, ref: OrgRef | None, create: bool | None = None) -> int | None:
|
||||
if ref is None:
|
||||
return None
|
||||
allow_create = ref.create if create is None else create
|
||||
cache_key = (tuple(sorted(ref.ids.items())), ref.slug, ref.name, ref.short_name)
|
||||
if cache_key in self._org_cache and (self._org_cache[cache_key] is not None or not allow_create):
|
||||
return self._org_cache[cache_key]
|
||||
found = self._org_lookup(ref)
|
||||
if found is None and allow_create and (ref.name or ref.short_name):
|
||||
found = self._org_create(ref)
|
||||
elif found is not None:
|
||||
self._add_org_ids(found, ref)
|
||||
self._org_cache[cache_key] = found
|
||||
return found
|
||||
|
||||
def _org_lookup(self, ref: OrgRef) -> int | None:
|
||||
for scheme, value in ref.ids.items():
|
||||
if scheme == "politrack_org": # internal id (enrichers)
|
||||
row = self.conn.execute("SELECT id FROM organization WHERE id = %s", (int(value),)).fetchone()
|
||||
if row:
|
||||
return row["id"]
|
||||
continue
|
||||
row = self.conn.execute(
|
||||
"SELECT entity_id FROM identifier WHERE entity_type = 'organization' AND scheme = %s AND value = %s",
|
||||
(scheme, str(value)),
|
||||
).fetchone()
|
||||
if row:
|
||||
return row["entity_id"]
|
||||
if ref.slug:
|
||||
row = self.conn.execute("SELECT id FROM organization WHERE slug = %s", (ref.slug,)).fetchone()
|
||||
if row:
|
||||
return row["id"]
|
||||
for label in (ref.short_name, ref.name):
|
||||
if not label:
|
||||
continue
|
||||
for norm in {normalize(label), normalize_org_name(label)} - {""}:
|
||||
rows = self.conn.execute(
|
||||
"SELECT DISTINCT entity_id FROM alias WHERE entity_type = 'organization' AND alias_norm = %s",
|
||||
(norm,),
|
||||
).fetchall()
|
||||
if len(rows) == 1:
|
||||
return rows[0]["entity_id"]
|
||||
return None
|
||||
|
||||
def _org_create(self, ref: OrgRef) -> int:
|
||||
base = ref.slug or make_slug(ref.short_name or ref.name or "org")[:60] or "org"
|
||||
slug, n = base, 1
|
||||
while self.conn.execute("SELECT 1 FROM organization WHERE slug = %s", (slug,)).fetchone():
|
||||
n += 1
|
||||
slug = f"{base}-{n}"
|
||||
org_id = self.conn.execute(
|
||||
"INSERT INTO organization (slug, name, short_name, kind) VALUES (%s, %s, %s, %s) RETURNING id",
|
||||
(slug, ref.name or ref.short_name, ref.short_name, ref.kind if ref.kind else "other"),
|
||||
).fetchone()["id"]
|
||||
self.add_org_aliases(org_id, [ref.name, ref.short_name])
|
||||
self._add_org_ids(org_id, ref)
|
||||
return org_id
|
||||
|
||||
def add_org_aliases(self, org_id: int, names: list[str | None]) -> None:
|
||||
for name in names:
|
||||
if not name:
|
||||
continue
|
||||
for norm in {normalize(name), normalize_org_name(name)} - {""}:
|
||||
self.conn.execute(
|
||||
"INSERT INTO alias (entity_type, entity_id, alias, alias_norm) VALUES ('organization', %s, %s, %s) "
|
||||
"ON CONFLICT DO NOTHING",
|
||||
(org_id, name, norm),
|
||||
)
|
||||
|
||||
def _add_org_ids(self, org_id: int, ref: OrgRef) -> None:
|
||||
for scheme, value in ref.ids.items():
|
||||
if value:
|
||||
self.conn.execute(
|
||||
"INSERT INTO identifier (entity_type, entity_id, scheme, value) VALUES ('organization', %s, %s, %s) "
|
||||
"ON CONFLICT DO NOTHING",
|
||||
(org_id, scheme, str(value)),
|
||||
)
|
||||
@@ -0,0 +1,232 @@
|
||||
"""Runs adapters and enrichers.
|
||||
|
||||
fetch mode: start() -> fetch (policy, archive) -> parse() -> Loader, one transaction per response
|
||||
reparse mode: replay the latest archived response of every request the adapter ever made
|
||||
test mode: parse a given response without DB (used by golden tests)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import traceback
|
||||
from collections import deque
|
||||
from collections.abc import Iterable
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from psycopg import Connection
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from politrack.core import registry
|
||||
from politrack.core.adapter import Adapter, Enricher, Request, Response, RunContext, StateStore
|
||||
from politrack.core.archive import Archive
|
||||
from politrack.core.fetch import Disallowed, Fetcher
|
||||
from politrack.core.loader import Loader, source_row
|
||||
from politrack.db import connection
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
ENRICH_COMMIT_EVERY = 200
|
||||
|
||||
|
||||
def run(name: str, *, mode: str = "fetch", since: datetime | None = None, params: dict[str, Any] | None = None,
|
||||
max_requests: int | None = None) -> dict:
|
||||
cls = registry.get(name)
|
||||
with connection() as conn:
|
||||
run_id = conn.execute("INSERT INTO run (adapter, mode) VALUES (%s, %s) RETURNING id", (name, mode)).fetchone()["id"]
|
||||
conn.commit()
|
||||
ctx = RunContext(adapter=name, spec=cls.spec, state=StateStore(conn, name), since=since, params=params or {},
|
||||
mode=mode, conn=conn, creates_persons=cls.creates_persons,
|
||||
log=logging.getLogger(f"politrack.{name}"))
|
||||
status, error = "ok", None
|
||||
try:
|
||||
if issubclass(cls, Enricher):
|
||||
_run_enricher(cls(), ctx, conn)
|
||||
elif mode == "reparse":
|
||||
_reparse(cls(), ctx, conn)
|
||||
else:
|
||||
_fetch_run(cls(), ctx, conn, max_requests)
|
||||
if ctx.stats.get("errors"):
|
||||
status = "partial"
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
status, error = "error", traceback.format_exc()
|
||||
ctx.log.exception("%s failed", name)
|
||||
conn.execute("UPDATE run SET finished_at = now(), status = %s, stats = %s, error = %s WHERE id = %s",
|
||||
(status, Jsonb(dict(ctx.stats)), error, run_id))
|
||||
conn.commit()
|
||||
ctx.log.info("%s %s: %s %s", name, mode, status, dict(ctx.stats))
|
||||
if status == "error":
|
||||
raise RuntimeError(f"{name} failed, see run {run_id}")
|
||||
return {"status": status, **dict(ctx.stats)}
|
||||
|
||||
|
||||
def _fetch_run(adapter: Adapter, ctx: RunContext, conn: Connection, max_requests: int | None) -> None:
|
||||
fetcher = Fetcher()
|
||||
loader = Loader(conn, ctx)
|
||||
queue: deque[Request] = deque(adapter.start(ctx))
|
||||
seen: set[str] = set()
|
||||
try:
|
||||
while queue:
|
||||
req = queue.popleft()
|
||||
if req.fingerprint in seen:
|
||||
continue
|
||||
seen.add(req.fingerprint)
|
||||
if max_requests is not None and ctx.stats["requests"] >= max_requests:
|
||||
ctx.stats["stopped_at_max_requests"] = 1
|
||||
break
|
||||
ctx.stats["requests"] += 1
|
||||
# Multi-publisher adapters (news feeds, GDELT) name the publisher per request.
|
||||
spec = registry.SPECS.get(req.meta.get("source") or adapter.spec.slug, adapter.spec)
|
||||
try:
|
||||
resp = fetcher.fetch(conn, req, adapter=adapter.name, spec=spec,
|
||||
source_id=source_row(conn, spec.slug)["id"])
|
||||
except Disallowed:
|
||||
ctx.stats["disallowed_by_robots"] += 1
|
||||
ctx.log.warning("robots.txt disallows %s", req.url)
|
||||
continue
|
||||
except Exception as exc:
|
||||
conn.rollback()
|
||||
ctx.stats["fetch_errors"] += 1
|
||||
ctx.stats["errors"] += 1
|
||||
ctx.log.warning("fetch failed %s: %s", req.url, exc)
|
||||
continue
|
||||
if req.skip_unchanged and _unchanged(conn, resp, adapter.parser_version):
|
||||
# Same body, same parser: its records are already loaded. Still collect follow-up requests,
|
||||
# so children an interrupted run never fetched are picked up (they have their own checks).
|
||||
ctx.stats["unchanged"] += 1
|
||||
queue.extend(_follow_ups(adapter, ctx, resp))
|
||||
continue
|
||||
queue.extend(_parse_and_load(adapter, ctx, conn, loader, resp))
|
||||
finally:
|
||||
fetcher.close()
|
||||
|
||||
|
||||
def _unchanged(conn: Connection, resp: Response, parser_version: int) -> bool:
|
||||
if resp.fetch_id is None:
|
||||
return False
|
||||
row = conn.execute(
|
||||
"""SELECT 1 FROM fetch_log WHERE request_fp = %s AND parsed_sha256 = %s AND parser_version = %s LIMIT 1""",
|
||||
(resp.request.fingerprint, resp.sha256, parser_version),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
|
||||
def _follow_ups(adapter: Adapter, ctx: RunContext, resp: Response) -> list[Request]:
|
||||
ctx.response = resp
|
||||
try:
|
||||
return [item for item in adapter.parse(ctx, resp) if isinstance(item, Request)]
|
||||
except Exception:
|
||||
ctx.stats["parse_errors"] += 1
|
||||
ctx.log.exception("parse failed for unchanged %s", resp.url)
|
||||
return []
|
||||
finally:
|
||||
ctx.response = None
|
||||
|
||||
|
||||
def _parse_and_load(adapter: Adapter, ctx: RunContext, conn: Connection, loader: Loader,
|
||||
resp: Response) -> list[Request]:
|
||||
follow: list[Request] = []
|
||||
ctx.response = resp
|
||||
try:
|
||||
for item in adapter.parse(ctx, resp):
|
||||
if isinstance(item, Request):
|
||||
follow.append(item)
|
||||
else:
|
||||
loader.load(item)
|
||||
if resp.fetch_id is not None:
|
||||
Archive().mark_parsed(conn, resp.fetch_id, resp.sha256, adapter.parser_version)
|
||||
conn.commit()
|
||||
except Exception as exc:
|
||||
conn.rollback()
|
||||
ctx.stats["errors"] += 1
|
||||
ctx.stats["parse_errors"] += 1
|
||||
ctx.log.exception("parse/load failed for %s", resp.url)
|
||||
if resp.fetch_id is not None:
|
||||
Archive().mark_error(conn, resp.fetch_id, f"{type(exc).__name__}: {exc}")
|
||||
conn.commit()
|
||||
loader.resolver = type(loader.resolver)(conn) # drop caches that may reference rolled-back rows
|
||||
finally:
|
||||
ctx.response = None
|
||||
return follow
|
||||
|
||||
|
||||
def _reparse(adapter: Adapter, ctx: RunContext, conn: Connection) -> None:
|
||||
archive = Archive()
|
||||
loader = Loader(conn, ctx)
|
||||
started = conn.execute("SELECT now() AS t").fetchone()["t"]
|
||||
rows = archive.latest_stored(conn, adapter.name)
|
||||
ctx.log.info("reparsing %s archived responses", len(rows))
|
||||
# Parse list/index responses before detail responses: records referencing others (vote positions ->
|
||||
# vote events) are emitted by the parent's parse() in order, so the original order is preserved
|
||||
# by sorting on fetch time.
|
||||
for row in sorted(rows, key=lambda r: r["fetched_at"]):
|
||||
body = archive.read_blob(row["sha256"])
|
||||
if body is None:
|
||||
ctx.stats["missing_blobs"] += 1
|
||||
continue
|
||||
req_info = row["request"] or {}
|
||||
req = Request(url=row["url"], kind=req_info.get("kind", "default"), method=row["method"],
|
||||
params=req_info.get("params"), json=req_info.get("json"), meta=req_info.get("meta") or {})
|
||||
resp = Response(request=req, status=row["status"], url=row["url"],
|
||||
headers={"content-type": row["content_type"] or ""}, body=body,
|
||||
fetched_at=row["fetched_at"], sha256=row["sha256"], fetch_id=row["id"],
|
||||
tdm_reserved=row["tdm_reserved"], from_archive=True)
|
||||
ctx.stats["responses"] += 1
|
||||
_parse_and_load(adapter, ctx, conn, loader, resp) # follow-ups are archived separately
|
||||
if not ctx.stats.get("errors") and not ctx.stats.get("missing_blobs"):
|
||||
_sweep_stale_claims(adapter, ctx, conn, started)
|
||||
|
||||
|
||||
def _sweep_stale_claims(adapter: Adapter, ctx: RunContext, conn: Connection, started: datetime) -> None:
|
||||
"""After a complete reparse, delete this adapter's claims that the current parser no longer produces
|
||||
(e.g. a fixed parser emits a corrected fact with a new identity) - unless another source backs them.
|
||||
Claims are reproducible from the archive, so nothing is lost; facets and evidence links cascade."""
|
||||
deleted = conn.execute(
|
||||
"""DELETE FROM claim c
|
||||
WHERE c.created_by = %(adapter)s AND c.updated_at < %(started)s
|
||||
AND NOT EXISTS (SELECT 1 FROM claim_evidence e JOIN document d ON d.id = e.document_id
|
||||
JOIN source s ON s.id = d.source_id
|
||||
WHERE e.claim_id = c.id AND s.slug <> %(source)s)""",
|
||||
{"adapter": adapter.name, "started": started, "source": adapter.spec.slug},
|
||||
).rowcount
|
||||
conn.commit()
|
||||
ctx.stats["stale_claims_deleted"] = deleted
|
||||
|
||||
|
||||
def _run_enricher(enricher: Enricher, ctx: RunContext, conn: Connection) -> None:
|
||||
loader = Loader(conn, ctx)
|
||||
result = enricher.run(ctx)
|
||||
pending = 0
|
||||
for record in result or []:
|
||||
try:
|
||||
loader.load(record)
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
ctx.stats["errors"] += 1
|
||||
ctx.log.exception("enricher %s failed to load a record", enricher.name)
|
||||
loader = Loader(conn, ctx)
|
||||
continue
|
||||
pending += 1
|
||||
if pending >= ENRICH_COMMIT_EVERY:
|
||||
conn.commit()
|
||||
pending = 0
|
||||
conn.commit()
|
||||
|
||||
|
||||
def parse_offline(adapter: Adapter, response: Response, params: dict[str, Any] | None = None) -> Iterable[Any]:
|
||||
"""Run parse() without a database (golden tests, debugging)."""
|
||||
ctx = RunContext(adapter=adapter.name, spec=adapter.spec, state=StateStore(None, adapter.name), mode="test",
|
||||
params=params or {})
|
||||
ctx.response = response
|
||||
return list(adapter.parse(ctx, response))
|
||||
|
||||
|
||||
def make_response(url: str, body: bytes, *, kind: str = "default", meta: dict | None = None,
|
||||
content_type: str = "", method: str = "GET", json_body: Any = None,
|
||||
params: dict | None = None) -> Response:
|
||||
"""Build a Response for tests/fixtures."""
|
||||
import hashlib
|
||||
|
||||
req = Request(url=url, kind=kind, meta=meta or {}, method=method, json=json_body, params=params)
|
||||
return Response(request=req, status=200, url=url, headers={"content-type": content_type}, body=body,
|
||||
fetched_at=datetime(2026, 1, 1, tzinfo=timezone.utc), sha256=hashlib.sha256(body).hexdigest())
|
||||
@@ -0,0 +1,49 @@
|
||||
"""SourceSpec: the single source of truth about a publisher we collect from.
|
||||
|
||||
Specs live next to the adapters that use them (backend/politrack/sources/*.py) and are synced
|
||||
into the `source` table by `politrack migrate` / `politrack sources sync`.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
|
||||
SourceKind = Literal["official", "ngo", "media", "social", "aggregator", "user", "derived"]
|
||||
AccessMethod = Literal["api", "bulk", "rss", "scrape", "manual", "derived"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceSpec:
|
||||
slug: str
|
||||
name: str
|
||||
kind: SourceKind
|
||||
homepage: str
|
||||
license: str
|
||||
access: AccessMethod
|
||||
trust: int
|
||||
#: may we keep body text (articles, posts, protocols)? False -> title, short excerpt and link only
|
||||
store_fulltext: bool = True
|
||||
#: may raw responses be archived? False for TDM-reserved publishers or restrictive terms
|
||||
keep_raw: bool = True
|
||||
#: minimum seconds between two requests to the same host (robots.txt crawl-delay can raise it)
|
||||
min_interval: float = 2.0
|
||||
#: URL prefixes of documented APIs whose operators allow programmatic use although robots.txt
|
||||
#: lists them (justify in tos_notes). Everything else always obeys robots.txt.
|
||||
robots_exempt: tuple[str, ...] = ()
|
||||
tos_notes: str = ""
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not 0 <= self.trust <= 100:
|
||||
raise ValueError(f"{self.slug}: trust must be 0-100")
|
||||
|
||||
|
||||
# Sources used by the built-in enrichers for derived documents/claims.
|
||||
DERIVED = SourceSpec(
|
||||
slug="politrack",
|
||||
name="Politrack (abgeleitet)",
|
||||
kind="derived",
|
||||
homepage="",
|
||||
license="derived from cited sources",
|
||||
access="derived",
|
||||
trust=50,
|
||||
tos_notes="Claims computed by Politrack from other sources; each cites its original evidence.",
|
||||
)
|
||||
@@ -0,0 +1,31 @@
|
||||
import atexit
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from functools import lru_cache
|
||||
|
||||
from psycopg import Connection
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
from politrack.config import get_settings
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_pool() -> ConnectionPool:
|
||||
pool = ConnectionPool(
|
||||
get_settings().database_url,
|
||||
min_size=1,
|
||||
max_size=10,
|
||||
kwargs={"row_factory": dict_row},
|
||||
open=False,
|
||||
)
|
||||
pool.open()
|
||||
atexit.register(pool.close)
|
||||
return pool
|
||||
|
||||
|
||||
@contextmanager
|
||||
def connection() -> Iterator[Connection]:
|
||||
"""A pooled connection inside a transaction (committed on success)."""
|
||||
with get_pool().connection() as conn:
|
||||
yield conn
|
||||
Whitespace-only changes.
@@ -0,0 +1,244 @@
|
||||
"""Corroboration: find claims from different sources that state the same fact and merge them.
|
||||
|
||||
Merging keeps the oldest claim, copies the duplicate's evidence onto it and marks
|
||||
the duplicate with `merged_into`. Afterwards `refresh_claim_status` marks the
|
||||
survivor "corroborated" when at least two *independent sources* support it.
|
||||
|
||||
Matchers:
|
||||
* roles: same person, equivalent body/office, overlapping or near-identical periods
|
||||
(e.g. Wikidata "Abgeordneter zum Nationalrat" == Parliament mandate)
|
||||
* stories: news articles about the same event (similar headlines, shared people,
|
||||
within 72 hours) are grouped; mentions of a person in one story are merged
|
||||
* events: trips/meetings of the same person to the same place/counterpart within 3 days
|
||||
"""
|
||||
|
||||
from collections import defaultdict
|
||||
from datetime import timedelta
|
||||
|
||||
from psycopg import Connection
|
||||
from rapidfuzz import fuzz
|
||||
|
||||
from politrack.core import Enricher, RunContext, register
|
||||
from politrack.core.spec import DERIVED
|
||||
from politrack.enrich.events import counterpart_anchor
|
||||
from politrack.text import normalize
|
||||
|
||||
REFRESH_STATUS = """
|
||||
UPDATE claim SET status = CASE
|
||||
WHEN status = 'retracted' THEN status
|
||||
WHEN EXISTS (SELECT 1 FROM claim_evidence e WHERE e.claim_id = claim.id AND e.stance = 'contradicts') THEN 'disputed'
|
||||
WHEN (SELECT count(DISTINCT d.source_id) FROM claim_evidence e JOIN document d ON d.id = e.document_id
|
||||
WHERE e.claim_id = claim.id AND e.stance IN ('supports', 'mentions')) >= 2 THEN 'corroborated'
|
||||
ELSE 'reported' END
|
||||
WHERE id = %s
|
||||
"""
|
||||
|
||||
|
||||
def merge_claims(conn: Connection, keep_id: int, dup_id: int) -> None:
|
||||
if keep_id == dup_id:
|
||||
return
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO claim_evidence (claim_id, document_id, stance, quote, locator, extracted_by, confidence)
|
||||
SELECT %s, document_id, stance, quote, locator, extracted_by, confidence
|
||||
FROM claim_evidence WHERE claim_id = %s
|
||||
ON CONFLICT (claim_id, document_id, locator) DO NOTHING
|
||||
""",
|
||||
(keep_id, dup_id),
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO topic_assignment (topic_id, claim_id, confidence, method)
|
||||
SELECT topic_id, %s, confidence, method FROM topic_assignment WHERE claim_id = %s
|
||||
ON CONFLICT DO NOTHING
|
||||
""",
|
||||
(keep_id, dup_id),
|
||||
)
|
||||
conn.execute("UPDATE claim SET merged_into = %s, updated_at = now() WHERE id = %s", (keep_id, dup_id))
|
||||
conn.execute("UPDATE claim SET merged_into = %s WHERE merged_into = %s", (keep_id, dup_id))
|
||||
conn.execute(REFRESH_STATUS, (keep_id,))
|
||||
|
||||
|
||||
# -- roles ---------------------------------------------------------------------
|
||||
|
||||
BODY_KEYWORDS = {
|
||||
"nationalrat": ("nationalrat",),
|
||||
"bundesrat": ("bundesrat", "bundesrates"),
|
||||
}
|
||||
|
||||
|
||||
def _same_office(a: dict, b: dict) -> bool:
|
||||
"""a: authoritative role (Parliament), b: other source (Wikidata)."""
|
||||
if a["role"] == "mp" and b["role"] == "mp":
|
||||
return a["body_slug"] == b["body_slug"]
|
||||
label = normalize(b["role"])
|
||||
if a["role"] == "mp" and a["body_slug"] in BODY_KEYWORDS:
|
||||
return any(k in label for k in BODY_KEYWORDS[a["body_slug"]]) and "prasident" not in label
|
||||
return fuzz.token_set_ratio(normalize(a["role"]), label) >= 80
|
||||
|
||||
|
||||
def _periods_match(a: dict, b: dict) -> bool:
|
||||
a_start, b_start = a["start_date"], b["start_date"]
|
||||
a_end, b_end = a["end_date"], b["end_date"]
|
||||
if a_start and b_start and abs((a_start - b_start).days) <= 60:
|
||||
return True
|
||||
# Wikidata often only records one term for several consecutive mandates: accept containment.
|
||||
if a_start and b_start and b_start <= a_start and (b_end is None or a_end is None or a_end <= b_end + timedelta(days=60)):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def merge_roles(conn: Connection) -> int:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT r.claim_id, r.person_id, r.role, r.start_date, r.end_date, c.created_by, o.slug AS body_slug
|
||||
FROM role r JOIN claim c ON c.id = r.claim_id LEFT JOIN organization o ON o.id = r.organization_id
|
||||
WHERE c.merged_into IS NULL AND r.level <> 'party'
|
||||
AND (c.created_by LIKE 'parlament%%' OR c.created_by LIKE 'wikidata%%' OR c.created_by LIKE 'europarl%%')
|
||||
ORDER BY r.person_id, r.start_date NULLS LAST
|
||||
"""
|
||||
).fetchall()
|
||||
by_person: dict[int, dict[str, list[dict]]] = defaultdict(lambda: defaultdict(list))
|
||||
for r in rows:
|
||||
family = "wikidata" if r["created_by"].startswith("wikidata") else "parlament"
|
||||
by_person[r["person_id"]][family].append(r)
|
||||
merged = 0
|
||||
for roles in by_person.values():
|
||||
for other in roles.get("wikidata", []):
|
||||
match = next((a for a in roles.get("parlament", []) if _same_office(a, other) and _periods_match(a, other)),
|
||||
None)
|
||||
if match:
|
||||
merge_claims(conn, match["claim_id"], other["claim_id"])
|
||||
merged += 1
|
||||
conn.commit()
|
||||
return merged
|
||||
|
||||
|
||||
# -- news stories -------------------------------------------------------------
|
||||
|
||||
def cluster_stories(conn: Connection, days: int = 14) -> int:
|
||||
"""Group recent news documents about the same thing into stories."""
|
||||
docs = conn.execute(
|
||||
"""
|
||||
SELECT d.id, d.title, d.published_at, d.story_id, d.source_id, d.meta->'people' AS people
|
||||
FROM document d JOIN source s ON s.id = d.source_id
|
||||
WHERE (s.kind = 'media' OR s.slug = 'gdelt') AND d.published_at > now() - make_interval(days => %s)
|
||||
AND d.title IS NOT NULL
|
||||
ORDER BY d.published_at
|
||||
""",
|
||||
(days,),
|
||||
).fetchall()
|
||||
stories: list[dict] = [] # {id, docs: [...]}
|
||||
for doc in docs:
|
||||
title = normalize(doc["title"])
|
||||
people = set(doc["people"] or [])
|
||||
best, best_score = None, 0
|
||||
for story in stories[-400:]:
|
||||
lead = story["docs"][0]
|
||||
if doc["published_at"] - lead["published_at"] > timedelta(hours=72):
|
||||
continue
|
||||
score = fuzz.token_set_ratio(title, lead["norm"])
|
||||
shared = people & lead["people"]
|
||||
if score >= 82 or (shared and score >= 60):
|
||||
if score > best_score:
|
||||
best, best_score = story, score
|
||||
entry = {"id": doc["id"], "norm": title, "people": people, "published_at": doc["published_at"],
|
||||
"source_id": doc["source_id"], "story_id": doc["story_id"], "title": doc["title"]}
|
||||
if best:
|
||||
best["docs"].append(entry)
|
||||
else:
|
||||
stories.append({"docs": [entry]})
|
||||
|
||||
assigned = 0
|
||||
for story in stories:
|
||||
if len(story["docs"]) < 2:
|
||||
continue
|
||||
existing = next((d["story_id"] for d in story["docs"] if d["story_id"]), None)
|
||||
lead = story["docs"][0]
|
||||
if existing is None:
|
||||
existing = conn.execute(
|
||||
"INSERT INTO story (title, first_seen) VALUES (%s, %s) RETURNING id", (lead["title"], lead["published_at"])
|
||||
).fetchone()["id"]
|
||||
ids = [d["id"] for d in story["docs"]]
|
||||
conn.execute("UPDATE document SET story_id = %s WHERE id = ANY(%s)", (existing, ids))
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE story SET last_seen = sub.last_seen, document_count = sub.n, source_count = sub.sources
|
||||
FROM (SELECT max(published_at) AS last_seen, count(*) AS n, count(DISTINCT source_id) AS sources
|
||||
FROM document WHERE story_id = %s) sub
|
||||
WHERE story.id = %s
|
||||
""",
|
||||
(existing, existing),
|
||||
)
|
||||
assigned += len(ids)
|
||||
conn.commit()
|
||||
return assigned
|
||||
|
||||
|
||||
def merge_story_mentions(conn: Connection) -> int:
|
||||
"""One media_mention claim per (person or party, story); other outlets become additional evidence."""
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT c.id, c.person_id, c.organization_id, d.story_id, c.occurred_at
|
||||
FROM claim c
|
||||
JOIN claim_evidence e ON e.claim_id = c.id
|
||||
JOIN document d ON d.id = e.document_id
|
||||
WHERE c.claim_type = 'media_mention' AND c.merged_into IS NULL AND d.story_id IS NOT NULL
|
||||
ORDER BY c.occurred_at NULLS LAST, c.id
|
||||
"""
|
||||
).fetchall()
|
||||
keep: dict[tuple, int] = {}
|
||||
merged = 0
|
||||
for r in rows:
|
||||
key = (r["person_id"], r["organization_id"], r["story_id"])
|
||||
if key in keep:
|
||||
if keep[key] != r["id"]:
|
||||
merge_claims(conn, keep[key], r["id"])
|
||||
merged += 1
|
||||
else:
|
||||
keep[key] = r["id"]
|
||||
conn.commit()
|
||||
return merged
|
||||
|
||||
|
||||
# -- events --------------------------------------------------------------------
|
||||
|
||||
def merge_events(conn: Connection) -> int:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT c.id, c.person_id, c.claim_type, c.occurred_at, c.data->>'country' AS country,
|
||||
c.data->>'counterpart' AS counterpart
|
||||
FROM claim c
|
||||
WHERE c.claim_type IN ('travel', 'meeting') AND c.merged_into IS NULL
|
||||
ORDER BY c.person_id, c.claim_type, c.occurred_at
|
||||
"""
|
||||
).fetchall()
|
||||
merged = 0
|
||||
previous: dict[tuple, dict] = {}
|
||||
for r in rows:
|
||||
anchor = r["country"] if r["claim_type"] == "travel" else counterpart_anchor(r["counterpart"] or "")
|
||||
key = (r["person_id"], r["claim_type"], anchor)
|
||||
prev = previous.get(key)
|
||||
if prev and r["occurred_at"] and prev["occurred_at"] and r["occurred_at"] - prev["occurred_at"] <= timedelta(days=3):
|
||||
merge_claims(conn, prev["id"], r["id"])
|
||||
merged += 1
|
||||
continue
|
||||
previous[key] = r
|
||||
conn.commit()
|
||||
return merged
|
||||
|
||||
|
||||
@register
|
||||
class Corroborator(Enricher):
|
||||
spec = DERIVED
|
||||
name = "enrich.corroborate"
|
||||
description = "Merge near-duplicate claims from different sources; cluster news stories"
|
||||
schedule = "9-59/10 * * * *"
|
||||
|
||||
def run(self, ctx: RunContext) -> None:
|
||||
conn = ctx.conn
|
||||
ctx.stats["roles_merged"] = merge_roles(conn)
|
||||
ctx.stats["story_documents"] = cluster_stories(conn)
|
||||
ctx.stats["story_mentions_merged"] = merge_story_mentions(conn)
|
||||
ctx.stats["events_merged"] = merge_events(conn)
|
||||
return None
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Curated lists maintained by the operator in config/ (accounts that no official register lists).
|
||||
|
||||
* config.social_accounts - config/social_accounts.yaml -> social_account rows for persons/organisations,
|
||||
e.g. party Telegram channels or Bluesky accounts linked from party websites. The social adapters then
|
||||
collect their public posts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Enricher, RunContext, SourceSpec, register
|
||||
from politrack.core.records import OrgRef, PersonRef, SocialAccount
|
||||
|
||||
CURATED = SourceSpec(
|
||||
slug="curated",
|
||||
name="Kuratierte Listen (config/)",
|
||||
kind="derived",
|
||||
homepage="",
|
||||
license="operator-maintained",
|
||||
access="manual",
|
||||
trust=60,
|
||||
tos_notes="Entries maintained by the operator; each should name the public page that links the account.",
|
||||
)
|
||||
|
||||
|
||||
def person_ref(ctx: RunContext, entry: dict[str, Any]) -> PersonRef | None:
|
||||
"""`person: <politrack slug>`, `person: {wikidata: Q..}` / `{parlament: ..}` or `name: <full name>`."""
|
||||
person = entry.get("person")
|
||||
if isinstance(person, dict):
|
||||
return PersonRef(ids={k: str(v) for k, v in person.items()})
|
||||
if isinstance(person, str):
|
||||
row = ctx.conn.execute("SELECT id FROM person WHERE slug = %s", (person,)).fetchone()
|
||||
return PersonRef(ids={"politrack": str(row["id"])}) if row else None
|
||||
if entry.get("name"):
|
||||
return PersonRef(name=entry["name"])
|
||||
return None
|
||||
|
||||
|
||||
def org_ref(entry: dict[str, Any]) -> OrgRef | None:
|
||||
"""`org: <slug>` for existing organisations, or `org: {slug, name, kind}` to create one."""
|
||||
org = entry.get("org")
|
||||
if isinstance(org, str):
|
||||
return OrgRef(slug=org, create=False)
|
||||
if isinstance(org, dict):
|
||||
return OrgRef(slug=org.get("slug"), name=org.get("name"), short_name=org.get("short_name"),
|
||||
kind=org.get("kind", "other"), create=bool(org.get("name")))
|
||||
return None
|
||||
|
||||
|
||||
@register
|
||||
class CuratedSocialAccounts(Enricher):
|
||||
spec = CURATED
|
||||
name = "config.social_accounts"
|
||||
description = "Import social accounts from config/social_accounts.yaml"
|
||||
schedule = "5 3 * * *"
|
||||
|
||||
def run(self, ctx: RunContext) -> Iterator[SocialAccount]:
|
||||
path = get_settings().config_dir / "social_accounts.yaml"
|
||||
if not path.exists():
|
||||
return
|
||||
for entry in (yaml.safe_load(path.read_text(encoding="utf-8")) or {}).get("accounts") or []:
|
||||
person = person_ref(ctx, entry)
|
||||
org = None if person else org_ref(entry)
|
||||
if person is None and org is None:
|
||||
ctx.stats["entries_without_owner"] += 1
|
||||
ctx.log.warning("social_accounts.yaml: no person/org for %s", entry.get("url"))
|
||||
continue
|
||||
yield SocialAccount(platform=entry["platform"], url=entry["url"], handle=entry.get("handle"),
|
||||
person=person, org=org, verified=bool(entry.get("verified")))
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Chunking + embeddings for retrieval (RAG).
|
||||
|
||||
Every claim gets one chunk (its summary plus evidence titles) so agents can
|
||||
retrieve facts *with citations*; documents with licensed full text are split
|
||||
into ~1,200 character chunks. Embeddings are computed by an Ollama server
|
||||
(model bge-m3, multilingual, 1024 dims) if POLITRACK_OLLAMA_URL is set;
|
||||
otherwise only German full-text search is available.
|
||||
"""
|
||||
|
||||
import httpx
|
||||
from psycopg import Connection
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Enricher, RunContext, register
|
||||
from politrack.core.spec import DERIVED
|
||||
|
||||
CHUNK_SIZE = 1200
|
||||
UPSERT_CHUNK = """
|
||||
INSERT INTO chunk (claim_id, document_id, chunk_index, text, lang) VALUES (%s, %s, %s, %s, 'de')
|
||||
ON CONFLICT (COALESCE(document_id, 0), COALESCE(claim_id, 0), chunk_index)
|
||||
DO UPDATE SET text = EXCLUDED.text, embedding = CASE WHEN chunk.text = EXCLUDED.text THEN chunk.embedding END
|
||||
"""
|
||||
|
||||
|
||||
@register
|
||||
class Chunker(Enricher):
|
||||
spec = DERIVED
|
||||
name = "enrich.embed"
|
||||
description = "Build search chunks for claims/documents and embed them (Ollama, optional)"
|
||||
schedule = "*/15 * * * *"
|
||||
|
||||
def run(self, ctx: RunContext) -> None:
|
||||
while (n := build_chunks(ctx.conn)):
|
||||
ctx.stats["chunks"] += n
|
||||
ctx.stats["embedded"] = embed_pending(ctx.conn)
|
||||
return None
|
||||
|
||||
|
||||
def split_text(text: str, size: int = CHUNK_SIZE) -> list[str]:
|
||||
paragraphs, chunks, current = [p.strip() for p in text.split("\n") if p.strip()], [], ""
|
||||
for p in paragraphs:
|
||||
if current and len(current) + len(p) > size:
|
||||
chunks.append(current)
|
||||
current = ""
|
||||
current = f"{current}\n{p}".strip()
|
||||
while len(current) > size * 1.5:
|
||||
chunks.append(current[:size])
|
||||
current = current[size:]
|
||||
if current:
|
||||
chunks.append(current)
|
||||
return chunks
|
||||
|
||||
|
||||
def build_chunks(conn: Connection, batch: int = 5000) -> int:
|
||||
claims = conn.execute(
|
||||
"""
|
||||
SELECT c.id, c.claim_type, c.summary_de, c.occurred_at, p.full_name, o.name AS org_name,
|
||||
string_agg(DISTINCT s.name || ': ' || COALESCE(d.title, d.url), ' | ') AS sources
|
||||
FROM claim c
|
||||
LEFT JOIN person p ON p.id = c.person_id
|
||||
LEFT JOIN organization o ON o.id = c.organization_id
|
||||
JOIN claim_evidence e ON e.claim_id = c.id
|
||||
JOIN document d ON d.id = e.document_id
|
||||
JOIN source s ON s.id = d.source_id
|
||||
WHERE c.merged_into IS NULL AND c.review_status = 'published'
|
||||
AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.claim_id = c.id)
|
||||
GROUP BY c.id, p.full_name, o.name
|
||||
LIMIT %s
|
||||
""",
|
||||
(batch,),
|
||||
).fetchall()
|
||||
for c in claims:
|
||||
who = c["full_name"] or c["org_name"] or ""
|
||||
when = c["occurred_at"].date().isoformat() if c["occurred_at"] else ""
|
||||
text = f"[{c['claim_type']}] {when} {who}\n{c['summary_de']}\nQuellen: {c['sources']}"
|
||||
conn.execute(UPSERT_CHUNK, (c["id"], None, 0, text.strip()))
|
||||
|
||||
docs = conn.execute(
|
||||
"""
|
||||
SELECT d.id, d.title, d.full_text FROM document d
|
||||
WHERE d.full_text IS NOT NULL AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.document_id = d.id)
|
||||
LIMIT %s
|
||||
""",
|
||||
(batch // 10,),
|
||||
).fetchall()
|
||||
for d in docs:
|
||||
for i, part in enumerate(split_text(d["full_text"])):
|
||||
conn.execute(UPSERT_CHUNK, (None, d["id"], i, f"{d['title'] or ''}\n{part}".strip()))
|
||||
conn.commit()
|
||||
return len(claims) + len(docs)
|
||||
|
||||
|
||||
def embed_texts(texts: list[str]) -> list[list[float]]:
|
||||
settings = get_settings()
|
||||
resp = httpx.post(f"{settings.ollama_url}/api/embed", json={"model": settings.embedding_model, "input": texts},
|
||||
timeout=300)
|
||||
resp.raise_for_status()
|
||||
return resp.json()["embeddings"]
|
||||
|
||||
|
||||
def embed_pending(conn: Connection, batch: int = 64, max_batches: int = 50) -> int:
|
||||
settings = get_settings()
|
||||
if not settings.ollama_url:
|
||||
return 0
|
||||
total = 0
|
||||
for _ in range(max_batches):
|
||||
rows = conn.execute("SELECT id, text FROM chunk WHERE embedding IS NULL ORDER BY id LIMIT %s", (batch,)).fetchall()
|
||||
if not rows:
|
||||
break
|
||||
vectors = embed_texts([r["text"] for r in rows])
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"UPDATE chunk SET embedding = %s::vector, embedding_model = %s WHERE id = %s",
|
||||
[(str(v), settings.embedding_model, r["id"]) for r, v in zip(rows, vectors)],
|
||||
)
|
||||
conn.commit()
|
||||
total += len(rows)
|
||||
return total
|
||||
@@ -0,0 +1,179 @@
|
||||
"""Extract trips and meetings from official press releases and news headlines.
|
||||
|
||||
Rule-based and conservative. A sentence becomes an event only if it names a
|
||||
known politician and contains a travel or meeting trigger:
|
||||
|
||||
travel: "reist nach Brüssel", "Arbeitsbesuch in Israel", "beim Gipfel in Montenegro", "in China"
|
||||
meeting: "empfängt den tschechischen Außenminister", "Treffen mit ...", "traf ... zu einem Gespräch"
|
||||
|
||||
Sources:
|
||||
* Parlamentskorrespondenz and government news: title + full text (official works / CC BY),
|
||||
* news headlines: title only (we do not store article text).
|
||||
|
||||
The event date is the publication date of the source (`data.date_precision =
|
||||
"publication"`). The claim key is (kind, person, place/counterpart, ISO week),
|
||||
so reports from different sources about the same trip attach to the same claim
|
||||
as additional evidence and corroborate it. corroborate.merge_events() also
|
||||
merges near-duplicates whose dates differ by a few days.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from politrack.core import Enricher, RunContext, register
|
||||
from politrack.core.records import Claim, Event, Evidence, Participation, PersonRef
|
||||
from politrack.core.spec import DERIVED
|
||||
from politrack.enrich.gazetteer import AUSTRIA, PLACES
|
||||
from politrack.enrich.linker import MentionLinker
|
||||
from politrack.text import excerpt, fingerprint, normalize
|
||||
|
||||
# (?<!-)/(?!-): skip names inside compounds such as "Berlin-Prozess" or "EU-Brüssel-Treffen".
|
||||
PLACE_RE = re.compile(
|
||||
r"(?<![-\w])(" + "|".join(sorted((re.escape(p) for p in PLACES), key=len, reverse=True)) + r")(?![-\w])"
|
||||
)
|
||||
DESTINATION_RE = re.compile(r"\b(?:in|nach|aus)\s+(?:de[mnr]\s+|die\s+)?$")
|
||||
STRONG_TRAVEL = re.compile(r"\b(reist\w*|Reise\w*|\w*[Bb]esuch\w*|\w*Gipfel\w*|Delegation\w*|Konferenz\w*|Tagung\w*)\b")
|
||||
MEETING_RE = re.compile(
|
||||
r"\b(?:empfängt|empfing|empfangen|trifft|traf|getroffen|Treffen mit|Gespräch mit|Arbeitsgespräch mit|"
|
||||
r"Unterredung mit|tauschte sich mit|im Austausch mit|zusammengetroffen mit|kam mit)\s+"
|
||||
r"(?P<counterpart>(?:de[mnr]|die|das|seine[mnr]?|ihre[mnr]?)?\s*[^.,;:]{3,90})",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
COUNTERPART_ROLE = re.compile(
|
||||
r"(minister|präsident|kanzler|botschafter|delegation|amtskolleg|vertreter|bürgermeister|abgeordnet|chef|"
|
||||
r"premier|könig|papst|kommissar|generalsekretär|regierungschef|staatssekretär|landeshauptm|gouverneur|"
|
||||
r"bischof|kardinal|vorsitzend|ceo|gewerkschaft|sozialpartner|außenminister)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
FULL_NAME = re.compile(r"\b[A-ZÄÖÜ][a-zäöüß]+(?:-[A-ZÄÖÜ][a-zäöüß]+)?\s+[A-ZÄÖÜ][a-zäöüß]+")
|
||||
SENTENCE_SPLIT = re.compile(r"(?<=[.!?])\s+(?=[A-ZÄÖÜ„\"])|\n+")
|
||||
|
||||
|
||||
FILLER = re.compile(r"^(?:(?:heute|gestern|morgen|am|vorgestern|Montag|Dienstag|Mittwoch|Donnerstag|Freitag|"
|
||||
r"Samstag|Sonntag|de[mnr]|die|das|seine[mnr]?|ihre[mnr]?|eine[mnr]?)\s+)+", re.IGNORECASE)
|
||||
|
||||
|
||||
def clean_counterpart(text: str) -> str:
|
||||
return FILLER.sub("", text.strip()).strip()
|
||||
|
||||
|
||||
def counterpart_anchor(text: str) -> str:
|
||||
"""Stable key for a counterpart: its last capitalised word (usually the surname or country)."""
|
||||
words = re.findall(r"[A-ZÄÖÜ][\wäöüß-]+", clean_counterpart(text))
|
||||
return normalize(words[-1]) if words else normalize(text)[:25]
|
||||
|
||||
|
||||
def find_places(text: str) -> list[tuple[str, str]]:
|
||||
"""Places in order of preference: those introduced by "in/nach" first (the destination)."""
|
||||
matches = list(PLACE_RE.finditer(text))
|
||||
preferred = [m for m in matches if DESTINATION_RE.search(text[: m.start()])]
|
||||
ordered = preferred + [m for m in matches if m not in preferred]
|
||||
return [(m.group(1), PLACES[m.group(1)]) for m in ordered]
|
||||
|
||||
|
||||
def detect(text: str) -> list[dict]:
|
||||
"""Return candidate events found in one sentence (no person resolution)."""
|
||||
events = []
|
||||
places = find_places(text)
|
||||
foreign = [(p, iso) for p, iso in places if iso != "AT"]
|
||||
if foreign and STRONG_TRAVEL.search(text):
|
||||
place, iso = foreign[0]
|
||||
events.append({"kind": "foreign_trip", "claim_type": "travel", "place": place, "country": iso})
|
||||
meeting = MEETING_RE.search(text)
|
||||
if meeting:
|
||||
counterpart = re.split(r"\s+(?:in|im|am|zu|über|bei|und|sowie)\s+", meeting.group("counterpart").strip())[0]
|
||||
place, iso = (places[0] if places else (None, None))
|
||||
# "traf eine Entscheidung" is not a meeting: require a role word or a person-like name.
|
||||
if COUNTERPART_ROLE.search(counterpart) or FULL_NAME.search(counterpart):
|
||||
events.append({"kind": "meeting", "claim_type": "meeting", "counterpart": clean_counterpart(counterpart),
|
||||
"place": place, "country": iso})
|
||||
return events
|
||||
|
||||
|
||||
@register
|
||||
class EventExtractor(Enricher):
|
||||
spec = DERIVED
|
||||
name = "enrich.events"
|
||||
description = "Trips and meetings from press releases, government news and headlines"
|
||||
schedule = "3-59/10 * * * *"
|
||||
|
||||
def run(self, ctx: RunContext) -> Iterator[Any]:
|
||||
conn = ctx.conn
|
||||
linker = MentionLinker(conn)
|
||||
while True:
|
||||
docs = conn.execute(
|
||||
"""SELECT d.id, d.url, d.title, d.full_text, d.published_at, d.meta, s.slug AS source, s.kind AS source_kind
|
||||
FROM document d JOIN source s ON s.id = d.source_id
|
||||
WHERE NOT (d.meta ? 'events_at') AND d.published_at IS NOT NULL
|
||||
AND (d.doc_type = 'press_release' OR s.kind = 'media' OR s.slug = 'gdelt')
|
||||
ORDER BY d.id LIMIT 300"""
|
||||
).fetchall()
|
||||
if not docs:
|
||||
return
|
||||
for doc in docs:
|
||||
found = 0
|
||||
for record in self.document_events(conn, linker, doc):
|
||||
found += isinstance(record, Claim)
|
||||
yield record
|
||||
conn.execute("UPDATE document SET meta = meta || %s WHERE id = %s",
|
||||
(Jsonb({"events_at": datetime.now().isoformat(), "events": found}), doc["id"]))
|
||||
ctx.stats["documents"] += 1
|
||||
|
||||
def document_events(self, conn, linker: MentionLinker, doc: dict) -> Iterator[Any]:
|
||||
official = doc["source_kind"] == "official"
|
||||
title = doc["title"] or ""
|
||||
body = (doc["full_text"] or "")[:4000] if official else ""
|
||||
pads = (doc["meta"] or {}).get("person_pads") or []
|
||||
doc_people: set[int] = set()
|
||||
if pads:
|
||||
doc_people = {r["entity_id"] for r in conn.execute(
|
||||
"SELECT entity_id FROM identifier WHERE entity_type = 'person' AND scheme = 'parlament' AND value = ANY(%s)",
|
||||
([str(p) for p in pads],))}
|
||||
sentences = [title] + [s for s in SENTENCE_SPLIT.split(body) if s][:25]
|
||||
for i, sentence in enumerate(sentences):
|
||||
candidates = detect(sentence)
|
||||
if not candidates:
|
||||
continue
|
||||
people = linker.all_people(sentence)
|
||||
# In the headline of an official release the actors are usually the people it links/names.
|
||||
if not people and i == 0:
|
||||
people = doc_people or linker.all_people(f"{title} {body[:600]}")
|
||||
for person_id in people:
|
||||
for ev in candidates:
|
||||
yield from event_records(conn, doc, person_id, ev, sentence, official)
|
||||
|
||||
|
||||
def event_records(conn, doc: dict, person_id: int, ev: dict, sentence: str, official: bool) -> Iterator[Any]:
|
||||
published = doc["published_at"]
|
||||
week = published.strftime("%G-W%V")
|
||||
anchor = ev.get("country") if ev["kind"] == "foreign_trip" else counterpart_anchor(ev.get("counterpart") or "")
|
||||
row = conn.execute("SELECT full_name, family_name FROM person WHERE id = %s", (person_id,)).fetchone()
|
||||
# A sentence like "X empfing Y" yields the meeting for X and Y; never record "Y met Y".
|
||||
if ev["kind"] == "meeting" and row["family_name"] and normalize(row["family_name"]) in normalize(ev["counterpart"]):
|
||||
return
|
||||
if ev["kind"] == "foreign_trip":
|
||||
summary_de, summary_en = f"Reise/Aufenthalt in {ev['place']}", f"Trip to {ev['place']}"
|
||||
else:
|
||||
where = f" ({ev['place']})" if ev.get("place") else ""
|
||||
summary_de = f"Treffen mit {ev['counterpart']}{where}"
|
||||
summary_en = f"Meeting with {ev['counterpart']}{where}"
|
||||
event_key = fingerprint("event", ev["kind"], person_id, anchor, week)
|
||||
yield Event(key=event_key, kind=ev["kind"], title=f"{row['full_name']}: {summary_de}", starts_at=published,
|
||||
location=ev.get("place"), country=ev.get("country"), url=doc["url"], description=excerpt(sentence, 400))
|
||||
foreign = ev.get("country") and ev["country"] not in AUSTRIA.values()
|
||||
yield Claim(
|
||||
claim_type=ev["claim_type"], person=PersonRef(ids={"politrack": str(person_id)}), occurred_at=published,
|
||||
summary_de=summary_de, summary_en=summary_en,
|
||||
data={"place": ev.get("place"), "country": ev.get("country"), "counterpart": ev.get("counterpart"),
|
||||
"date_precision": "publication", "extracted": "rules"},
|
||||
evidence=[Evidence(url=doc["url"], source=doc["source"], title=doc["title"], published_at=published,
|
||||
quote=excerpt(sentence, 400), confidence=0.8 if official else 0.6)],
|
||||
topics=("Außenpolitik",) if foreign else (),
|
||||
facet=Participation(event_key=event_key),
|
||||
)
|
||||
@@ -0,0 +1,52 @@
|
||||
"""German place names -> ISO 3166-1 alpha-2, for detecting trips in German texts.
|
||||
|
||||
Countries in nominative form (as used after "nach"/"in"); cities that are
|
||||
frequent destinations of Austrian politicians. Austrian places map to "AT" so
|
||||
domestic visits are recognised but not counted as trips abroad.
|
||||
"""
|
||||
|
||||
COUNTRIES = {
|
||||
"Ägypten": "EG", "Albanien": "AL", "Algerien": "DZ", "Argentinien": "AR", "Armenien": "AM", "Aserbaidschan": "AZ",
|
||||
"Äthiopien": "ET", "Australien": "AU", "Belgien": "BE", "Bosnien und Herzegowina": "BA", "Bosnien": "BA",
|
||||
"Brasilien": "BR", "Bulgarien": "BG", "Chile": "CL", "China": "CN", "Dänemark": "DK", "Deutschland": "DE",
|
||||
"Estland": "EE", "Finnland": "FI", "Frankreich": "FR", "Georgien": "GE", "Ghana": "GH", "Griechenland": "GR",
|
||||
"Großbritannien": "GB", "Vereinigten Königreich": "GB", "Vereinigtes Königreich": "GB", "Indien": "IN",
|
||||
"Indonesien": "ID", "Irak": "IQ", "Iran": "IR", "Irland": "IE", "Island": "IS", "Israel": "IL", "Italien": "IT",
|
||||
"Japan": "JP", "Jordanien": "JO", "Kanada": "CA", "Kasachstan": "KZ", "Katar": "QA", "Kenia": "KE",
|
||||
"Kolumbien": "CO", "Kosovo": "XK", "Kroatien": "HR", "Kuba": "CU", "Lettland": "LV", "Libanon": "LB",
|
||||
"Libyen": "LY", "Liechtenstein": "LI", "Litauen": "LT", "Luxemburg": "LU", "Malta": "MT", "Marokko": "MA",
|
||||
"Mexiko": "MX", "Moldau": "MD", "Moldawien": "MD", "Monaco": "MC", "Mongolei": "MN", "Montenegro": "ME",
|
||||
"Neuseeland": "NZ", "Niederlande": "NL", "Nigeria": "NG", "Nordmazedonien": "MK", "Norwegen": "NO",
|
||||
"Oman": "OM", "Pakistan": "PK", "Palästina": "PS", "Polen": "PL", "Portugal": "PT", "Ruanda": "RW",
|
||||
"Rumänien": "RO", "Russland": "RU", "Saudi-Arabien": "SA", "Schweden": "SE", "Schweiz": "CH", "Senegal": "SN",
|
||||
"Serbien": "RS", "Singapur": "SG", "Slowakei": "SK", "Slowenien": "SI", "Spanien": "ES", "Südafrika": "ZA",
|
||||
"Südkorea": "KR", "Syrien": "SY", "Taiwan": "TW", "Thailand": "TH", "Tschechien": "CZ", "Tunesien": "TN",
|
||||
"Türkei": "TR", "Ukraine": "UA", "Ungarn": "HU", "USA": "US", "Vereinigten Staaten": "US",
|
||||
"Vereinigte Staaten": "US", "Usbekistan": "UZ", "Vatikan": "VA", "Venezuela": "VE",
|
||||
"Vereinigten Arabischen Emiraten": "AE", "Vereinigte Arabische Emirate": "AE", "Vietnam": "VN",
|
||||
"Weißrussland": "BY", "Belarus": "BY", "Zypern": "CY", "Andorra": "AD", "San Marino": "SM",
|
||||
}
|
||||
|
||||
CITIES = {
|
||||
"Brüssel": "BE", "Straßburg": "FR", "Luxemburg": "LU", "Berlin": "DE", "München": "DE", "Frankfurt": "DE",
|
||||
"Paris": "FR", "Rom": "IT", "Mailand": "IT", "Madrid": "ES", "Lissabon": "PT", "London": "GB", "Dublin": "IE",
|
||||
"Amsterdam": "NL", "Den Haag": "NL", "Kopenhagen": "DK", "Stockholm": "SE", "Oslo": "NO", "Helsinki": "FI",
|
||||
"Warschau": "PL", "Prag": "CZ", "Bratislava": "SK", "Pressburg": "SK", "Budapest": "HU", "Ljubljana": "SI",
|
||||
"Laibach": "SI", "Zagreb": "HR", "Belgrad": "RS", "Sarajevo": "BA", "Podgorica": "ME", "Skopje": "MK",
|
||||
"Tirana": "AL", "Pristina": "XK", "Sofia": "BG", "Bukarest": "RO", "Chișinău": "MD", "Kiew": "UA",
|
||||
"Kyjiw": "UA", "Moskau": "RU", "Minsk": "BY", "Athen": "GR", "Nikosia": "CY", "Valletta": "MT",
|
||||
"Ankara": "TR", "Istanbul": "TR", "Bern": "CH", "Genf": "CH", "Zürich": "CH", "Davos": "CH", "Vaduz": "LI",
|
||||
"Washington": "US", "New York": "US", "Ottawa": "CA", "Peking": "CN", "Beijing": "CN", "Shanghai": "CN",
|
||||
"Tokio": "JP", "Neu-Delhi": "IN", "Jerusalem": "IL", "Tel Aviv": "IL", "Kairo": "EG", "Amman": "JO",
|
||||
"Riad": "SA", "Doha": "QA", "Abu Dhabi": "AE", "Dubai": "AE", "Teheran": "IR", "Baku": "AZ", "Tiflis": "GE",
|
||||
"Eriwan": "AM", "Astana": "KZ", "Taschkent": "UZ", "Brdo": "SI", "Göttweig": "AT",
|
||||
}
|
||||
|
||||
AUSTRIA = {
|
||||
"Österreich": "AT", "Wien": "AT", "Graz": "AT", "Linz": "AT", "Salzburg": "AT", "Innsbruck": "AT",
|
||||
"Klagenfurt": "AT", "Villach": "AT", "Wels": "AT", "St. Pölten": "AT", "Dornbirn": "AT", "Bregenz": "AT",
|
||||
"Eisenstadt": "AT", "Niederösterreich": "AT", "Oberösterreich": "AT", "Steiermark": "AT", "Kärnten": "AT",
|
||||
"Tirol": "AT", "Vorarlberg": "AT", "Burgenland": "AT", "Alpbach": "AT",
|
||||
}
|
||||
|
||||
PLACES = {**AUSTRIA, **COUNTRIES, **CITIES}
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Detect reports about legal proceedings in news headlines (and WKStA press releases).
|
||||
|
||||
Everything found here is a *proposal*: claims are created with review=True (hidden until a human approves
|
||||
them via `politrack review`). Headline wording is ambiguous, the data is specially protected (Art. 10 GDPR)
|
||||
and wrong attributions can be defamatory. Official parliament immunity data does not go through this path.
|
||||
Several outlets reporting the same person + stage in the same month attach to one claim (corroboration).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime
|
||||
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from politrack.core import Enricher, RunContext, register
|
||||
from politrack.core.records import Claim, Evidence, Legal, PersonRef
|
||||
from politrack.core.spec import DERIVED
|
||||
|
||||
# Most specific first.
|
||||
STAGES = [
|
||||
# "nicht rechtskräftig verurteilt" must never be read as a final conviction.
|
||||
("convicted_final", re.compile(r"(?<!nicht )\brechtskräftig\s+(verurteilt|schuldig)", re.I)),
|
||||
("convicted_not_final", re.compile(r"\b(nicht rechtskräftig\b.*\bverurteilt|verurteilt\b.*nicht rechtskräftig|"
|
||||
r"schuldig gesprochen|Schuldspruch|\bverurteilt\b)", re.I)),
|
||||
("acquitted", re.compile(r"\b(freigesprochen|Freispruch)\b", re.I)),
|
||||
("diversion", re.compile(r"\bDiversion\b", re.I)),
|
||||
("proceedings_dropped", re.compile(r"\b(Verfahren|Ermittlungen)\b.*\beingestellt\b|\beingestellt\b.*\bVerfahren\b", re.I)),
|
||||
("trial", re.compile(r"\b(Prozess|Strafprozess|vor Gericht|Prozessauftakt|Hauptverhandlung)\b", re.I)),
|
||||
("indicted", re.compile(r"\b(angeklagt|Anklage|Strafantrag)\b", re.I)),
|
||||
("investigation", re.compile(r"\b(ermittelt|Ermittlungen|Ermittlungsverfahren|Beschuldigte[rn]?|Hausdurchsuchung|"
|
||||
r"WKStA|Staatsanwaltschaft|Razzia)\b", re.I)),
|
||||
]
|
||||
STAGE_DE = {
|
||||
"convicted_final": "rechtskräftige Verurteilung (laut Bericht)",
|
||||
"convicted_not_final": "nicht rechtskräftige Verurteilung (laut Bericht)",
|
||||
"acquitted": "Freispruch (laut Bericht)",
|
||||
"diversion": "Diversion (laut Bericht)",
|
||||
"proceedings_dropped": "Verfahren eingestellt (laut Bericht)",
|
||||
"trial": "Gerichtsverfahren (laut Bericht)",
|
||||
"indicted": "Anklage (laut Bericht)",
|
||||
"investigation": "Ermittlungen (laut Bericht)",
|
||||
}
|
||||
|
||||
|
||||
def classify(headline: str) -> str | None:
|
||||
for stage, pattern in STAGES:
|
||||
if pattern.search(headline):
|
||||
return stage
|
||||
return None
|
||||
|
||||
|
||||
@register
|
||||
class LegalDetector(Enricher):
|
||||
spec = DERIVED
|
||||
name = "enrich.legal"
|
||||
description = "Propose legal-proceeding claims from headlines (human review required)"
|
||||
schedule = "6-59/10 * * * *"
|
||||
|
||||
def run(self, ctx: RunContext) -> Iterator[Claim]:
|
||||
conn = ctx.conn
|
||||
while True:
|
||||
rows = conn.execute(
|
||||
"""SELECT c.id, c.person_id, c.occurred_at, d.url, d.title, d.published_at, s.slug AS source
|
||||
FROM claim c
|
||||
JOIN claim_evidence e ON e.claim_id = c.id
|
||||
JOIN document d ON d.id = e.document_id
|
||||
JOIN source s ON s.id = d.source_id
|
||||
WHERE c.claim_type = 'media_mention' AND c.person_id IS NOT NULL AND NOT (c.data ? 'legal_checked')
|
||||
AND (s.kind IN ('media', 'aggregator') OR s.slug = 'wksta')
|
||||
ORDER BY c.id LIMIT 1000"""
|
||||
).fetchall()
|
||||
if not rows:
|
||||
return
|
||||
for row in rows:
|
||||
stage = classify(row["title"] or "")
|
||||
if stage:
|
||||
when = row["published_at"] or row["occurred_at"] or datetime.now()
|
||||
yield Claim(
|
||||
claim_type="legal_proceeding", person=PersonRef(ids={"politrack": str(row["person_id"])}),
|
||||
key=("legal-news", row["person_id"], stage, when.strftime("%Y-%m")), occurred_at=when,
|
||||
summary_de=STAGE_DE[stage], summary_en=f"Reported legal matter ({stage.replace('_', ' ')})",
|
||||
data={"detected_from": "headline"}, sensitive=True, review=True,
|
||||
evidence=[Evidence(url=row["url"], source=row["source"], title=row["title"], published_at=when,
|
||||
quote=row["title"], confidence=0.5)],
|
||||
topics=("Inneres und Recht",),
|
||||
facet=Legal(stage=stage, stage_date=when.date()),
|
||||
)
|
||||
ctx.stats["proposed"] += 1
|
||||
conn.execute("UPDATE claim SET data = data || %s WHERE id = %s", (Jsonb({"legal_checked": True}), row["id"]))
|
||||
@@ -0,0 +1,153 @@
|
||||
"""Link documents (headlines, press releases, government news) to the politicians and parties they mention.
|
||||
|
||||
Conservative by design: full names (>= 2 tokens) only, plus "office title + surname" of *current* office
|
||||
holders ("Bundeskanzler Stocker"); an ambiguous name is only linked if exactly one candidate is active.
|
||||
False links are worse than missed ones on a site like this.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from psycopg import Connection
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
from politrack.core import Enricher, RunContext, register
|
||||
from politrack.core.records import Claim, Evidence, OrgRef, PersonRef
|
||||
from politrack.core.spec import DERIVED
|
||||
from politrack.text import normalize, strip_titles
|
||||
|
||||
TOKEN = re.compile(r"[\w'-]+")
|
||||
OFFICE_TITLE = re.compile(
|
||||
r"\b(?:Bundeskanzler(?:in)?|Vizekanzler(?:in)?|[A-ZÄÖÜ][a-zäöüß]*minister(?:in)?|Minister(?:in)?|"
|
||||
r"Staatssekretär(?:in)?|Nationalratspräsident(?:in)?|Bundesratspräsident(?:in)?|Bundespräsident(?:in)?|"
|
||||
r"Landeshauptmann|Landeshauptfrau|Landesrät(?:in)?|Landesrat|Bürgermeister(?:in)?|Klubobmann|Klubobfrau|"
|
||||
r"Klubchef(?:in)?|Parteichef(?:in)?|Generalsekretär(?:in)?|Abgeordnete[rn]?|EU-Abgeordnete[rn]?|"
|
||||
r"EU-Kommissar(?:in)?|Präsident(?:in)?)\s+(?:(?:Dr|Mag|DI|Ing)\.\s+)?(?P<surname>[A-ZÄÖÜ][a-zäöüß]+(?:-[A-ZÄÖÜ][a-zäöüß]+)?)\b"
|
||||
# "Nationalratspräsident Walter Rosenkranz": a following capitalised word means we matched a first name.
|
||||
r"(?!\s+(?:von\s+)?[A-ZÄÖÜ][a-zäöüß]+)"
|
||||
)
|
||||
CONTEXT_PATTERNS = [
|
||||
("interview", re.compile(r"\binterview\b", re.I)),
|
||||
("tv", re.compile(r"\b(ZIB ?2|ZiB ?2|Pressestunde|Im Zentrum|Sommergespräch|Report|Puls ?24|ServusTV|Talk im Hangar|"
|
||||
r"Milborn|Pro und Contra)\b", re.I)),
|
||||
("radio", re.compile(r"\b(Ö1|Morgenjournal|Mittagsjournal|Journal zu Gast)\b", re.I)),
|
||||
("op_ed", re.compile(r"\b(Gastkommentar|Kommentar der anderen)\b", re.I)),
|
||||
]
|
||||
|
||||
|
||||
def tokens(text: str) -> list[str]:
|
||||
return TOKEN.findall(normalize(text))
|
||||
|
||||
|
||||
def detect_context(text: str) -> str | None:
|
||||
for name, pattern in CONTEXT_PATTERNS:
|
||||
if pattern.search(text):
|
||||
return name
|
||||
return None
|
||||
|
||||
|
||||
class MentionLinker:
|
||||
def __init__(self, conn: Connection) -> None:
|
||||
self.person_index: dict[tuple[str, ...], set[int]] = defaultdict(set)
|
||||
for row in conn.execute("SELECT entity_id, alias FROM alias WHERE entity_type = 'person'"):
|
||||
key = tuple(tokens(strip_titles(row["alias"])))
|
||||
if len(key) >= 2:
|
||||
self.person_index[key].add(row["entity_id"])
|
||||
self.active = {r["person_id"] for r in conn.execute("SELECT person_id FROM person_status WHERE status = 'active'")}
|
||||
self.active_surnames: dict[str, set[int]] = defaultdict(set)
|
||||
for row in conn.execute(
|
||||
"SELECT p.id, p.family_name FROM person p JOIN person_status s ON s.person_id = p.id "
|
||||
"WHERE s.status = 'active' AND p.family_name IS NOT NULL"
|
||||
):
|
||||
self.active_surnames[normalize(row["family_name"])].add(row["id"])
|
||||
self.org_index: dict[tuple[str, ...], int] = {}
|
||||
for row in conn.execute(
|
||||
"""SELECT o.id, a.alias FROM organization o JOIN alias a ON a.entity_type = 'organization' AND a.entity_id = o.id
|
||||
WHERE o.kind = 'party'
|
||||
UNION SELECT id, short_name FROM organization WHERE kind = 'party' AND short_name IS NOT NULL"""
|
||||
):
|
||||
key = tuple(tokens(row["alias"]))
|
||||
if key and len("".join(key)) >= 3:
|
||||
self.org_index[key] = row["id"]
|
||||
self.max_len = max((len(k) for k in self.person_index), default=2)
|
||||
|
||||
def people(self, text: str) -> set[int]:
|
||||
toks = tokens(text)
|
||||
found: set[int] = set()
|
||||
for n in range(2, self.max_len + 1):
|
||||
for i in range(len(toks) - n + 1):
|
||||
candidates = self.person_index.get(tuple(toks[i : i + n]))
|
||||
if not candidates:
|
||||
continue
|
||||
if len(candidates) == 1:
|
||||
found |= candidates
|
||||
elif len(active := candidates & self.active) == 1:
|
||||
found |= active
|
||||
return found
|
||||
|
||||
def people_by_title(self, text: str) -> set[int]:
|
||||
found: set[int] = set()
|
||||
for m in OFFICE_TITLE.finditer(text):
|
||||
candidates = self.active_surnames.get(normalize(m.group("surname")), set())
|
||||
if len(candidates) == 1:
|
||||
found |= candidates
|
||||
return found
|
||||
|
||||
def all_people(self, text: str) -> set[int]:
|
||||
return self.people(text) | self.people_by_title(text)
|
||||
|
||||
def parties(self, text: str) -> set[int]:
|
||||
toks = tokens(text)
|
||||
found = set()
|
||||
for n in (1, 2, 3):
|
||||
for i in range(len(toks) - n + 1):
|
||||
if org := self.org_index.get(tuple(toks[i : i + n])):
|
||||
found.add(org)
|
||||
return found
|
||||
|
||||
|
||||
@register
|
||||
class Linker(Enricher):
|
||||
spec = DERIVED
|
||||
name = "enrich.linker"
|
||||
description = "Link news headlines, press releases and government news to politicians and parties"
|
||||
schedule = "*/10 * * * *"
|
||||
|
||||
def run(self, ctx: RunContext) -> Iterator[Claim]:
|
||||
conn = ctx.conn
|
||||
linker = MentionLinker(conn)
|
||||
while True:
|
||||
docs = conn.execute(
|
||||
"""SELECT d.id, d.url, d.title, d.excerpt, left(d.full_text, 3000) AS full_text, d.published_at,
|
||||
s.slug AS source, s.name AS source_name, s.kind AS source_kind,
|
||||
COALESCE((d.meta->>'tdm_reserved')::boolean, false) AS tdm_reserved
|
||||
FROM document d JOIN source s ON s.id = d.source_id
|
||||
WHERE (s.kind IN ('media', 'user') OR s.slug IN ('gdelt', 'bundesregierung', 'wksta'))
|
||||
AND NOT (d.meta ? 'linked_at') AND d.doc_type IN ('article', 'press_release')
|
||||
ORDER BY d.id LIMIT 500"""
|
||||
).fetchall()
|
||||
if not docs:
|
||||
return
|
||||
for doc in docs:
|
||||
title = doc["title"] or ""
|
||||
text = title if doc["tdm_reserved"] else f"{title} {doc['full_text'] or doc['excerpt'] or ''}"
|
||||
context = detect_context(title)
|
||||
people, parties = linker.all_people(text), linker.parties(title)
|
||||
evidence = Evidence(url=doc["url"], source=doc["source"], title=title, quote=title, stance="mentions",
|
||||
confidence=0.8, published_at=doc["published_at"])
|
||||
for person_id in people:
|
||||
yield Claim(claim_type="media_mention", person=PersonRef(ids={"politrack": str(person_id)}),
|
||||
occurred_at=doc["published_at"], summary_de=f"{doc['source_name']}: {title}",
|
||||
data={"context": context, "source": doc["source"]}, evidence=[evidence])
|
||||
for org_id in parties:
|
||||
yield Claim(claim_type="media_mention", org=OrgRef(ids={"politrack_org": str(org_id)}, create=False),
|
||||
occurred_at=doc["published_at"], summary_de=f"{doc['source_name']}: {title}",
|
||||
data={"context": context, "source": doc["source"]}, evidence=[evidence])
|
||||
conn.execute("UPDATE document SET meta = meta || %s WHERE id = %s",
|
||||
(Jsonb({"linked_at": datetime.now(timezone.utc).isoformat(), "people": sorted(people),
|
||||
"parties": sorted(parties)}), doc["id"]))
|
||||
ctx.stats["documents_linked"] += 1
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Housekeeping enrichers that work directly on canonical rows (no new facts, only derived structure).
|
||||
|
||||
* enrich.roles - give open-ended offices the party of the holder's party membership at that time
|
||||
(Wikidata offices carry no party), close open roles at the holder's date of death
|
||||
* enrich.rollcall - derive each faction's majority position from individual roll-call votes, so party
|
||||
pages show roll calls too ("ÖVP mostly for: ... 70 yes, 2 no")
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter, defaultdict
|
||||
from collections.abc import Iterator
|
||||
|
||||
from politrack.core import Enricher, RunContext, register
|
||||
from politrack.core.records import Claim, Evidence, OrgRef, VotePosition
|
||||
from politrack.core.spec import DERIVED
|
||||
|
||||
|
||||
@register
|
||||
class RoleMaintenance(Enricher):
|
||||
spec = DERIVED
|
||||
name = "enrich.roles"
|
||||
description = "Assign parties to offices from party memberships; close roles at death"
|
||||
schedule = "50 5 * * *"
|
||||
|
||||
def run(self, ctx: RunContext) -> None:
|
||||
conn = ctx.conn
|
||||
ctx.stats["roles_with_party"] = conn.execute(
|
||||
"""UPDATE role r SET party_id = pm.party_id
|
||||
FROM role pm JOIN claim c ON c.id = pm.claim_id
|
||||
WHERE r.party_id IS NULL AND r.level <> 'party' AND pm.role = 'party_member' AND c.merged_into IS NULL
|
||||
AND pm.person_id = r.person_id
|
||||
AND (pm.start_date IS NULL OR r.end_date IS NULL OR pm.start_date <= r.end_date)
|
||||
AND (pm.end_date IS NULL OR r.start_date IS NULL OR pm.end_date >= r.start_date)"""
|
||||
).rowcount
|
||||
ctx.stats["roles_closed_at_death"] = conn.execute(
|
||||
"""UPDATE role r SET end_date = p.death_date FROM person p
|
||||
WHERE p.id = r.person_id AND r.end_date IS NULL AND p.death_date IS NOT NULL"""
|
||||
).rowcount
|
||||
return None
|
||||
|
||||
|
||||
@register
|
||||
class RollCallFactions(Enricher):
|
||||
spec = DERIVED
|
||||
name = "enrich.rollcall"
|
||||
description = "Faction majority positions derived from individual roll-call votes"
|
||||
schedule = "55 4 * * *"
|
||||
|
||||
def run(self, ctx: RunContext) -> Iterator[Claim]:
|
||||
rows = ctx.conn.execute(
|
||||
"""SELECT ve.event_key, ve.item_title, ve.voted_on, ve.url, ch.slug AS chamber, vp.party_id, vp.position,
|
||||
o.short_name, o.name
|
||||
FROM vote_event ve JOIN vote_position vp ON vp.vote_event_id = ve.id
|
||||
JOIN organization ch ON ch.id = ve.chamber_id
|
||||
JOIN organization o ON o.id = vp.party_id
|
||||
JOIN claim c ON c.id = vp.claim_id AND c.merged_into IS NULL
|
||||
WHERE ve.roll_call AND vp.person_id IS NOT NULL"""
|
||||
).fetchall()
|
||||
tally: dict[tuple, Counter] = defaultdict(Counter)
|
||||
info: dict[tuple, dict] = {}
|
||||
for r in rows:
|
||||
key = (r["event_key"], r["party_id"])
|
||||
tally[key][r["position"]] += 1
|
||||
info[key] = r
|
||||
for key, counts in tally.items():
|
||||
r = info[key]
|
||||
position, _ = counts.most_common(1)[0]
|
||||
party = r["short_name"] or r["name"]
|
||||
yield Claim(
|
||||
claim_type="vote", org=OrgRef(ids={"politrack_org": str(r["party_id"])}, create=False),
|
||||
counterpart_org=OrgRef(slug=r["chamber"]), occurred_at=r["voted_on"],
|
||||
summary_de=f"{party} stimmte mehrheitlich {'für' if position == 'for' else 'gegen'}: {r['item_title']} "
|
||||
f"(namentliche Abstimmung, {counts['for']} Ja, {counts['against']} Nein)",
|
||||
summary_en=f"{party} mostly voted {'for' if position == 'for' else 'against'}: {r['item_title']} "
|
||||
f"(roll call, {counts['for']} yes / {counts['against']} no)",
|
||||
data={"position": position, "roll_call": True, "yes": counts["for"], "no": counts["against"]},
|
||||
evidence=[Evidence(url=r["url"], source="parlament", doc_type="api_record", title=r["item_title"],
|
||||
published_at=r["voted_on"], locator=f"rollcall:{r['voted_on']}")],
|
||||
facet=VotePosition(event_key=r["event_key"], position=position),
|
||||
)
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Keyword-based topic tagging for claims that did not get topics from their source.
|
||||
|
||||
Parliament items carry official topics; news mentions, donations and posts
|
||||
are tagged here with the keyword lists of the `topic` table. An optional LLM
|
||||
classifier can replace this later (see docs/roadmap.md).
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
from politrack.core import Enricher, RunContext, register
|
||||
from politrack.core.spec import DERIVED
|
||||
|
||||
|
||||
def keyword_pattern(keywords: list[str]) -> re.Pattern:
|
||||
"""Keywords match as word prefixes ("Klima" -> "Klimaschutz"); short ones (<= 3 chars) only as whole words."""
|
||||
parts = [re.escape(k) + (r"\b" if len(k) <= 3 else "") for k in keywords]
|
||||
return re.compile(r"\b(" + "|".join(parts) + ")", re.IGNORECASE)
|
||||
|
||||
|
||||
@register
|
||||
class TopicTagger(Enricher):
|
||||
spec = DERIVED
|
||||
name = "enrich.topics"
|
||||
description = "Keyword topics for claims without official topics"
|
||||
schedule = "8-59/10 * * * *"
|
||||
|
||||
def run(self, ctx: RunContext) -> None:
|
||||
while (n := tag_claims(ctx.conn)):
|
||||
ctx.stats["claims_tagged"] += n
|
||||
return None
|
||||
|
||||
|
||||
def tag_claims(conn, batch: int = 2000) -> int:
|
||||
topics = [
|
||||
(row["id"], keyword_pattern(row["keywords"]))
|
||||
for row in conn.execute("SELECT id, keywords FROM topic WHERE cardinality(keywords) > 0")
|
||||
]
|
||||
claims = conn.execute(
|
||||
"""
|
||||
SELECT c.id, c.summary_de || ' ' || COALESCE(string_agg(d.title, ' '), '') AS text
|
||||
FROM claim c
|
||||
LEFT JOIN claim_evidence e ON e.claim_id = c.id
|
||||
LEFT JOIN document d ON d.id = e.document_id
|
||||
WHERE c.claim_type IN ('media_mention', 'statement', 'donation', 'side_job', 'appearance', 'meeting', 'travel',
|
||||
'money_flow', 'government_decision', 'wiki_edit', 'career')
|
||||
AND NOT (c.data ? 'topics_tagged')
|
||||
GROUP BY c.id ORDER BY c.id LIMIT %s
|
||||
""",
|
||||
(batch,),
|
||||
).fetchall()
|
||||
for claim in claims:
|
||||
for topic_id, pattern in topics:
|
||||
if pattern.search(claim["text"]):
|
||||
conn.execute(
|
||||
"INSERT INTO topic_assignment (topic_id, claim_id, confidence, method) "
|
||||
"VALUES (%s, %s, 0.5, 'keyword') ON CONFLICT DO NOTHING",
|
||||
(topic_id, claim["id"]),
|
||||
)
|
||||
conn.execute("UPDATE claim SET data = data || '{\"topics_tagged\": true}' WHERE id = %s", (claim["id"],))
|
||||
conn.commit()
|
||||
return len(claims)
|
||||
@@ -0,0 +1,189 @@
|
||||
"""MCP (Model Context Protocol) server so AI agents can query Politrack with citations.
|
||||
|
||||
Connect an MCP client (Claude Desktop/Code, etc.) to http://<host>:8001/mcp
|
||||
(streamable HTTP transport). All tools return evidence URLs; agents should cite them.
|
||||
Tools are thin wrappers around politrack.queries - the same functions the REST API uses.
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import date
|
||||
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
|
||||
from politrack import queries
|
||||
from politrack.db import connection
|
||||
from politrack.queries import SourceFilter
|
||||
|
||||
mcp = FastMCP(
|
||||
"politrack",
|
||||
instructions=(
|
||||
"Evidence-backed data on Austrian politicians and parties. Every claim has an `evidence` list of URLs: "
|
||||
"always cite them. status 'corroborated' = confirmed by at least two independent sources. "
|
||||
"Claims of type legal_proceeding: unless legal_stage is 'convicted_final' (and the claim is not "
|
||||
"'disputed'), the presumption of innocence applies - always state 'Es gilt die Unschuldsvermutung' / "
|
||||
"'presumed innocent'. Parliamentary votes in Austria are mostly recorded per faction; roll_call=true "
|
||||
"votes contain individual positions. Travel/meeting dates are publication dates of the source. "
|
||||
"Money flows (donations, public contracts, government ads, party funding) come from official registers; "
|
||||
"a flow to an organisation connected to a person is a connection, not an accusation. "
|
||||
"Social accounts marked verified=false come from Wikidata and were not confirmed by an official source."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _dump(rows) -> str:
|
||||
return json.dumps(rows, default=str, ensure_ascii=False)
|
||||
|
||||
|
||||
def _date(value: str | None) -> date | None:
|
||||
return date.fromisoformat(value) if value else None
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def search(query: str, limit: int = 10, since: str | None = None, until: str | None = None,
|
||||
exclude_sources: list[str] | None = None) -> str:
|
||||
"""Hybrid (keyword + semantic) search over all claims and documents. Dates as YYYY-MM-DD."""
|
||||
with connection() as conn:
|
||||
return _dump(queries.search(conn, query, sources=SourceFilter(exclude_sources or []), since=_date(since),
|
||||
until=_date(until), limit=limit))
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def find_person(name: str, status: str = "all") -> str:
|
||||
"""Find politicians by (part of) their name. status: active | former | party_member | deceased | all."""
|
||||
with connection() as conn:
|
||||
return _dump(queries.list_persons(conn, q=name, status=None if status == "all" else status, limit=20))
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def get_person(slug: str) -> str:
|
||||
"""Profile: activity status, roles/mandates over time (with sources), social accounts, claim counts."""
|
||||
with connection() as conn:
|
||||
return _dump(queries.get_person(conn, slug))
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def person_network(slug: str) -> str:
|
||||
"""Organisations connected to a person (side jobs, career, party, meetings, lobby contacts) with the
|
||||
money those organisations paid or received (donations, public contracts, government ads)."""
|
||||
with connection() as conn:
|
||||
return _dump(queries.person_network(conn, slug))
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def list_parties() -> str:
|
||||
"""All parties with number of currently active officials."""
|
||||
with connection() as conn:
|
||||
return _dump(queries.list_parties(conn))
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def get_party(slug: str) -> str:
|
||||
"""Party overview: current officials, former members, money per year, finance reports, votes, dissenters,
|
||||
affiliated organisations and holdings."""
|
||||
with connection() as conn:
|
||||
return _dump(queries.get_party(conn, slug))
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def find_organization(name: str, kind: str | None = None) -> str:
|
||||
"""Find organisations (companies, associations, public bodies, media, lobby groups) by name.
|
||||
kind: party | company | association | public_body | ministry | media | ngo | eu_institution | other."""
|
||||
with connection() as conn:
|
||||
return _dump(queries.list_organizations(conn, q=name, kind=kind, limit=20))
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def get_organization(slug: str) -> str:
|
||||
"""Organisation profile: identifiers (Firmenbuch, UID, EU transparency register), money paid/received by
|
||||
kind, top counterparties and connected people."""
|
||||
with connection() as conn:
|
||||
return _dump(queries.get_organization(conn, slug))
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def list_claims(
|
||||
person: str | None = None,
|
||||
party: str | None = None,
|
||||
organization: str | None = None,
|
||||
claim_types: list[str] | None = None,
|
||||
topics: list[str] | None = None,
|
||||
since: str | None = None,
|
||||
until: str | None = None,
|
||||
limit: int = 30,
|
||||
exclude_sources: list[str] | None = None,
|
||||
min_trust: int = 0,
|
||||
) -> str:
|
||||
"""Claims about a person/party/organisation, newest first. claim_types: role, vote, donation, money_flow,
|
||||
statement, appearance, meeting, travel, side_job, legal_proceeding, lobbying, media_mention, career,
|
||||
parliamentary_question, government_decision, wiki_edit, declaration, ownership, affiliation, party_finance.
|
||||
Dates as YYYY-MM-DD."""
|
||||
with connection() as conn:
|
||||
return _dump(queries.list_claims(
|
||||
conn, sources=SourceFilter(exclude_sources or [], min_trust), person=person, party=party,
|
||||
organization=organization, claim_types=claim_types, topics=topics, since=_date(since),
|
||||
until=_date(until), limit=limit))
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def money_flows(kinds: list[str] | None = None, organization: str | None = None, person: str | None = None,
|
||||
query: str | None = None, since: str | None = None, until: str | None = None,
|
||||
min_amount: float | None = None, limit: int = 30) -> str:
|
||||
"""Money flows from official registers. kinds: donation, contract, government_ad, political_ad, subsidy,
|
||||
party_funding, party_ad, loan, side_income, sponsorship. `person` = flows of organisations connected to
|
||||
that person. Dates as YYYY-MM-DD."""
|
||||
with connection() as conn:
|
||||
return _dump(queries.list_money(conn, sources=SourceFilter(), kinds=kinds, org=organization, person=person,
|
||||
q=query, since=_date(since), until=_date(until), min_amount=min_amount,
|
||||
limit=limit))
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def social_posts(person: str | None = None, party: str | None = None, platforms: list[str] | None = None,
|
||||
limit: int = 30) -> str:
|
||||
"""Public posts from politicians' accounts. platforms: bluesky, mastodon, telegram, youtube."""
|
||||
with connection() as conn:
|
||||
return _dump(queries.social_feed(conn, sources=SourceFilter(), person=person, party=party,
|
||||
platforms=platforms, limit=limit))
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def wikipedia_edits(limit: int = 50) -> str:
|
||||
"""Wikipedia edits made from IP ranges of parliament/government (anonymous edits only)."""
|
||||
with connection() as conn:
|
||||
return _dump(queries.wiki_edits(conn, limit=limit))
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def calendar(start: str, end: str, person: str | None = None, party: str | None = None) -> str:
|
||||
"""Events (sessions, hearings, council of ministers, trips, meetings) and per-day activity between two dates."""
|
||||
with connection() as conn:
|
||||
return _dump(queries.calendar(conn, sources=SourceFilter(), start=date.fromisoformat(start),
|
||||
end=date.fromisoformat(end), person=person, party=party))
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def person_votes(slug: str, topics: list[str] | None = None, limit: int = 50) -> str:
|
||||
"""Votes: individual roll-call votes (with dissent flag) and faction votes during the person's mandates.
|
||||
`spoke` links a speech by the person on the same item (say vs. vote)."""
|
||||
with connection() as conn:
|
||||
return _dump(queries.person_votes(conn, slug, topics=topics, limit=limit))
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def former_politicians(party: str | None = None, limit: int = 30) -> str:
|
||||
"""Former politicians with their latest news coverage and career entries."""
|
||||
with connection() as conn:
|
||||
return _dump(queries.former_overview(conn, party=party, limit=limit))
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
def list_topics() -> str:
|
||||
"""Topic taxonomy (slugs usable as filters)."""
|
||||
with connection() as conn:
|
||||
return _dump(queries.list_topics(conn))
|
||||
|
||||
|
||||
def serve(host: str, port: int) -> None:
|
||||
mcp.settings.host = host
|
||||
mcp.settings.port = port
|
||||
mcp.run(transport="streamable-http")
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Minimal forward-only SQL migration runner (files in backend/migrations)."""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
from psycopg import sql
|
||||
|
||||
from politrack.config import get_settings
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations"
|
||||
|
||||
|
||||
def migrate() -> None:
|
||||
settings = get_settings()
|
||||
with psycopg.connect(settings.database_url, autocommit=False) as conn:
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations ("
|
||||
" name TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT now())"
|
||||
)
|
||||
applied = {r[0] for r in conn.execute("SELECT name FROM schema_migrations")}
|
||||
for path in sorted(MIGRATIONS_DIR.glob("*.sql")):
|
||||
if path.name in applied:
|
||||
continue
|
||||
log.info("applying %s", path.name)
|
||||
conn.execute(path.read_text(encoding="utf-8"))
|
||||
conn.execute("INSERT INTO schema_migrations (name) VALUES (%s)", (path.name,))
|
||||
conn.commit()
|
||||
|
||||
if settings.reader_password:
|
||||
conn.execute(
|
||||
sql.SQL("ALTER ROLE politrack_reader PASSWORD {}").format(sql.Literal(settings.reader_password))
|
||||
)
|
||||
conn.commit()
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Generate TypeScript types from the OpenAPI schema (no npm needed).
|
||||
|
||||
Supports the JSON-Schema subset FastAPI/Pydantic emit: objects, arrays, enums/Literals, anyOf with null,
|
||||
$ref, additionalProperties, primitive formats. Output: one `export type X = {...}` per component schema
|
||||
plus `Paths` mapping GET endpoints to their response types.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
HEADER = "// Generated by `politrack openapi --ts` from the FastAPI schema. Do not edit by hand.\n\n"
|
||||
|
||||
|
||||
def ts_type(schema: dict[str, Any]) -> str:
|
||||
if "$ref" in schema:
|
||||
return schema["$ref"].rsplit("/", 1)[-1]
|
||||
if "const" in schema:
|
||||
return _literal(schema["const"])
|
||||
if "enum" in schema:
|
||||
return " | ".join(_literal(v) for v in schema["enum"])
|
||||
for key in ("anyOf", "oneOf"):
|
||||
if key in schema:
|
||||
parts = []
|
||||
for sub in schema[key]:
|
||||
part = ts_type(sub)
|
||||
if part not in parts:
|
||||
parts.append(part)
|
||||
return " | ".join(parts)
|
||||
kind = schema.get("type")
|
||||
if kind == "array":
|
||||
inner = ts_type(schema.get("items", {}))
|
||||
return f"({inner})[]" if "|" in inner else f"{inner}[]"
|
||||
if kind == "object":
|
||||
if "properties" in schema:
|
||||
return _object(schema)
|
||||
extra = schema.get("additionalProperties")
|
||||
value = ts_type(extra) if isinstance(extra, dict) and extra else "unknown"
|
||||
return f"Record<string, {value}>"
|
||||
if kind in ("integer", "number"):
|
||||
return "number"
|
||||
if kind == "boolean":
|
||||
return "boolean"
|
||||
if kind == "string":
|
||||
return "string"
|
||||
if kind == "null":
|
||||
return "null"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _literal(value: Any) -> str:
|
||||
return f"'{value}'" if isinstance(value, str) else ("null" if value is None else str(value).lower())
|
||||
|
||||
|
||||
def _object(schema: dict[str, Any], indent: str = "") -> str:
|
||||
required = set(schema.get("required", []))
|
||||
lines = ["{"]
|
||||
for name, prop in schema.get("properties", {}).items():
|
||||
optional = "" if name in required else "?"
|
||||
lines.append(f"{indent} {name}{optional}: {ts_type(prop)};")
|
||||
lines.append(f"{indent}}}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def to_typescript(openapi: dict[str, Any]) -> str:
|
||||
out = [HEADER]
|
||||
for name, schema in sorted(openapi.get("components", {}).get("schemas", {}).items()):
|
||||
if name in ("HTTPValidationError", "ValidationError"):
|
||||
continue
|
||||
out.append(f"export type {name} = {ts_type(schema)};\n")
|
||||
out.append("\nexport type Paths = {")
|
||||
for path, ops in sorted(openapi.get("paths", {}).items()):
|
||||
get = ops.get("get")
|
||||
if not get:
|
||||
continue
|
||||
content = get.get("responses", {}).get("200", {}).get("content", {}).get("application/json", {})
|
||||
out.append(f" '{path}': {ts_type(content.get('schema', {})) if content.get('schema') else 'unknown'};")
|
||||
out.append("};\n")
|
||||
return "\n".join(out)
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Read queries shared by the REST API and the MCP server (one module per domain)."""
|
||||
|
||||
from politrack.queries.admin import list_sources, list_topics, pending_claims, recent_runs, set_review, stats
|
||||
from politrack.queries.claims import calendar, list_claims, timeline_histogram
|
||||
from politrack.queries.common import SourceFilter
|
||||
from politrack.queries.money import list_money, money_summary, social_feed, wiki_edits
|
||||
from politrack.queries.organizations import get_organization, get_party, list_organizations, list_parties
|
||||
from politrack.queries.people import former_overview, get_person, list_persons, person_network
|
||||
from politrack.queries.search import search
|
||||
from politrack.queries.votes import list_votes, person_votes, vote_detail
|
||||
|
||||
__all__ = [
|
||||
"SourceFilter", "calendar", "former_overview", "get_organization", "get_party", "get_person", "list_claims",
|
||||
"list_money", "list_organizations", "list_parties", "list_persons", "list_sources", "list_topics", "list_votes",
|
||||
"money_summary", "pending_claims", "recent_runs", "person_network", "person_votes", "search", "set_review", "social_feed",
|
||||
"stats", "timeline_histogram", "vote_detail", "wiki_edits",
|
||||
]
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Sources, topics, statistics, moderation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from psycopg import Connection
|
||||
|
||||
|
||||
def list_sources(conn: Connection) -> list[dict]:
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT s.slug, s.name, s.kind, s.homepage, s.license, s.access_method, s.default_trust, s.tos_notes,
|
||||
(SELECT count(*) FROM document d WHERE d.source_id = s.id) AS documents
|
||||
FROM source s ORDER BY s.kind, s.default_trust DESC, s.name
|
||||
"""
|
||||
).fetchall()
|
||||
|
||||
|
||||
def list_topics(conn: Connection) -> list[dict]:
|
||||
return conn.execute("SELECT slug, name_de, name_en FROM topic ORDER BY name_de").fetchall()
|
||||
|
||||
|
||||
def pending_claims(conn: Connection, limit: int = 50) -> list[dict]:
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT c.id, c.claim_type, c.summary_de, c.occurred_at, p.full_name,
|
||||
(SELECT jsonb_agg(jsonb_build_object('url', d.url, 'title', d.title))
|
||||
FROM claim_evidence e JOIN document d ON d.id = e.document_id WHERE e.claim_id = c.id) AS evidence
|
||||
FROM claim c LEFT JOIN person p ON p.id = c.person_id
|
||||
WHERE c.review_status = 'pending' AND c.merged_into IS NULL
|
||||
ORDER BY c.occurred_at DESC NULLS LAST LIMIT %s
|
||||
""",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
|
||||
|
||||
def set_review(conn: Connection, claim_id: int, decision: str, reviewer: str) -> None:
|
||||
if decision not in ("published", "rejected"):
|
||||
raise ValueError("decision must be 'published' or 'rejected'")
|
||||
conn.execute(
|
||||
"UPDATE claim SET review_status = %s, reviewed_by = %s, reviewed_at = now() WHERE id = %s",
|
||||
(decision, reviewer, claim_id),
|
||||
)
|
||||
|
||||
|
||||
def stats(conn: Connection) -> dict:
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT (SELECT count(*) FROM person) AS persons,
|
||||
(SELECT count(*) FROM person_status WHERE status = 'active') AS active_persons,
|
||||
(SELECT count(*) FROM organization WHERE kind = 'party') AS parties,
|
||||
(SELECT count(*) FROM claim WHERE merged_into IS NULL AND review_status = 'published') AS claims,
|
||||
(SELECT count(*) FROM claim WHERE merged_into IS NULL AND status = 'corroborated') AS corroborated,
|
||||
(SELECT count(*) FROM claim WHERE review_status = 'pending') AS pending_review,
|
||||
(SELECT count(*) FROM document) AS documents,
|
||||
(SELECT count(*) FROM vote_event) AS vote_events,
|
||||
(SELECT count(*) FROM vote_event WHERE roll_call) AS roll_call_votes,
|
||||
(SELECT count(*) FROM event) AS events,
|
||||
(SELECT count(*) FROM money_flow) AS money_flows,
|
||||
(SELECT max(finished_at) FROM run WHERE status IN ('ok', 'partial')) AS last_update
|
||||
"""
|
||||
).fetchone()
|
||||
|
||||
|
||||
def recent_runs(conn: Connection, limit: int = 50) -> list[dict]:
|
||||
return conn.execute(
|
||||
"SELECT adapter, mode, status, started_at, finished_at, stats, left(error, 2000) AS error FROM run "
|
||||
"ORDER BY id DESC LIMIT %s",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
@@ -0,0 +1,157 @@
|
||||
"""Claims (the generic fact list), timeline histogram and calendar."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
from psycopg import Connection
|
||||
|
||||
from politrack.queries.common import VISIBLE, SourceFilter
|
||||
|
||||
|
||||
def list_claims(
|
||||
conn: Connection,
|
||||
*,
|
||||
sources: SourceFilter,
|
||||
person: str | None = None,
|
||||
party: str | None = None,
|
||||
organization: str | None = None,
|
||||
claim_types: list[str] | None = None,
|
||||
topics: list[str] | None = None,
|
||||
since: date | None = None,
|
||||
until: date | None = None,
|
||||
q: str | None = None,
|
||||
include_pending: bool = False,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> list[dict]:
|
||||
source_sql, params = sources.sql("s")
|
||||
params.update({
|
||||
"person": person, "party": party, "organization": organization, "claim_types": claim_types, "topics": topics, "since": since,
|
||||
"until": until, "q": q, "limit": limit, "offset": offset, "include_pending": include_pending,
|
||||
})
|
||||
return conn.execute(
|
||||
f"""
|
||||
SELECT c.id, c.claim_type, c.summary_de, c.summary_en, c.occurred_at, c.valid_from, c.valid_to, c.status,
|
||||
c.sensitive, c.review_status, c.data,
|
||||
p.slug AS person_slug, p.full_name AS person_name,
|
||||
o.slug AS organization_slug, o.short_name AS organization_short, o.name AS organization_name,
|
||||
lp.stage AS legal_stage, lp.offense AS legal_offense, lp.authority AS legal_authority,
|
||||
cp.slug AS counterpart_slug, cp.name AS counterpart_name, cp.kind AS counterpart_kind,
|
||||
cpp.slug AS counterpart_person_slug, cpp.full_name AS counterpart_person_name,
|
||||
to_jsonb(mf) - 'claim_id' AS money, st.platform AS platform,
|
||||
(SELECT jsonb_agg(jsonb_build_object('slug', t.slug, 'name_de', t.name_de, 'name_en', t.name_en))
|
||||
FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id WHERE ta.claim_id = c.id) AS topics,
|
||||
ev.evidence, ev.source_count
|
||||
FROM claim c
|
||||
LEFT JOIN person p ON p.id = c.person_id
|
||||
LEFT JOIN organization o ON o.id = c.organization_id
|
||||
LEFT JOIN legal_proceeding lp ON lp.claim_id = c.id
|
||||
LEFT JOIN organization cp ON cp.id = c.counterpart_org_id
|
||||
LEFT JOIN person cpp ON cpp.id = c.counterpart_person_id
|
||||
LEFT JOIN money_flow mf ON mf.claim_id = c.id
|
||||
LEFT JOIN statement st ON st.claim_id = c.id
|
||||
CROSS JOIN LATERAL (
|
||||
SELECT jsonb_agg(jsonb_build_object(
|
||||
'url', d.url, 'title', d.title, 'source', s.slug, 'source_name', s.name, 'source_kind', s.kind,
|
||||
'published_at', d.published_at, 'quote', e.quote, 'stance', e.stance, 'archive_url', d.archive_url)
|
||||
ORDER BY s.default_trust DESC, d.published_at) AS evidence,
|
||||
count(DISTINCT s.id) AS source_count
|
||||
FROM claim_evidence e JOIN document d ON d.id = e.document_id JOIN source s ON s.id = d.source_id
|
||||
WHERE e.claim_id = c.id AND {source_sql}
|
||||
) ev
|
||||
WHERE ev.evidence IS NOT NULL AND {VISIBLE}
|
||||
AND (%(person)s::text IS NULL OR p.slug = %(person)s OR cpp.slug = %(person)s)
|
||||
AND (%(party)s::text IS NULL OR o.slug = %(party)s
|
||||
OR c.counterpart_org_id = (SELECT id FROM organization WHERE slug = %(party)s))
|
||||
AND (%(organization)s::text IS NULL OR o.slug = %(organization)s
|
||||
OR c.counterpart_org_id = (SELECT id FROM organization WHERE slug = %(organization)s))
|
||||
AND (%(claim_types)s::text[] IS NULL OR c.claim_type = ANY(%(claim_types)s))
|
||||
AND (%(topics)s::text[] IS NULL OR c.claim_type IN ('role', 'career') OR EXISTS (
|
||||
SELECT 1 FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id
|
||||
WHERE ta.claim_id = c.id AND t.slug = ANY(%(topics)s)))
|
||||
AND (%(since)s::date IS NULL OR COALESCE(c.occurred_at::date, c.valid_to, c.valid_from, current_date) >= %(since)s)
|
||||
AND (%(until)s::date IS NULL OR COALESCE(c.occurred_at::date, c.valid_from) <= %(until)s)
|
||||
AND (%(q)s::text IS NULL OR c.summary_de ILIKE '%%' || %(q)s || '%%')
|
||||
ORDER BY COALESCE(c.occurred_at, c.valid_from::timestamptz, c.created_at) DESC, c.id DESC
|
||||
LIMIT %(limit)s OFFSET %(offset)s
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
|
||||
|
||||
def timeline_histogram(conn: Connection, *, sources: SourceFilter, person: str | None = None,
|
||||
party: str | None = None) -> list[dict]:
|
||||
"""Claim counts per month and type - drives the timeline overview."""
|
||||
source_sql, params = sources.sql("s")
|
||||
params.update({"person": person, "party": party, "include_pending": False})
|
||||
return conn.execute(
|
||||
f"""
|
||||
SELECT to_char(date_trunc('month', COALESCE(c.occurred_at, c.valid_from::timestamptz)), 'YYYY-MM') AS month,
|
||||
c.claim_type, count(DISTINCT c.id) AS count
|
||||
FROM claim c
|
||||
LEFT JOIN person p ON p.id = c.person_id
|
||||
LEFT JOIN person cpp ON cpp.id = c.counterpart_person_id
|
||||
LEFT JOIN organization o ON o.id = c.organization_id
|
||||
JOIN claim_evidence e ON e.claim_id = c.id JOIN document d ON d.id = e.document_id
|
||||
JOIN source s ON s.id = d.source_id
|
||||
WHERE {VISIBLE} AND {source_sql} AND COALESCE(c.occurred_at, c.valid_from::timestamptz) IS NOT NULL
|
||||
AND (%(person)s::text IS NULL OR p.slug = %(person)s OR cpp.slug = %(person)s)
|
||||
AND (%(party)s::text IS NULL OR o.slug = %(party)s)
|
||||
AND c.claim_type <> 'role'
|
||||
GROUP BY 1, 2 ORDER BY 1
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
|
||||
|
||||
def calendar(conn: Connection, *, sources: SourceFilter, start: date, end: date, person: str | None = None,
|
||||
party: str | None = None, kinds: list[str] | None = None) -> dict:
|
||||
"""Events (sessions, trips, meetings, appearances) plus per-day claim counts in [start, end]."""
|
||||
source_sql, params = sources.sql("s")
|
||||
params.update({"start": start, "end": end, "person": person, "party": party, "kinds": kinds,
|
||||
"include_pending": False})
|
||||
events = conn.execute(
|
||||
"""
|
||||
SELECT ev.id, ev.kind, ev.title, ev.starts_at, ev.ends_at, ev.location_name, ev.country, ev.url,
|
||||
ev.description, o.name AS body,
|
||||
COALESCE(jsonb_agg(DISTINCT jsonb_build_object('slug', p.slug, 'name', p.full_name))
|
||||
FILTER (WHERE p.id IS NOT NULL), '[]'::jsonb) AS participants
|
||||
FROM event ev
|
||||
LEFT JOIN organization o ON o.id = ev.organization_id
|
||||
LEFT JOIN event_participant ep ON ep.event_id = ev.id
|
||||
LEFT JOIN claim c ON c.id = ep.claim_id
|
||||
LEFT JOIN person p ON p.id = ep.person_id AND c.merged_into IS NULL
|
||||
WHERE ev.starts_at::date BETWEEN %(start)s AND %(end)s
|
||||
AND (%(kinds)s::text[] IS NULL OR ev.kind = ANY(%(kinds)s))
|
||||
AND (%(person)s::text IS NULL OR EXISTS (
|
||||
SELECT 1 FROM event_participant ep2 JOIN person p2 ON p2.id = ep2.person_id
|
||||
JOIN claim c2 ON c2.id = ep2.claim_id AND c2.merged_into IS NULL
|
||||
WHERE ep2.event_id = ev.id AND p2.slug = %(person)s))
|
||||
AND (%(party)s::text IS NULL OR EXISTS (
|
||||
SELECT 1 FROM event_participant ep3 JOIN person_status ps3 ON ps3.person_id = ep3.person_id
|
||||
JOIN organization o3 ON o3.id = ANY(ps3.current_party_ids)
|
||||
WHERE ep3.event_id = ev.id AND o3.slug = %(party)s))
|
||||
GROUP BY ev.id, o.name
|
||||
ORDER BY ev.starts_at
|
||||
LIMIT 2000
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
days = conn.execute(
|
||||
f"""
|
||||
SELECT c.occurred_at::date AS day, c.claim_type, count(DISTINCT c.id) AS count
|
||||
FROM claim c
|
||||
LEFT JOIN person p ON p.id = c.person_id
|
||||
LEFT JOIN person cpp ON cpp.id = c.counterpart_person_id
|
||||
LEFT JOIN organization o ON o.id = c.organization_id
|
||||
JOIN claim_evidence e ON e.claim_id = c.id JOIN document d ON d.id = e.document_id
|
||||
JOIN source s ON s.id = d.source_id
|
||||
WHERE {VISIBLE} AND {source_sql} AND c.occurred_at::date BETWEEN %(start)s AND %(end)s
|
||||
AND (%(person)s::text IS NULL OR p.slug = %(person)s OR cpp.slug = %(person)s)
|
||||
AND (%(party)s::text IS NULL OR o.slug = %(party)s)
|
||||
GROUP BY 1, 2 ORDER BY 1
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
return {"events": events, "days": days}
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Rules shared by all read queries.
|
||||
|
||||
* Source filtering happens at the evidence level: a claim is returned only if at least one document comes
|
||||
from an allowed source, and only allowed evidence is shown.
|
||||
* Claims merged into another claim (corroboration) are hidden; the survivor carries all evidence.
|
||||
* Claims pending human review are hidden unless include_pending is set (moderation).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
VISIBLE = ("c.merged_into IS NULL AND (c.review_status = 'published' OR "
|
||||
"(%(include_pending)s AND c.review_status = 'pending'))")
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceFilter:
|
||||
exclude_sources: list[str] = field(default_factory=list)
|
||||
min_trust: int = 0
|
||||
|
||||
def sql(self, alias: str = "s") -> tuple[str, dict[str, Any]]:
|
||||
return (
|
||||
f"(NOT ({alias}.slug = ANY(%(exclude_sources)s)) AND {alias}.default_trust >= %(min_trust)s)",
|
||||
{"exclude_sources": self.exclude_sources, "min_trust": self.min_trust},
|
||||
)
|
||||
|
||||
|
||||
def evidence_json(source_sql: str) -> str:
|
||||
"""Lateral subquery aggregating the allowed evidence of claim `c`."""
|
||||
return f"""
|
||||
CROSS JOIN LATERAL (
|
||||
SELECT jsonb_agg(jsonb_build_object(
|
||||
'url', d.url, 'title', d.title, 'source', s.slug, 'source_name', s.name, 'source_kind', s.kind,
|
||||
'published_at', d.published_at, 'quote', e.quote, 'stance', e.stance, 'archive_url', d.archive_url)
|
||||
ORDER BY s.default_trust DESC, d.published_at) AS evidence,
|
||||
count(DISTINCT s.id) AS source_count
|
||||
FROM claim_evidence e JOIN document d ON d.id = e.document_id JOIN source s ON s.id = d.source_id
|
||||
WHERE e.claim_id = c.id AND {source_sql}
|
||||
) ev"""
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Money flows (donations, contracts, government ads, party funding, political ads), social posts, wiki edits."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
from psycopg import Connection
|
||||
|
||||
from politrack.queries.common import VISIBLE, SourceFilter, evidence_json
|
||||
|
||||
|
||||
def list_money(conn: Connection, *, sources: SourceFilter, kinds: list[str] | None = None, org: str | None = None,
|
||||
person: str | None = None, q: str | None = None, since: date | None = None, until: date | None = None,
|
||||
min_amount: float | None = None, limit: int = 50, offset: int = 0) -> list[dict]:
|
||||
"""Money flows, filterable by kind, organisation (either side), connected person and period.
|
||||
|
||||
`person` = flows of organisations connected to that person (side jobs, careers, party) - the
|
||||
"follow the money" view.
|
||||
"""
|
||||
source_sql, params = sources.sql("s")
|
||||
params.update({"kinds": kinds, "org": org, "person": person, "q": q, "since": since, "until": until,
|
||||
"min_amount": min_amount, "limit": limit, "offset": offset, "include_pending": False})
|
||||
return conn.execute(
|
||||
f"""
|
||||
WITH person_orgs AS (
|
||||
SELECT c.counterpart_org_id AS org_id FROM claim c JOIN person p ON p.id = c.person_id
|
||||
WHERE p.slug = %(person)s AND c.counterpart_org_id IS NOT NULL AND c.merged_into IS NULL
|
||||
UNION SELECT r.party_id FROM role r JOIN person p ON p.id = r.person_id WHERE p.slug = %(person)s
|
||||
)
|
||||
SELECT c.id, c.summary_de, c.summary_en, c.occurred_at, c.status, c.data, mf.kind, mf.amount, mf.amount_category,
|
||||
mf.currency, mf.paid_on, mf.period, mf.reference, mf.category, mf.payer_name, mf.payee_name,
|
||||
pa.slug AS payer_slug, pa.kind AS payer_kind, pe.slug AS payee_slug, pe.kind AS payee_kind,
|
||||
ev.evidence, ev.source_count
|
||||
FROM money_flow mf
|
||||
JOIN claim c ON c.id = mf.claim_id
|
||||
LEFT JOIN organization pa ON pa.id = mf.payer_org_id
|
||||
LEFT JOIN organization pe ON pe.id = mf.payee_org_id
|
||||
{evidence_json(source_sql)}
|
||||
WHERE ev.evidence IS NOT NULL AND {VISIBLE}
|
||||
AND (%(kinds)s::text[] IS NULL OR mf.kind = ANY(%(kinds)s))
|
||||
AND (%(org)s::text IS NULL OR pa.slug = %(org)s OR pe.slug = %(org)s)
|
||||
AND (%(person)s::text IS NULL OR mf.payer_org_id IN (SELECT org_id FROM person_orgs)
|
||||
OR mf.payee_org_id IN (SELECT org_id FROM person_orgs))
|
||||
AND (%(q)s::text IS NULL OR mf.payer_name ILIKE '%%' || %(q)s || '%%' OR mf.payee_name ILIKE '%%' || %(q)s || '%%'
|
||||
OR c.summary_de ILIKE '%%' || %(q)s || '%%')
|
||||
AND (%(since)s::date IS NULL OR COALESCE(mf.paid_on, c.occurred_at::date, c.valid_from) >= %(since)s)
|
||||
AND (%(until)s::date IS NULL OR COALESCE(mf.paid_on, c.occurred_at::date, c.valid_from) <= %(until)s)
|
||||
AND (%(min_amount)s::numeric IS NULL OR mf.amount >= %(min_amount)s)
|
||||
ORDER BY COALESCE(mf.paid_on, c.occurred_at::date, c.valid_from) DESC NULLS LAST, mf.amount DESC NULLS LAST
|
||||
LIMIT %(limit)s OFFSET %(offset)s
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
|
||||
|
||||
def money_summary(conn: Connection, *, kinds: list[str] | None = None, since: date | None = None) -> list[dict]:
|
||||
"""Top payers/payees per kind (for the money overview page)."""
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT mf.kind, o.slug, o.name, o.kind AS org_kind,
|
||||
CASE WHEN o.id = mf.payer_org_id THEN 'payer' ELSE 'payee' END AS side,
|
||||
sum(mf.amount) AS total, count(*) AS count
|
||||
FROM money_flow mf JOIN claim c ON c.id = mf.claim_id AND c.merged_into IS NULL
|
||||
JOIN organization o ON o.id IN (mf.payer_org_id, mf.payee_org_id)
|
||||
WHERE (%(kinds)s::text[] IS NULL OR mf.kind = ANY(%(kinds)s))
|
||||
AND (%(since)s::date IS NULL OR COALESCE(mf.paid_on, c.valid_from) >= %(since)s)
|
||||
GROUP BY 1, 2, 3, 4, 5
|
||||
ORDER BY total DESC NULLS LAST LIMIT 100
|
||||
""",
|
||||
{"kinds": kinds, "since": since},
|
||||
).fetchall()
|
||||
|
||||
|
||||
def social_feed(conn: Connection, *, sources: SourceFilter, person: str | None = None, party: str | None = None,
|
||||
platforms: list[str] | None = None, limit: int = 50, offset: int = 0) -> list[dict]:
|
||||
"""Public posts of politicians and of party/institution channels (`party` = the party's own channels
|
||||
plus posts of its current members)."""
|
||||
source_sql, params = sources.sql("s")
|
||||
params.update({"person": person, "party": party, "platforms": platforms, "limit": limit, "offset": offset,
|
||||
"include_pending": False})
|
||||
return conn.execute(
|
||||
f"""
|
||||
SELECT c.id, st.platform, st.said_at, st.excerpt, st.url, st.title, c.data, p.slug AS person_slug,
|
||||
p.full_name AS person_name, o.slug AS org_slug, o.name AS org_name, ev.evidence
|
||||
FROM statement st JOIN claim c ON c.id = st.claim_id
|
||||
LEFT JOIN person p ON p.id = st.person_id
|
||||
LEFT JOIN organization o ON o.id = c.organization_id AND st.person_id IS NULL
|
||||
{evidence_json(source_sql)}
|
||||
WHERE st.context = 'social_post' AND ev.evidence IS NOT NULL AND {VISIBLE}
|
||||
AND (%(person)s::text IS NULL OR p.slug = %(person)s)
|
||||
AND (%(platforms)s::text[] IS NULL OR st.platform = ANY(%(platforms)s))
|
||||
AND (%(party)s::text IS NULL OR o.slug = %(party)s OR EXISTS (
|
||||
SELECT 1 FROM person_status ps JOIN organization po ON po.id = ANY(ps.current_party_ids)
|
||||
WHERE ps.person_id = p.id AND po.slug = %(party)s))
|
||||
ORDER BY st.said_at DESC NULLS LAST
|
||||
LIMIT %(limit)s OFFSET %(offset)s
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
|
||||
|
||||
def wiki_edits(conn: Connection, *, limit: int = 100, offset: int = 0) -> list[dict]:
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT c.id, c.occurred_at, c.summary_de, c.summary_en, c.data, o.slug AS org_slug, o.name AS org_name,
|
||||
p.slug AS person_slug, p.full_name AS person_name,
|
||||
(SELECT d.url FROM claim_evidence e JOIN document d ON d.id = e.document_id WHERE e.claim_id = c.id LIMIT 1) AS url
|
||||
FROM claim c JOIN organization o ON o.id = c.organization_id
|
||||
LEFT JOIN person p ON p.id = c.counterpart_person_id
|
||||
WHERE c.claim_type = 'wiki_edit' AND c.merged_into IS NULL
|
||||
ORDER BY c.occurred_at DESC NULLS LAST LIMIT %s OFFSET %s
|
||||
""",
|
||||
(limit, offset),
|
||||
).fetchall()
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Parties and organisations (companies, public bodies, media owners, lobby groups)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from psycopg import Connection
|
||||
|
||||
|
||||
def list_parties(conn: Connection) -> list[dict]:
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT o.slug, o.name, o.short_name, o.color, o.logo_url,
|
||||
(SELECT count(*) FROM person_status ps WHERE ps.status = 'active' AND o.id = ANY(ps.current_party_ids))
|
||||
AS active_officials,
|
||||
count(DISTINCT r.person_id) AS total_people
|
||||
FROM organization o LEFT JOIN role r ON r.party_id = o.id
|
||||
WHERE o.kind = 'party'
|
||||
GROUP BY o.id
|
||||
ORDER BY active_officials DESC, total_people DESC, o.name
|
||||
"""
|
||||
).fetchall()
|
||||
|
||||
|
||||
def get_party(conn: Connection, slug: str) -> dict | None:
|
||||
party = conn.execute(
|
||||
"SELECT id, slug, name, short_name, color, logo_url, kind FROM organization WHERE slug = %s", (slug,)
|
||||
).fetchone()
|
||||
if not party:
|
||||
return None
|
||||
pid = party["id"]
|
||||
party["members"] = conn.execute(
|
||||
"""
|
||||
SELECT DISTINCT ON (p.id) p.slug, p.full_name, p.photo_url, r.role, r.level, r.region, r.start_date,
|
||||
ch.name AS body
|
||||
FROM role r
|
||||
JOIN claim c ON c.id = r.claim_id AND c.merged_into IS NULL
|
||||
JOIN person p ON p.id = r.person_id
|
||||
JOIN person_status ps ON ps.person_id = p.id AND ps.status = 'active'
|
||||
LEFT JOIN organization ch ON ch.id = r.organization_id
|
||||
WHERE r.party_id = %s AND (r.end_date IS NULL OR r.end_date >= current_date) AND r.level <> 'party'
|
||||
AND r.role = ANY(ps.current_roles)
|
||||
ORDER BY p.id, CASE r.level WHEN 'eu' THEN 0 WHEN 'federal' THEN 1 WHEN 'state' THEN 2 ELSE 3 END
|
||||
""",
|
||||
(pid,),
|
||||
).fetchall()
|
||||
party["former_members"] = conn.execute(
|
||||
"""
|
||||
SELECT p.slug, p.full_name, ps.last_office_end,
|
||||
(SELECT max(c.occurred_at) FROM claim c WHERE c.person_id = p.id AND c.claim_type = 'media_mention')
|
||||
AS last_mention
|
||||
FROM person p JOIN person_status ps ON ps.person_id = p.id
|
||||
WHERE ps.status IN ('former', 'party_member') AND EXISTS (SELECT 1 FROM role r WHERE r.person_id = p.id AND r.party_id = %s)
|
||||
ORDER BY last_mention DESC NULLS LAST, ps.last_office_end DESC NULLS LAST LIMIT 50
|
||||
""",
|
||||
(pid,),
|
||||
).fetchall()
|
||||
party["money_by_year"] = conn.execute(
|
||||
"""
|
||||
SELECT COALESCE(extract(year FROM mf.paid_on)::int, left(mf.period, 4)::int) AS year, mf.kind,
|
||||
sum(mf.amount) AS total, count(*) AS count
|
||||
FROM money_flow mf JOIN claim c ON c.id = mf.claim_id AND c.merged_into IS NULL
|
||||
WHERE (mf.payee_org_id = %(id)s OR c.organization_id = %(id)s)
|
||||
AND mf.kind IN ('donation', 'sponsorship', 'party_ad', 'party_funding', 'loan')
|
||||
AND COALESCE(mf.paid_on::text, mf.period) ~ '^\\d{4}'
|
||||
GROUP BY 1, 2 ORDER BY 1 DESC, 2
|
||||
""",
|
||||
{"id": pid},
|
||||
).fetchall()
|
||||
party["finance"] = conn.execute(
|
||||
"""SELECT c.data, c.valid_from FROM claim c WHERE c.organization_id = %s AND c.claim_type = 'party_finance'
|
||||
AND c.merged_into IS NULL ORDER BY c.valid_from DESC NULLS LAST LIMIT 5""",
|
||||
(pid,),
|
||||
).fetchall()
|
||||
party["vote_stats"] = conn.execute(
|
||||
"SELECT vp.position, count(*) AS count FROM vote_position vp WHERE vp.organization_id = %s GROUP BY 1", (pid,)
|
||||
).fetchall()
|
||||
party["dissent"] = conn.execute(
|
||||
"""
|
||||
SELECT p.slug, p.full_name, count(*) AS dissenting_votes
|
||||
FROM vote_position vp
|
||||
JOIN person p ON p.id = vp.person_id
|
||||
JOIN vote_position fp ON fp.vote_event_id = vp.vote_event_id AND fp.organization_id = vp.party_id
|
||||
WHERE vp.party_id = %s AND vp.position <> fp.position
|
||||
GROUP BY p.id ORDER BY dissenting_votes DESC LIMIT 20
|
||||
""",
|
||||
(pid,),
|
||||
).fetchall()
|
||||
party["related"] = conn.execute(
|
||||
"""SELECT c.claim_type, o.slug, o.name, o.kind FROM claim c JOIN organization o ON o.id = c.counterpart_org_id
|
||||
WHERE c.organization_id = %s AND c.claim_type IN ('ownership', 'affiliation') AND c.merged_into IS NULL
|
||||
ORDER BY c.claim_type, o.name""",
|
||||
(pid,),
|
||||
).fetchall()
|
||||
return party
|
||||
|
||||
|
||||
def list_organizations(conn: Connection, *, q: str | None = None, kind: str | None = None, limit: int = 50,
|
||||
offset: int = 0) -> list[dict]:
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT o.slug, o.name, o.short_name, o.kind,
|
||||
(SELECT sum(amount) FROM money_flow WHERE payer_org_id = o.id) AS paid,
|
||||
(SELECT sum(amount) FROM money_flow WHERE payee_org_id = o.id) AS received
|
||||
FROM organization o
|
||||
WHERE (%(q)s::text IS NULL OR o.name ILIKE '%%' || %(q)s || '%%'
|
||||
OR EXISTS (SELECT 1 FROM alias a WHERE a.entity_type = 'organization' AND a.entity_id = o.id
|
||||
AND a.alias_norm LIKE '%%' || lower(unaccent(%(q)s)) || '%%'))
|
||||
AND (%(kind)s::text IS NULL OR o.kind = %(kind)s)
|
||||
ORDER BY COALESCE((SELECT sum(amount) FROM money_flow WHERE payer_org_id = o.id OR payee_org_id = o.id), 0) DESC,
|
||||
o.name
|
||||
LIMIT %(limit)s OFFSET %(offset)s
|
||||
""",
|
||||
{"q": q, "kind": kind, "limit": limit, "offset": offset},
|
||||
).fetchall()
|
||||
|
||||
|
||||
def get_organization(conn: Connection, slug: str) -> dict | None:
|
||||
org = conn.execute(
|
||||
"""SELECT o.id, o.slug, o.name, o.short_name, o.kind, parent.slug AS parent_slug, parent.name AS parent_name,
|
||||
(SELECT jsonb_object_agg(i.scheme, i.value) FROM identifier i
|
||||
WHERE i.entity_type = 'organization' AND i.entity_id = o.id) AS identifiers
|
||||
FROM organization o LEFT JOIN organization parent ON parent.id = o.parent_id WHERE o.slug = %s""",
|
||||
(slug,),
|
||||
).fetchone()
|
||||
if not org:
|
||||
return None
|
||||
oid = org["id"]
|
||||
org["money"] = conn.execute(
|
||||
"""
|
||||
SELECT mf.kind, CASE WHEN mf.payer_org_id = %(id)s THEN 'paid' ELSE 'received' END AS direction,
|
||||
sum(mf.amount) AS total, count(*) AS count, min(COALESCE(mf.paid_on::text, mf.period)) AS first,
|
||||
max(COALESCE(mf.paid_on::text, mf.period)) AS last
|
||||
FROM money_flow mf JOIN claim c ON c.id = mf.claim_id AND c.merged_into IS NULL
|
||||
WHERE mf.payer_org_id = %(id)s OR mf.payee_org_id = %(id)s
|
||||
GROUP BY 1, 2 ORDER BY 3 DESC NULLS LAST
|
||||
""",
|
||||
{"id": oid},
|
||||
).fetchall()
|
||||
org["counterparties"] = conn.execute(
|
||||
"""
|
||||
SELECT o.slug, o.name, o.kind, mf.kind AS money_kind,
|
||||
CASE WHEN mf.payer_org_id = %(id)s THEN 'paid_to' ELSE 'received_from' END AS direction,
|
||||
sum(mf.amount) AS total, count(*) AS count
|
||||
FROM money_flow mf JOIN claim c ON c.id = mf.claim_id AND c.merged_into IS NULL
|
||||
JOIN organization o ON o.id = CASE WHEN mf.payer_org_id = %(id)s THEN mf.payee_org_id ELSE mf.payer_org_id END
|
||||
WHERE mf.payer_org_id = %(id)s OR mf.payee_org_id = %(id)s
|
||||
GROUP BY 1, 2, 3, 4, 5 ORDER BY total DESC NULLS LAST LIMIT 50
|
||||
""",
|
||||
{"id": oid},
|
||||
).fetchall()
|
||||
org["people"] = conn.execute(
|
||||
"""
|
||||
SELECT DISTINCT ON (p.id, c.claim_type) p.slug, p.full_name, c.claim_type AS via, c.summary_de
|
||||
FROM claim c JOIN person p ON p.id = c.person_id
|
||||
WHERE c.counterpart_org_id = %(id)s AND c.merged_into IS NULL AND c.review_status = 'published'
|
||||
AND c.claim_type IN ('side_job', 'career', 'meeting', 'role', 'lobbying')
|
||||
ORDER BY p.id, c.claim_type LIMIT 100
|
||||
""",
|
||||
{"id": oid},
|
||||
).fetchall()
|
||||
return org
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Persons: lists, profiles, former politicians, connections (network)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from psycopg import Connection
|
||||
|
||||
|
||||
def list_persons(conn: Connection, *, q: str | None = None, party: str | None = None, level: str | None = None,
|
||||
status: str | None = "active", limit: int = 50, offset: int = 0) -> list[dict]:
|
||||
"""status: active | former | party_member | deceased | None (all)."""
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT p.slug, p.full_name, p.photo_url, ps.status, ps.verified AS status_verified, ps.last_office_end,
|
||||
ps.current_roles,
|
||||
(SELECT jsonb_agg(DISTINCT jsonb_build_object('slug', o.slug, 'short_name', o.short_name, 'color', o.color))
|
||||
FROM organization o WHERE o.id = ANY(ps.current_party_ids)) AS parties
|
||||
FROM person p
|
||||
JOIN person_status ps ON ps.person_id = p.id
|
||||
WHERE (%(status)s::text IS NULL OR ps.status = %(status)s)
|
||||
AND (%(q)s::text IS NULL OR p.full_name ILIKE '%%' || %(q)s || '%%'
|
||||
OR EXISTS (SELECT 1 FROM alias a WHERE a.entity_type = 'person' AND a.entity_id = p.id
|
||||
AND a.alias_norm LIKE '%%' || lower(unaccent(%(q)s)) || '%%'))
|
||||
AND (%(party)s::text IS NULL OR EXISTS (
|
||||
SELECT 1 FROM role r JOIN organization o ON o.id = r.party_id
|
||||
WHERE r.person_id = p.id AND o.slug = %(party)s
|
||||
AND (ps.status <> 'active' OR o.id = ANY(ps.current_party_ids))))
|
||||
AND (%(level)s::text IS NULL OR EXISTS (
|
||||
SELECT 1 FROM role r WHERE r.person_id = p.id AND r.level = %(level)s
|
||||
AND (ps.status <> 'active' OR r.role = ANY(ps.current_roles))))
|
||||
AND EXISTS (SELECT 1 FROM role r WHERE r.person_id = p.id)
|
||||
ORDER BY p.family_name NULLS LAST, p.full_name
|
||||
LIMIT %(limit)s OFFSET %(offset)s
|
||||
""",
|
||||
{"q": q, "party": party, "level": level, "status": status, "limit": limit, "offset": offset},
|
||||
).fetchall()
|
||||
|
||||
|
||||
def get_person(conn: Connection, slug: str) -> dict | None:
|
||||
person = conn.execute(
|
||||
"""
|
||||
SELECT p.id, p.slug, p.full_name, p.given_name, p.family_name, p.honorific_prefix, p.birth_date, p.death_date,
|
||||
p.photo_url, pi.wikidata_qid, pi.parlament_pad, pi.ep_id, p.summary_de, p.summary_en, ps.status,
|
||||
ps.verified AS status_verified, ps.last_office_end
|
||||
FROM person p JOIN person_status ps ON ps.person_id = p.id JOIN person_identifier pi ON pi.person_id = p.id
|
||||
WHERE p.slug = %s
|
||||
""",
|
||||
(slug,),
|
||||
).fetchone()
|
||||
if not person:
|
||||
return None
|
||||
person["roles"] = conn.execute(
|
||||
"""
|
||||
SELECT r.role, r.level, r.region, r.start_date, r.end_date, o.name AS body, o.slug AS body_slug,
|
||||
pa.slug AS party_slug, pa.short_name AS party_short, pa.color AS party_color, c.status AS claim_status,
|
||||
(SELECT jsonb_agg(DISTINCT jsonb_build_object('url', d.url, 'source', s.slug))
|
||||
FROM claim_evidence e JOIN document d ON d.id = e.document_id JOIN source s ON s.id = d.source_id
|
||||
WHERE e.claim_id = r.claim_id) AS evidence
|
||||
FROM role r
|
||||
JOIN claim c ON c.id = r.claim_id AND c.merged_into IS NULL
|
||||
LEFT JOIN organization o ON o.id = r.organization_id
|
||||
LEFT JOIN organization pa ON pa.id = r.party_id
|
||||
WHERE r.person_id = %s
|
||||
ORDER BY r.end_date DESC NULLS FIRST, r.start_date DESC NULLS LAST
|
||||
""",
|
||||
(person["id"],),
|
||||
).fetchall()
|
||||
person["social_accounts"] = conn.execute(
|
||||
"SELECT platform, handle, url, verified FROM social_account WHERE person_id = %s ORDER BY verified DESC, platform",
|
||||
(person["id"],),
|
||||
).fetchall()
|
||||
person["claim_counts"] = {
|
||||
r["claim_type"]: r["count"]
|
||||
for r in conn.execute(
|
||||
"SELECT claim_type, count(*) AS count FROM claim WHERE person_id = %s AND merged_into IS NULL "
|
||||
"AND review_status = 'published' GROUP BY claim_type",
|
||||
(person["id"],),
|
||||
)
|
||||
}
|
||||
return person
|
||||
|
||||
|
||||
def former_overview(conn: Connection, *, party: str | None = None, limit: int = 50, offset: int = 0) -> list[dict]:
|
||||
"""Former politicians, most recently in the news first ("Was macht eigentlich ...?")."""
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT p.slug, p.full_name, p.photo_url, ps.status, ps.last_office_end,
|
||||
(SELECT r.role FROM role r JOIN claim c ON c.id = r.claim_id AND c.merged_into IS NULL
|
||||
WHERE r.person_id = p.id AND r.level <> 'party' ORDER BY r.end_date DESC NULLS LAST LIMIT 1) AS last_role,
|
||||
latest.summary_de AS latest_news, latest.occurred_at AS latest_news_at, latest.url AS latest_news_url,
|
||||
career.summary_de AS latest_career
|
||||
FROM person p
|
||||
JOIN person_status ps ON ps.person_id = p.id
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT c.summary_de, c.occurred_at, d.url FROM claim c
|
||||
JOIN claim_evidence e ON e.claim_id = c.id JOIN document d ON d.id = e.document_id
|
||||
WHERE c.person_id = p.id AND c.claim_type IN ('media_mention', 'statement') AND c.merged_into IS NULL
|
||||
ORDER BY c.occurred_at DESC NULLS LAST LIMIT 1) latest ON true
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT c.summary_de FROM claim c WHERE c.person_id = p.id AND c.claim_type = 'career'
|
||||
AND c.merged_into IS NULL ORDER BY c.valid_from DESC NULLS LAST LIMIT 1) career ON true
|
||||
WHERE ps.status IN ('former', 'party_member')
|
||||
AND (%(party)s::text IS NULL OR EXISTS (
|
||||
SELECT 1 FROM role r JOIN organization o ON o.id = r.party_id
|
||||
WHERE r.person_id = p.id AND o.slug = %(party)s))
|
||||
AND (latest.occurred_at IS NOT NULL OR career.summary_de IS NOT NULL)
|
||||
ORDER BY latest.occurred_at DESC NULLS LAST, ps.last_office_end DESC NULLS LAST
|
||||
LIMIT %(limit)s OFFSET %(offset)s
|
||||
""",
|
||||
{"party": party, "limit": limit, "offset": offset},
|
||||
).fetchall()
|
||||
|
||||
|
||||
def person_network(conn: Connection, slug: str) -> dict:
|
||||
"""Organisations connected to a person and what flows through them.
|
||||
|
||||
Connections: employers/boards from side-job disclosures and careers, parties, orgs met (meetings).
|
||||
For each organisation: donations it made, contracts it won, ads/funding it received or paid.
|
||||
"""
|
||||
person = conn.execute("SELECT id FROM person WHERE slug = %s", (slug,)).fetchone()
|
||||
if not person:
|
||||
return {"organizations": []}
|
||||
orgs = conn.execute(
|
||||
"""
|
||||
WITH links AS (
|
||||
SELECT c.counterpart_org_id AS org_id, c.claim_type AS via, max(c.summary_de) AS label
|
||||
FROM claim c
|
||||
WHERE c.person_id = %(pid)s AND c.counterpart_org_id IS NOT NULL AND c.merged_into IS NULL
|
||||
AND c.review_status = 'published'
|
||||
AND c.claim_type IN ('side_job', 'career', 'meeting', 'role', 'lobbying')
|
||||
GROUP BY 1, 2
|
||||
UNION
|
||||
SELECT r.party_id, 'party', NULL FROM role r WHERE r.person_id = %(pid)s AND r.party_id IS NOT NULL
|
||||
)
|
||||
SELECT o.id, o.slug, o.name, o.kind, array_agg(DISTINCT l.via) AS via, max(l.label) AS label,
|
||||
(SELECT jsonb_object_agg(k, v) FROM (
|
||||
SELECT mf.kind || CASE WHEN mf.payer_org_id = o.id THEN '_paid' ELSE '_received' END AS k,
|
||||
jsonb_build_object('total', sum(mf.amount), 'count', count(*)) AS v
|
||||
FROM money_flow mf JOIN claim mc ON mc.id = mf.claim_id AND mc.merged_into IS NULL
|
||||
WHERE mf.payer_org_id = o.id OR mf.payee_org_id = o.id GROUP BY 1) x) AS money
|
||||
FROM links l JOIN organization o ON o.id = l.org_id
|
||||
GROUP BY o.id
|
||||
ORDER BY (o.kind = 'party') DESC, o.name
|
||||
""",
|
||||
{"pid": person["id"]},
|
||||
).fetchall()
|
||||
return {"organizations": orgs}
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Hybrid search (German full text + optional vector similarity)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
import httpx
|
||||
from psycopg import Connection
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.queries.common import VISIBLE, SourceFilter
|
||||
|
||||
|
||||
def search(conn: Connection, q: str, *, sources: SourceFilter, since: date | None = None, until: date | None = None,
|
||||
limit: int = 20) -> list[dict]:
|
||||
"""Hybrid search: German full-text + (if configured) vector similarity, fused by reciprocal rank."""
|
||||
settings = get_settings()
|
||||
source_sql, params = sources.sql("s")
|
||||
params.update({"q": q, "limit": limit * 3, "since": since, "until": until, "include_pending": False})
|
||||
base = f"""
|
||||
SELECT ch.id, ch.text, ch.claim_id, ch.document_id, c.summary_de, c.claim_type, c.occurred_at,
|
||||
p.slug AS person_slug, p.full_name AS person_name, o.slug AS organization_slug,
|
||||
dd.url AS document_url, dd.title AS document_title, dd.published_at AS document_date,
|
||||
(SELECT jsonb_agg(DISTINCT jsonb_build_object('url', d.url, 'title', d.title, 'source', s.slug))
|
||||
FROM claim_evidence e JOIN document d ON d.id = e.document_id JOIN source s ON s.id = d.source_id
|
||||
WHERE e.claim_id = ch.claim_id AND {source_sql}) AS evidence
|
||||
FROM chunk ch
|
||||
LEFT JOIN claim c ON c.id = ch.claim_id
|
||||
LEFT JOIN person p ON p.id = c.person_id
|
||||
LEFT JOIN organization o ON o.id = c.organization_id
|
||||
LEFT JOIN document dd ON dd.id = ch.document_id
|
||||
LEFT JOIN source ds ON ds.id = dd.source_id
|
||||
WHERE (ch.claim_id IS NULL OR ({VISIBLE}))
|
||||
AND (ch.document_id IS NULL OR (NOT (ds.slug = ANY(%(exclude_sources)s)) AND ds.default_trust >= %(min_trust)s))
|
||||
AND (%(since)s::date IS NULL OR COALESCE(c.occurred_at::date, c.valid_from, dd.published_at::date) >= %(since)s)
|
||||
AND (%(until)s::date IS NULL OR COALESCE(c.occurred_at::date, c.valid_from, dd.published_at::date) <= %(until)s)
|
||||
"""
|
||||
fts = conn.execute(
|
||||
base + " AND ch.tsv @@ websearch_to_tsquery('german', %(q)s)"
|
||||
" ORDER BY ts_rank_cd(ch.tsv, websearch_to_tsquery('german', %(q)s)) DESC LIMIT %(limit)s",
|
||||
params,
|
||||
).fetchall()
|
||||
ranked: dict[int, tuple[float, dict]] = {}
|
||||
for rank, row in enumerate(fts):
|
||||
ranked[row["id"]] = (1 / (60 + rank), row)
|
||||
|
||||
if settings.ollama_url:
|
||||
try:
|
||||
resp = httpx.post(f"{settings.ollama_url}/api/embed",
|
||||
json={"model": settings.embedding_model, "input": [q]}, timeout=30)
|
||||
resp.raise_for_status()
|
||||
params["vec"] = str(resp.json()["embeddings"][0])
|
||||
vec_rows = conn.execute(
|
||||
base + " AND ch.embedding IS NOT NULL ORDER BY ch.embedding <=> %(vec)s::vector LIMIT %(limit)s",
|
||||
params,
|
||||
).fetchall()
|
||||
for rank, row in enumerate(vec_rows):
|
||||
score = ranked.get(row["id"], (0.0, row))[0] + 1 / (60 + rank)
|
||||
ranked[row["id"]] = (score, row)
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
|
||||
results = [row | {"score": score} for score, row in sorted(ranked.values(), key=lambda x: -x[0])]
|
||||
# Hide results whose only evidence comes from sources the user excluded.
|
||||
return [r for r in results if r["evidence"] or r["claim_id"] is None][:limit]
|
||||
|
||||
|
||||
# -- moderation -----------------------------------------------------------------
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Votes: per person (individual roll calls + faction votes during mandates, with "say vs. vote"), lists, details."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
from psycopg import Connection
|
||||
|
||||
# Did the person speak on this item on the day of the vote? Speech titles are the item titles of the
|
||||
# parliament's speech list; vote_event titles come from the item page - similar, not identical.
|
||||
SPOKE = """
|
||||
(SELECT jsonb_build_object('url', st.url, 'title', st.title) FROM statement st
|
||||
JOIN claim sc ON sc.id = st.claim_id AND sc.merged_into IS NULL
|
||||
WHERE st.person_id = me.id AND st.context = 'parliament_speech' AND st.said_at::date = ve.voted_on
|
||||
AND similarity(st.title, ve.item_title) > 0.35
|
||||
ORDER BY similarity(st.title, ve.item_title) DESC LIMIT 1)
|
||||
"""
|
||||
|
||||
|
||||
def person_votes(conn: Connection, slug: str, *, topics: list[str] | None = None, limit: int = 100,
|
||||
offset: int = 0) -> list[dict]:
|
||||
"""Votes concerning a person.
|
||||
|
||||
* individual=true: roll-call votes with the person's own vote; dissent=true when it differs from the
|
||||
majority of their faction (vote_position.party_id = their faction that day).
|
||||
* individual=false: faction votes cast while the person held a mandate for that faction.
|
||||
* spoke: the person's speech on that item on the same day ("say vs. vote"), if any.
|
||||
"""
|
||||
return conn.execute(
|
||||
f"""
|
||||
WITH me AS (SELECT id FROM person WHERE slug = %(slug)s)
|
||||
SELECT ve.id, ve.item_title, ve.item_ref, ve.stage, ve.voted_on, ve.result, ve.url, ve.protocol_url,
|
||||
ve.roll_call, vp.position, fo.short_name AS faction, ch.name AS chamber, true AS individual,
|
||||
(fp.position IS NOT NULL AND fp.position <> vp.position) AS dissent, {SPOKE} AS spoke,
|
||||
(SELECT jsonb_agg(DISTINCT t.slug) FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id
|
||||
WHERE ta.claim_id = vp.claim_id) AS topics
|
||||
FROM vote_position vp
|
||||
JOIN me ON me.id = vp.person_id
|
||||
JOIN claim c ON c.id = vp.claim_id AND c.merged_into IS NULL
|
||||
JOIN vote_event ve ON ve.id = vp.vote_event_id
|
||||
JOIN organization ch ON ch.id = ve.chamber_id
|
||||
LEFT JOIN organization fo ON fo.id = vp.party_id
|
||||
LEFT JOIN vote_position fp ON fp.vote_event_id = ve.id AND fp.organization_id = vp.party_id
|
||||
WHERE (%(topics)s::text[] IS NULL OR EXISTS (
|
||||
SELECT 1 FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id
|
||||
WHERE ta.claim_id = vp.claim_id AND t.slug = ANY(%(topics)s)))
|
||||
UNION ALL
|
||||
SELECT ve.id, ve.item_title, ve.item_ref, ve.stage, ve.voted_on, ve.result, ve.url, ve.protocol_url,
|
||||
ve.roll_call, vp.position, o.short_name, ch.name, false, false, {SPOKE},
|
||||
(SELECT jsonb_agg(DISTINCT t.slug) FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id
|
||||
WHERE ta.claim_id = vp.claim_id)
|
||||
FROM me
|
||||
JOIN role r ON r.person_id = me.id AND r.role = 'mp'
|
||||
JOIN claim rc ON rc.id = r.claim_id AND rc.merged_into IS NULL
|
||||
JOIN vote_event ve ON ve.chamber_id = r.organization_id AND NOT ve.roll_call
|
||||
AND ve.voted_on BETWEEN r.start_date AND COALESCE(r.end_date, 'infinity'::date)
|
||||
JOIN vote_position vp ON vp.vote_event_id = ve.id AND vp.organization_id = r.party_id
|
||||
JOIN organization o ON o.id = vp.organization_id
|
||||
JOIN organization ch ON ch.id = ve.chamber_id
|
||||
WHERE (%(topics)s::text[] IS NULL OR EXISTS (
|
||||
SELECT 1 FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id
|
||||
WHERE ta.claim_id = vp.claim_id AND t.slug = ANY(%(topics)s)))
|
||||
ORDER BY voted_on DESC NULLS LAST
|
||||
LIMIT %(limit)s OFFSET %(offset)s
|
||||
""",
|
||||
{"slug": slug, "topics": topics, "limit": limit, "offset": offset},
|
||||
).fetchall()
|
||||
|
||||
|
||||
def list_votes(conn: Connection, *, party: str | None = None, topics: list[str] | None = None,
|
||||
roll_call: bool | None = None, since: date | None = None, until: date | None = None,
|
||||
limit: int = 50, offset: int = 0) -> list[dict]:
|
||||
return conn.execute(
|
||||
"""
|
||||
SELECT ve.id, ve.item_title, ve.item_ref, ve.stage, ve.voted_on, ve.result, ve.url, ve.protocol_url,
|
||||
ve.raw_text, ve.roll_call, ch.name AS chamber,
|
||||
COALESCE(jsonb_agg(jsonb_build_object('party', o.short_name, 'slug', o.slug, 'color', o.color,
|
||||
'position', vp.position) ORDER BY o.short_name)
|
||||
FILTER (WHERE o.id IS NOT NULL), '[]'::jsonb) AS positions,
|
||||
(SELECT count(*) FROM vote_position x WHERE x.vote_event_id = ve.id AND x.person_id IS NOT NULL)
|
||||
AS individual_votes
|
||||
FROM vote_event ve
|
||||
JOIN organization ch ON ch.id = ve.chamber_id
|
||||
LEFT JOIN vote_position vp ON vp.vote_event_id = ve.id AND vp.organization_id IS NOT NULL
|
||||
LEFT JOIN organization o ON o.id = vp.organization_id
|
||||
WHERE (%(party)s::text IS NULL OR EXISTS (
|
||||
SELECT 1 FROM vote_position v2 JOIN organization o2 ON o2.id = v2.organization_id
|
||||
WHERE v2.vote_event_id = ve.id AND o2.slug = %(party)s))
|
||||
AND (%(topics)s::text[] IS NULL OR EXISTS (
|
||||
SELECT 1 FROM vote_position v3 JOIN topic_assignment ta ON ta.claim_id = v3.claim_id
|
||||
JOIN topic t ON t.id = ta.topic_id WHERE v3.vote_event_id = ve.id AND t.slug = ANY(%(topics)s)))
|
||||
AND (%(roll_call)s::boolean IS NULL OR ve.roll_call = %(roll_call)s)
|
||||
AND (%(since)s::date IS NULL OR ve.voted_on >= %(since)s)
|
||||
AND (%(until)s::date IS NULL OR ve.voted_on <= %(until)s)
|
||||
GROUP BY ve.id, ch.name
|
||||
ORDER BY ve.voted_on DESC NULLS LAST, ve.id DESC
|
||||
LIMIT %(limit)s OFFSET %(offset)s
|
||||
""",
|
||||
{"party": party, "topics": topics, "roll_call": roll_call, "since": since, "until": until,
|
||||
"limit": limit, "offset": offset},
|
||||
).fetchall()
|
||||
|
||||
|
||||
def vote_detail(conn: Connection, vote_id: int) -> dict | None:
|
||||
vote = conn.execute(
|
||||
"SELECT ve.*, ch.name AS chamber FROM vote_event ve JOIN organization ch ON ch.id = ve.chamber_id WHERE ve.id = %s",
|
||||
(vote_id,),
|
||||
).fetchone()
|
||||
if not vote:
|
||||
return None
|
||||
vote["positions"] = conn.execute(
|
||||
"""SELECT o.short_name AS party, o.slug, o.color, vp.position FROM vote_position vp
|
||||
JOIN organization o ON o.id = vp.organization_id WHERE vp.vote_event_id = %s ORDER BY o.short_name""",
|
||||
(vote_id,),
|
||||
).fetchall()
|
||||
vote["individual"] = conn.execute(
|
||||
"""
|
||||
SELECT p.slug, p.full_name, vp.position, o.short_name AS faction,
|
||||
(fp.position IS NOT NULL AND fp.position <> vp.position) AS dissent
|
||||
FROM vote_position vp JOIN person p ON p.id = vp.person_id
|
||||
LEFT JOIN organization o ON o.id = vp.party_id
|
||||
LEFT JOIN vote_position fp ON fp.vote_event_id = vp.vote_event_id AND fp.organization_id = vp.party_id
|
||||
WHERE vp.vote_event_id = %s ORDER BY o.short_name, p.family_name
|
||||
""",
|
||||
(vote_id,),
|
||||
).fetchall()
|
||||
return vote
|
||||
@@ -0,0 +1,5 @@
|
||||
"""Built-in source adapters. Every module in this package (and sub-packages) is imported by
|
||||
politrack.core.registry.load_builtin(); modules register their adapters with @register.
|
||||
|
||||
To add a source: create one module here (see docs/contributing/adapters.md).
|
||||
"""
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Bundeskanzleramt: government news archive and Council of Ministers minutes (Ministerratsprotokolle).
|
||||
|
||||
Official works (§7 UrhG); robots.txt only blocks /401.
|
||||
|
||||
* News ("Nachrichten der Bundesregierung", one index per year): stored with full text; the event
|
||||
extractor turns summits, trips abroad and bilateral meetings into travel/meeting claims.
|
||||
* Council of Ministers minutes (Beschlussprotokolle): every agenda item names the reporting office(s)
|
||||
("Bericht der Bundesministerin für Justiz, Zahl ..., betreffend ..."). Each becomes a
|
||||
government_decision claim for the office holder on that date (resolved via the office hint).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Document, Event, Evidence, OrgRef, Participation, PersonRef
|
||||
from politrack.text import excerpt, fingerprint
|
||||
|
||||
BASE = "https://www.bundeskanzleramt.gv.at"
|
||||
NEWS_INDEX = BASE + "/bundeskanzleramt/nachrichten-der-bundesregierung/{year}.html"
|
||||
MR_INDEX = BASE + "/bundeskanzleramt/die-bundesregierung/ministerratsprotokolle.html"
|
||||
SPEC = SourceSpec(
|
||||
slug="bundesregierung",
|
||||
name="Bundeskanzleramt – Bundesregierung",
|
||||
kind="official",
|
||||
homepage=BASE,
|
||||
license="Amtliches Werk (§7 UrhG)",
|
||||
access="scrape",
|
||||
trust=90,
|
||||
min_interval=2.0,
|
||||
tos_notes="Government news archive and Council of Ministers minutes. robots.txt only blocks /401.",
|
||||
)
|
||||
|
||||
MONTHS = {m: i for i, m in enumerate(["jänner", "februar", "märz", "april", "mai", "juni", "juli", "august",
|
||||
"september", "oktober", "november", "dezember"], start=1)}
|
||||
MONTHS["januar"] = 1
|
||||
GERMAN_DATE = re.compile(r"(\d{1,2})\.\s*([A-Za-zäÄ]+)\s+(\d{4})")
|
||||
# Office names may contain commas ("für Wohnen, Kunst, Kultur, Medien und Sport"); file numbers sometimes use
|
||||
# non-breaking hyphens or spaces ("2026 0.040.056"); subjects span lines (joined with " | ") and end at the
|
||||
# decision ("Der Ministerrat beschließt ...") or at the next item.
|
||||
ITEM = re.compile(r"(?:(?:Gemeinsamer\s+)?Bericht|Vortrag|Antrag)\s+(?P<offices>de[sr]\s+[^|]+?),?\s+"
|
||||
r"Zahl\s+(?P<zahl>\d[\d./\-\u2010-\u2015 ]*?)\s*,\s*betreffend\s+(?P<subject>.+?)"
|
||||
r"(?=\s*\|?\s*Der Ministerrat\b"
|
||||
r"|\s*\|\s*(?:\d+(?:\.\d+)*\.?\s+)?(?:Gemeinsamer\s+)?(?:Bericht|Vortrag|Antrag)\s+de[sr]\s|$)")
|
||||
OFFICE_SPLIT = re.compile(r",\s*(?=de[sr]\s)|\s+und\s+(?=de[sr]\s)")
|
||||
GENITIVE = [(r"\bBundeskanzlers\b", "Bundeskanzler"), (r"\bVizekanzlers\b", "Vizekanzler"),
|
||||
(r"\bBundesministers\b", "Bundesminister"), (r"\bStaatssekretärs\b", "Staatssekretär")]
|
||||
|
||||
|
||||
def german_date(text: str) -> date | None:
|
||||
m = GERMAN_DATE.search(text)
|
||||
if not m or m.group(2).lower() not in MONTHS:
|
||||
return None
|
||||
return date(int(m.group(3)), MONTHS[m.group(2).lower()], int(m.group(1)))
|
||||
|
||||
|
||||
def period_start(path: str) -> int:
|
||||
"""'.../ministerratsprotokolle-der-xxvi-regierungsperiode-2017-2018.html' -> 2018 (latest year named)."""
|
||||
return max((int(y) for y in re.findall(r"(20\d\d)", path)), default=0)
|
||||
|
||||
|
||||
def offices(text: str) -> list[str]:
|
||||
"""'des Bundeskanzlers, des Bundesministers für Wohnen, Kunst, Kultur, Medien und Sport und der
|
||||
Bundesministerin für Justiz' -> ['Bundeskanzler', 'Bundesminister für Wohnen, ...', 'Bundesministerin für Justiz']."""
|
||||
result = []
|
||||
for part in OFFICE_SPLIT.split(text):
|
||||
part = re.sub(r"^\s*de[sr]\s+", "", part).strip(" ,")
|
||||
for pattern, repl in GENITIVE:
|
||||
part = re.sub(pattern, repl, part)
|
||||
if part:
|
||||
result.append(part)
|
||||
return result
|
||||
|
||||
|
||||
def main_text(response: Response) -> tuple[str, str]:
|
||||
soup = response.soup
|
||||
main = soup.find("main") or soup
|
||||
title_el = main.find(class_="title") or main.find("h1")
|
||||
title = title_el.get_text(" ", strip=True) if title_el else (soup.title.string if soup.title else response.url)
|
||||
for noise in main.find_all(["nav", "aside", "script", "style", "form", "footer"]):
|
||||
noise.decompose()
|
||||
parts = [p.get_text(" ", strip=True) for p in main.find_all(["p", "li", "h2", "h3"])]
|
||||
return title, "\n".join(p for p in parts if p)
|
||||
|
||||
|
||||
@register
|
||||
class GovernmentNews(Adapter):
|
||||
spec = SPEC
|
||||
name = "bundesregierung.news"
|
||||
description = "Government news archive (summits, trips, meetings of government members)"
|
||||
schedule = "10 */3 * * *"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
first = int(ctx.params.get("since_year", date.today().year - 1))
|
||||
for year in range(first, date.today().year + 1):
|
||||
yield Request(url=NEWS_INDEX.format(year=year), kind="index", meta={"year": year})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
if response.request.kind == "index":
|
||||
year = response.meta["year"]
|
||||
for path in sorted(set(re.findall(
|
||||
rf'href="(/bundeskanzleramt/nachrichten-der-bundesregierung/{year}/[^"]+\.html)"', response.text))):
|
||||
yield Request(url=BASE + path, kind="article")
|
||||
return
|
||||
title, text = main_text(response)
|
||||
time_el = response.soup.find("time")
|
||||
yield Document(Evidence(url=response.request.url, doc_type="press_release", title=title,
|
||||
published_at=time_el.get("datetime") if time_el else None,
|
||||
excerpt=excerpt(text, 300), full_text=f"{title}\n{text}"))
|
||||
|
||||
|
||||
@register
|
||||
class CouncilOfMinisters(Adapter):
|
||||
spec = SPEC
|
||||
name = "bundesregierung.ministerrat"
|
||||
description = "Council of Ministers minutes: decisions per reporting minister"
|
||||
schedule = "20 6 * * 3,4"
|
||||
parser_version = 2
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
yield Request(url=MR_INDEX, kind="index")
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
kind = response.request.kind
|
||||
if kind == "index":
|
||||
periods = sorted(set(re.findall(r'href="(/bundeskanzleramt/die-bundesregierung/ministerratsprotokolle/'
|
||||
r'[^"?]+\.html)"', response.text)), key=period_start)
|
||||
# Period pages are named after the year a government took office ("...-xxviii-2025") and list all
|
||||
# its sessions, so the latest period that started before `since_year` may still contain newer ones.
|
||||
since = int(ctx.params.get("since_year", date.today().year - 1))
|
||||
started_before = [period_start(p) for p in periods if period_start(p) < since]
|
||||
keep_from = max(started_before, default=0)
|
||||
for path in reversed([p for p in periods if period_start(p) >= keep_from]): # newest first
|
||||
yield Request(url=BASE + path, kind="period")
|
||||
elif kind == "period":
|
||||
base = response.request.url.split("?")[0]
|
||||
for href in sorted(set(re.findall(r'href="([^"]+)"', response.text))):
|
||||
url = urljoin(response.url, href)
|
||||
if url.startswith(base.removesuffix(".html") + "/"):
|
||||
yield Request(url=url, kind="protocol")
|
||||
elif url.startswith(base + "?currentPage="):
|
||||
yield Request(url=url, kind="period")
|
||||
else:
|
||||
yield from self.parse_protocol(response)
|
||||
|
||||
def parse_protocol(self, response: Response) -> Iterator[Any]:
|
||||
title, text = main_text(response)
|
||||
held_on = german_date(title) or german_date(text[:300])
|
||||
url = response.request.url
|
||||
yield Document(Evidence(url=url, doc_type="protocol", title=title, published_at=held_on,
|
||||
excerpt=excerpt(text, 300), full_text=f"{title}\n{text}"))
|
||||
event_key = fingerprint("ministerrat", url)
|
||||
yield Event(key=event_key, kind="government_meeting", title=title, starts_at=held_on, location="Wien",
|
||||
country="AT", url=url, body=OrgRef(slug="bundesregierung"))
|
||||
flat = " | ".join(text.split("\n"))
|
||||
for n, m in enumerate(ITEM.finditer(flat)):
|
||||
subject = re.sub(r"\s*\|\s*", " ", m.group("subject")).strip(" .")
|
||||
zahl = re.sub(r"[\s\u2010-\u2015]+", "-", m.group("zahl").strip())
|
||||
for office in offices(m.group("offices")):
|
||||
yield Claim(
|
||||
claim_type="government_decision",
|
||||
person=PersonRef(name=None, hints={"office": office, "on": held_on}),
|
||||
key=("ministerrat", url, zahl, office), occurred_at=held_on,
|
||||
summary_de=f"Bericht im Ministerrat ({office}): {excerpt(subject, 220)}",
|
||||
summary_en=f"Report to the Council of Ministers ({office}): {excerpt(subject, 220)}",
|
||||
data={"office": office, "zahl": zahl, "item": n + 1},
|
||||
evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=held_on,
|
||||
quote=excerpt(re.sub(r"\s*\|\s*", " ", m.group(0)), 500),
|
||||
locator=f"item:{zahl}")],
|
||||
)
|
||||
# Participation of the reporting ministers in this session (calendar)
|
||||
for office in offices(m.group("offices")):
|
||||
yield Claim(
|
||||
claim_type="appearance", person=PersonRef(hints={"office": office, "on": held_on}),
|
||||
occurred_at=held_on, summary_de=f"Teilnahme am Ministerrat ({title})",
|
||||
evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=held_on)],
|
||||
facet=Participation(event_key=event_key, participation="reporting_minister"),
|
||||
)
|
||||
@@ -0,0 +1,193 @@
|
||||
"""EU institutions: Commission lobby meetings and the European Parliament (Austrian MEPs, declarations).
|
||||
|
||||
* European Commission - meetings with interest representatives (open data XLSX per Commission term).
|
||||
Every row names the cabinet ("Cabinet of Commissioner Magnus Brunner"). Only meetings of Austrian
|
||||
Commissioners' cabinets are kept (names passed from start(): Commission roles in the DB + a fallback list).
|
||||
* European Parliament Open Data API v2 (CC BY 4.0): current Austrian MEPs (authoritative mandate data),
|
||||
and their declarations (financial interests, conflicts of interest, gifts, ...).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import (
|
||||
Claim,
|
||||
Event,
|
||||
Evidence,
|
||||
OrgRef,
|
||||
Participation,
|
||||
PersonRecord,
|
||||
PersonRef,
|
||||
Role,
|
||||
)
|
||||
from politrack.text import excerpt, fingerprint, normalize
|
||||
|
||||
EC_SPEC = SourceSpec(
|
||||
slug="ec-meetings", name="Europäische Kommission – Treffen mit Interessenvertretern", kind="official",
|
||||
homepage="https://commission.europa.eu/about/service-standards-and-principles/transparency_en",
|
||||
license="EU open data (reuse permitted, attribution)", access="bulk", trust=90, min_interval=2.0,
|
||||
tos_notes="Open data export of meetings of Commissioners and their cabinets with interest representatives.",
|
||||
)
|
||||
EP_SPEC = SourceSpec(
|
||||
slug="europarl", name="Europäisches Parlament Open Data", kind="official",
|
||||
homepage="https://data.europarl.europa.eu", license="CC BY 4.0", access="api", trust=95, min_interval=1.0,
|
||||
tos_notes="EP Open Data API v2 (limit 500 requests / 5 min / endpoint).",
|
||||
)
|
||||
EC_EXPORT = "https://ec.europa.eu/transparency-initiative/meetings/data/meetings/dataxlsx"
|
||||
EC_TERMS = ("meetingscommissionrepresentatives2429", "meetingscommissionrepresentatives1924")
|
||||
EP_API = "https://data.europarl.europa.eu/api/v2"
|
||||
# Austrian Commissioners in the terms covered by EC_TERMS (fallback if the DB has no Commission roles yet).
|
||||
AUSTRIAN_COMMISSIONERS = ("Johannes Hahn", "Magnus Brunner")
|
||||
CABINET = re.compile(r"Cabinet of (?:Executive Vice-President|Vice-President|Commissioner|President)\s+(?P<name>.+)$")
|
||||
|
||||
|
||||
@register
|
||||
class CommissionMeetings(Adapter):
|
||||
spec = EC_SPEC
|
||||
name = "eu.commission_meetings"
|
||||
description = "Meetings of (Austrian) EU Commissioners and their cabinets with interest representatives"
|
||||
schedule = "0 5 * * 1"
|
||||
queue = "bulk"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
names = set(AUSTRIAN_COMMISSIONERS)
|
||||
if ctx.conn is not None:
|
||||
names |= {row["full_name"] for row in ctx.conn.execute(
|
||||
"""SELECT DISTINCT p.full_name FROM role r JOIN person p ON p.id = r.person_id
|
||||
JOIN organization o ON o.id = r.organization_id WHERE o.slug = 'europaeische-kommission'""")}
|
||||
for term in EC_TERMS:
|
||||
yield Request(url=EC_EXPORT, params={"name": term}, kind="xlsx", meta={"commissioners": sorted(names)})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
import openpyxl
|
||||
|
||||
wb = openpyxl.load_workbook(io.BytesIO(response.body), read_only=True, data_only=True)
|
||||
rows = list(wb.worksheets[0].iter_rows(values_only=True))
|
||||
header_index = next((i for i, r in enumerate(rows) if r and r[0] == "Name of cabinet"), None)
|
||||
if header_index is None:
|
||||
ctx.stats["unexpected_layout"] += 1
|
||||
return
|
||||
header = [str(h) for h in rows[header_index]]
|
||||
url = f"{EC_EXPORT}?name={response.request.params['name']}"
|
||||
wanted = {normalize(n) for n in response.meta.get("commissioners") or AUSTRIAN_COMMISSIONERS}
|
||||
for row in rows[header_index + 1:]:
|
||||
rec = dict(zip(header, row))
|
||||
cabinet = rec.get("Name of cabinet") or ""
|
||||
m = CABINET.search(cabinet.strip())
|
||||
if not m or not rec.get("Date of meeting"):
|
||||
continue
|
||||
commissioner = m.group("name").strip()
|
||||
if normalize(commissioner) not in wanted:
|
||||
ctx.stats["other_commissioners"] += 1
|
||||
continue
|
||||
org_name = (rec.get("Name of interest representative") or "").strip()
|
||||
held = str(rec["Date of meeting"])[:10]
|
||||
subject = (rec.get("Subject of the meeting") or "").strip()
|
||||
reps = rec.get("Name of EC representative") or ""
|
||||
titles = rec.get("Title of EC representative") or ""
|
||||
register_id = (rec.get("Transparency register ID") or "").strip()
|
||||
event_key = fingerprint("ec-meeting", cabinet, held, org_name, subject)
|
||||
yield Event(key=event_key, kind="meeting", title=f"{cabinet}: {org_name}", starts_at=held,
|
||||
location=rec.get("Location"), url=url, description=subject,
|
||||
body=OrgRef(slug="europaeische-kommission"))
|
||||
yield Claim(
|
||||
claim_type="meeting", person=PersonRef(name=commissioner), occurred_at=held,
|
||||
counterpart_org=OrgRef(name=org_name, kind="lobby",
|
||||
ids={"eu_transparency": register_id} if register_id else {}) if org_name else None,
|
||||
summary_de=f"Treffen ({'Kommissar persönlich' if 'Commissioner' in titles else 'Kabinett'}) mit "
|
||||
f"{org_name}: {excerpt(subject, 160)}",
|
||||
summary_en=f"Meeting ({'Commissioner' if 'Commissioner' in titles else 'cabinet'}) with {org_name}: "
|
||||
f"{excerpt(subject, 160)}",
|
||||
data={"cabinet": cabinet, "representatives": reps, "titles": titles, "location": rec.get("Location"),
|
||||
"register_id": register_id, "date_precision": "day"},
|
||||
evidence=[Evidence(url=url, doc_type="dataset_row", title=f"EU-Kommission: Treffen {cabinet}",
|
||||
published_at=held, quote=f"{held} | {org_name} | {subject}",
|
||||
locator=f"{held}:{org_name}:{subject[:60]}")],
|
||||
topics=("Europäische Union",),
|
||||
facet=Participation(event_key=event_key, participation="host"),
|
||||
)
|
||||
|
||||
|
||||
@register
|
||||
class EuropeanParliament(Adapter):
|
||||
spec = EP_SPEC
|
||||
name = "europarl.meps"
|
||||
description = "Current Austrian MEPs (mandates) and their declarations"
|
||||
schedule = "30 4 * * 2"
|
||||
creates_persons = True
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
yield Request(url=f"{EP_API}/meps/show-current", kind="meps",
|
||||
params={"country-of-representation": "AT", "format": "application/ld+json", "limit": 100})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
data = response.json().get("data") or []
|
||||
if response.request.kind == "meps":
|
||||
for mep in data:
|
||||
mep_id = str(mep.get("identifier"))
|
||||
given, family = mep.get("givenName"), mep.get("familyName")
|
||||
name = f"{given} {family}" if given and family else mep.get("label")
|
||||
ref = PersonRef(ids={"ep": mep_id}, name=name)
|
||||
yield PersonRecord(ref=ref, full_name=name, given_name=given, family_name=family,
|
||||
aliases=(mep.get("label") or "",), match_by_name=True)
|
||||
yield Request(url=f"{EP_API}/meps/{mep_id}", params={"format": "application/ld+json"}, kind="mep",
|
||||
meta={"mep_id": mep_id, "name": name, "group": mep.get("api:political-group")})
|
||||
yield Request(url=f"{EP_API}/meps-declarations", kind="declarations",
|
||||
params={"person-id": mep_id, "format": "application/ld+json", "limit": 100},
|
||||
meta={"mep_id": mep_id, "name": name})
|
||||
elif response.request.kind == "mep":
|
||||
yield from self.parse_mep(response, data[0] if data else {})
|
||||
else:
|
||||
yield from self.parse_declarations(response, data)
|
||||
|
||||
def parse_mep(self, response: Response, mep: dict) -> Iterator[Any]:
|
||||
meta = response.meta
|
||||
ref = PersonRef(ids={"ep": meta["mep_id"]})
|
||||
if mep.get("img"):
|
||||
yield PersonRecord(ref=ref, full_name=meta["name"], photo_url=mep["img"],
|
||||
birth_date=date.fromisoformat(mep["bday"][:10]) if mep.get("bday") else None)
|
||||
starts = [m.get("memberDuring", {}).get("startDate") for m in mep.get("hasMembership") or []
|
||||
if "EU_PARLIAMENT" in str(m.get("membershipClassification", ""))]
|
||||
starts = sorted(s for s in starts if s)
|
||||
body = OrgRef(slug="europaeisches-parlament")
|
||||
start = date.fromisoformat(starts[-1]) if starts else None
|
||||
url = f"https://www.europarl.europa.eu/meps/de/{meta['mep_id']}"
|
||||
yield Claim(
|
||||
claim_type="role", person=ref, org=body, valid_from=start,
|
||||
summary_de=f"Mitglied des Europäischen Parlaments (Fraktion {meta.get('group') or '?'})",
|
||||
summary_en=f"Member of the European Parliament ({meta.get('group') or '?'})",
|
||||
data={"ep_group": meta.get("group")},
|
||||
evidence=[Evidence(url=url, doc_type="api_record", title=f"Europäisches Parlament: {meta['name']}")],
|
||||
facet=Role(role="mp", level="eu", body=body, start=start),
|
||||
)
|
||||
|
||||
def parse_declarations(self, response: Response, docs: list[dict]) -> Iterator[Claim]:
|
||||
meta = response.meta
|
||||
for doc in docs:
|
||||
titles = []
|
||||
pdf = None
|
||||
for expression in doc.get("is_realized_by") or []:
|
||||
title = (expression.get("title") or {})
|
||||
titles.extend(v for v in title.values() if v)
|
||||
for manifestation in expression.get("is_embodied_by") or []:
|
||||
if "pdf" in str(manifestation.get("media_type", "")).lower() and not pdf:
|
||||
pdf = manifestation.get("is_exemplified_by")
|
||||
title = next((t for t in titles if t), doc.get("id"))
|
||||
url = f"https://www.europarl.europa.eu/{pdf}" if pdf and not pdf.startswith("http") else (
|
||||
pdf or f"https://data.europarl.europa.eu/{doc.get('id')}")
|
||||
kind = str(doc.get("work_type", "")).rsplit("/", 1)[-1]
|
||||
yield Claim(
|
||||
claim_type="declaration", person=PersonRef(ids={"ep": meta["mep_id"]}),
|
||||
key=("ep-declaration", doc.get("id")), occurred_at=doc.get("document_date"),
|
||||
summary_de=f"Erklärung im Europäischen Parlament: {excerpt(title, 200)}",
|
||||
summary_en=f"Declaration filed with the European Parliament: {excerpt(title, 200)}",
|
||||
data={"work_type": kind},
|
||||
evidence=[Evidence(url=url, doc_type="pdf", title=title, published_at=doc.get("document_date"))],
|
||||
topics=("Transparenz und Korruption",),
|
||||
)
|
||||
@@ -0,0 +1,70 @@
|
||||
"""GDELT DOC 2.0 API: news coverage of politicians across many more outlets (metadata only).
|
||||
|
||||
Free, open, max. one request every 5 s (we use 6 s and back off on the plain-text throttle notice).
|
||||
Coverage of German-language Austrian media is thin, so this is a supplement. Queries: active
|
||||
federal/EU politicians daily, former politicians in a weekly rotation ("what are they up to?").
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Document, Evidence
|
||||
|
||||
API = "https://api.gdeltproject.org/api/v2/doc/doc"
|
||||
SPEC = SourceSpec(
|
||||
slug="gdelt", name="Weitere Medien (via GDELT)", kind="aggregator", homepage="https://www.gdeltproject.org",
|
||||
license="GDELT open data (citation required)", access="api", trust=55, store_fulltext=False, keep_raw=False,
|
||||
min_interval=6.0, tos_notes="GDELT DOC 2.0 API: article metadata (URL, title, date). Max 1 request / 5 s.",
|
||||
)
|
||||
DOMAIN_SOURCES = {
|
||||
"orf.at": "orf", "derstandard.at": "derstandard", "diepresse.com": "diepresse", "kurier.at": "kurier",
|
||||
"profil.at": "profil", "falter.at": "falter", "krone.at": "krone", "sn.at": "sn", "kleinezeitung.at": "kleinezeitung",
|
||||
"ots.at": "apa-ots", "vol.at": "vol", "vienna.at": "vol",
|
||||
}
|
||||
|
||||
|
||||
@register
|
||||
class Gdelt(Adapter):
|
||||
spec = SPEC
|
||||
name = "gdelt.coverage"
|
||||
description = "Supplementary news coverage via GDELT (active and former politicians)"
|
||||
schedule = "0 1 * * *"
|
||||
queue = "bulk"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
if ctx.conn is None:
|
||||
return
|
||||
rows = ctx.conn.execute(
|
||||
"""SELECT p.id, p.given_name, p.family_name, s.status FROM person p
|
||||
JOIN person_status s ON s.person_id = p.id
|
||||
WHERE p.given_name IS NOT NULL AND p.family_name IS NOT NULL AND (
|
||||
(s.status = 'active' AND EXISTS (SELECT 1 FROM role r WHERE r.person_id = p.id
|
||||
AND r.level IN ('federal', 'eu') AND r.end_date IS NULL))
|
||||
OR (s.status IN ('former', 'party_member') AND s.last_office_end > current_date - interval '25 years'
|
||||
AND p.id %% 7 = %s))
|
||||
ORDER BY s.status, p.id LIMIT %s""",
|
||||
(date.today().weekday(), int(ctx.params.get("max_queries", 400))),
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
name = f"{row['given_name']} {row['family_name']}"
|
||||
yield Request(url=API, kind="artlist", meta={"person_id": row["id"]}, conditional=False,
|
||||
params={"query": f'"{name}" sourcecountry:AU', "mode": "artlist", "format": "json",
|
||||
"maxrecords": 50, "timespan": "7d" if row["status"] == "active" else "1m",
|
||||
"sort": "datedesc"})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
if not response.text.strip().startswith("{"):
|
||||
ctx.stats["throttled"] += 1 # plain-text rate-limit notice
|
||||
return
|
||||
for article in json.loads(response.text, strict=False).get("articles", []):
|
||||
domain = (article.get("domain") or "").removeprefix("www.")
|
||||
seen = article.get("seendate") or ""
|
||||
published = f"{seen[:4]}-{seen[4:6]}-{seen[6:8]}T{seen[9:11]}:{seen[11:13]}:00Z" if len(seen) >= 13 else None
|
||||
yield Document(Evidence(url=article["url"], source=DOMAIN_SOURCES.get(domain, "gdelt"),
|
||||
title=article.get("title"), doc_type="article", published_at=published,
|
||||
meta={"via": "gdelt", "domain": domain}))
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Press releases of the Wirtschafts- und Korruptionsstaatsanwaltschaft (WKStA, anti-corruption prosecutors).
|
||||
|
||||
Official statements on high-profile proceedings. They usually anonymise suspects ("ein ehemaliger
|
||||
Bundesminister"), so we store them as official documents (full text, searchable, RAG) and let the
|
||||
legal-matter detector propose claims - which, as always for legal matters, need human review.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Document, Evidence
|
||||
from politrack.sources.bundesregierung import german_date
|
||||
from politrack.text import excerpt
|
||||
|
||||
BASE = "https://www.justiz.gv.at"
|
||||
START = BASE + "/wksta/wirtschafts-und-korruptionsstaatsanwaltschaft.312.de.html"
|
||||
SPEC = SourceSpec(
|
||||
slug="wksta", name="WKStA – Pressemitteilungen", kind="official", homepage=START,
|
||||
license="Amtliches Werk (§7 UrhG)", access="scrape", trust=90, min_interval=3.0,
|
||||
tos_notes="Press releases of the anti-corruption prosecutors. robots.txt allows all.",
|
||||
)
|
||||
RELEASE = re.compile(r"/medienstelle/pressemitteilungen/pressemitteilungen-(\d{4})/")
|
||||
LISTING = re.compile(r"/medienstelle(?:/pressemitteilungen(?:/pressemitteilungen-\d{4})?)?\.[0-9a-f]+\.de\.html")
|
||||
|
||||
|
||||
def clean(url: str) -> str:
|
||||
return re.sub(r";jsessionid=[^?#]*", "", url)
|
||||
|
||||
|
||||
@register
|
||||
class WkstaPress(Adapter):
|
||||
spec = SPEC
|
||||
name = "justiz.wksta"
|
||||
description = "WKStA press releases (official documents for search and reviewed legal matters)"
|
||||
schedule = "40 8 * * *"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
yield Request(url=START, kind="listing", meta={"depth": 0})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
if response.request.kind == "listing":
|
||||
since = int(ctx.params.get("since_year", date.today().year - 2))
|
||||
for a in response.soup.find_all("a", href=True):
|
||||
url = clean(urljoin(response.url, a["href"]))
|
||||
year = RELEASE.search(url)
|
||||
if year and int(year.group(1)) >= since:
|
||||
yield Request(url=url, kind="release")
|
||||
elif LISTING.search(url) and response.meta["depth"] < 2:
|
||||
yield Request(url=url, kind="listing", meta={"depth": response.meta["depth"] + 1})
|
||||
return
|
||||
main = response.soup.find("main") or response.soup.find(id="content") or response.soup
|
||||
title_el = main.find("h1")
|
||||
title = title_el.get_text(" ", strip=True) if title_el else response.url
|
||||
text = "\n".join(p.get_text(" ", strip=True) for p in main.find_all(["p", "li"]) if p.get_text(strip=True))
|
||||
yield Document(Evidence(url=clean(response.request.url), doc_type="press_release", title=title,
|
||||
published_at=german_date(text[:400]) or german_date(title), excerpt=excerpt(text, 300),
|
||||
full_text=f"{title}\n{text}"))
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Headline monitoring via publishers' RSS feeds and Google-News sitemaps (listed in robots.txt).
|
||||
|
||||
Policy (docs/sources.md): only feeds/sitemaps publishers offer for automated discovery; store headline,
|
||||
teaser/keywords and link - never article text (store_fulltext=False, keep_raw=False). If a site declares
|
||||
a TDM reservation, the Loader drops even the teaser and linking uses the headline only.
|
||||
|
||||
The feed list lives in config/feeds.yaml (user-editable); every `source` there needs a SourceSpec below.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime, timezone
|
||||
from time import mktime
|
||||
from typing import Any
|
||||
|
||||
import feedparser
|
||||
import yaml
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register, register_spec
|
||||
from politrack.core.records import Document, Evidence
|
||||
from politrack.text import html_to_text
|
||||
|
||||
|
||||
def media(slug: str, name: str, homepage: str, trust: int, notes: str = "RSS/news sitemap only; headline + link.",
|
||||
license: str = "All rights reserved") -> SourceSpec:
|
||||
return register_spec(SourceSpec(slug=slug, name=name, kind="media", homepage=homepage, license=license,
|
||||
access="rss", trust=trust, store_fulltext=False, keep_raw=False,
|
||||
min_interval=2.0, tos_notes=notes))
|
||||
|
||||
|
||||
ORF = media("orf", "ORF.at", "https://orf.at", 75, "Public RSS feeds (rss.orf.at). Headline, teaser, link.")
|
||||
media("derstandard", "DER STANDARD", "https://www.derstandard.at", 75,
|
||||
"RSS + news sitemap. robots.txt blocks AI crawlers by name; we only use feed/sitemap and link out.")
|
||||
media("diepresse", "Die Presse", "https://www.diepresse.com", 75)
|
||||
media("kurier", "KURIER", "https://kurier.at", 70)
|
||||
media("profil", "profil", "https://www.profil.at", 75)
|
||||
media("falter", "FALTER", "https://www.falter.at", 75)
|
||||
media("krone", "Kronen Zeitung", "https://www.krone.at", 55)
|
||||
media("sn", "Salzburger Nachrichten", "https://www.sn.at", 75)
|
||||
media("kleinezeitung", "Kleine Zeitung", "https://www.kleinezeitung.at", 75)
|
||||
media("vol", "Vorarlberg Online", "https://www.vol.at", 65)
|
||||
media("apa-ots", "APA-OTS Presseaussendungen", "https://www.ots.at", 60,
|
||||
"tdmrep.json declares tdm-reservation=1 for the whole site: headline + link only.")
|
||||
|
||||
NS = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9", "news": "http://www.google.com/schemas/sitemap-news/0.9"}
|
||||
|
||||
|
||||
@register
|
||||
class NewsFeeds(Adapter):
|
||||
spec = ORF
|
||||
name = "news.feeds"
|
||||
description = "Headlines from outlets' RSS feeds and Google-News sitemaps (config/feeds.yaml)"
|
||||
schedule = "*/20 * * * *"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
config = yaml.safe_load((get_settings().config_dir / "feeds.yaml").read_text(encoding="utf-8"))
|
||||
for feed in config.get("feeds") or []:
|
||||
yield Request(url=feed["url"], kind=feed.get("type", "rss"), meta={"source": feed["source"]},
|
||||
skip_unchanged=True)
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
source = response.meta["source"]
|
||||
doc_type = "press_release" if source == "apa-ots" else "article"
|
||||
if response.request.kind == "sitemap":
|
||||
root = ET.fromstring(response.body)
|
||||
for node in root.findall("sm:url", NS):
|
||||
link, title = node.findtext("sm:loc", namespaces=NS), node.findtext("news:news/news:title", namespaces=NS)
|
||||
if not link or not title:
|
||||
continue
|
||||
keywords = node.findtext("news:news/news:keywords", namespaces=NS)
|
||||
yield Document(Evidence(url=link.strip(), source=source, title=html_to_text(title), doc_type=doc_type,
|
||||
published_at=node.findtext("news:news/news:publication_date", namespaces=NS),
|
||||
excerpt=f"Schlagworte: {keywords}" if keywords else None,
|
||||
meta={"sitemap": response.request.url}))
|
||||
return
|
||||
for entry in feedparser.parse(response.body).entries:
|
||||
if not entry.get("link"):
|
||||
continue
|
||||
published = (datetime.fromtimestamp(mktime(entry.published_parsed), tz=timezone.utc)
|
||||
if entry.get("published_parsed") else None)
|
||||
yield Document(Evidence(url=entry["link"], source=source, title=html_to_text(entry.get("title")),
|
||||
doc_type=doc_type, published_at=published, excerpt=html_to_text(entry.get("summary")),
|
||||
meta={"feed": response.request.url}))
|
||||
@@ -0,0 +1 @@
|
||||
"""Austrian Parliament adapters: persons, items (votes, immunity, questions, hearings), press, calendar."""
|
||||
@@ -0,0 +1,199 @@
|
||||
"""Shared spec and pure parsing helpers for the Austrian Parliament adapters (CC BY 4.0).
|
||||
|
||||
API (https://www.parlament.gv.at/recherchieren/open-data/):
|
||||
* list exports: POST /Filter/api/filter/data/<listeId>?js=eval&showAll=true (JSON body = filter)
|
||||
409 persons since 1918 · 101 parliamentary items · 251 speeches of a person
|
||||
110 Parlamentskorrespondenz (press releases) · 600 calendar (Termine)
|
||||
* detail pages: <page URL>?json=TRUE
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import date, datetime
|
||||
from typing import Any
|
||||
|
||||
from politrack.core import Request, SourceSpec, register_spec
|
||||
from politrack.text import html_to_text
|
||||
|
||||
BASE = "https://www.parlament.gv.at"
|
||||
CHAMBER_SLUG = {"NR": "nationalrat", "BR": "bundesrat"}
|
||||
CHAMBER_NAME_DE = {"NR": "Nationalrat", "BR": "Bundesrat"}
|
||||
GOVERNMENT_RE = re.compile(r"Bundeskanzler|Vizekanzler|Bundesminister|Staatssekretär", re.IGNORECASE)
|
||||
|
||||
SPEC = register_spec(SourceSpec(
|
||||
slug="parlament",
|
||||
name="Parlament Österreich (Open Data)",
|
||||
kind="official",
|
||||
homepage="https://www.parlament.gv.at/recherchieren/open-data/",
|
||||
license="CC BY 4.0",
|
||||
access="api",
|
||||
trust=95,
|
||||
store_fulltext=True,
|
||||
min_interval=1.0,
|
||||
tos_notes="Official JSON API (?json=TRUE, /Filter/api/filter/data/<list>). Attribution required. The "
|
||||
"Parlamentsdirektion may prohibit uses incompatible with the dignity of parliament.",
|
||||
))
|
||||
|
||||
|
||||
def list_request(list_id: int, body: dict, kind: str, meta: dict | None = None) -> Request:
|
||||
return Request(url=f"{BASE}/Filter/api/filter/data/{list_id}?js=eval&showAll=true", method="POST", json=body,
|
||||
kind=kind, meta=meta or {})
|
||||
|
||||
|
||||
def detail_request(path: str, kind: str, meta: dict | None = None) -> Request:
|
||||
return Request(url=f"{BASE}{path}", params={"json": "TRUE"}, kind=kind, meta=meta or {})
|
||||
|
||||
|
||||
def list_rows(data: dict) -> list[dict[str, Any]]:
|
||||
"""Turn a filter-API export into dicts keyed by column label (duplicates get a _<index> suffix)."""
|
||||
labels: list[str] = []
|
||||
for i, header in enumerate(data.get("header", [])):
|
||||
label = header.get("label") or header.get("feld_name") or f"col{i}"
|
||||
labels.append(label if label not in labels else f"{label}_{i}")
|
||||
return [dict(zip(labels, row)) for row in data.get("rows", [])]
|
||||
|
||||
|
||||
def parse_date(value: str | None) -> date | None:
|
||||
if not value:
|
||||
return None
|
||||
for fmt in ("%d.%m.%Y", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d"):
|
||||
try:
|
||||
return datetime.strptime(value[:19], fmt).date()
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def walk_dicts(obj: Any, required_key: str):
|
||||
"""Yield every nested dict that contains `required_key`."""
|
||||
if isinstance(obj, dict):
|
||||
if required_key in obj:
|
||||
yield obj
|
||||
for value in obj.values():
|
||||
yield from walk_dicts(value, required_key)
|
||||
elif isinstance(obj, list):
|
||||
for value in obj:
|
||||
yield from walk_dicts(value, required_key)
|
||||
|
||||
|
||||
# -- faction votes ------------------------------------------------------------------------
|
||||
|
||||
PARTY_TOKEN = re.compile(r"^[A-ZÄÖÜ][A-Za-zÄÖÜäöüß.\- ]{0,30}$")
|
||||
# Split votes ("teilweise", "bzw.", per-part results in parentheses) can't be attributed to factions.
|
||||
SPLIT_VOTE = re.compile(r"\(|\bbzw\.|\bteilweise\b|\btlw\.|\bmehrstimmig\b|\bgetrennt", re.IGNORECASE)
|
||||
|
||||
|
||||
def parse_party_list(value: str | None) -> list[str]:
|
||||
if not value:
|
||||
return []
|
||||
parts = [p.strip(" .;") for p in re.split(r",|\bund\b", value)]
|
||||
return [p for p in parts if PARTY_TOKEN.match(p)]
|
||||
|
||||
|
||||
def parse_vote_text(text: str) -> dict[str, Any] | None:
|
||||
"""'137. Sitzung …: Gesetzesvorschlag in dritter Lesung angenommen Dafür: ÖVP, SPÖ, dagegen: FPÖ'"""
|
||||
m = re.search(r"\b(angenommen|abgelehnt)\b", text, re.IGNORECASE)
|
||||
if not m:
|
||||
return None
|
||||
rest = text[m.end():]
|
||||
roll_call = "namentliche abstimmung" in text.lower()
|
||||
split = bool(SPLIT_VOTE.search(rest)) and not roll_call
|
||||
pro = re.search(r"Dafür:\s*(.*?)(?:,?\s*dagegen:|$)", rest, re.IGNORECASE)
|
||||
contra = re.search(r"dagegen:\s*(.*)$", rest, re.IGNORECASE)
|
||||
return {
|
||||
"stage": text[: m.start()].strip(" :,"),
|
||||
"result": "accepted" if m.group(1).lower() == "angenommen" else "rejected",
|
||||
"for": [] if split or roll_call else parse_party_list(pro.group(1) if pro else None),
|
||||
"against": [] if split or roll_call else parse_party_list(contra.group(1) if contra else None),
|
||||
"unanimous": "einstimmig" in text.lower(),
|
||||
"split": split,
|
||||
"roll_call": roll_call,
|
||||
}
|
||||
|
||||
|
||||
# -- roll-call votes ----------------------------------------------------------------------
|
||||
|
||||
ROLL_CALL_HEADER = re.compile(r"Mit\s*[„\"“]?\s*(Ja|Nein)\s*[“\"”]?\s*stimmten[^:]*:", re.IGNORECASE)
|
||||
NEW_VOTE = re.compile(r"\b(gelangen|kommen)\s+(?:\w+\s+){0,2}zur\s+Abstimmung\b", re.IGNORECASE)
|
||||
PERSON_LINK = re.compile(r"<a[^>]*href=\"[^\"]*/person/(\d+)\"[^>]*>.*?</a>", re.IGNORECASE | re.DOTALL)
|
||||
|
||||
|
||||
def parse_roll_call(html_text: str, anchor: str | None) -> dict[str, list[tuple[str, int | None]]]:
|
||||
"""Extract the "Mit „Ja“ stimmten …" / "Mit „Nein“ stimmten …" lists that follow `anchor`.
|
||||
|
||||
Returns {"for": [(label, pad|None)], "against": [...]}. Names are mostly surnames; where a surname is
|
||||
ambiguous the protocol links the person, which gives us the parliament id directly.
|
||||
"""
|
||||
start = 0
|
||||
if anchor:
|
||||
m = re.search(r'id="%s"' % re.escape(anchor), html_text)
|
||||
if not m:
|
||||
return {}
|
||||
start = m.start()
|
||||
first = ROLL_CALL_HEADER.search(html_text, start)
|
||||
if not first:
|
||||
return {}
|
||||
# The lists must belong to this anchor: they follow within a few numbered paragraphs and no new vote
|
||||
# may be called in between.
|
||||
between = html_text[start + 10 : first.start()]
|
||||
if anchor and (len(re.findall(r'class="[^"]*randnummer', between)) > 3 or NEW_VOTE.search(html_to_text(between))):
|
||||
return {}
|
||||
end_match = re.compile(r"<hr|class=\"[^\"]*randnummer").search(html_text, first.end())
|
||||
segment = html_text[first.start() : end_match.start() if end_match else len(html_text)]
|
||||
segment = PERSON_LINK.sub(lambda m: f" @@{m.group(1)}@@ ", segment)
|
||||
text = html_to_text(segment)
|
||||
|
||||
result: dict[str, list[tuple[str, int | None]]] = {"for": [], "against": []}
|
||||
headers = list(ROLL_CALL_HEADER.finditer(text))
|
||||
for i, header in enumerate(headers):
|
||||
block = text[header.end() : headers[i + 1].start() if i + 1 < len(headers) else len(text)]
|
||||
key = "for" if header.group(1).lower() == "ja" else "against"
|
||||
for token in re.split(r"[,;]|\.\s*$", block):
|
||||
token = token.strip(" .")
|
||||
if not token:
|
||||
continue
|
||||
link = re.fullmatch(r"@@(\d+)@@", token)
|
||||
result[key].append((token, int(link.group(1))) if link else (token, None))
|
||||
return result
|
||||
|
||||
|
||||
# -- immunity -----------------------------------------------------------------------------
|
||||
|
||||
STGB_OFFENSES = {
|
||||
"111": "Üble Nachrede", "115": "Beleidigung", "146": "Betrug", "153": "Untreue", "165": "Geldwäscherei",
|
||||
"201": "Vergewaltigung", "283": "Verhetzung", "288": "Falsche Beweisaussage", "293": "Fälschung eines Beweismittels",
|
||||
"295": "Unterdrückung eines Beweismittels", "297": "Verleumdung", "302": "Missbrauch der Amtsgewalt",
|
||||
"304": "Bestechlichkeit", "305": "Vorteilsannahme", "306": "Vorteilsannahme zur Beeinflussung",
|
||||
"307": "Bestechung", "308": "Verbotene Intervention", "310": "Verletzung des Amtsgeheimnisses",
|
||||
}
|
||||
|
||||
|
||||
def parse_immunity_report(text: str) -> dict[str, Any]:
|
||||
authority = re.search(r"Ersuchen (?:der|des) (.+?),\s*(?:do\.|GZ|Zl)", text)
|
||||
case_ref = re.search(r"\bGZ\.?\s*([^,]+?),", text)
|
||||
# "wegen des Verdachtes einer strafbaren Handlung nach § 283 Abs. 1 Z 1 und Z 2, Abs. 2 StGB"
|
||||
offense = re.search(r"wegen[^§]{0,90}?(§§? ?\d+.{0,120}?\b(?:StGB|VerbotsG|FinStrG|[A-Z][A-Za-z]*G)\b)", text)
|
||||
offense_text = offense.group(1).strip() if offense else None
|
||||
if offense_text:
|
||||
section = re.search(r"§ ?(\d+)", offense_text)
|
||||
if section and section.group(1) in STGB_OFFENSES and "StGB" in offense_text:
|
||||
offense_text = f"{offense_text} ({STGB_OFFENSES[section.group(1)]})"
|
||||
decision = re.search(r"(?:wird|werde)\s+(nicht\s+)?zugestimmt|(nicht\s+)?zuzustimmen", text)
|
||||
recommendation = None
|
||||
if decision:
|
||||
recommendation = "refuse" if (decision.group(1) or decision.group(2)) else "consent"
|
||||
connection = None
|
||||
if re.search(r"\bkein Zusammenhang\b", text):
|
||||
connection = False
|
||||
elif re.search(r"\bein Zusammenhang\b", text):
|
||||
connection = True
|
||||
quote = re.search(r"Als Ergebnis seiner Beratungen.*?(?:zugestimmt|nicht zugestimmt)\s*\.", text)
|
||||
return {
|
||||
"authority": authority.group(1).strip() if authority else None,
|
||||
"case_ref": case_ref.group(1).strip() if case_ref else None,
|
||||
"offense": offense_text,
|
||||
"recommendation": recommendation,
|
||||
"political_connection": connection,
|
||||
"quote": quote.group(0) if quote else None,
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
"""Parliament items (Gegenstände) of the configured legislative periods.
|
||||
|
||||
From one list export per period (list 101) we derive:
|
||||
* votes -> item detail -> VoteEvent + faction VotePositions
|
||||
roll calls -> stenographic protocol -> individual VotePositions
|
||||
* immunity requests ("Ersuchen um Zustimmung zur behördlichen Verfolgung") -> committee report -> Legal claim
|
||||
* written questions and answers (ITYP J / AB) -> parliamentary_question claims (directly from the list)
|
||||
* investigative-committee hearings ("öffentliche Befragung der Auskunftsperson ...") -> hearing Event +
|
||||
appearance claim for politicians who testified (other witnesses are not stored)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Adapter, Request, Response, RunContext, register
|
||||
from politrack.core.records import (
|
||||
Claim,
|
||||
Event,
|
||||
Evidence,
|
||||
Legal,
|
||||
OrgRef,
|
||||
Participation,
|
||||
PersonRef,
|
||||
VoteEvent,
|
||||
VotePosition,
|
||||
)
|
||||
from politrack.sources.parlament.common import (
|
||||
BASE,
|
||||
CHAMBER_SLUG,
|
||||
SPEC,
|
||||
detail_request,
|
||||
list_request,
|
||||
list_rows,
|
||||
parse_date,
|
||||
parse_immunity_report,
|
||||
parse_roll_call,
|
||||
parse_vote_text,
|
||||
walk_dicts,
|
||||
)
|
||||
from politrack.text import fingerprint, html_to_text, strip_titles
|
||||
|
||||
HEARING = re.compile(r"Befragung der Auskunftsperson\s+(?P<name>.+)$")
|
||||
WITNESS_TITLES = re.compile(r"^(?:(?:Oberstaatsanwalt|Staatsanwalt|Sektionschef(?:in)?|MinR|Generalsekretär(?:in)?|"
|
||||
r"Bundesminister(?:in)?|Abgeordnete[rn]?|Klubobmann|Klubobfrau|Bundeskanzler|"
|
||||
r"Vizekanzler|Präsident(?:in)?)\s+)+")
|
||||
|
||||
|
||||
@register
|
||||
class ParlamentItems(Adapter):
|
||||
spec = SPEC
|
||||
name = "parlament.items"
|
||||
description = "Votes (faction + roll call), immunity requests, written questions, U-committee hearings"
|
||||
schedule = "45 3 * * *"
|
||||
parser_version = 2
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
for gp in ctx.params.get("periods") or get_settings().parlament_periods:
|
||||
yield list_request(101, {"GP_CODE": [gp], "NRBR": ["NR"]}, kind="items", meta={"gp": gp})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
handler = {
|
||||
"items": self.parse_items,
|
||||
"votes": self.parse_votes,
|
||||
"rollcall": self.parse_rollcall,
|
||||
"immunity": self.parse_immunity_item,
|
||||
"immunity_report": self.parse_immunity_report,
|
||||
"hearing": self.parse_hearing_item,
|
||||
}[response.request.kind]
|
||||
yield from handler(ctx, response)
|
||||
|
||||
# -- list ---------------------------------------------------------------------------
|
||||
|
||||
def parse_items(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
for row in list_rows(response.json()):
|
||||
path, subject, kind = row.get("HIS_URL"), row.get("Betreff") or "", row.get("ITYP")
|
||||
if not path:
|
||||
continue
|
||||
if row.get("Dafür") or row.get("Dagegen") or row.get("Abstimmungstext"):
|
||||
# The list row changes when new votes happen, so the detail is re-fetched only then
|
||||
# (conditional request + unchanged-body detection do the rest).
|
||||
yield detail_request(path, kind="votes", meta={"title": subject, "ref": row.get("Nummer")})
|
||||
if subject.startswith("Ersuchen") and "behördlichen Verfolgung" in subject:
|
||||
yield detail_request(path, kind="immunity",
|
||||
meta={"title": subject, "date": row.get("Datum"), "status": row.get("Status")})
|
||||
if kind in ("J", "AB"):
|
||||
yield from self.question_claims(row, path)
|
||||
if kind == "KOMM" and HEARING.search(subject):
|
||||
yield detail_request(path, kind="hearing", meta={"title": subject, "date": row.get("Datum")})
|
||||
|
||||
def question_claims(self, row: dict, path: str) -> Iterator[Claim]:
|
||||
pads = row.get("Personen") or "[]"
|
||||
pads = json.loads(pads) if isinstance(pads, str) else pads
|
||||
subject = row.get("Betreff") or ""
|
||||
is_question = row.get("ITYP") == "J"
|
||||
asked = parse_date(row.get("Datum"))
|
||||
topics = tuple(json.loads(row.get("THEMEN") or "[]")) if isinstance(row.get("THEMEN"), str) else ()
|
||||
for pad in pads:
|
||||
if not str(pad).strip():
|
||||
continue
|
||||
verb_de = "stellte die schriftliche Anfrage" if is_question else "beantwortete die Anfrage"
|
||||
verb_en = "submitted the written question" if is_question else "answered the question"
|
||||
yield Claim(
|
||||
claim_type="parliamentary_question", person=PersonRef(ids={"parlament": str(pad)}),
|
||||
key=("question", path, str(pad)), occurred_at=asked,
|
||||
summary_de=f"{verb_de[0].upper()}{verb_de[1:]} „{subject}“ ({row.get('Nummer')})",
|
||||
summary_en=f"{verb_en[0].upper()}{verb_en[1:]} \"{subject}\" ({row.get('Nummer')})",
|
||||
data={"kind": "question" if is_question else "answer", "number": row.get("Nummer")},
|
||||
evidence=[Evidence(url=f"{BASE}{path}", doc_type="api_record", title=subject, published_at=asked)],
|
||||
topics=topics,
|
||||
)
|
||||
|
||||
# -- votes ----------------------------------------------------------------------------
|
||||
|
||||
def parse_votes(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
content = response.json().get("content") or {}
|
||||
path = response.request.url.removeprefix(BASE)
|
||||
title = html_to_text(content.get("title")) or response.meta.get("title") or path
|
||||
item_ref = html_to_text(content.get("zitation")) or response.meta.get("ref")
|
||||
item_url = f"{BASE}{path}"
|
||||
topics = tuple(b.get("label") for b in ((content.get("topics") or {}).get("data") or {}).get("bubbles") or [])
|
||||
eurovoc = [b.get("label") for b in ((content.get("eurovoc") or {}).get("data") or {}).get("bubbles") or []]
|
||||
|
||||
for stage in walk_dicts(content.get("phase") or content, "text"):
|
||||
text = html_to_text(stage.get("text"))
|
||||
if "angenommen" not in text and "abgelehnt" not in text:
|
||||
continue
|
||||
parsed = parse_vote_text(text)
|
||||
if not parsed or not (parsed["for"] or parsed["against"] or parsed["unanimous"] or parsed["split"]
|
||||
or parsed["roll_call"]):
|
||||
continue
|
||||
voted_on = parse_date(stage.get("date"))
|
||||
refs = stage.get("fsth") or []
|
||||
protocol_url = f"{BASE}{refs[0]['url']}" if refs and refs[0].get("url") else None
|
||||
chamber = "BR" if (refs and refs[0].get("gp_code") == "BR") or "Bundesrat" in text else "NR"
|
||||
chamber_ref = OrgRef(slug=CHAMBER_SLUG[chamber])
|
||||
event_key = fingerprint("vote", path, voted_on, parsed["stage"])
|
||||
yield VoteEvent(key=event_key, chamber=chamber_ref, title=title, voted_on=voted_on, result=parsed["result"],
|
||||
stage=parsed["stage"], item_ref=item_ref, roll_call=parsed["roll_call"], url=item_url,
|
||||
protocol_url=protocol_url, raw_text=text)
|
||||
evidence = [Evidence(url=item_url, doc_type="api_record", title=title, published_at=voted_on, quote=text,
|
||||
locator=f"stage:{voted_on}:{parsed['stage'][:80]}")]
|
||||
if protocol_url:
|
||||
evidence.append(Evidence(url=protocol_url, doc_type="protocol", published_at=voted_on,
|
||||
title=refs[0].get("title") or "Stenographisches Protokoll"))
|
||||
outcome = "angenommen" if parsed["result"] == "accepted" else "abgelehnt"
|
||||
for party, position in [(p, "for") for p in parsed["for"]] + [(p, "against") for p in parsed["against"]]:
|
||||
verb_de, verb_en = ("stimmte für", "voted for") if position == "for" else ("stimmte gegen", "voted against")
|
||||
yield Claim(
|
||||
claim_type="vote", org=OrgRef(name=party, short_name=party, kind="parliamentary_group"),
|
||||
counterpart_org=chamber_ref, occurred_at=voted_on,
|
||||
summary_de=f"{party} {verb_de}: {title} ({parsed['stage']}) – {outcome}",
|
||||
summary_en=f"{party} {verb_en}: {title} ({parsed['stage']})",
|
||||
data={"position": position, "result": parsed["result"], "item_ref": item_ref, "chamber": chamber,
|
||||
"eurovoc": eurovoc},
|
||||
evidence=evidence, topics=topics, facet=VotePosition(event_key=event_key, position=position),
|
||||
)
|
||||
if parsed["roll_call"]:
|
||||
# Individual votes are listed in the protocol after one of the referenced anchors; the
|
||||
# result is usually behind the last one, sometimes in another fragment (session break).
|
||||
refs_list = [r["url"].split("#", 1) for r in reversed(refs) if r.get("url") and "#" in r["url"]]
|
||||
if refs_list:
|
||||
yield Request(url=f"{BASE}{refs_list[0][0]}", kind="rollcall",
|
||||
meta={"event_key": event_key, "refs": refs_list, "tried": [refs_list[0][0]],
|
||||
"title": title, "chamber": chamber, "date": str(voted_on),
|
||||
"item_url": item_url, "topics": list(topics)})
|
||||
|
||||
def parse_rollcall(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
meta = response.meta
|
||||
doc = response.request.url.removeprefix(BASE)
|
||||
names: dict[str, list] = {}
|
||||
anchor_used = None
|
||||
for ref_doc, anchor in meta["refs"]:
|
||||
if ref_doc != doc:
|
||||
continue
|
||||
names = parse_roll_call(response.text, anchor)
|
||||
if names.get("for") or names.get("against"):
|
||||
anchor_used = anchor
|
||||
break
|
||||
if anchor_used is None:
|
||||
remaining = [d for d, _ in meta["refs"] if d not in meta["tried"]]
|
||||
if remaining:
|
||||
ctx.stats["rollcall_followup"] += 1
|
||||
yield Request(url=f"{BASE}{remaining[0]}", kind="rollcall",
|
||||
meta={**meta, "tried": [*meta["tried"], remaining[0]]})
|
||||
else:
|
||||
ctx.stats["rollcall_unparsed"] += 1
|
||||
return
|
||||
voted_on = parse_date(meta["date"])
|
||||
body = CHAMBER_SLUG[meta["chamber"]]
|
||||
source_url = f"{response.request.url}#{anchor_used}"
|
||||
evidence = [
|
||||
Evidence(url=source_url, doc_type="protocol", published_at=voted_on,
|
||||
title="Stenographisches Protokoll – Ergebnis der namentlichen Abstimmung", locator=anchor_used),
|
||||
Evidence(url=meta["item_url"], doc_type="api_record", title=meta["title"], published_at=voted_on,
|
||||
locator=f"rollcall:{voted_on}"),
|
||||
]
|
||||
for position, entries in names.items():
|
||||
answer = "Ja" if position == "for" else "Nein"
|
||||
for label, pad in entries:
|
||||
ref = (PersonRef(ids={"parlament": str(pad)}) if pad is not None else
|
||||
PersonRef(name=strip_titles(label), hints={"surname_in_body": body, "on": voted_on}))
|
||||
yield Claim(
|
||||
claim_type="vote", person=ref, counterpart_org=OrgRef(slug=body), occurred_at=voted_on,
|
||||
summary_de=f"Stimmte in namentlicher Abstimmung mit „{answer}“: {meta['title']}",
|
||||
summary_en=f"Voted \"{'yes' if position == 'for' else 'no'}\" in a roll-call vote: {meta['title']}",
|
||||
data={"position": position, "roll_call": True, "chamber": meta["chamber"], "label": label},
|
||||
evidence=evidence, topics=tuple(meta.get("topics") or ()),
|
||||
facet=VotePosition(event_key=meta["event_key"], position=position),
|
||||
)
|
||||
|
||||
# -- immunity -----------------------------------------------------------------------
|
||||
|
||||
def parse_immunity_item(self, ctx: RunContext, response: Response) -> Iterator[Request]:
|
||||
content = response.json().get("content") or {}
|
||||
report = next((d["link"] for group in content.get("documents") or [] for d in group.get("documents") or []
|
||||
if d.get("type") == "HTML"), None)
|
||||
if report:
|
||||
yield Request(url=f"{BASE}{report}", kind="immunity_report",
|
||||
meta={**response.meta, "item": response.request.url.removeprefix(BASE),
|
||||
"title": html_to_text(content.get("title")) or response.meta.get("title")})
|
||||
|
||||
def parse_immunity_report(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||||
meta = response.meta
|
||||
text = html_to_text(response.text)
|
||||
details = parse_immunity_report(text)
|
||||
title = meta["title"]
|
||||
name = re.sub(r"^.*?(Abgeordneten zum Nationalrat|Mitglied(es)? des Bundesrates)\s*(Klubobmann|Klubobfrau)?",
|
||||
"", title).strip()
|
||||
decided = str(meta.get("status")) == "5"
|
||||
stage = {"consent": "immunity_waived" if decided else "immunity_waiver_requested",
|
||||
"refuse": "immunity_refused" if decided else "immunity_waiver_requested"}.get(
|
||||
details["recommendation"] or "", "immunity_waiver_requested")
|
||||
stage_de = {"immunity_waived": "Nationalrat stimmte der behördlichen Verfolgung zu (Immunität aufgehoben)",
|
||||
"immunity_refused": "Nationalrat verweigerte die Zustimmung zur behördlichen Verfolgung",
|
||||
"immunity_waiver_requested": "Ersuchen um Aufhebung der Immunität"}[stage]
|
||||
offense = details["offense"]
|
||||
requested = parse_date(meta.get("date"))
|
||||
yield Claim(
|
||||
claim_type="legal_proceeding", person=PersonRef(name=strip_titles(name)),
|
||||
key=("immunity", meta["item"]), occurred_at=requested, sensitive=True,
|
||||
summary_de=stage_de + (f" – Verdacht nach {offense}" if offense else "")
|
||||
+ (f" (Ersuchen: {details['authority']})" if details["authority"] else ""),
|
||||
data={"political_connection": details["political_connection"],
|
||||
"recommendation": details["recommendation"], "item": meta["item"]},
|
||||
evidence=[
|
||||
Evidence(url=response.request.url, doc_type="register_entry", published_at=requested,
|
||||
title=f"Bericht des Immunitätsausschusses: {title}", quote=details["quote"], full_text=text),
|
||||
Evidence(url=f"{BASE}{meta['item']}", doc_type="api_record", title=title, published_at=requested),
|
||||
],
|
||||
topics=("Inneres und Recht",),
|
||||
facet=Legal(stage=stage, offense=offense, authority=details["authority"], case_ref=details["case_ref"],
|
||||
stage_date=requested),
|
||||
)
|
||||
|
||||
# -- investigative-committee hearings ----------------------------------------------
|
||||
|
||||
def parse_hearing_item(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
content = response.json().get("content") or {}
|
||||
title = html_to_text(content.get("title")) or response.meta["title"]
|
||||
m = HEARING.search(title)
|
||||
if not m:
|
||||
return
|
||||
witness = WITNESS_TITLES.sub("", strip_titles(m.group("name"))).strip()
|
||||
published = parse_date(response.meta.get("date"))
|
||||
protocol = next((d["link"] for group in content.get("documents") or [] for d in group.get("documents") or []
|
||||
if d.get("type") == "HTML"), None)
|
||||
url = f"{BASE}{protocol}" if protocol else response.request.url.split("?")[0]
|
||||
event_key = fingerprint("hearing", response.request.url)
|
||||
yield Event(key=event_key, kind="hearing", title=f"Befragung im Untersuchungsausschuss: {witness}",
|
||||
starts_at=published, location="Parlament, Wien", country="AT", url=url,
|
||||
description=title, body=OrgRef(slug="nationalrat"))
|
||||
yield Claim(
|
||||
claim_type="appearance", person=PersonRef(name=witness), occurred_at=published,
|
||||
summary_de=f"Befragung als Auskunftsperson im Untersuchungsausschuss ({title})",
|
||||
summary_en="Questioned as a witness by a parliamentary investigative committee",
|
||||
data={"context": "investigative_committee"},
|
||||
evidence=[Evidence(url=url, doc_type="protocol", title=title, published_at=published)],
|
||||
topics=("Transparenz und Korruption",),
|
||||
facet=Participation(event_key=event_key, participation="witness"),
|
||||
)
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Parliament press releases (Parlamentskorrespondenz, list 110) and calendar (Termine, list 600). CC BY 4.0.
|
||||
|
||||
Press releases are stored with full text; the people they link (/person/<id>) get a mention claim.
|
||||
The event extractor later turns visits, receptions and trips described in them into meeting/travel
|
||||
claims. The calendar fills the calendar view with sessions, committee meetings, hearings and events.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import date, timedelta
|
||||
from typing import Any
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, register
|
||||
from politrack.core.records import Claim, Document, Event, Evidence, OrgRef, PersonRef
|
||||
from politrack.sources.parlament.common import BASE, CHAMBER_SLUG, SPEC, list_request, list_rows, parse_date
|
||||
from politrack.text import excerpt, fingerprint, html_to_text
|
||||
|
||||
ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}")
|
||||
SKIP_TERMIN = re.compile(r"Guided Tour|Führung|Galeriebesuch|Besuch einer Plenarsitzung|Demokratiebildung", re.I)
|
||||
TERMIN_KIND = [
|
||||
(re.compile(r"Plenarsitzung|Bundesversammlung|Fest-|Gedenksitzung", re.I), "parliament_session"),
|
||||
(re.compile(r"Untersuchungsausschuss", re.I), "hearing"),
|
||||
(re.compile(r"Ausschuss|Enquete", re.I), "committee"),
|
||||
(re.compile(r"Pressekonferenz", re.I), "press_conference"),
|
||||
(re.compile(r"Internationales|Konferenz", re.I), "meeting"),
|
||||
]
|
||||
|
||||
|
||||
def release_body(content: list) -> str:
|
||||
"""The detail JSON is a list of render blocks; the article is the one wrapped in <div class=pk>."""
|
||||
for block in content:
|
||||
text = (block.get("data") or {}).get("text") if isinstance(block, dict) else None
|
||||
if isinstance(text, str) and re.search(r"class=\"?pk\b", text):
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
@register
|
||||
class ParlamentPress(Adapter):
|
||||
spec = SPEC
|
||||
name = "parlament.press"
|
||||
description = "Parlamentskorrespondenz: official press releases with linked people"
|
||||
schedule = "5 */2 * * *"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
last = ctx.state.get("last_date")
|
||||
backfill = int(ctx.params.get("backfill_days", 365))
|
||||
since = (date.fromisoformat(last) - timedelta(days=2)) if last else date.today() - timedelta(days=backfill)
|
||||
yield list_request(110, {"DATERANGE": [since.isoformat(), date.today().isoformat()]}, kind="list",
|
||||
meta={"since": since.isoformat()})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
if response.request.kind == "list":
|
||||
newest = response.meta["since"]
|
||||
for row in list_rows(response.json()):
|
||||
if not row.get("Link"):
|
||||
continue
|
||||
published = parse_date(row.get("Datum")) # "23.10.2026" (German) in list 110
|
||||
if published:
|
||||
newest = max(newest, published.isoformat())
|
||||
yield Request(url=f"{BASE}{row['Link']}", params={"json": "TRUE"}, kind="release",
|
||||
meta={k: row.get(k) for k in ("Titel", "Untertitel", "Datum", "Nr.", "Abothemen",
|
||||
"Stichworte")})
|
||||
if ctx.mode != "test":
|
||||
ctx.state.set("last_date", newest)
|
||||
return
|
||||
yield from self.parse_release(response)
|
||||
|
||||
def parse_release(self, response: Response) -> Iterator[Any]:
|
||||
meta = response.meta
|
||||
url = response.request.url
|
||||
body_html = release_body(response.json().get("content") or [])
|
||||
pads = sorted({p for p in re.findall(r"/person/(\d+)", body_html)})
|
||||
text = html_to_text(body_html)
|
||||
title, subtitle = meta.get("Titel") or "", meta.get("Untertitel") or ""
|
||||
topics_raw = meta.get("Abothemen")
|
||||
topics = tuple(json.loads(topics_raw)) if isinstance(topics_raw, str) and topics_raw.startswith("[") else ()
|
||||
published = parse_date(meta.get("Datum"))
|
||||
yield Document(Evidence(url=url, doc_type="press_release", title=title, published_at=published,
|
||||
excerpt=subtitle or excerpt(text, 300), full_text=f"{title}\n{subtitle}\n{text}",
|
||||
meta={"pk_number": meta.get("Nr."), "person_pads": pads, "topics": list(topics)}))
|
||||
for pad in pads:
|
||||
yield Claim(
|
||||
claim_type="media_mention", person=PersonRef(ids={"parlament": pad}), occurred_at=published,
|
||||
summary_de=f"Parlamentskorrespondenz: {title}",
|
||||
data={"context": "parlamentskorrespondenz"},
|
||||
evidence=[Evidence(url=url, doc_type="press_release", title=title, published_at=published,
|
||||
quote=subtitle or None, stance="mentions")],
|
||||
topics=topics,
|
||||
)
|
||||
|
||||
|
||||
@register
|
||||
class ParlamentCalendar(Adapter):
|
||||
spec = SPEC
|
||||
name = "parlament.calendar"
|
||||
description = "Parliament calendar: plenary sessions, committees, hearings, events (tours skipped)"
|
||||
schedule = "30 2 * * *"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
yield list_request(600, {}, kind="list")
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Event]:
|
||||
since = str(ctx.params.get("since_year", date.today().year - 2))
|
||||
for row in list_rows(response.json()):
|
||||
kind_label = (row.get("Terminart") or "").replace("<BR>", " ")
|
||||
starts = next((v for k, v in row.items() if k.startswith("Datum") and k != "Datum Bis"
|
||||
and isinstance(v, str) and ISO_DATE.match(v)), None)
|
||||
if not starts or starts[:4] < since or SKIP_TERMIN.search(f"{kind_label} {row.get('Bezeichnung') or ''}"):
|
||||
continue
|
||||
title = row.get("Bezeichnung") or kind_label
|
||||
kind = next((k for pattern, k in TERMIN_KIND if pattern.search(f"{kind_label} {title}")), "public_event")
|
||||
chamber = {"Nationalrat": "NR", "Bundesrat": "BR"}.get(row.get("Gremium") or "")
|
||||
ends = next((v for k, v in row.items() if k.startswith("Datum Bis") and isinstance(v, str)
|
||||
and ISO_DATE.match(v)), None)
|
||||
yield Event(
|
||||
key=fingerprint("termin", row.get("WENTRY_ID") or row.get("Link")), kind=kind, title=title,
|
||||
starts_at=starts, ends_at=ends, location=(row.get("Ort") or "").replace(" | ", ", ") or "Parlament, Wien",
|
||||
country="AT", url=f"{BASE}{row['Link']}" if row.get("Link") else None, description=kind_label,
|
||||
body=OrgRef(slug=CHAMBER_SLUG[chamber]) if chamber else None,
|
||||
)
|
||||
@@ -0,0 +1,239 @@
|
||||
"""Parliament: people, mandates, government functions, side-job disclosures, social links, speeches.
|
||||
|
||||
Flow: list 409 (all MPs since 1918) -> PersonRecord + mandate roles
|
||||
-> detail /person/<id> for members of the configured periods
|
||||
detail -> photo, social accounts, functions, side jobs (§6 Unv-Transparenz-G)
|
||||
-> list 251 (speeches of that person)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Adapter, Request, Response, RunContext, register
|
||||
from politrack.core.records import Claim, Evidence, OrgRef, PersonRecord, PersonRef, Role, SocialAccount, Statement
|
||||
from politrack.sources.parlament.common import (
|
||||
BASE,
|
||||
CHAMBER_NAME_DE,
|
||||
CHAMBER_SLUG,
|
||||
GOVERNMENT_RE,
|
||||
SPEC,
|
||||
detail_request,
|
||||
list_request,
|
||||
list_rows,
|
||||
parse_date,
|
||||
walk_dicts,
|
||||
)
|
||||
from politrack.text import firmenbuch_number, html_to_text, strip_titles
|
||||
|
||||
SOCIAL_PLATFORMS = {"TWITTER": "twitter", "FACEBOOK": "facebook", "INSTAGRAM": "instagram", "TIKTOK": "tiktok",
|
||||
"YOUTUBE": "youtube", "LINKEDIN": "linkedin", "WEBSITE": "website", "BLUESKY": "bluesky",
|
||||
"MASTODON": "mastodon", "TELEGRAM": "telegram", "THREADS": "threads"}
|
||||
|
||||
|
||||
FN_IN_TEXT = re.compile(r"\bFN\s*\d{1,6}\s?[a-zA-Z]\b")
|
||||
POSTCODE_CITY = re.compile(r"(?:(?<=\d)|\b)\d{4}\s+[A-ZÄÖÜ][\wäöüß .\-]*$")
|
||||
LEGAL_FORM_END = re.compile(r"(?i:gesmbh|gmbh|ges\.\s?m\.\s?b\.\s?h\.?|m\.\s?b\.\s?h\.?|aktiengesellschaft|\bag|\bkg|\bog|"
|
||||
r"e\.\s?u\.|\bse|privatstiftung|genossenschaft|sparkasse)(?=[A-ZÄÖÜ\s,]|$)")
|
||||
|
||||
|
||||
STREET_WORD = r"\S*(?:straße|strasse|str\.|gasse|platz|weg|kai|ring|allee|zeile|lände|markt|steig)"
|
||||
|
||||
|
||||
def strip_street(text: str) -> str:
|
||||
"""'Verein Musterhilfe Hauptstraße 12' -> 'Verein Musterhilfe' (greedy name, shortest street suffix)."""
|
||||
m = (re.match(rf"^(.*\S)\s+{STREET_WORD}\s*\d+[a-z]?(?:/\d+)*\s*$", text, re.IGNORECASE)
|
||||
or re.match(r"^(.*\S)\s+\S+\s*\d+[a-z]?(?:/\d+)*\s*$", text))
|
||||
return m.group(1) if m else text
|
||||
|
||||
|
||||
def employer(cell: str) -> tuple[str | None, str | None]:
|
||||
"""Employer name without business address, plus its company register number.
|
||||
|
||||
'Hypo Tirol Bank AG, Meraner Straße 8, 6020 Innsbruck' -> ('Hypo Tirol Bank AG', None)
|
||||
'25 PersonaldienstleistungsgesmbHFranz Josef Kai 95020 Salzburg FN306527a' (parliament's JSON glues
|
||||
the lines together) -> ('25 PersonaldienstleistungsgesmbH', '306527a')
|
||||
"""
|
||||
cell = re.sub(r"^lit\.\s*\w\s*:\s*", "", cell).strip()
|
||||
fn_match = FN_IN_TEXT.search(cell)
|
||||
fn = firmenbuch_number(fn_match.group(0)) if fn_match else None
|
||||
name = FN_IN_TEXT.sub("", cell).split(",")[0].strip(" *")
|
||||
address = POSTCODE_CITY.search(name)
|
||||
if address:
|
||||
head = name[:address.start()]
|
||||
ends = [m.end() for m in LEGAL_FORM_END.finditer(head)]
|
||||
# cut after the last legal form; otherwise drop "<street> <number>" before the postcode
|
||||
name = head[:ends[-1]] if ends else strip_street(head)
|
||||
name = name.strip(" ,-*")
|
||||
return (name if len(name) >= 3 and not re.fullmatch(r"[\d\s.€-]+", name) else None), fn
|
||||
|
||||
|
||||
@register
|
||||
class ParlamentPersons(Adapter):
|
||||
spec = SPEC
|
||||
name = "parlament.persons"
|
||||
description = "Members of Nationalrat and Bundesrat since 1918, mandates, functions, side jobs, speeches"
|
||||
schedule = "15 3 * * *"
|
||||
creates_persons = True
|
||||
parser_version = 3
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
yield list_request(409, {"NRBR": ["NR", "BR"]}, kind="persons")
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
kind = response.request.kind
|
||||
if kind == "persons":
|
||||
yield from self.parse_persons(ctx, response)
|
||||
elif kind == "person":
|
||||
yield from self.parse_person(ctx, response)
|
||||
elif kind == "speeches":
|
||||
yield from self.parse_speeches(ctx, response)
|
||||
|
||||
# -- persons & mandates -------------------------------------------------------------
|
||||
|
||||
def parse_persons(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
periods = set(ctx.params.get("periods") or get_settings().parlament_periods)
|
||||
for row in list_rows(response.json()):
|
||||
attr = row.get("Attribute") or {}
|
||||
if isinstance(attr, str):
|
||||
attr = json.loads(attr)
|
||||
m = re.search(r"/person/(\d+)", attr.get("uri") or "")
|
||||
if not m:
|
||||
continue
|
||||
pad = m.group(1)
|
||||
ref = PersonRef(ids={"parlament": pad})
|
||||
display = attr.get("zit") or strip_titles(row.get("Name") or "")
|
||||
yield PersonRecord(ref=ref, full_name=display, given_name=row.get("vorname"), family_name=row.get("nachname"),
|
||||
gender={"M": "male", "W": "female"}.get(row.get("GESCHL_CODE") or attr.get("geschlecht")))
|
||||
mandates = attr.get("mandate_detail") or []
|
||||
for mandate in mandates:
|
||||
claim = self.mandate_claim(pad, display, mandate)
|
||||
if claim:
|
||||
yield claim
|
||||
if any(md.get("gp_code") in periods for md in mandates):
|
||||
yield detail_request(f"/person/{pad}", kind="person", meta={"pad": pad, "name": display})
|
||||
|
||||
def mandate_claim(self, pad: str, name: str, m: dict) -> Claim | None:
|
||||
code = m.get("mand_code") or "NR"
|
||||
if code not in CHAMBER_SLUG:
|
||||
return None
|
||||
party_code = m.get("wahlpartei_code")
|
||||
party_name = m.get("wahlpartei_full_txt") or m.get("wahlpartei_txt") or party_code
|
||||
party = OrgRef(name=party_name, short_name=party_code, kind="party") if party_name else None
|
||||
start, end = parse_date(m.get("mandat_von")), parse_date(m.get("mandat_bis"))
|
||||
gp = m.get("gp_code") or ""
|
||||
region = m.get("wahlkreis_bundesland")
|
||||
label = party_code or party_name or "?"
|
||||
body = OrgRef(slug=CHAMBER_SLUG[code])
|
||||
return Claim(
|
||||
claim_type="role",
|
||||
person=PersonRef(ids={"parlament": pad}),
|
||||
org=body,
|
||||
counterpart_org=party,
|
||||
valid_from=start,
|
||||
valid_to=end,
|
||||
summary_de=f"{name}: Mandat im {CHAMBER_NAME_DE[code]} ({gp}. GP) für {label}"
|
||||
+ (f", Wahlkreis {region}" if region else ""),
|
||||
summary_en=f"{name}: member of the {CHAMBER_NAME_DE[code]} ({gp} legislative period) for {label}",
|
||||
data={"gp": gp, "party_code": party_code, "faction_code": m.get("fraktionscode"), "region": region},
|
||||
evidence=[Evidence(url=f"{BASE}/person/{pad}", doc_type="api_record", title=f"Parlament: {name}",
|
||||
locator=f"mandat:{code}:{gp}:{start}")],
|
||||
facet=Role(role="mp", level="federal", body=body, party=party, region=region, start=start, end=end),
|
||||
)
|
||||
|
||||
# -- person detail ----------------------------------------------------------------
|
||||
|
||||
def parse_person(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
data = response.json()
|
||||
content = data.get("content") or {}
|
||||
pad, name = response.meta["pad"], response.meta["name"]
|
||||
ref = PersonRef(ids={"parlament": pad})
|
||||
url = f"{BASE}/person/{pad}"
|
||||
photo = ((data.get("meta") or {}).get("openGraph") or {}).get("image")
|
||||
if photo:
|
||||
yield PersonRecord(ref=ref, full_name=name, photo_url=photo)
|
||||
|
||||
for account in (content.get("kontakt") or {}).get("socialMedia") or []:
|
||||
platform = SOCIAL_PLATFORMS.get((account.get("type") or "").upper(), (account.get("type") or "website").lower())
|
||||
yield SocialAccount(platform=platform, url=account["url"], person=ref, verified=True)
|
||||
|
||||
for fn in walk_dicts(content.get("biografie") or {}, "funktion_text"):
|
||||
if fn.get("gremium") in CHAMBER_SLUG:
|
||||
continue
|
||||
text = fn.get("funktion_text") or fn.get("bez")
|
||||
if not text:
|
||||
continue
|
||||
start, end = parse_date(fn.get("funktion_von")), parse_date(fn.get("funktion_bis"))
|
||||
is_gov = bool(GOVERNMENT_RE.search(text))
|
||||
body = OrgRef(slug="bundesregierung") if is_gov else None
|
||||
yield Claim(
|
||||
claim_type="role", person=ref, org=body, valid_from=start, valid_to=end,
|
||||
summary_de=f"{name}: {text}",
|
||||
evidence=[Evidence(url=url, doc_type="api_record", title=f"Parlament: {name}",
|
||||
locator=f"funktion:{text}:{start}")],
|
||||
facet=Role(role=text, level="federal" if is_gov else "other", body=body, start=start, end=end),
|
||||
)
|
||||
|
||||
yield from self.side_jobs(ref, pad, name, content.get("unvtrans") or {})
|
||||
yield list_request(251, {"PAD_INTERN": [int(pad)]}, kind="speeches", meta={"pad": pad, "name": name})
|
||||
|
||||
def side_jobs(self, ref: PersonRef, pad: str, name: str, unv: dict) -> Iterator[Claim]:
|
||||
"""Disclosures under §6 Unv-Transparenz-G: leading positions, paid side jobs, honorary positions."""
|
||||
tables = unv.get("unvtransData") or {}
|
||||
pdf = next((d["link"] for group in unv.get("linkliste") or [] for d in group.get("documents") or []
|
||||
if d.get("link", "").endswith(f"/person/{pad}/BezBegrBVGPar9.pdf")), None)
|
||||
for key, table in tables.items():
|
||||
m = re.fullmatch(r"(z\d+)Table", key)
|
||||
if not m or not isinstance(table, dict):
|
||||
continue
|
||||
header = html_to_text(tables.get(f"{m.group(1)}Header")) or key
|
||||
for row in (table.get("data") or {}).get("rows") or []:
|
||||
cells = [c for c in (html_to_text(c) for c in row if isinstance(c, str)) if c]
|
||||
if not cells:
|
||||
continue
|
||||
name_, fn = employer(cells[0]) if m.group(1) in ("z1", "z2", "z3") else (None, None)
|
||||
if name_: # never keep the (business) address that parliament publishes with the employer
|
||||
cells = [name_ + (f" (FN {fn})" if fn else ""), *cells[1:]]
|
||||
text = " – ".join(cells)
|
||||
yield Claim(
|
||||
claim_type="side_job", person=ref, key=("unvtrans", pad, m.group(1), text),
|
||||
counterpart_org=OrgRef(name=name_, kind="company" if LEGAL_FORM_END.search(name_) else "other",
|
||||
ids={"firmenbuch": fn} if fn else {}) if name_ else None,
|
||||
summary_de=f"{name} – {header.split(' - ')[0]}: {text}",
|
||||
data={"category": header, "cells": cells, "unpaid": "*" in "".join(str(c) for c in row),
|
||||
"employer": name_, "firmenbuch": fn},
|
||||
evidence=[Evidence(url=f"{BASE}{pdf}" if pdf else f"{BASE}/person/{pad}", doc_type="register_entry",
|
||||
title=f"Unvereinbarkeits-/Transparenzliste {name}", quote=text,
|
||||
locator=f"unvtrans:{m.group(1)}")],
|
||||
topics=("Transparenz und Korruption",),
|
||||
)
|
||||
|
||||
# -- speeches -----------------------------------------------------------------------
|
||||
|
||||
def parse_speeches(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||||
periods = set(ctx.params.get("periods") or get_settings().parlament_periods)
|
||||
pad, name = response.meta["pad"], response.meta["name"]
|
||||
for row in list_rows(response.json()):
|
||||
if row.get("GP") not in periods and row.get("GREMIUM") != "B":
|
||||
continue
|
||||
html_cells = " ".join(str(v) for v in row.values() if isinstance(v, str))
|
||||
protocol = re.search(r'href="(/dokument/[^"]+)"', html_cells)
|
||||
session = html_to_text(row.get("Sitzung"))
|
||||
topic = html_to_text(row.get("Rede"))
|
||||
said_at = row.get("fromDate")
|
||||
url = f"{BASE}{protocol.group(1)}" if protocol else f"{BASE}/person/{pad}"
|
||||
chamber = "Bundesrat" if row.get("GREMIUM") == "B" else "Nationalrat"
|
||||
yield Claim(
|
||||
claim_type="statement", person=PersonRef(ids={"parlament": pad}),
|
||||
key=("parl-speech", pad, row.get("Sortier"), topic), occurred_at=said_at,
|
||||
summary_de=f"{name} sprach im {chamber} zu: {topic}",
|
||||
summary_en=f"{name} spoke in the {chamber} on: {topic}",
|
||||
data={"session": session, "gp": row.get("GP"), "item_title": topic},
|
||||
evidence=[Evidence(url=url, doc_type="protocol", title=session, published_at=parse_date(said_at),
|
||||
locator=url.split("#", 1)[-1] if "#" in url else "")],
|
||||
facet=Statement(context="parliament_speech", url=url, said_at=said_at, medium=chamber, title=topic),
|
||||
)
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Political advertising on online platforms.
|
||||
|
||||
* Google Political Ads Transparency Report - public download bundle (zip with CSVs, ~300 MB). We keep
|
||||
ads served in Austria (Regions contains "AT"): advertiser, period, impressions bracket, spend range.
|
||||
Note: Google stopped serving political ads in the EU in October 2025 (EU TTPA regulation), so this
|
||||
is historical data (2019-2025).
|
||||
* Meta Ad Library API (political and issue ads reaching Austria) - requires an identity-verified developer
|
||||
token (POLITRACK_META_AD_LIBRARY_TOKEN) and page ids in config/meta_pages.yaml; skipped otherwise.
|
||||
|
||||
Advertisers are resolved to parties/organisations by name where possible (e.g. "FPÖ", "SPÖ Wien").
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import zipfile
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Evidence, Money, OrgRef
|
||||
from politrack.text import format_eur_de
|
||||
|
||||
GOOGLE_BUNDLE = "https://storage.googleapis.com/political-csv/google-political-ads-transparency-bundle.zip"
|
||||
GOOGLE_SPEC = SourceSpec(
|
||||
slug="google-political-ads", name="Google Political Ads Transparency Report", kind="aggregator",
|
||||
homepage="https://adstransparency.google.com/political", license="Google transparency data (public download)",
|
||||
access="bulk", trust=85, keep_raw=False, min_interval=5.0,
|
||||
tos_notes="Public CSV bundle of political ads. ~300 MB; fetched monthly with conditional requests.",
|
||||
)
|
||||
META_SPEC = SourceSpec(
|
||||
slug="meta-adlibrary", name="Meta Ad Library", kind="social", homepage="https://www.facebook.com/ads/library/api",
|
||||
license="Meta Platform Terms", access="api", trust=80, min_interval=2.0,
|
||||
tos_notes="Official Ad Library API for political and issue ads (identity-verified developer token required).",
|
||||
)
|
||||
META_API = "https://graph.facebook.com/v21.0/ads_archive"
|
||||
|
||||
|
||||
def col(row: dict[str, str], *names: str) -> str | None:
|
||||
lowered = {k.lower(): v for k, v in row.items()}
|
||||
for name in names:
|
||||
if lowered.get(name.lower()) not in (None, ""):
|
||||
return lowered[name.lower()]
|
||||
return None
|
||||
|
||||
|
||||
@register
|
||||
class GooglePoliticalAds(Adapter):
|
||||
spec = GOOGLE_SPEC
|
||||
name = "ads.google"
|
||||
description = "Political ads served in Austria on Google/YouTube (historical, until 10/2025)"
|
||||
schedule = "0 3 1 * *"
|
||||
queue = "bulk"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
yield Request(url=GOOGLE_BUNDLE, kind="bundle")
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||||
since = ctx.params.get("since") or get_settings().ad_archive_since
|
||||
with zipfile.ZipFile(io.BytesIO(response.body)) as bundle:
|
||||
name = next((n for n in bundle.namelist() if n.endswith("creative-stats.csv")), None)
|
||||
if name is None:
|
||||
ctx.stats["creative_stats_missing"] += 1
|
||||
return
|
||||
with bundle.open(name) as fh:
|
||||
for row in csv.DictReader(io.TextIOWrapper(fh, encoding="utf-8", errors="replace")):
|
||||
regions = (col(row, "Regions") or "").replace(" ", "").split(",")
|
||||
if "AT" not in regions:
|
||||
continue
|
||||
start = col(row, "Date_Range_Start") or ""
|
||||
if start < since:
|
||||
continue
|
||||
advertiser = col(row, "Advertiser_Name") or "?"
|
||||
low, high = col(row, "Spend_Range_Min_EUR"), col(row, "Spend_Range_Max_EUR")
|
||||
ad_url = col(row, "Ad_URL") or GOOGLE_BUNDLE
|
||||
payer = OrgRef(name=advertiser, kind="other", ids={"google_advertiser": col(row, "Advertiser_ID") or ""})
|
||||
spend = f"{format_eur_de(low)}–{format_eur_de(high)} €" if low and high else None
|
||||
yield Claim(
|
||||
claim_type="money_flow", org=payer, occurred_at=start,
|
||||
summary_de=f"Politische Google-Werbung von {advertiser} ({start} – {col(row, 'Date_Range_End')})"
|
||||
+ (f", Ausgaben {spend}" if spend else ""),
|
||||
summary_en=f"Political Google ad by {advertiser} ({start} – {col(row, 'Date_Range_End')})",
|
||||
data={"impressions": col(row, "Impressions"), "ad_type": col(row, "Ad_Type"),
|
||||
"age_targeting": col(row, "Age_Targeting"), "gender_targeting": col(row, "Gender_Targeting"),
|
||||
"geo_included": col(row, "Geo_Targeting_Included")},
|
||||
evidence=[Evidence(url=ad_url, doc_type="dataset_row", title=f"Google Ads Transparency: {advertiser}",
|
||||
published_at=start)],
|
||||
facet=Money(kind="political_ad", payer=payer, payer_name=advertiser,
|
||||
payee=OrgRef(name="Google", kind="company"), payee_name="Google",
|
||||
paid_on=date.fromisoformat(start[:10]) if start else None,
|
||||
amount_category=spend, reference=col(row, "Ad_ID"), category=col(row, "Ad_Type")),
|
||||
)
|
||||
|
||||
|
||||
@register
|
||||
class MetaPoliticalAds(Adapter):
|
||||
spec = META_SPEC
|
||||
name = "ads.meta"
|
||||
description = "Political/issue ads reaching Austria from Meta's Ad Library (token + page list required)"
|
||||
schedule = "0 4 * * 4"
|
||||
queue = "bulk"
|
||||
|
||||
FIELDS = ("id,page_id,page_name,bylines,ad_delivery_start_time,ad_delivery_stop_time,spend,impressions,"
|
||||
"currency,ad_snapshot_url,ad_creative_bodies")
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
token = get_settings().meta_ad_library_token
|
||||
path = get_settings().config_dir / "meta_pages.yaml"
|
||||
if not token or not path.exists():
|
||||
ctx.stats["skipped_not_configured"] = 1
|
||||
return
|
||||
pages = [str(p["page_id"]) for p in (yaml.safe_load(path.read_text(encoding="utf-8")) or {}).get("pages", [])]
|
||||
for i in range(0, len(pages), 10):
|
||||
yield Request(url=META_API, kind="ads", conditional=False, params={
|
||||
"access_token": token, "ad_reached_countries": '["AT"]', "ad_type": "POLITICAL_AND_ISSUE_ADS",
|
||||
"search_page_ids": ",".join(pages[i:i + 10]), "fields": self.FIELDS, "limit": 250,
|
||||
"ad_delivery_date_min": ctx.params.get("since") or get_settings().ad_archive_since})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
data = response.json()
|
||||
next_url = (data.get("paging") or {}).get("next")
|
||||
if next_url:
|
||||
yield Request(url=next_url, kind="ads", conditional=False)
|
||||
for ad in data.get("data") or []:
|
||||
spend = ad.get("spend") or {}
|
||||
payer_name = ad.get("bylines") or ad.get("page_name") or "?"
|
||||
payer = OrgRef(name=ad.get("page_name") or payer_name, kind="other", ids={"facebook_page": ad.get("page_id")})
|
||||
rng = f"{spend.get('lower_bound')}–{spend.get('upper_bound')} {ad.get('currency') or 'EUR'}" if spend else None
|
||||
start = (ad.get("ad_delivery_start_time") or "")[:10]
|
||||
yield Claim(
|
||||
claim_type="money_flow", org=payer, occurred_at=start or None,
|
||||
summary_de=f"Politische Meta-Werbung von {payer_name} ab {start}" + (f", Ausgaben {rng}" if rng else ""),
|
||||
data={"impressions": ad.get("impressions"), "bodies": (ad.get("ad_creative_bodies") or [])[:2]},
|
||||
evidence=[Evidence(url=ad.get("ad_snapshot_url") or f"https://www.facebook.com/ads/library/?id={ad['id']}",
|
||||
doc_type="dataset_row", title=f"Meta Ad Library: {payer_name}", published_at=start or None)],
|
||||
facet=Money(kind="political_ad", payer=payer, payer_name=payer_name,
|
||||
payee=OrgRef(name="Meta Platforms", kind="company"), payee_name="Meta",
|
||||
paid_on=date.fromisoformat(start) if start else None, amount_category=rng,
|
||||
reference=ad.get("id")),
|
||||
)
|
||||
@@ -0,0 +1,254 @@
|
||||
"""Public procurement: contract award notices published as open "core data" (Kerndaten, §62/64 BVergG 2018).
|
||||
|
||||
Every contracting authority (or the platform it uses) publishes a "Kerndatenquelle" (KDQ): an XML index
|
||||
<kdq><item id lastmod><url>…</url></item>…</kdq> listed as a dataset on data.gv.at. Each item is one
|
||||
notice; since 2023/24 notices use the EU eForms UBL schema. We keep ContractAwardNotice documents
|
||||
(who bought what from whom, for how much) and skip calls for tenders.
|
||||
|
||||
Correlations this enables: suppliers that are also party donors, sponsors or side-job employers of
|
||||
politicians; contracts by ministries led by a given minister (organisation pages).
|
||||
|
||||
Volume: tens of thousands of notices; first run backfills from POLITRACK_PROCUREMENT_SINCE, later runs only
|
||||
fetch items whose lastmod changed. Runs on the `bulk` queue.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Evidence, Money, OrgRef
|
||||
from politrack.text import firmenbuch_number, format_eur_de
|
||||
|
||||
HUB = "https://www.data.gv.at/api/hub/search/search"
|
||||
SPEC = SourceSpec(
|
||||
slug="vergabe", name="Öffentliche Aufträge (Kerndaten, data.gv.at)", kind="official",
|
||||
homepage="https://www.data.gv.at", license="Open data (free licence required by §64 BVergG 2018)", access="bulk",
|
||||
trust=90, min_interval=1.0,
|
||||
tos_notes="Contracting authorities must publish award core data as open, machine-readable data under a free "
|
||||
"licence (BVergG 2018). We read each publisher's Kerndatenquelle (KDQ) XML and its eForms notices.",
|
||||
)
|
||||
NS = {
|
||||
"cbc": "urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2",
|
||||
"cac": "urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2",
|
||||
"efac": "http://data.europa.eu/p27/eforms-ubl-extension-aggregate-components/1",
|
||||
"efbc": "http://data.europa.eu/p27/eforms-ubl-extension-basic-components/1",
|
||||
"efext": "http://data.europa.eu/p27/eforms-ubl-extensions/1",
|
||||
"ext": "urn:oasis:names:specification:ubl:schema:xsd:CommonExtensionComponents-2",
|
||||
}
|
||||
CATALOG_QUERIES = ("Kerndaten", "Kerndatenquelle", "KDQ BVergG")
|
||||
|
||||
|
||||
def local(tag: str) -> str:
|
||||
return tag.rsplit("}", 1)[-1]
|
||||
|
||||
|
||||
def text(node: ET.Element | None, path: str) -> str | None:
|
||||
if node is None:
|
||||
return None
|
||||
found = node.find(path, NS)
|
||||
return found.text.strip() if found is not None and found.text else None
|
||||
|
||||
|
||||
def parse_kerndaten(root: ET.Element) -> dict[str, Any] | None:
|
||||
"""Austrian core-data format (BVergG 2018, schema brz.gv.at/eproc/kd): KD_8_2_* = awarded contracts.
|
||||
|
||||
Still used by many publishers (e-shops, below-threshold awards). Contractor NATIONALID is usually the
|
||||
company register number ("183286p"), occasionally the VAT id.
|
||||
"""
|
||||
if not local(root.tag).startswith("KD_8_2"):
|
||||
return None
|
||||
|
||||
def t(node: ET.Element | None, path: str) -> str | None:
|
||||
found = node.find(path) if node is not None else None
|
||||
return found.text.strip() if found is not None and found.text else None
|
||||
|
||||
buyer = root.find("{*}CONTRACTING_BODY/{*}ADDRESS_CONTRACTING_BODY")
|
||||
title = " ".join(p.text.strip() for p in root.iterfind("{*}OBJECT_CONTRACT/{*}TITLE/{*}P") if p.text)
|
||||
cpv = root.find("{*}OBJECT_CONTRACT/{*}CPV_MAIN/{*}CPV_CODE")
|
||||
contracts = []
|
||||
for awarded in root.iterfind(".//{*}AWARD_CONTRACT/{*}AWARDED_CONTRACT"):
|
||||
contracts.append({
|
||||
"id": None,
|
||||
"title": None,
|
||||
"awarded": t(awarded, "{*}DATE_CONCLUSION_CONTRACT"),
|
||||
"amount": t(awarded, "{*}VAL_TOTAL"),
|
||||
"winners": [{"name": t(c, "{*}ADDRESS_CONTRACTOR/{*}OFFICIALNAME"),
|
||||
"company_id": t(c, "{*}ADDRESS_CONTRACTOR/{*}NATIONALID")}
|
||||
for c in awarded.iterfind("{*}CONTRACTOR")],
|
||||
})
|
||||
return {
|
||||
"notice_id": t(root, "{*}EXTENSION[@KEY='Referenz']") or t(root, "{*}OBJECT_CONTRACT/{*}REFERENCE_NUMBER"),
|
||||
"issued": None,
|
||||
"title": title or None,
|
||||
"cpv": cpv.get("CODE") if cpv is not None else None,
|
||||
"total": None,
|
||||
"buyers": [{"name": t(buyer, "{*}OFFICIALNAME"), "company_id": None}] if buyer is not None else [],
|
||||
"contracts": contracts,
|
||||
}
|
||||
|
||||
|
||||
def parse_award_notice(xml: bytes) -> dict[str, Any] | None:
|
||||
"""Extract buyer, winners, amounts, title, CPV and dates from an award notice: eForms
|
||||
ContractAwardNotice, or the Austrian core-data format (parse_kerndaten)."""
|
||||
root = ET.fromstring(xml)
|
||||
if local(root.tag).startswith("KD_"):
|
||||
return parse_kerndaten(root)
|
||||
if local(root.tag) != "ContractAwardNotice":
|
||||
return None
|
||||
orgs: dict[str, dict[str, Any]] = {}
|
||||
for org in root.iterfind(".//efac:Organizations/efac:Organization/efac:Company", NS):
|
||||
org_id = text(org, "cac:PartyIdentification/cbc:ID")
|
||||
if org_id:
|
||||
orgs[org_id] = {"name": text(org, "cac:PartyName/cbc:Name"),
|
||||
"company_id": text(org, "cac:PartyLegalEntity/cbc:CompanyID"),
|
||||
"city": text(org, "cac:PostalAddress/cbc:CityName")}
|
||||
buyer_ids = [e.text for e in root.iterfind("cac:ContractingParty/cac:Party/cac:PartyIdentification/cbc:ID", NS)
|
||||
if e.text]
|
||||
result = root.find(".//efac:NoticeResult", NS)
|
||||
tender_amounts = {text(t, "cbc:ID"): text(t, "cac:LegalMonetaryTotal/cbc:PayableAmount")
|
||||
for t in (result.iterfind("efac:LotTender", NS) if result is not None else [])}
|
||||
tendering_parties = {text(tp, "cbc:ID"): [e.text for e in tp.iterfind("efac:Tenderer/cbc:ID", NS) if e.text]
|
||||
for tp in (result.iterfind("efac:TenderingParty", NS) if result is not None else [])}
|
||||
lot_tender_party = {text(t, "cbc:ID"): text(t, "efac:TenderingParty/cbc:ID")
|
||||
for t in (result.iterfind("efac:LotTender", NS) if result is not None else [])}
|
||||
contracts = []
|
||||
for contract in (result.iterfind("efac:SettledContract", NS) if result is not None else []):
|
||||
tender_id = text(contract, "efac:LotTender/cbc:ID")
|
||||
winners = tendering_parties.get(lot_tender_party.get(tender_id), [])
|
||||
contracts.append({
|
||||
"id": text(contract, "cbc:ID"),
|
||||
"title": text(contract, "cbc:Title"),
|
||||
"awarded": text(contract, "cbc:AwardDate") or text(contract, "cbc:IssueDate"),
|
||||
"amount": tender_amounts.get(tender_id),
|
||||
"winners": [orgs.get(w, {"name": None}) | {"ref": w} for w in winners],
|
||||
})
|
||||
project_title = text(root, "cac:ProcurementProject/cbc:Name")
|
||||
return {
|
||||
"notice_id": text(root, "cbc:ID"),
|
||||
"issued": text(root, "cbc:IssueDate"),
|
||||
"title": project_title,
|
||||
"cpv": text(root, "cac:ProcurementProject/cac:MainCommodityClassification/cbc:ItemClassificationCode"),
|
||||
"total": text(root, ".//efac:NoticeResult/cbc:TotalAmount"),
|
||||
"buyers": [orgs.get(b, {"name": None}) | {"ref": b} for b in buyer_ids],
|
||||
"contracts": contracts,
|
||||
}
|
||||
|
||||
|
||||
def org_ref(org: dict[str, Any], kind: str) -> OrgRef | None:
|
||||
if not org.get("name"):
|
||||
return None
|
||||
company_id = (org.get("company_id") or "").replace(" ", "")
|
||||
ids = {}
|
||||
if company_id.upper().startswith("ATU"):
|
||||
ids["uid"] = company_id.upper()
|
||||
elif fn := firmenbuch_number(company_id):
|
||||
ids["firmenbuch"] = fn
|
||||
return OrgRef(name=org["name"], kind=kind, ids=ids)
|
||||
|
||||
|
||||
def as_date(value: str | None) -> date | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return date.fromisoformat(value[:10])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
@register
|
||||
class PublicContracts(Adapter):
|
||||
spec = SPEC
|
||||
name = "vergabe.awards"
|
||||
description = "Contract awards of Austrian public bodies (eForms and BVergG core data via data.gv.at)"
|
||||
schedule = "0 2 * * *"
|
||||
queue = "bulk"
|
||||
parser_version = 2
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
for query in CATALOG_QUERIES:
|
||||
yield Request(url=HUB, params={"q": query, "limit": 100, "page": 0}, kind="catalog", conditional=False)
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
kind = response.request.kind
|
||||
if kind == "catalog":
|
||||
result = response.json().get("result") or {}
|
||||
for dataset in result.get("results") or []:
|
||||
for dist in dataset.get("distributions") or []:
|
||||
url = (dist.get("access_url") or dist.get("download_url") or [None])
|
||||
url = url[0] if isinstance(url, list) else url
|
||||
if url and url.startswith("http") and "data.gv.at/katalog" not in url:
|
||||
if ctx.params.get("only_url") and ctx.params["only_url"] not in url:
|
||||
continue # targeted runs: -p only_url=bbg.gv.at
|
||||
title = (dataset.get("title") or {}).get("de") or ""
|
||||
yield Request(url=url, kind="kdq", meta={"publisher": title.split("von", 1)[-1].strip(" '")})
|
||||
if result.get("count", 0) > (response.request.params["page"] + 1) * 100:
|
||||
yield Request(url=HUB, params={**response.request.params, "page": response.request.params["page"] + 1},
|
||||
kind="catalog", conditional=False)
|
||||
elif kind == "kdq":
|
||||
yield from self.parse_kdq(ctx, response)
|
||||
elif kind == "notice":
|
||||
yield from self.parse_notice(ctx, response)
|
||||
|
||||
def parse_kdq(self, ctx: RunContext, response: Response) -> Iterator[Request]:
|
||||
try:
|
||||
root = ET.fromstring(response.body)
|
||||
except ET.ParseError:
|
||||
ctx.stats["kdq_not_xml"] += 1
|
||||
return
|
||||
state_key = f"kdq:{response.request.url}"
|
||||
since = ctx.params.get("since") or ctx.state.get(state_key) or get_settings().procurement_since
|
||||
limit = int(ctx.params.get("max_notices_per_source", 3000))
|
||||
items = [(item.get("lastmod") or "", item.findtext("{*}url") or item.findtext("url"))
|
||||
for item in root.iter() if local(item.tag) == "item"]
|
||||
# Oldest first so an interrupted backfill resumes where it stopped (cursor = newest lastmod handled).
|
||||
items = sorted(i for i in items if i[1] and i[0] > since)[:limit]
|
||||
for lastmod, url in items:
|
||||
# lastmod in meta (not in the URL) + conditional requests: unchanged notices cost a 304 at most.
|
||||
# The cursor advances only when a notice has been parsed (see advance_cursor), so an interrupted
|
||||
# run resumes with the first notice it did not get to.
|
||||
yield Request(url=url.strip(), kind="notice", meta={"publisher": response.meta.get("publisher"),
|
||||
"lastmod": lastmod, "state_key": state_key})
|
||||
|
||||
@staticmethod
|
||||
def advance_cursor(ctx: RunContext, response: Response) -> None:
|
||||
key, lastmod = response.meta.get("state_key"), response.meta.get("lastmod")
|
||||
if key and lastmod and lastmod > (ctx.state.get(key) or ""):
|
||||
ctx.state.set(key, lastmod)
|
||||
|
||||
def parse_notice(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||||
self.advance_cursor(ctx, response) # notices arrive oldest-first per publisher
|
||||
try:
|
||||
notice = parse_award_notice(response.body)
|
||||
except ET.ParseError:
|
||||
ctx.stats["notice_not_xml"] += 1
|
||||
return
|
||||
if notice is None:
|
||||
ctx.stats["not_an_award"] += 1
|
||||
return
|
||||
buyer = next((org_ref(b, "public_body") for b in notice["buyers"] if b.get("name")), None)
|
||||
if buyer is None:
|
||||
return
|
||||
for contract in notice["contracts"]:
|
||||
for winner in contract["winners"]:
|
||||
supplier = org_ref(winner, "company")
|
||||
if supplier is None:
|
||||
continue
|
||||
amount = contract["amount"] or (notice["total"] if len(notice["contracts"]) == 1 else None)
|
||||
awarded = as_date(contract["awarded"]) or as_date(notice["issued"])
|
||||
title = contract["title"] or notice["title"] or "Auftrag"
|
||||
yield Claim(
|
||||
claim_type="money_flow", org=buyer, counterpart_org=supplier, occurred_at=awarded,
|
||||
summary_de=f"Auftrag „{title}“ an {supplier.name}" + (f" – {format_eur_de(amount)} €" if amount else ""),
|
||||
summary_en=f"Contract \"{title}\" awarded to {supplier.name}" + (f" – EUR {amount}" if amount else ""),
|
||||
data={"notice_id": notice["notice_id"], "cpv": notice["cpv"], "publisher": response.meta.get("publisher")},
|
||||
evidence=[Evidence(url=response.request.url, doc_type="register_entry", title=f"Vergabe: {title}",
|
||||
published_at=awarded, quote=f"{buyer.name} → {supplier.name}: {title}")],
|
||||
facet=Money(kind="contract", amount=amount, payer=buyer, payer_name=buyer.name, payee=supplier,
|
||||
payee_name=supplier.name, paid_on=awarded,
|
||||
reference=f"{notice['notice_id']}/{contract['id']}", category=notice["cpv"]),
|
||||
)
|
||||
@@ -0,0 +1,551 @@
|
||||
"""Rechnungshof (Court of Audit): published party donations and parties' annual financial reports.
|
||||
|
||||
Official works (§7 UrhG); no robots restrictions.
|
||||
|
||||
Donations: HTML pages + one CSV per year/election. The CSVs come in UTF-8, Windows-1252 and Mac Roman,
|
||||
with or without title rows, and one older file without header (date; amount; donor; address; party).
|
||||
|
||||
Annual reports (Rechenschaftsberichte, 2023+ as XLSX): income statement per organisation level
|
||||
(public funding "Fördermittel", membership fees, donations, sponsoring, ad income...) and named lists:
|
||||
donations > EUR 500, sponsors, advertisers in party media, party-owned companies, loans, affiliated
|
||||
organisations.
|
||||
|
||||
Data minimisation: donor addresses and postcodes are never mapped; private donors are stored as names
|
||||
only (no person entity is created); company donors become organisation entities (contract/donor correlation).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Document, Evidence, Money, OrgRef
|
||||
from politrack.text import firmenbuch_number, format_eur_de, normalize, normalize_org_name
|
||||
|
||||
BASE = "https://www.rechnungshof.gv.at"
|
||||
SPEC = SourceSpec(
|
||||
slug="rechnungshof",
|
||||
name="Rechnungshof Österreich",
|
||||
kind="official",
|
||||
homepage=BASE,
|
||||
license="Amtliches Werk (§7 UrhG)",
|
||||
access="scrape",
|
||||
trust=95,
|
||||
min_interval=2.0,
|
||||
tos_notes="Party donation lists and audited annual party reports. Official works; no robots restrictions.",
|
||||
)
|
||||
|
||||
DONATION_INDEX = [
|
||||
"/rh/home/was-wir-tun/was-wir-tun_5/Kontrolle_der_Parteien.html",
|
||||
"/rh/home/was-wir-tun/was-wir-tun_5/was-wir-tun_5/Parteispenden/Parteispenden_2024.html",
|
||||
]
|
||||
REPORTS_INDEX = "/rh/home/was-wir-tun/was-wir-tun_5/was-wir-tun_6/Kontrolle_der_Parteien.html"
|
||||
SECTION_PREFIX = "/rh/home/was-wir-tun/"
|
||||
DONATION_LINK = re.compile(r"spende", re.IGNORECASE)
|
||||
PARTY_PREFIXES = {"ovp": "oevp", "osterreichische volkspartei": "oevp", "die neue volkspartei": "oevp",
|
||||
"spo": "spoe", "sozialdemokratische": "spoe", "fpo": "fpoe", "freiheitliche": "fpoe",
|
||||
"die freiheitliche": "fpoe", "neos": "neos", "die grunen": "gruene", "grune": "gruene",
|
||||
"kpo": "kpoe", "kommunistische": "kpoe", "bier": "bier", "die bierpartei": "bier", "mfg": "mfg"}
|
||||
COMPANY = re.compile(r"\b(GmbH|AG|KG|OG|e\.U\.|GesmbH|Ges\.m\.b\.H|Gesellschaft|Verein|Stiftung|Bank|Holding|"
|
||||
r"Genossenschaft|Verband|Kammer|Sparkasse|Versicherung|Privatstiftung)\b", re.IGNORECASE)
|
||||
|
||||
|
||||
# -- shared helpers -----------------------------------------------------------------------
|
||||
|
||||
def decode(raw: bytes) -> str:
|
||||
"""UTF-8, Windows-1252 or Mac Roman: pick the decoding that yields real umlauts."""
|
||||
try:
|
||||
return raw.decode("utf-8-sig")
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
candidates = [raw.decode(enc, errors="replace") for enc in ("cp1252", "mac_roman")]
|
||||
return max(candidates, key=lambda t: sum(t.count(c) for c in "äöüÄÖÜß"))
|
||||
|
||||
|
||||
def parse_amount(value: Any) -> Decimal | None:
|
||||
if isinstance(value, (int, float, Decimal)):
|
||||
return Decimal(str(value)).quantize(Decimal("0.01"))
|
||||
value = (value or "").strip().replace(".", "").replace(",", ".").replace("€", "").strip()
|
||||
try:
|
||||
return Decimal(value)
|
||||
except InvalidOperation:
|
||||
return None
|
||||
|
||||
|
||||
def parse_de_date(value: str) -> date | None:
|
||||
for fmt in ("%d.%m.%Y", "%d.%m.%y"):
|
||||
try:
|
||||
return datetime.strptime((value or "").strip(), fmt).date()
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def format_de(amount: Decimal) -> str:
|
||||
return format_eur_de(amount)
|
||||
|
||||
|
||||
POSTAL_ADDRESS = re.compile(r"^\d{4}\s")
|
||||
|
||||
|
||||
def clean_name(value: str) -> str:
|
||||
"""Guard against malformed cells ('"2522 Ort Straße 4;Partei"') - an address never ends up in a name."""
|
||||
value = (value or "").split(";")[-1].strip()
|
||||
return "" if POSTAL_ADDRESS.match(value) else value
|
||||
|
||||
|
||||
def party_ref(label: str) -> OrgRef | None:
|
||||
if not label:
|
||||
return None
|
||||
norm = normalize(label)
|
||||
for prefix, slug in PARTY_PREFIXES.items():
|
||||
if norm.startswith(prefix):
|
||||
return OrgRef(slug=slug)
|
||||
return OrgRef(name=label.strip(), kind="party")
|
||||
|
||||
|
||||
def donor_ref(name: str) -> OrgRef | None:
|
||||
"""Organisations become entities (for correlations); private persons stay a plain name."""
|
||||
return OrgRef(name=name, kind="company") if COMPANY.search(name) else None
|
||||
|
||||
|
||||
def column(header: list[str], *needles: str) -> int | None:
|
||||
for i, h in enumerate(header):
|
||||
h = normalize(h)
|
||||
if any(n in h for n in needles):
|
||||
return i
|
||||
return None
|
||||
|
||||
|
||||
DATE_CELL = re.compile(r"^\d{1,2}\.\d{1,2}\.\d{2,4}$")
|
||||
AMOUNT_CELL = re.compile(r"^[\d.]+(,\d{1,2})?$")
|
||||
|
||||
|
||||
def find_columns(rows: list[list[str]]) -> tuple[int, dict[str, int | None]]:
|
||||
"""Locate the header row (older files have a title row above it) and map the columns we use.
|
||||
Address/postcode columns are never mapped."""
|
||||
for i, row in enumerate(rows[:5]):
|
||||
norm = [normalize(c) for c in row]
|
||||
if any("betrag" in c for c in norm) and any("name" in c or "spender" in c for c in norm):
|
||||
return i + 1, {
|
||||
"party": column(row, "partei"),
|
||||
"date": column(row, "eingang") if column(row, "eingang") is not None else column(row, "datum"),
|
||||
"donor": column(row, "name", "spender"),
|
||||
"amount": column(row, "betrag"),
|
||||
"currency": column(row, "wahrung", "waehrung"),
|
||||
"recipient": column(row, "empfang", "empfaeng"),
|
||||
"note": column(row, "anmerk"),
|
||||
}
|
||||
first = next((r for r in rows if any(c.strip() for c in r)), [])
|
||||
date_i = next((i for i, c in enumerate(first) if DATE_CELL.match(c.strip())), None)
|
||||
amount_i = next((i for i, c in enumerate(first) if AMOUNT_CELL.match(c.strip()) and i != date_i), None)
|
||||
if date_i is None or amount_i is None:
|
||||
return 0, {}
|
||||
return 0, {"party": len(first) - 1, "date": date_i, "donor": amount_i + 1, "amount": amount_i,
|
||||
"currency": None, "recipient": len(first) - 1, "note": None}
|
||||
|
||||
|
||||
def read_rows(text: str) -> tuple[dict[str, int | None], list[list[str]]]:
|
||||
"""Parse the semicolon CSV; repairs rows broken by stray line breaks."""
|
||||
reader = list(csv.reader(io.StringIO(text), delimiter=";"))
|
||||
if not reader:
|
||||
return {}, []
|
||||
start, idx = find_columns(reader)
|
||||
if not idx:
|
||||
return {}, []
|
||||
width = max(i for i in idx.values() if i is not None) + 1
|
||||
rows, pending = [], []
|
||||
for row in reader[start:]:
|
||||
if pending:
|
||||
pending[-1] += " " + (row[0] if row else "")
|
||||
pending.extend(row[1:])
|
||||
else:
|
||||
pending = list(row)
|
||||
if len(pending) >= width - 1:
|
||||
rows.append(pending)
|
||||
pending = []
|
||||
return idx, [r for r in rows if any(c.strip() for c in r)]
|
||||
|
||||
|
||||
# -- donations ----------------------------------------------------------------------------
|
||||
|
||||
@register
|
||||
class RechnungshofDonations(Adapter):
|
||||
spec = SPEC
|
||||
name = "rechnungshof.donations"
|
||||
description = "Published party donations (CSV per year / election)"
|
||||
schedule = "30 5 * * *"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
for path in DONATION_INDEX:
|
||||
yield Request(url=urljoin(BASE, path), kind="page", meta={"depth": 0})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
if response.request.kind == "page":
|
||||
depth = response.meta.get("depth", 0)
|
||||
for a in response.soup.find_all("a", href=True):
|
||||
href = urljoin(response.url, a["href"])
|
||||
path = href.removeprefix(BASE)
|
||||
if href.lower().endswith(".csv") and DONATION_LINK.search(href):
|
||||
yield Request(url=href, kind="csv", meta={"page": response.url})
|
||||
elif (depth < 2 and path.startswith(SECTION_PREFIX) and path.endswith(".html")
|
||||
and DONATION_LINK.search(path)):
|
||||
yield Request(url=href, kind="page", meta={"depth": depth + 1})
|
||||
return
|
||||
yield from self.parse_csv(ctx, response)
|
||||
|
||||
def parse_csv(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||||
idx, rows = read_rows(decode(response.body))
|
||||
if idx.get("donor") is None or idx.get("amount") is None:
|
||||
ctx.stats["unparsed_files"] += 1
|
||||
return
|
||||
|
||||
def cell(row: list[str], key: str) -> str:
|
||||
i = idx[key]
|
||||
value = row[i].strip() if i is not None and i < len(row) else ""
|
||||
return clean_name(value) if key in ("party", "donor", "recipient") else value
|
||||
|
||||
for n, row in enumerate(rows):
|
||||
party_label, donor, recipient = cell(row, "party"), cell(row, "donor"), cell(row, "recipient")
|
||||
amount, received = parse_amount(cell(row, "amount")), parse_de_date(cell(row, "date"))
|
||||
if not donor or amount is None:
|
||||
continue
|
||||
currency = cell(row, "currency") or "EUR"
|
||||
currency = "EUR" if currency.lower().startswith("eur") else currency
|
||||
party = party_ref(party_label) or party_ref(recipient)
|
||||
line = " | ".join(c for c in (cell(row, "date"), party_label, donor, cell(row, "amount"), currency,
|
||||
recipient) if c)
|
||||
yield Claim(
|
||||
claim_type="donation", org=party, occurred_at=received,
|
||||
summary_de=f"{recipient or party_label} erhielt {format_de(amount)} {currency} von {donor}",
|
||||
summary_en=f"{recipient or party_label} received {currency} {amount:,.2f} from {donor}",
|
||||
data={"party_label": party_label, "recipient": recipient, "note": cell(row, "note")},
|
||||
evidence=[
|
||||
Evidence(url=response.meta["page"], doc_type="register_entry",
|
||||
title="Rechnungshof: veröffentlichte Parteispenden", quote=line, locator=f"row:{line}"),
|
||||
Evidence(url=response.request.url, doc_type="register_entry",
|
||||
title="Rechnungshof: Parteispenden (CSV)", locator=f"row:{n + 1}"),
|
||||
],
|
||||
topics=("Transparenz und Korruption",),
|
||||
facet=Money(kind="donation", amount=amount, currency=currency, payer=donor_ref(donor), payer_name=donor,
|
||||
payee=party, payee_name=recipient or party_label, paid_on=received),
|
||||
)
|
||||
|
||||
|
||||
# -- annual party reports -------------------------------------------------------------------
|
||||
|
||||
# Lines of the income statement (§ 5 Abs. 4/5 PartG), matched on the normalised label. Order matters:
|
||||
# the first matching pattern wins ("Inserate und Werbeeinschaltungen" is an expense, "Inserate" income).
|
||||
FINANCE_LINES = [
|
||||
(re.compile(r"^(gesamt)?summe (der )?ertrage"), "total_income"),
|
||||
(re.compile(r"^(gesamt)?summe (der )?aufwendungen"), "total_expenses"),
|
||||
(re.compile(r"^fordermittel"), "public_funding"),
|
||||
(re.compile(r"^mitgliedsbeitrage"), "membership_fees"),
|
||||
(re.compile(r"^beitrage der .*angehorenden mandatare"), "mandate_holder_levies"),
|
||||
(re.compile(r"^geldspenden"), "donations"),
|
||||
(re.compile(r"^sponsoring"), "sponsoring"),
|
||||
(re.compile(r"^inserate und werbeeinschaltungen"), "ad_spending"),
|
||||
(re.compile(r"^inserate\b"), "ad_income"),
|
||||
(re.compile(r"^ertrage aus nahestehenden organisationen"), "from_affiliates"),
|
||||
(re.compile(r"^personalaufwand"), "staff_costs"),
|
||||
(re.compile(r"^au(ss|ß)enwerbung"), "outdoor_advertising"),
|
||||
(re.compile(r"^direktwerbung"), "direct_marketing"),
|
||||
]
|
||||
|
||||
|
||||
def finance_lines(rows: list[list[Any]], year: str) -> dict[str, Decimal]:
|
||||
"""Income/expense lines of one sheet, summed over all blocks in it.
|
||||
|
||||
Some parties stack one block per organisation in a sheet (e.g. every state below the other); each block
|
||||
has its own header row with the years. The value is taken from the reporting-year column of the latest
|
||||
header (columns are "2024, 2023" in some files and "2023, 2024" in others); without a header, the first
|
||||
number after the label.
|
||||
"""
|
||||
total: dict[str, Decimal] = {}
|
||||
block: dict[str, Decimal] = {}
|
||||
year_col: int | None = None
|
||||
for row in rows:
|
||||
years = [i for i, c in enumerate(row) if str(c).strip() in (year, f"{year}.0")]
|
||||
if years and sum(1 for c in row if re.fullmatch(r"(19|20)\d\d(\.0)?", str(c).strip() if c is not None else "")) >= 1:
|
||||
year_col = years[0]
|
||||
for i, cell in enumerate(row):
|
||||
if not isinstance(cell, str) or len(cell.strip()) < 4:
|
||||
continue
|
||||
label = re.sub(r"\s+", " ", normalize(cell))
|
||||
key = next((k for pattern, k in FINANCE_LINES if pattern.search(label)), None)
|
||||
if key:
|
||||
value = number(row[year_col]) if year_col is not None and year_col > i and year_col < len(row) else None
|
||||
if value is None:
|
||||
value = next((number(c) for c in row[i + 1:] if number(c) is not None), None)
|
||||
if value is not None:
|
||||
if key in block: # the same line again: a new block (next organisation) starts
|
||||
for k, v in block.items():
|
||||
total[k] = total.get(k, Decimal(0)) + v
|
||||
block = {}
|
||||
block[key] = value
|
||||
break # only the first text cell of a row is its label
|
||||
for k, v in block.items():
|
||||
total[k] = total.get(k, Decimal(0)) + v
|
||||
return total
|
||||
|
||||
|
||||
def sum_levels(levels: dict[str, dict[str, Decimal]]) -> tuple[dict[str, Decimal], dict[str, dict[str, Decimal]]]:
|
||||
"""Sum all organisation levels (federal, states, capitals, districts ...) -> (totals, levels used).
|
||||
|
||||
Overview sheets that add up other levels are left out: titles listing several levels ("BO&LOs&LH-St.&NTG")
|
||||
and sheets whose public funding equals the sum of all the others. Totals (income, expenses) are only
|
||||
reported when every level states them - a partial sum would be misleading.
|
||||
"""
|
||||
levels = {t: v for t, v in levels.items() if "&" not in t}
|
||||
funding = {title: v.get("public_funding", Decimal(0)) for title, v in levels.items()}
|
||||
for title, value in funding.items():
|
||||
others = sum(v for t, v in funding.items() if t != title)
|
||||
if value > 0 and len(funding) > 2 and abs(value - others) <= Decimal(1):
|
||||
levels = {t: v for t, v in levels.items() if t != title}
|
||||
break
|
||||
totals: dict[str, Decimal] = {}
|
||||
for values in levels.values():
|
||||
for key, value in values.items():
|
||||
totals[key] = totals.get(key, Decimal(0)) + value
|
||||
for key in ("total_income", "total_expenses"):
|
||||
if key in totals and not all(key in v for v in levels.values()):
|
||||
del totals[key]
|
||||
return totals, levels
|
||||
|
||||
|
||||
AFFILIATE_SKIP = re.compile(
|
||||
r"^(anlage|liste|ertrag|gem\.|gemass|\(?§|name$|zvr|reg\.? ?nr|organisationseinheit|nahestehende|"
|
||||
r"bundesorganisation|landesorganisation|zweigvereine:?$|[a-z]\)|[a-z]\. anlage|keine|leermeldung|rechtsform|zweck)")
|
||||
ZVR = re.compile(r"\(?ZVR(?:-Zahl)?:?\s*(\d{6,10})\)?", re.IGNORECASE)
|
||||
|
||||
|
||||
def affiliate_from_row(row: list[Any]) -> tuple[str, str | None] | None:
|
||||
"""Name (and association-register number) of an affiliated organisation from one row of the list."""
|
||||
cells = [str(c).strip() for c in row if c is not None and str(c).strip()]
|
||||
number_cell = next((c for c in cells if re.fullmatch(r"\d{6,10}", c)), None)
|
||||
for cell in cells:
|
||||
if re.fullmatch(r"[\d\s./-]+", cell) or cell.lower() in ("partei", "verein", "gmbh", "stiftung"):
|
||||
continue
|
||||
if AFFILIATE_SKIP.search(normalize(cell)) or len(cell) <= 3:
|
||||
return None
|
||||
zvr = ZVR.search(cell)
|
||||
name = ZVR.sub("", cell)
|
||||
name = re.sub(r'"?,?\s*Kurzbezeichnung:?\s*"?([^"]+)"?\s*$', r" (\1)", name).strip(' "„“,;-')
|
||||
return (name, (zvr.group(1) if zvr else number_cell)) if len(name) > 3 else None
|
||||
return None
|
||||
|
||||
|
||||
def transliterate(text: str) -> str:
|
||||
"""'Kaernten Koefer Gruenen' -> 'Kärnten Köfer Grünen' (file names; not after vowels: 'Neue', 'Steuer')."""
|
||||
for plain, umlaut in (("ae", "ä"), ("oe", "ö"), ("ue", "ü")):
|
||||
text = re.sub(rf"(?<![aeiouqAEIOUQ]){plain}", umlaut, text)
|
||||
text = re.sub(rf"(?<![aeiouqAEIOUQ]){plain.capitalize()}", umlaut.upper(), text)
|
||||
return text
|
||||
|
||||
|
||||
def report_party_name(sheets: dict[str, list[list[Any]]], url: str) -> str:
|
||||
"""Cover sheet, else the 'Territoriale Gliederung ... : <party>' line, else the file name."""
|
||||
cover = [str(c).strip(" -–") for row in sheets.get("Deckblatt", []) for c in row if c and str(c).strip(" -–")]
|
||||
names = [c for c in cover if len(c) > 2 and not re.search(
|
||||
r"(?i)rechenschaftsbericht|bericht|kalenderjahr|jahr\b|partg|gemäß|name der|politische partei$|^\d", c)]
|
||||
if names:
|
||||
return names[0]
|
||||
for rows in sheets.values():
|
||||
for row in rows:
|
||||
for cell in row:
|
||||
if isinstance(cell, str) and (m := re.search(r"Territoriale Gliederung[^:]*:\s*(.+)$", cell.strip())):
|
||||
return m.group(1).strip()
|
||||
stem = url.rsplit("/", 1)[-1].rsplit(".", 1)[0]
|
||||
stem = re.sub(r"(?i)^Rechenschaftsbericht[_-]?\d{4}[_-]?|[_-](gepr(ue|ü)ft|Anlagen|Verm(oe|ö)gensaufstellung)$", "", stem)
|
||||
return transliterate(re.sub(r"[_-]+", " ", stem)).strip()
|
||||
|
||||
|
||||
def sheets_titled(sheets: dict[str, list[list[Any]]], *keywords: str) -> list[list[Any]]:
|
||||
"""All rows of every sheet whose (normalised) title contains one of the keywords."""
|
||||
return [row for title, rows in sheets.items() if any(k in normalize(title) for k in keywords) for row in rows]
|
||||
|
||||
|
||||
def number(value: Any) -> Decimal | None:
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, (int, float, Decimal)):
|
||||
return Decimal(str(value)).quantize(Decimal("0.01"))
|
||||
text = str(value).strip()
|
||||
if re.fullmatch(r"-?\d+\.\d{1,2}", text): # '301561.19' - plain decimal point (some exports)
|
||||
return Decimal(text).quantize(Decimal("0.01"))
|
||||
return parse_amount(text) if re.fullmatch(r"[\d.,\s€-]+", text or "x") else None
|
||||
|
||||
|
||||
def sheet_lists(rows: list[list[Any]], header_needles: tuple[str, ...]) -> Iterator[dict[str, Any]]:
|
||||
"""Yield dict rows of every table in a sheet whose header contains all needles (case-insensitive)."""
|
||||
header: list[str] | None = None
|
||||
for row in rows:
|
||||
cells = ["" if c is None else str(c).strip() for c in row]
|
||||
lowered = [normalize(c) for c in cells]
|
||||
if all(any(n in c for c in lowered) for n in header_needles):
|
||||
header = lowered
|
||||
continue
|
||||
if header is None or not any(cells):
|
||||
continue
|
||||
if any("leermeldung" in c or "keine" in c for c in lowered[:2]):
|
||||
header = None
|
||||
continue
|
||||
record = {header[i]: row[i] for i in range(min(len(header), len(row))) if header[i]}
|
||||
amount = next((number(row[i]) for i in range(len(row) - 1, -1, -1) if number(row[i]) is not None), None)
|
||||
if amount is not None and any(isinstance(v, str) and v.strip() for v in record.values()):
|
||||
record["__amount"] = amount
|
||||
yield record
|
||||
|
||||
|
||||
def pick(record: dict[str, Any], *needles: str) -> str:
|
||||
for key, value in record.items():
|
||||
if any(n in key for n in needles) and isinstance(value, str) and value.strip():
|
||||
return value.strip()
|
||||
return ""
|
||||
|
||||
|
||||
@register
|
||||
class RechnungshofPartyReports(Adapter):
|
||||
spec = SPEC
|
||||
name = "rechnungshof.party_reports"
|
||||
description = "Parties' annual financial reports: funding, income, sponsors, advertisers, companies, loans"
|
||||
schedule = "0 6 * * 1"
|
||||
parser_version = 3
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
yield Request(url=f"{BASE}{REPORTS_INDEX}", kind="index")
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
if response.request.kind == "index":
|
||||
for a in response.soup.find_all("a", href=True):
|
||||
href = urljoin(response.url, a["href"])
|
||||
if re.search(r"Rechenschaftsbericht[_-]?\d{4}.*\.xlsx$", href, re.I):
|
||||
yield Request(url=href, kind="xlsx")
|
||||
elif re.search(r"(Ergebnis[_-]der[_-]Pruefung|Rechenschaftsbericht).*\.pdf$", href, re.I):
|
||||
yield Document(Evidence(url=href, doc_type="pdf", title=a.get_text(" ", strip=True) or href.rsplit("/", 1)[-1],
|
||||
meta={"kind": "party_report_pdf"}))
|
||||
return
|
||||
yield from self.parse_report(ctx, response)
|
||||
|
||||
def parse_report(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
import openpyxl
|
||||
|
||||
wb = openpyxl.load_workbook(io.BytesIO(response.body), data_only=True, read_only=True)
|
||||
sheets = {ws.title.strip(): [list(r) for r in ws.iter_rows(values_only=True)] for ws in wb.worksheets}
|
||||
url = response.request.url
|
||||
year_match = re.search(r"Rechenschaftsbericht[_-]?(\d{4})", url) or re.search(r"(\d{4})", url)
|
||||
year = year_match.group(1) if year_match else "?"
|
||||
audited = bool(re.search(r"gepr(ue|ü)ft", url, re.IGNORECASE))
|
||||
party_name = report_party_name(sheets, url)
|
||||
party = party_ref(party_name)
|
||||
version_de = "vom Rechnungshof geprüft" if audited else "eingereicht"
|
||||
evidence = [Evidence(url=url, doc_type="register_entry",
|
||||
title=f"Rechenschaftsbericht {year} ({version_de}): {party_name}")]
|
||||
|
||||
# Income statement per organisation level (federal, states, capitals, districts...), then summed.
|
||||
levels = {title: lines for title, rows in sheets.items()
|
||||
if (lines := finance_lines(rows, year)) and ("public_funding" in lines or "total_income" in lines)}
|
||||
totals, levels = sum_levels(levels)
|
||||
if totals:
|
||||
income = totals.get("total_income")
|
||||
yield Claim(
|
||||
claim_type="party_finance", org=party,
|
||||
key=("rh-report", normalize_org_name(party_name), year, "audited" if audited else "submitted"),
|
||||
valid_from=date(int(year), 1, 1) if year.isdigit() else None,
|
||||
valid_to=date(int(year), 12, 31) if year.isdigit() else None,
|
||||
summary_de=f"Rechenschaftsbericht {year} ({version_de}): "
|
||||
+ (f"Erträge {format_de(income)} €, " if income is not None else "")
|
||||
+ f"davon Fördermittel {format_de(totals.get('public_funding', Decimal(0)))} €"
|
||||
+ " (Summe aller ausgewiesenen Gliederungen)",
|
||||
summary_en=f"Annual report {year} ({'audited' if audited else 'submitted'}): "
|
||||
+ (f"income EUR {income:,.2f}, " if income is not None else "")
|
||||
+ f"public funding EUR {totals.get('public_funding', 0):,.2f}",
|
||||
data={"year": year, "audited": audited, **{k: str(v) for k, v in totals.items()},
|
||||
"levels": {t: {k: str(v) for k, v in lines.items()} for t, lines in levels.items()}},
|
||||
evidence=evidence, topics=("Transparenz und Korruption",),
|
||||
)
|
||||
if totals.get("public_funding"):
|
||||
yield Claim(
|
||||
claim_type="money_flow", org=party, valid_from=date(int(year), 1, 1) if year.isdigit() else None,
|
||||
summary_de=f"{party_name} erhielt {year} öffentliche Fördermittel von {format_de(totals['public_funding'])} €",
|
||||
summary_en=f"{party_name} received EUR {totals['public_funding']:,.2f} in public funding in {year}",
|
||||
evidence=evidence, topics=("Transparenz und Korruption",),
|
||||
facet=Money(kind="party_funding", amount=totals["public_funding"], payee=party,
|
||||
payee_name=party_name, payer_name="Öffentliche Parteienförderung", period=year),
|
||||
)
|
||||
else:
|
||||
ctx.stats["reports_without_income_statement"] += 1
|
||||
|
||||
def money_claims(rows: list[list[Any]], needles: tuple[str, ...], kind: str, who_keys: tuple[str, ...],
|
||||
label_de: str) -> Iterator[Claim]:
|
||||
for rec in sheet_lists(rows, needles):
|
||||
who = clean_name(pick(rec, *who_keys))
|
||||
if not who:
|
||||
continue
|
||||
recipient = pick(rec, "empfanger", "empfager")
|
||||
medium = pick(rec, "medium")
|
||||
amount = rec["__amount"]
|
||||
yield Claim(
|
||||
claim_type="donation" if kind == "donation" else "money_flow", org=party,
|
||||
valid_from=date(int(year), 1, 1) if year.isdigit() else None,
|
||||
summary_de=f"{label_de}: {who} → {recipient or party_name}"
|
||||
+ (f" ({medium})" if medium else "") + f", {format_de(amount)} € ({year})",
|
||||
data={"year": year, "recipient": recipient, "medium": medium},
|
||||
evidence=[Evidence(url=url, doc_type="register_entry", title=evidence[0].title,
|
||||
quote=f"{who} | {recipient} | {medium} | {amount}", locator=f"{kind}:{who}:{amount}")],
|
||||
topics=("Transparenz und Korruption",),
|
||||
facet=Money(kind=kind, amount=amount, payer=donor_ref(who), payer_name=who, payee=party,
|
||||
payee_name=recipient or party_name, period=year, category=medium or None),
|
||||
)
|
||||
|
||||
# List sheets are named differently by every party ("Spenden", "Spendenliste (§ 6 PartG)", "Spenden,
|
||||
# Sponsoring, Inserate" ...): select by title keyword, find the tables by their header.
|
||||
yield from money_claims(sheets_titled(sheets, "spende"), ("empfanger", "name", "plz"), "donation", ("name",),
|
||||
"Spende laut Rechenschaftsbericht")
|
||||
yield from money_claims(sheets_titled(sheets, "sponsor"), ("sponsor",), "sponsorship", ("sponsor",), "Sponsoring")
|
||||
yield from money_claims(sheets_titled(sheets, "inserat"), ("inserent",), "party_ad", ("inserent",),
|
||||
"Inserat in Parteimedium")
|
||||
for rec in sheet_lists(sheets_titled(sheets, "kredit", "darlehen", "verbindlichkeit"), ("kreditgeber",)):
|
||||
lender = clean_name(pick(rec, "kreditgeber"))
|
||||
if lender:
|
||||
yield Claim(
|
||||
claim_type="money_flow", org=party,
|
||||
summary_de=f"Kredit/Darlehen von {lender} an {party_name}: {format_de(rec['__amount'])} € ({year})",
|
||||
evidence=evidence, facet=Money(kind="loan", amount=rec["__amount"], payer=OrgRef(name=lender, kind="company"),
|
||||
payer_name=lender, payee=party, payee_name=party_name, period=year),
|
||||
)
|
||||
for row in sheets_titled(sheets, "beteiligung"):
|
||||
cells = [str(c).strip() for c in row if c is not None and str(c).strip()]
|
||||
fn = next((f for f in (firmenbuch_number(c) for c in cells[1:3]) if f), None)
|
||||
if fn:
|
||||
company = re.sub(r"\s*\d\)$", "", cells[0]) # footnote markers: "Werbeagentur 1)"
|
||||
yield Claim(
|
||||
claim_type="ownership", org=party,
|
||||
counterpart_org=OrgRef(name=company, kind="company", ids={"firmenbuch": fn}),
|
||||
key=("rh-ownership", normalize_org_name(party_name), year, fn),
|
||||
summary_de=f"{party_name} hält eine Beteiligung an {company} (FN {fn}) – laut Rechenschaftsbericht {year}",
|
||||
evidence=evidence, topics=("Transparenz und Korruption",),
|
||||
)
|
||||
for title, rows in sheets.items():
|
||||
# list of affiliated organisations (not the sheet of income received from them)
|
||||
if "nahestehend" not in normalize(title) or "ertr" in normalize(title):
|
||||
continue
|
||||
for row in rows:
|
||||
affiliate = affiliate_from_row(row)
|
||||
if affiliate:
|
||||
name, zvr = affiliate
|
||||
yield Claim(
|
||||
claim_type="affiliation", org=party,
|
||||
counterpart_org=OrgRef(name=name, kind="ngo", ids={"zvr": zvr} if zvr else {}),
|
||||
key=("rh-affiliate", normalize_org_name(party_name), year, normalize_org_name(name)),
|
||||
summary_de=f"Nahestehende Organisation von {party_name}: {name} ({year})",
|
||||
evidence=evidence,
|
||||
)
|
||||
ctx.stats["reports"] += 1
|
||||
@@ -0,0 +1,106 @@
|
||||
"""RTR / KommAustria media transparency (MedKF-TG): advertising and media subsidies paid by public bodies.
|
||||
|
||||
Open data API (no usage restrictions, attribution): https://data.rtr.at/api/v1/tables/<table>.json
|
||||
* mt-bekanntgabe2024 (half-yearly since 2024): payer, medium, media owner, campaign, amount, ad sujet link
|
||||
* mt-bekanntgabe (quarterly until 2023): payer, medium/owner, amount
|
||||
Each row -> money_flow (kind government_ad for advertising, subsidy for media funding) from the public body
|
||||
(ministry, state government, municipality, public company...) to the media owner.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Evidence, Money, OrgRef
|
||||
from politrack.text import format_eur_de
|
||||
|
||||
API = "https://data.rtr.at/api/v1/tables"
|
||||
PAGE_SIZE = 5000
|
||||
SPEC = SourceSpec(
|
||||
slug="rtr-medientransparenz", name="RTR/KommAustria Medientransparenz", kind="official",
|
||||
homepage="https://www.rtr.at/medien/was_wir_tun/medientransparenz/",
|
||||
license="Amtliches Werk (§7 UrhG); attribution \"KommAustria – data.rtr.at\"", access="api",
|
||||
trust=95, min_interval=2.0,
|
||||
# data.rtr.at/robots.txt disallows everything for crawlers, but RTR documents this API as "der Endpunkt zur
|
||||
# automatischen Abfrage der Daten" (rtr.at/rtr/service/opendata/OD_Schnittstellenbeschreibung.de.html).
|
||||
robots_exempt=(API + "/",),
|
||||
tos_notes="Government advertising and media funding reported under MedKF-TG. The REST API is documented by "
|
||||
"RTR for automated queries (OD_Schnittstellenbeschreibung); robots.txt on data.rtr.at blocks "
|
||||
"crawlers only. Terms (OD_Nutzungsbedingungen): free works under §7 UrhG, attribution requested, "
|
||||
"no use for defamatory/discriminatory purposes.",
|
||||
)
|
||||
KIND = {2: "government_ad", 4: "subsidy"}
|
||||
|
||||
|
||||
def half_years(since_year: int) -> list[str]:
|
||||
today = date.today()
|
||||
return [f"{y}{h}" for y in range(since_year, today.year + 1) for h in (1, 2)
|
||||
if (y, h) <= (today.year, 1 if today.month <= 6 else 2)]
|
||||
|
||||
|
||||
def quarters(since_year: int, until_year: int = 2023) -> list[str]:
|
||||
return [f"{y}{q}" for y in range(since_year, until_year + 1) for q in (1, 2, 3, 4)]
|
||||
|
||||
|
||||
@register
|
||||
class MediaTransparency(Adapter):
|
||||
spec = SPEC
|
||||
name = "rtr.medientransparenz"
|
||||
description = "Government advertising and media subsidies per public body and media owner"
|
||||
schedule = "0 4 * * 2"
|
||||
queue = "bulk"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
for period in half_years(int(ctx.params.get("since_year", 2024))):
|
||||
yield Request(url=f"{API}/mt-bekanntgabe2024.json", params={"halbjahr": period, "size": PAGE_SIZE, "page": 0},
|
||||
kind="halbjahr", meta={"period": period})
|
||||
for period in quarters(int(ctx.params.get("legacy_since_year", 2021))):
|
||||
yield Request(url=f"{API}/mt-bekanntgabe.json", params={"quartal": period, "size": PAGE_SIZE, "page": 0},
|
||||
kind="quartal", meta={"period": period})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
rows = response.json().get("data") or []
|
||||
params = response.request.params or {}
|
||||
if len(rows) >= PAGE_SIZE:
|
||||
yield Request(url=response.request.url, params={**params, "page": params.get("page", 0) + 1},
|
||||
kind=response.request.kind, meta=response.meta)
|
||||
period = response.meta["period"]
|
||||
label = f"{period[:4]}-H{period[4:]}" if response.request.kind == "halbjahr" else f"{period[:4]}-Q{period[4:]}"
|
||||
min_euro = float(ctx.params.get("min_euro", 0))
|
||||
for row in rows:
|
||||
amount = row.get("euro")
|
||||
if row.get("leermeldung") or amount is None or float(amount) < min_euro:
|
||||
continue
|
||||
kind = KIND.get(row.get("bekanntgabe"), "other")
|
||||
payer_name = (row.get("rechtstraeger") or "").strip().strip('"')
|
||||
owner = (row.get("medieninhaber") or row.get("mediumMedieninhaber") or "").strip()
|
||||
medium = (row.get("medium") or row.get("mediumMedieninhaber") or "").strip()
|
||||
if not payer_name or not owner:
|
||||
continue
|
||||
payer = OrgRef(name=payer_name, kind="public_body",
|
||||
ids={"rtr": str(row["rechtstraeger_id"])} if row.get("rechtstraeger_id") else {})
|
||||
payee = OrgRef(name=owner, kind="media")
|
||||
campaign = row.get("kampagne")
|
||||
what = "Werbung" if kind == "government_ad" else "Förderung"
|
||||
evidence_url = (f"{API}/{'mt-bekanntgabe2024' if response.request.kind == 'halbjahr' else 'mt-bekanntgabe'}"
|
||||
f".json?{'halbjahr' if response.request.kind == 'halbjahr' else 'quartal'}={period}"
|
||||
+ (f"&rechtstraeger_id={row['rechtstraeger_id']}" if row.get("rechtstraeger_id") else ""))
|
||||
yield Claim(
|
||||
claim_type="money_flow", org=payer, counterpart_org=payee,
|
||||
summary_de=f"{payer_name}: {what} in {medium} ({owner}), {format_eur_de(amount)} € ({label})"
|
||||
+ (f" – Kampagne „{campaign}“" if campaign else ""),
|
||||
summary_en=f"{payer_name}: {'advertising' if kind == 'government_ad' else 'funding'} in {medium} "
|
||||
f"({owner}), EUR {amount:,.2f} ({label})",
|
||||
data={"medium": medium, "category": row.get("kategorie"), "subcategory": row.get("subkategorie"),
|
||||
"campaign": campaign, "sujet": row.get("sujet"), "period": label},
|
||||
evidence=[Evidence(url=evidence_url, doc_type="dataset_row",
|
||||
title=f"Medientransparenz {label}: {payer_name}",
|
||||
quote=f"{payer_name} → {medium} ({owner}): {amount} €")],
|
||||
topics=("Information und Medien",),
|
||||
facet=Money(kind=kind, amount=amount, payer=payer, payer_name=payer_name, payee=payee, payee_name=owner,
|
||||
period=label, reference=f"{campaign or ''}|{medium}|{row.get('sujet_dateiname') or ''}",
|
||||
category=row.get("kategorie")),
|
||||
)
|
||||
@@ -0,0 +1,279 @@
|
||||
"""Public posts of politicians on open platforms: Bluesky, Mastodon, Telegram (public channels), YouTube,
|
||||
and X via the official (paid) API when a token is configured.
|
||||
|
||||
Accounts come from the `social_account` table: parliament profiles (verified), Wikidata
|
||||
P2002/P12361/P4033/P3789/P2397 (unverified until confirmed) and the curated config/social_accounts.yaml
|
||||
(enricher `config.social_accounts`). Accounts belong to a person or to an organisation (party channels).
|
||||
|
||||
Not collected (see docs/sources.md): X without the official API (ToS forbid scraping and proxies such as
|
||||
Nitter), Facebook/Instagram (ToS), WhatsApp channels (no public read access, ToS forbid automated
|
||||
collection), Signal (private by design, no public channels), private Telegram groups.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Evidence, OrgRef, PersonRef, Statement
|
||||
from politrack.text import excerpt, html_to_text
|
||||
|
||||
|
||||
def social_spec(slug: str, name: str, homepage: str, notes: str, robots_exempt: tuple[str, ...] = ()) -> SourceSpec:
|
||||
return SourceSpec(slug=slug, name=name, kind="social", homepage=homepage, license="Per-post copyright of the author",
|
||||
access="api", trust=40, store_fulltext=True, keep_raw=True, min_interval=2.0,
|
||||
robots_exempt=robots_exempt, tos_notes=notes)
|
||||
|
||||
|
||||
BLUESKY = social_spec("bluesky", "Bluesky", "https://bsky.app",
|
||||
"Public AT Protocol AppView API, no auth for public reads. Posts by the politicians themselves.")
|
||||
MASTODON = social_spec("mastodon", "Mastodon / Fediverse", "https://joinmastodon.org",
|
||||
"Public REST API of the account's instance (no auth for public statuses). Respect instance rules.")
|
||||
TELEGRAM = social_spec("telegram", "Telegram (öffentliche Kanäle)", "https://telegram.org",
|
||||
"Public web preview of *public broadcast channels* (t.me/s/<channel>); no robots.txt restrictions. "
|
||||
"Private groups/chats are never accessed. Low frequency, latest posts only.")
|
||||
YOUTUBE = social_spec("youtube", "YouTube", "https://www.youtube.com",
|
||||
"YouTube Data API v3 (free quota, API key required via POLITRACK_YOUTUBE_API_KEY).")
|
||||
X_API = "https://api.x.com/2/"
|
||||
X = SourceSpec(slug="x", name="X (Twitter)", kind="social", homepage="https://x.com",
|
||||
license="Per-post copyright of the author; X Developer Agreement", access="api", trust=40,
|
||||
store_fulltext=False, keep_raw=False, min_interval=2.0, robots_exempt=(X_API,),
|
||||
tos_notes="Official X API v2 only (paid tier, POLITRACK_X_BEARER_TOKEN). Scraping x.com or using "
|
||||
"mirrors/proxies (Nitter etc.) violates the ToS and is not done. The Developer Agreement requires "
|
||||
"removing posts that were deleted on X; no raw archive or full text is kept, but the "
|
||||
"deletion sync is still open (docs/roadmap.md).")
|
||||
|
||||
PLATFORM_LABELS = {"bluesky": "Bluesky", "mastodon": "Mastodon", "telegram": "Telegram", "youtube": "YouTube",
|
||||
"x": "X"}
|
||||
|
||||
|
||||
def accounts(ctx: RunContext, platform: str, url_pattern: str | None = None) -> list[dict[str, Any]]:
|
||||
"""Accounts of persons or organisations on one platform; each row carries an `owner` dict for Request.meta.
|
||||
|
||||
`url_pattern` also picks up accounts stored as plain websites, so it must match only that platform's
|
||||
own host (never a generic shape like "https://host/@user", which TikTok and others share).
|
||||
"""
|
||||
if ctx.conn is None:
|
||||
return []
|
||||
rows = ctx.conn.execute(
|
||||
"""SELECT person_id, organization_id, url, handle, verified FROM social_account
|
||||
WHERE (platform = %s OR (%s::text IS NOT NULL AND url ~* %s))
|
||||
AND (person_id IS NOT NULL OR organization_id IS NOT NULL)
|
||||
ORDER BY verified DESC, id""",
|
||||
(platform, url_pattern, url_pattern),
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
row["owner"] = {"person_id": str(row["person_id"]) if row["person_id"] else None,
|
||||
"org_id": None if row["person_id"] else str(row["organization_id"]),
|
||||
"verified": row["verified"]}
|
||||
return rows
|
||||
|
||||
|
||||
def post_claim(*, owner: dict[str, Any], platform: str, url: str, text: str, created: Any,
|
||||
title: str | None = None, data: dict | None = None) -> Claim:
|
||||
"""A public post as a `statement` claim of the account owner (person, or organisation for party channels)."""
|
||||
label = PLATFORM_LABELS[platform]
|
||||
return Claim(
|
||||
claim_type="statement", occurred_at=created,
|
||||
person=PersonRef(ids={"politrack": owner["person_id"]}) if owner.get("person_id") else None,
|
||||
org=OrgRef(ids={"politrack_org": owner["org_id"]}, create=False) if owner.get("org_id") else None,
|
||||
summary_de=f"Auf {label}: „{excerpt(title or text, 200)}“",
|
||||
summary_en=f"On {label}: \"{excerpt(title or text, 200)}\"",
|
||||
data={"platform": platform, "account_verified": owner.get("verified", False), **(data or {})},
|
||||
evidence=[Evidence(url=url, source=platform, doc_type="social_post", title=excerpt(title or text, 120),
|
||||
published_at=created, full_text=text, excerpt=text, quote=excerpt(text, 500))],
|
||||
facet=Statement(context="social_post", url=url, said_at=created, platform=platform, title=title,
|
||||
excerpt=text),
|
||||
)
|
||||
|
||||
|
||||
@register
|
||||
class BlueskyPosts(Adapter):
|
||||
spec = BLUESKY
|
||||
name = "social.bluesky"
|
||||
description = "Public Bluesky posts of politicians"
|
||||
schedule = "7 * * * *"
|
||||
queue = "social"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
for acc in accounts(ctx, "bluesky", r"bsky\.app/profile/"):
|
||||
m = re.search(r"bsky\.app/profile/([^/?#]+)", acc["url"])
|
||||
if m:
|
||||
yield Request(url="https://public.api.bsky.app/xrpc/app.bsky.feed.getAuthorFeed",
|
||||
params={"actor": m.group(1), "limit": 50, "filter": "posts_no_replies"}, kind="feed",
|
||||
meta={**acc["owner"], "handle": m.group(1)})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||||
handle = response.meta["handle"]
|
||||
for item in response.json().get("feed", []):
|
||||
post = item.get("post") or {}
|
||||
if item.get("reason"): # repost of someone else's post
|
||||
continue
|
||||
record = post.get("record") or {}
|
||||
text = record.get("text") or ""
|
||||
author = (post.get("author") or {}).get("handle") or handle
|
||||
rkey = post.get("uri", "").rsplit("/", 1)[-1]
|
||||
yield post_claim(owner=response.meta, platform="bluesky",
|
||||
url=f"https://bsky.app/profile/{author}/post/{rkey}", text=text,
|
||||
created=record.get("createdAt"),
|
||||
data={"likes": post.get("likeCount"), "reposts": post.get("repostCount")})
|
||||
|
||||
|
||||
@register
|
||||
class MastodonPosts(Adapter):
|
||||
spec = MASTODON
|
||||
name = "social.mastodon"
|
||||
description = "Public Mastodon/Fediverse posts of politicians"
|
||||
schedule = "17 * * * *"
|
||||
queue = "social"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
for acc in accounts(ctx, "mastodon"): # instances have arbitrary hosts: platform only
|
||||
m = re.match(r"^https?://([^/]+)/@([^/@]+)", acc["url"])
|
||||
if m:
|
||||
host, user = m.groups()
|
||||
yield Request(url=f"https://{host}/api/v1/accounts/lookup", params={"acct": user}, kind="lookup",
|
||||
meta={**acc["owner"], "host": host})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
meta = response.meta
|
||||
if response.request.kind == "lookup":
|
||||
account_id = response.json().get("id")
|
||||
if account_id:
|
||||
yield Request(url=f"https://{meta['host']}/api/v1/accounts/{account_id}/statuses",
|
||||
params={"limit": 40, "exclude_replies": "true", "exclude_reblogs": "true"},
|
||||
kind="statuses", meta=meta)
|
||||
return
|
||||
for status in response.json():
|
||||
if status.get("visibility") not in (None, "public"):
|
||||
continue
|
||||
yield post_claim(owner=meta, platform="mastodon", url=status.get("url") or status.get("uri"),
|
||||
text=html_to_text(status.get("content")), created=status.get("created_at"),
|
||||
data={"favourites": status.get("favourites_count"), "reblogs": status.get("reblogs_count")})
|
||||
|
||||
|
||||
@register
|
||||
class TelegramChannels(Adapter):
|
||||
spec = TELEGRAM
|
||||
name = "social.telegram"
|
||||
description = "Public Telegram broadcast channels of politicians (web preview, latest posts)"
|
||||
schedule = "27 */2 * * *"
|
||||
queue = "social"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
for acc in accounts(ctx, "telegram", r"(t\.me|telegram\.me)/"):
|
||||
m = re.search(r"(?:t\.me|telegram\.me)/(?:s/)?([A-Za-z0-9_]{4,})", acc["url"])
|
||||
if m:
|
||||
yield Request(url=f"https://t.me/s/{m.group(1)}", kind="channel",
|
||||
meta={**acc["owner"], "channel": m.group(1)})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||||
soup = response.soup
|
||||
subscribers = soup.select_one(".tgme_channel_info_counter .counter_value")
|
||||
for msg in soup.select("div.tgme_widget_message[data-post]"):
|
||||
text_el = msg.select_one(".tgme_widget_message_text")
|
||||
time_el = msg.select_one(".tgme_widget_message_date time")
|
||||
text = text_el.get_text("\n", strip=True) if text_el else ""
|
||||
if not text:
|
||||
continue # media-only posts
|
||||
forwarded = msg.select_one(".tgme_widget_message_forwarded_from")
|
||||
views = msg.select_one(".tgme_widget_message_views")
|
||||
yield post_claim(
|
||||
owner=response.meta, platform="telegram", url=f"https://t.me/{msg['data-post']}",
|
||||
text=text, created=time_el.get("datetime") if time_el else None,
|
||||
data={"views": views.get_text(strip=True) if views else None,
|
||||
"forwarded_from": forwarded.get_text(" ", strip=True) if forwarded else None,
|
||||
"channel_subscribers": subscribers.get_text(strip=True) if subscribers else None},
|
||||
)
|
||||
|
||||
|
||||
@register
|
||||
class YouTubeVideos(Adapter):
|
||||
spec = YOUTUBE
|
||||
name = "social.youtube"
|
||||
description = "Latest YouTube uploads of politicians' channels (requires POLITRACK_YOUTUBE_API_KEY)"
|
||||
schedule = "37 */6 * * *"
|
||||
queue = "social"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
key = get_settings().youtube_api_key
|
||||
if not key:
|
||||
ctx.stats["skipped_no_api_key"] = 1
|
||||
return
|
||||
api = "https://www.googleapis.com/youtube/v3/channels"
|
||||
for acc in accounts(ctx, "youtube", r"youtube\.com/"):
|
||||
meta = acc["owner"]
|
||||
if m := re.search(r"youtube\.com/channel/(UC[\w-]+)", acc["url"]):
|
||||
params = {"id": m.group(1)}
|
||||
elif m := re.search(r"youtube\.com/@([\w.-]+)", acc["url"]):
|
||||
params = {"forHandle": f"@{m.group(1)}"}
|
||||
elif m := re.search(r"youtube\.com/(?:user/|c/)?([\w.-]+)/?$", acc["url"]):
|
||||
params = {"forUsername": m.group(1)}
|
||||
else:
|
||||
continue
|
||||
yield Request(url=api, params={**params, "part": "contentDetails", "key": key}, kind="channel", meta=meta)
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
data = response.json()
|
||||
meta = response.meta
|
||||
if response.request.kind == "channel":
|
||||
for item in data.get("items", []):
|
||||
uploads = item["contentDetails"]["relatedPlaylists"]["uploads"]
|
||||
yield Request(url="https://www.googleapis.com/youtube/v3/playlistItems", kind="uploads", meta=meta,
|
||||
params={"playlistId": uploads, "part": "snippet", "maxResults": 25,
|
||||
"key": get_settings().youtube_api_key})
|
||||
return
|
||||
for item in data.get("items", []):
|
||||
snippet = item.get("snippet") or {}
|
||||
video = (snippet.get("resourceId") or {}).get("videoId")
|
||||
if not video:
|
||||
continue
|
||||
yield post_claim(owner=meta, platform="youtube",
|
||||
url=f"https://www.youtube.com/watch?v={video}", title=snippet.get("title"),
|
||||
text=f"{snippet.get('title', '')}\n{snippet.get('description', '')}".strip(),
|
||||
created=snippet.get("publishedAt"))
|
||||
|
||||
|
||||
@register
|
||||
class XPosts(Adapter):
|
||||
spec = X
|
||||
name = "social.x"
|
||||
description = "Posts of politicians on X via the official API v2 (requires POLITRACK_X_BEARER_TOKEN)"
|
||||
schedule = "47 */6 * * *"
|
||||
queue = "social"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
token = get_settings().x_bearer_token
|
||||
if not token:
|
||||
ctx.stats["skipped_no_api_key"] = 1
|
||||
return
|
||||
for acc in accounts(ctx, "twitter", r"//(www\.)?(twitter|x)\.com/"):
|
||||
m = re.search(r"(?:twitter|x)\.com/(?:#!/)?@?(\w{1,15})/?$", acc["url"])
|
||||
if m:
|
||||
yield Request(url=f"{X_API}users/by/username/{m.group(1)}", kind="user", headers=self.auth(),
|
||||
meta=acc["owner"])
|
||||
|
||||
@staticmethod
|
||||
def auth() -> dict[str, str]:
|
||||
return {"Authorization": f"Bearer {get_settings().x_bearer_token}"}
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
data = response.json()
|
||||
meta = response.meta
|
||||
if response.request.kind == "user":
|
||||
user = data.get("data") or {}
|
||||
if user.get("id"):
|
||||
yield Request(url=f"{X_API}users/{user['id']}/tweets", kind="tweets", headers=self.auth(),
|
||||
params={"max_results": 50, "exclude": "replies,retweets",
|
||||
"tweet.fields": "created_at,public_metrics"},
|
||||
meta={**meta, "username": user.get("username")})
|
||||
return
|
||||
for tweet in data.get("data") or []:
|
||||
metrics = tweet.get("public_metrics") or {}
|
||||
yield post_claim(owner=meta, platform="x",
|
||||
url=f"https://x.com/{meta['username']}/status/{tweet['id']}", text=tweet.get("text", ""),
|
||||
created=tweet.get("created_at"),
|
||||
data={"likes": metrics.get("like_count"), "reposts": metrics.get("retweet_count")})
|
||||
@@ -0,0 +1,305 @@
|
||||
"""Wikidata (CC0) via the public SPARQL endpoint.
|
||||
|
||||
Fills what the federal parliament API does not cover: state parliaments (Landtage), state
|
||||
governments, mayors and other offices, party membership, careers (employers, positions abroad,
|
||||
EU roles) and social-media handles. People are merged with parliament records through P2280
|
||||
(Austrian Parliament ID = PAD_INTERN).
|
||||
|
||||
Seats in the Nationalrat/Bundesrat/European Parliament are mapped to the canonical role "mp" in
|
||||
that chamber, so a Wikidata mandate and a Parliament mandate with the same start date are the
|
||||
same claim (corroborated by construction).
|
||||
|
||||
Flow: list Austrian positions -> holders in batches of 40 positions -> for each batch of persons:
|
||||
party memberships, careers, social handles. Batching keeps queries within the endpoint's 60 s limit.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Evidence, OrgRef, PersonRecord, PersonRef, Role, SocialAccount
|
||||
|
||||
SPARQL = "https://query.wikidata.org/sparql"
|
||||
|
||||
SPEC = SourceSpec(
|
||||
slug="wikidata",
|
||||
name="Wikidata",
|
||||
kind="aggregator",
|
||||
homepage="https://www.wikidata.org",
|
||||
license="CC0",
|
||||
access="api",
|
||||
trust=65,
|
||||
min_interval=2.0,
|
||||
robots_exempt=(SPARQL,),
|
||||
tos_notes="SPARQL endpoint. robots.txt disallows /sparql for crawlers, but the Wikimedia User-Agent policy "
|
||||
"allows bots with a descriptive User-Agent to use the query service. Community-edited.",
|
||||
)
|
||||
|
||||
POSITION_LIST_QUERY = """
|
||||
SELECT DISTINCT ?position ?positionLabel WHERE {
|
||||
?position wdt:P17 wd:Q40 .
|
||||
[] ps:P39 ?position .
|
||||
SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". }
|
||||
}
|
||||
"""
|
||||
|
||||
POSITIONS_QUERY = """
|
||||
SELECT ?person ?personLabel ?givenLabel ?familyLabel ?birth ?death ?pad ?ep ?position ?positionLabel ?start ?end
|
||||
?group ?groupLabel WHERE {
|
||||
VALUES ?position { %(positions)s }
|
||||
?person p:P39 ?st .
|
||||
?st ps:P39 ?position .
|
||||
OPTIONAL { ?st pq:P580 ?start }
|
||||
OPTIONAL { ?st pq:P582 ?end }
|
||||
FILTER(!BOUND(?end) || ?end >= "%(since)s-01-01"^^xsd:dateTime)
|
||||
OPTIONAL { ?st pq:P4100 ?group }
|
||||
OPTIONAL { ?person wdt:P2280 ?pad }
|
||||
# Generic positions with country=Austria are also held by foreigners (e.g. "Bundessprecher" of the German
|
||||
# Green Youth): require an Austrian connection - parliament id, citizenship or an Austrian party.
|
||||
FILTER(BOUND(?pad) || EXISTS { ?person wdt:P27 wd:Q40 } || EXISTS { ?person wdt:P102 ?ap . ?ap wdt:P17 wd:Q40 })
|
||||
OPTIONAL { ?person wdt:P1186 ?ep }
|
||||
OPTIONAL { ?person wdt:P569 ?birth }
|
||||
OPTIONAL { ?person wdt:P570 ?death }
|
||||
OPTIONAL { ?person wdt:P735 ?given }
|
||||
OPTIONAL { ?person wdt:P734 ?family }
|
||||
SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". }
|
||||
}
|
||||
"""
|
||||
|
||||
PARTY_QUERY = """
|
||||
SELECT ?person ?party ?partyLabel ?partyShort ?start ?end WHERE {
|
||||
VALUES ?person { %(persons)s }
|
||||
?person p:P102 ?st .
|
||||
?st ps:P102 ?party .
|
||||
?party wdt:P17 wd:Q40 .
|
||||
OPTIONAL { ?st pq:P580 ?start }
|
||||
OPTIONAL { ?st pq:P582 ?end }
|
||||
OPTIONAL { ?party wdt:P1813 ?partyShort . FILTER(LANG(?partyShort) = "de") }
|
||||
SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". }
|
||||
}
|
||||
"""
|
||||
|
||||
CAREER_QUERY = """
|
||||
SELECT ?person ?org ?orgLabel ?start ?end ?kind WHERE {
|
||||
VALUES ?person { %(persons)s }
|
||||
{ ?person p:P108 ?st . ?st ps:P108 ?org . BIND("employer" AS ?kind) }
|
||||
UNION
|
||||
{ ?person p:P39 ?st . ?st ps:P39 ?org . FILTER NOT EXISTS { ?org wdt:P17 wd:Q40 } BIND("position" AS ?kind) }
|
||||
OPTIONAL { ?st pq:P580 ?start }
|
||||
OPTIONAL { ?st pq:P582 ?end }
|
||||
SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". }
|
||||
}
|
||||
"""
|
||||
|
||||
# Social-media properties: P2002 X/Twitter, P12361 Bluesky, P4033 Mastodon, P3789 Telegram,
|
||||
# P2397 YouTube channel id, P2003 Instagram, P2013 Facebook, P7085 TikTok
|
||||
SOCIAL_QUERY = """
|
||||
SELECT ?person ?prop ?value WHERE {
|
||||
VALUES ?person { %(persons)s }
|
||||
VALUES ?prop { wdt:P2002 wdt:P12361 wdt:P4033 wdt:P3789 wdt:P2397 wdt:P2003 wdt:P2013 wdt:P7085 }
|
||||
?person ?prop ?value .
|
||||
}
|
||||
"""
|
||||
SOCIAL_PROPS = {
|
||||
"P2002": ("twitter", "https://x.com/{}"),
|
||||
"P12361": ("bluesky", "https://bsky.app/profile/{}"),
|
||||
"P4033": ("mastodon", None),
|
||||
"P3789": ("telegram", "https://t.me/{}"),
|
||||
"P2397": ("youtube", "https://www.youtube.com/channel/{}"),
|
||||
"P2003": ("instagram", "https://www.instagram.com/{}"),
|
||||
"P2013": ("facebook", "https://www.facebook.com/{}"),
|
||||
"P7085": ("tiktok", "https://www.tiktok.com/@{}"),
|
||||
}
|
||||
|
||||
EU_POLITICAL = re.compile(r"Europäischen Parlament|Europaparlament|Europäischen Kommission|EU-Kommissar|"
|
||||
r"Member of the European Parliament", re.IGNORECASE)
|
||||
EXCLUDE = re.compile(
|
||||
r"professor|kommerzialrat|kommerzienrat|hofrat|ministerialrat|regierungsrat|gerichtshof|richter|"
|
||||
r"botschafter|bischof|rektor|ökonomierat|medizinalrat|schulrat|studienrat|baurat|veterinärrat|"
|
||||
r"bergrat|amtsrat|kammerrat|konsul",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
CHAMBER_LABELS = [
|
||||
(re.compile(r"Abgeordnete[rn]? zum Nationalrat|Mitglied des Nationalrates", re.I), "nationalrat", "federal"),
|
||||
(re.compile(r"Mitglied des (österreichischen )?Bundesrate?s", re.I), "bundesrat", "federal"),
|
||||
(re.compile(r"Mitglied des Europäischen Parlaments|Member of the European Parliament", re.I),
|
||||
"europaeisches-parlament", "eu"),
|
||||
]
|
||||
|
||||
|
||||
def classify_level(label: str) -> str:
|
||||
if re.search(r"europäischen parlament|europaabgeordnet|eu-kommiss", label, re.I):
|
||||
return "eu"
|
||||
if re.search(r"bürgermeister|gemeinderat|stadtrat|vizebürgermeister|gemeinde", label, re.I):
|
||||
return "municipal" if "Wiener Landtag" not in label else "state"
|
||||
if re.search(r"landtag|landesrat|landeshauptmann|landesregierung|landesamtsdirektor", label, re.I):
|
||||
return "state"
|
||||
if re.search(r"bezirk", label, re.I):
|
||||
return "district"
|
||||
return "federal"
|
||||
|
||||
|
||||
def wd_date(binding: dict, key: str) -> date | None:
|
||||
value = binding.get(key, {}).get("value")
|
||||
if not value or value.startswith("-"):
|
||||
return None
|
||||
try:
|
||||
return date.fromisoformat(value[:10])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def qid(binding: dict, key: str) -> str | None:
|
||||
value = binding.get(key, {}).get("value")
|
||||
return value.rsplit("/", 1)[-1] if value else None
|
||||
|
||||
|
||||
def label(binding: dict, key: str) -> str | None:
|
||||
value = binding.get(key, {}).get("value")
|
||||
return None if not value or re.fullmatch(r"Q\d+", value) else value
|
||||
|
||||
|
||||
def sparql(query: str, kind: str, meta: dict | None = None) -> Request:
|
||||
return Request(url=SPARQL, params={"query": query, "format": "json"}, kind=kind, meta=meta or {},
|
||||
headers={"Accept": "application/sparql-results+json"})
|
||||
|
||||
|
||||
def role_for(position_label: str) -> tuple[str, OrgRef | None, str]:
|
||||
for pattern, slug, level in CHAMBER_LABELS:
|
||||
if pattern.search(position_label):
|
||||
return "mp", OrgRef(slug=slug), level
|
||||
return position_label, None, classify_level(position_label)
|
||||
|
||||
|
||||
@register
|
||||
class WikidataPoliticians(Adapter):
|
||||
spec = SPEC
|
||||
name = "wikidata.politicians"
|
||||
description = "Austrian office holders (all levels), party membership, careers, social handles"
|
||||
schedule = "45 4 * * 0"
|
||||
creates_persons = True
|
||||
parser_version = 2
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
yield sparql(POSITION_LIST_QUERY, kind="positions")
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
bindings = json.loads(response.text, strict=False)["results"]["bindings"]
|
||||
kind = response.request.kind
|
||||
if kind == "positions":
|
||||
positions = [qid(b, "position") for b in bindings
|
||||
if label(b, "positionLabel") and not EXCLUDE.search(label(b, "positionLabel") or "")]
|
||||
since = ctx.params.get("since_year", 2000)
|
||||
for i in range(0, len(positions), 40):
|
||||
values = " ".join(f"wd:{q}" for q in positions[i:i + 40])
|
||||
yield sparql(POSITIONS_QUERY % {"positions": values, "since": since}, kind="holders")
|
||||
elif kind == "holders":
|
||||
yield from self.parse_holders(bindings)
|
||||
elif kind == "parties":
|
||||
yield from self.parse_parties(bindings)
|
||||
elif kind == "careers":
|
||||
yield from self.parse_careers(bindings)
|
||||
elif kind == "social":
|
||||
yield from self.parse_social(bindings)
|
||||
|
||||
def parse_holders(self, bindings: list[dict]) -> Iterator[Any]:
|
||||
seen: set[str] = set()
|
||||
for b in bindings:
|
||||
person_qid, name, position = qid(b, "person"), label(b, "personLabel"), label(b, "positionLabel")
|
||||
if not name or not position or EXCLUDE.search(position):
|
||||
continue
|
||||
pad = b.get("pad", {}).get("value")
|
||||
ids = {"wikidata": person_qid}
|
||||
if pad and pad.isdigit():
|
||||
ids["parlament"] = str(int(pad))
|
||||
if b.get("ep", {}).get("value"):
|
||||
ids["ep"] = b["ep"]["value"]
|
||||
ref = PersonRef(ids=ids)
|
||||
if person_qid not in seen:
|
||||
seen.add(person_qid)
|
||||
yield PersonRecord(ref=ref, full_name=name, given_name=label(b, "givenLabel"),
|
||||
family_name=label(b, "familyLabel"), birth_date=wd_date(b, "birth"),
|
||||
death_date=wd_date(b, "death"))
|
||||
start, end = wd_date(b, "start"), wd_date(b, "end") or wd_date(b, "death")
|
||||
group_label = label(b, "groupLabel")
|
||||
group = OrgRef(name=group_label, kind="parliamentary_group", ids={"wikidata": qid(b, "group")}) \
|
||||
if group_label else None
|
||||
role, body, level = role_for(position)
|
||||
yield Claim(
|
||||
claim_type="role", person=ref, org=body, counterpart_org=group, valid_from=start, valid_to=end,
|
||||
summary_de=f"{name}: {position}" + (f" ({group_label})" if group_label else ""),
|
||||
data={"position_qid": qid(b, "position"), "position_label": position},
|
||||
evidence=[Evidence(url=f"https://www.wikidata.org/wiki/{person_qid}", doc_type="api_record",
|
||||
title=f"Wikidata: {name}", locator=f"P39:{qid(b, 'position')}:{start}")],
|
||||
facet=Role(role=role, level=level, body=body, start=start, end=end),
|
||||
)
|
||||
persons = sorted(seen)
|
||||
for i in range(0, len(persons), 300):
|
||||
values = " ".join(f"wd:{q}" for q in persons[i:i + 300])
|
||||
yield sparql(PARTY_QUERY % {"persons": values}, kind="parties")
|
||||
yield sparql(CAREER_QUERY % {"persons": values}, kind="careers")
|
||||
yield sparql(SOCIAL_QUERY % {"persons": values}, kind="social")
|
||||
|
||||
def parse_parties(self, bindings: list[dict]) -> Iterator[Claim]:
|
||||
for b in bindings:
|
||||
party_label = label(b, "partyLabel")
|
||||
if not party_label:
|
||||
continue
|
||||
person_qid = qid(b, "person")
|
||||
party = OrgRef(name=party_label, short_name=b.get("partyShort", {}).get("value"), kind="party",
|
||||
ids={"wikidata": qid(b, "party")})
|
||||
start, end = wd_date(b, "start"), wd_date(b, "end")
|
||||
yield Claim(
|
||||
claim_type="role", person=PersonRef(ids={"wikidata": person_qid}), org=party,
|
||||
valid_from=start, valid_to=end,
|
||||
summary_de=f"Mitglied der Partei {party_label}", summary_en=f"Member of {party_label}",
|
||||
evidence=[Evidence(url=f"https://www.wikidata.org/wiki/{person_qid}", doc_type="api_record",
|
||||
title="Wikidata", locator=f"P102:{qid(b, 'party')}:{start}")],
|
||||
facet=Role(role="party_member", level="party", body=party, party=party, start=start, end=end),
|
||||
)
|
||||
|
||||
def parse_careers(self, bindings: list[dict]) -> Iterator[Claim]:
|
||||
for b in bindings:
|
||||
org_label = label(b, "orgLabel")
|
||||
if not org_label or EXCLUDE.search(org_label):
|
||||
continue
|
||||
person_qid = qid(b, "person")
|
||||
ref = PersonRef(ids={"wikidata": person_qid})
|
||||
start, end = wd_date(b, "start"), wd_date(b, "end")
|
||||
kind = b["kind"]["value"]
|
||||
evidence = [Evidence(url=f"https://www.wikidata.org/wiki/{person_qid}", doc_type="api_record",
|
||||
title="Wikidata", locator=f"{kind}:{qid(b, 'org')}:{start}")]
|
||||
if kind == "position" and EU_POLITICAL.search(org_label):
|
||||
role, body, level = role_for(org_label)
|
||||
yield Claim(claim_type="role", person=ref, org=body, valid_from=start, valid_to=end,
|
||||
summary_de=org_label, evidence=evidence,
|
||||
facet=Role(role=role, level="eu", body=body, start=start, end=end))
|
||||
continue
|
||||
employer = OrgRef(name=org_label, ids={"wikidata": qid(b, "org")}) if kind == "employer" else None
|
||||
yield Claim(
|
||||
claim_type="career", person=ref, counterpart_org=employer, valid_from=start, valid_to=end,
|
||||
occurred_at=start, key=("wikidata-career", person_qid, kind, qid(b, "org"), start),
|
||||
summary_de=f"{'Arbeitet(e) bei' if kind == 'employer' else 'Funktion:'} {org_label}",
|
||||
summary_en=f"{'Employed by' if kind == 'employer' else 'Position:'} {org_label}",
|
||||
data={"kind": kind, "org_qid": qid(b, "org")}, evidence=evidence,
|
||||
)
|
||||
|
||||
def parse_social(self, bindings: list[dict]) -> Iterator[SocialAccount]:
|
||||
for b in bindings:
|
||||
prop = b["prop"]["value"].rsplit("/", 1)[-1]
|
||||
value = b["value"]["value"].strip()
|
||||
platform, template = SOCIAL_PROPS[prop]
|
||||
if platform == "mastodon":
|
||||
user, _, host = value.lstrip("@").partition("@")
|
||||
if not host:
|
||||
continue
|
||||
url = f"https://{host}/@{user}"
|
||||
else:
|
||||
url = template.format(value)
|
||||
yield SocialAccount(platform=platform, url=url, handle=value,
|
||||
person=PersonRef(ids={"wikidata": qid(b, "person")}), verified=False)
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Wikipedia edits made from the networks of Parliament and federal government (classic transparency bot).
|
||||
|
||||
Anonymous Wikipedia edits are publicly attributed to the editor's IP address. We query the MediaWiki API
|
||||
(list=usercontribs, uciprange) for the IP ranges registered to public institutions in the RIPE database
|
||||
(config/ip_ranges.yaml) and record each edit as a `wiki_edit` claim of the institution, linked to the
|
||||
person whose article was edited when the title matches a known politician. Individual IP addresses are
|
||||
not displayed - only the institution owning the network.
|
||||
|
||||
The API is used per the MediaWiki API etiquette (descriptive User-Agent, sequential requests); robots.txt
|
||||
disallows /w/ for crawlers, which does not apply to API clients (robots_exempt below).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
from datetime import date, timedelta
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Evidence, OrgRef, PersonRef
|
||||
from politrack.text import excerpt
|
||||
|
||||
WIKIS = ("de.wikipedia.org", "en.wikipedia.org")
|
||||
SPEC = SourceSpec(
|
||||
slug="wikipedia", name="Wikipedia (Bearbeitungen aus Behördennetzen)", kind="aggregator",
|
||||
homepage="https://de.wikipedia.org", license="CC BY-SA 4.0", access="api", trust=80, min_interval=1.0,
|
||||
robots_exempt=tuple(f"https://{w}/w/api.php" for w in WIKIS),
|
||||
tos_notes="MediaWiki API (API etiquette: descriptive User-Agent, no parallel requests). Edits from IP ranges "
|
||||
"registered to Parliament / Bundeskanzleramt in the RIPE database.",
|
||||
)
|
||||
|
||||
|
||||
@register
|
||||
class InstitutionalWikiEdits(Adapter):
|
||||
spec = SPEC
|
||||
name = "wikipedia.institution_edits"
|
||||
description = "Wikipedia edits from the IP ranges of Parliament and federal government"
|
||||
schedule = "15 7 * * *"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
config = yaml.safe_load((get_settings().config_dir / "ip_ranges.yaml").read_text(encoding="utf-8"))
|
||||
days = int(ctx.params.get("days", 3650 if not ctx.state.get("initialised") else 30))
|
||||
end = (date.today() - timedelta(days=days)).isoformat() + "T00:00:00Z"
|
||||
for network in config.get("networks") or []:
|
||||
for cidr in network["ranges"]:
|
||||
for wiki in WIKIS:
|
||||
yield Request(
|
||||
url=f"https://{wiki}/w/api.php", kind="contribs",
|
||||
params={"action": "query", "list": "usercontribs", "uciprange": cidr, "uclimit": 500,
|
||||
"ucend": end, "ucprop": "ids|title|timestamp|comment|sizediff", "format": "json",
|
||||
"formatversion": 2},
|
||||
meta={"wiki": wiki, "org": network["org"], "label": network["label"]}, conditional=False)
|
||||
ctx.state.set("initialised", True)
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
data = response.json()
|
||||
meta = response.meta
|
||||
if "continue" in data:
|
||||
yield Request(url=response.request.url, kind="contribs", meta=meta, conditional=False,
|
||||
params={**response.request.params, **data["continue"]})
|
||||
for edit in (data.get("query") or {}).get("usercontribs") or []:
|
||||
title, revid = edit.get("title"), edit.get("revid")
|
||||
if not title or not revid or ":" in title.split(" ")[0] and not title.startswith("Liste"):
|
||||
continue # skip talk/user/meta pages ("Diskussion:", "Benutzer:")
|
||||
size = edit.get("sizediff")
|
||||
comment = edit.get("comment") or ""
|
||||
diff = f"https://{meta['wiki']}/w/index.php?diff={revid}"
|
||||
yield Claim(
|
||||
claim_type="wiki_edit", org=OrgRef(slug=meta["org"], create=False),
|
||||
counterpart_person=PersonRef(name=title, hints={}), key=("wiki", meta["wiki"], revid),
|
||||
occurred_at=edit.get("timestamp"),
|
||||
summary_de=f"Wikipedia-Bearbeitung aus dem Netz von {meta['label']}: „{title}“"
|
||||
+ (f" ({size:+d} Zeichen)" if isinstance(size, int) else "")
|
||||
+ (f" – {excerpt(comment, 120)}" if comment else ""),
|
||||
summary_en=f"Wikipedia edit from the network of {meta['label']}: \"{title}\""
|
||||
+ (f" ({size:+d} characters)" if isinstance(size, int) else ""),
|
||||
data={"wiki": meta["wiki"], "title": title, "sizediff": size, "comment": comment},
|
||||
evidence=[Evidence(url=diff, doc_type="web_page", title=f"Wikipedia: {title}",
|
||||
published_at=edit.get("timestamp"), quote=comment or None)],
|
||||
)
|
||||
@@ -0,0 +1,82 @@
|
||||
import hashlib
|
||||
import html
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from slugify import slugify
|
||||
|
||||
ACADEMIC_TITLES = re.compile(
|
||||
r"\b(Mag\.?(\s?\(FH\))?|Dr\.?|DI|Dipl\.-Ing\.?|Ing\.?|MMag\.?|Prof\.?|Univ\.-Prof\.?|BA|MA|BSc|MSc|MBA|LL\.?M\.?|"
|
||||
r"PhD|MAS|MPA|Bakk\.?|DDr\.?|KR|Abg\.?|Mmag\.?|MEd|BEd|Dipl\.-Päd\.?)(?=\s|,|$)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def normalize(text: str) -> str:
|
||||
"""Lowercase, strip accents and collapse whitespace (matches SQL lower(unaccent()))."""
|
||||
text = unicodedata.normalize("NFKD", text)
|
||||
text = "".join(c for c in text if not unicodedata.combining(c))
|
||||
return re.sub(r"\s+", " ", text).strip().lower()
|
||||
|
||||
|
||||
def strip_titles(name: str) -> str:
|
||||
name = ACADEMIC_TITLES.sub("", name)
|
||||
name = re.sub(r"\s*,\s*", ", ", name)
|
||||
return re.sub(r"\s+", " ", name).strip(" ,")
|
||||
|
||||
|
||||
def make_slug(*parts: str) -> str:
|
||||
return slugify(" ".join(p for p in parts if p), lowercase=True, replacements=[["ß", "ss"]])
|
||||
|
||||
|
||||
def html_to_text(fragment: str | None) -> str:
|
||||
if not fragment:
|
||||
return ""
|
||||
return re.sub(r"\s+", " ", BeautifulSoup(html.unescape(fragment), "html.parser").get_text(" ")).strip()
|
||||
|
||||
|
||||
def fingerprint(*parts: object) -> str:
|
||||
raw = "|".join(str(p) for p in parts)
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:40]
|
||||
|
||||
|
||||
LEGAL_FORMS = re.compile(
|
||||
r"\b(gesellschaft\s+m\.?\s?b\.?\s?h\.?|ges\.?\s?m\.?\s?b\.?\s?h\.?|gmbh|g\.m\.b\.h\.?|aktiengesellschaft|ag|"
|
||||
r"kommanditgesellschaft|kg|og|e\.?\s?u\.?|eg|e\.?\s?gen\.?|reg\.?\s?gen\.?\s?m\.?\s?b\.?\s?h\.?|se|ltd\.?|inc\.?|"
|
||||
r"llc|s\.?a\.?|b\.?v\.?|n\.?v\.?|& co\.?|co\.?|und co\.?|gesmbh|mbh|kgaa|plc)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def normalize_org_name(name: str) -> str:
|
||||
"""'HYPO TIROL BANK AG' == 'Hypo Tirol Bank Aktiengesellschaft' -> 'hypo tirol bank'.
|
||||
|
||||
Lowercase, strip accents, quotes, punctuation and legal-form suffixes, so the same company is
|
||||
recognised across sources (side-job lists, procurement notices, donation lists, ad reports).
|
||||
"""
|
||||
text = normalize(name)
|
||||
text = re.sub(r"[\"'„“”»«]", "", text)
|
||||
text = LEGAL_FORMS.sub(" ", text)
|
||||
text = re.sub(r"[^\w\s-]", " ", text)
|
||||
return re.sub(r"\s+", " ", text).strip(" -")
|
||||
|
||||
|
||||
def firmenbuch_number(value: str | None) -> str | None:
|
||||
"""'FN 306527 a', 'FN306527a', '306527a' -> '306527a' (Austrian company register number, normalised)."""
|
||||
m = re.fullmatch(r"\s*(?:FN\s*)?(\d{1,6})\s?([a-zA-Z])\s*", value or "", re.IGNORECASE)
|
||||
return f"{m.group(1)}{m.group(2).lower()}" if m else None
|
||||
|
||||
|
||||
def format_eur_de(amount: object) -> str:
|
||||
"""1234567.8 -> '1.234.567,80' (German number format; formats only the number, never the sentence)."""
|
||||
try:
|
||||
value = float(amount) # type: ignore[arg-type]
|
||||
except (TypeError, ValueError):
|
||||
return str(amount)
|
||||
return f"{value:,.2f}".replace(",", "X").replace(".", ",").replace("X", ".")
|
||||
|
||||
|
||||
def excerpt(text: str, limit: int = 400) -> str:
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
return text if len(text) <= limit else text[: limit - 1].rsplit(" ", 1)[0] + "…"
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Background jobs on Procrastinate (Postgres-backed queue; no Redis).
|
||||
|
||||
One periodic task is generated per registered adapter/enricher from its `schedule` (cron, UTC) and
|
||||
`queue`. Run workers per queue so heavy jobs don't block light ones:
|
||||
|
||||
procrastinate --app=politrack.worker.app worker --queues=scrape,social
|
||||
procrastinate --app=politrack.worker.app worker --queues=bulk
|
||||
procrastinate --app=politrack.worker.app worker --queues=enrich
|
||||
|
||||
Periodic tasks are deferred by every running worker and de-duplicated by Procrastinate; queueing locks
|
||||
prevent two runs of the same adapter at once. Workers can run on other machines that reach the DB.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from procrastinate import App, PsycopgConnector
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import registry
|
||||
from politrack.core.runner import run
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||||
|
||||
app = App(connector=PsycopgConnector(conninfo=get_settings().database_url))
|
||||
|
||||
|
||||
def _make_task(name: str, queue: str, schedule: str | None) -> None:
|
||||
def job(timestamp: int | None = None, **params) -> dict:
|
||||
return run(name, params=params or None)
|
||||
|
||||
job.__name__ = f"run_{name.replace('.', '_')}"
|
||||
task = app.task(name=f"politrack.run.{name}", queue=queue, queueing_lock=name, pass_context=False)(job)
|
||||
if schedule:
|
||||
app.periodic(cron=schedule, periodic_id=name)(task)
|
||||
|
||||
|
||||
registry.load_builtin()
|
||||
for _name, _cls in {**registry.ADAPTERS, **registry.ENRICHERS}.items():
|
||||
_make_task(_name, _cls.queue, _cls.schedule)
|
||||
Reference in new issue
Block a user