Compare commits

..
14 Commits
Author SHA1 Message Date
simonandClaude Opus 5.5 7bd0142734 Partial index for claims still awaiting topic tagging
Each enrich.topics batch scanned all claims for untagged ones; after the
history backfill a run took hours and blocked the enrich queue.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-14 12:23:27 +01:00
simonandClaude Opus 5.5 d03b9643e9 Index chunk.claim_id and chunk.document_id
enrich.embed looks up claims without a chunk on every batch. The existing
unique index is on COALESCE expressions, which that lookup can't use, so
after the history backfill each run took hours and starved the other
enrichers on the same queue.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-14 11:28:25 +01:00
simonandClaude Opus 5.5 ccf1a4c755 parlament.press: an explicit backfill_days overrides the cursor
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 00:46:57 +01:00
simonandClaude Opus 5.5 405c21a8c5 Backups, freshness monitoring, persistent logs, procurement cleanup
- backup service: nightly pg_dump with retention, recorded as a run
- GET /api/v1/status/sources: 503 when a scheduled source or the backup
  missed its last two slots (for Uptime Kuma)
- worker logs to ${DATA_DIR}/logs, so they survive redeploys
- procurement: reject implausible award dates (1900, contract end dates),
  skip placeholder contractors, credit framework agreements to the BBG

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 00:46:18 +01:00
simonandClaude Opus 5.5 79639e5319 Backfill: former MEPs, older elections, EC 2014-2019, social history paging
- europarl.meps also loads the Austrian MEPs of the 2019-2024 term; roles end
  with the last membership, so former MEPs don't show as current
- bmi.elections: National Council 2017 and EU 2019 preference votes
- eu.commission_meetings: the 2014-2019 Commission
- social.bluesky / social.mastodon: -p backfill_pages=N pages older posts

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 00:34:24 +01:00
simonandClaude Opus 5.5 4bd4e5b2db Add a phone-width audit script for the web app
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 00:00:07 +01:00
simonandClaude Opus 5.5 b7b7f2f648 Add a watchdog that repairs missed schedules and stalled jobs
Every daily and weekly source missed two days in November 2026: Procrastinate
skips a periodic slot that is more than ten minutes overdue, and jobs of
killed workers stayed "doing" forever. Every ten minutes the watchdog fails
jobs of dead workers, closes abandoned run records and defers any scheduled
task whose last slot passed without a run.

Also: EP vote references of joint motions and Council documents, and the
loader now refreshes a vote's reference and stage.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 23:42:19 +01:00
simonandClaude Opus 5.5 a59d475e6b Add official election results and candidates' preference votes (BMI)
National Council 2024 and 2019 and EU 2024: party results for Austria and
the states, and preference votes of candidates who resolve to a known
politician (exact name plus birth year where published). New claim type
election_result, new resolver hint birth_year.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 23:07:51 +01:00
simonandClaude Opus 5.5 b06f17e587 Add EP amendment and part votes, hidden from general lists by default
They come from the same responses as final votes. A sitting has over a
hundred, so they are flagged part_vote and only appear on request: the vote
lists have an option for them, feeds, counts and search chunks leave them out.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-09 13:17:10 +01:00
simonandClaude Opus 5.5 bf19e503f9 Use a raw string for the chunker's regex query
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-09 09:59:08 +01:00
simonandClaude Opus 5.5 1f945b8bad Stop the chunker looping forever on blank documents
A document whose text is only whitespace produces no chunks, so it was
selected again on every pass and enrich.embed never finished.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-09 09:56:58 +01:00
simonandClaude Opus 5.5 9c58bb3d8f Add committee memberships; tolerate empty EP declaration responses
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-09 02:19:25 +01:00
simonandClaude Opus 5.5 b4e26bb296 Let the web container write its fetch cache
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-09 02:15:06 +01:00
simonandClaude Opus 5.5 256906e9fe Prepare for going public: legal pages, corrections contact, portrait proxy
/impressum and /datenschutz read the operator's details from the environment
so they stay out of this repository, and show a visible notice until set.
Portraits go through an allowlisted /api/img so visitors' IPs stay with us.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-09 01:22:02 +01:00
74 changed files with 3053 additions and 105 deletions

No files matched your search

+11
View File
@@ -12,6 +12,13 @@ POLITRACK_READER_PASSWORD=change-me-too
POLITRACK_USER_AGENT=PolitrackBot/0.1 (+https://example.org/about; contact: you@example.org) POLITRACK_USER_AGENT=PolitrackBot/0.1 (+https://example.org/about; contact: you@example.org)
POLITRACK_MIN_REQUEST_INTERVAL=2.0 POLITRACK_MIN_REQUEST_INTERVAL=2.0
# Shown on /impressum and /datenschutz. Required before the site goes public (§ 5 ECG, §§ 24, 25 MedienG).
# POLITRACK_LEGAL_NAME=
# POLITRACK_LEGAL_ADDRESS=
# POLITRACK_LEGAL_EMAIL=
# Defaults to POLITRACK_LEGAL_EMAIL.
# POLITRACK_CORRECTIONS_EMAIL=
# Legislative periods of the Austrian parliament to ingest in detail (JSON list). # Legislative periods of the Austrian parliament to ingest in detail (JSON list).
POLITRACK_PARLAMENT_PERIODS=["XXVII","XXVIII"] POLITRACK_PARLAMENT_PERIODS=["XXVII","XXVIII"]
@@ -24,6 +31,10 @@ POLITRACK_API_CORS_ORIGINS=["http://localhost:3000"]
# Moderation endpoints (/api/v1/review, /api/v1/admin/runs). Leave empty to moderate via the CLI only. # Moderation endpoints (/api/v1/review, /api/v1/admin/runs). Leave empty to moderate via the CLI only.
# POLITRACK_ADMIN_TOKEN= # POLITRACK_ADMIN_TOKEN=
# Nightly database dumps (backup service). Ideally on a different disk or share than DATA_DIR.
# BACKUP_DIR=/mnt/user/backups/politrack
# BACKUP_KEEP_DAYS=14
# Raw-response archive inside the containers (mounted from ${DATA_DIR}/raw by docker-compose.yml). # Raw-response archive inside the containers (mounted from ${DATA_DIR}/raw by docker-compose.yml).
POLITRACK_RAW_DIR=/data/raw POLITRACK_RAW_DIR=/data/raw
+2
View File
@@ -1,2 +1,4 @@
# Golden fixtures are compared byte for byte; never convert their line endings. # Golden fixtures are compared byte for byte; never convert their line endings.
backend/tests/fixtures/** -text backend/tests/fixtures/** -text
# Shell scripts run in Linux containers: LF even in a Windows checkout.
*.sh text eol=lf
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE vote_event ADD COLUMN part_vote BOOLEAN NOT NULL DEFAULT FALSE;
COMMENT ON COLUMN vote_event.part_vote IS 'TRUE for a vote on an amendment or a single part of a text (European Parliament), FALSE for a vote on the text as a whole. Claims of part votes carry data.part_vote = true and are left out of general lists by default, because a sitting has over a hundred of them.';
@@ -0,0 +1,7 @@
ALTER TABLE claim DROP CONSTRAINT claim_claim_type_check;
ALTER TABLE claim ADD CONSTRAINT claim_claim_type_check CHECK (claim_type IN (
'role', 'vote', 'donation', 'money_flow', 'statement', 'appearance', 'meeting', 'travel',
'side_job', 'legal_proceeding', 'lobbying', 'media_mention', 'career',
'parliamentary_question', 'government_decision', 'wiki_edit', 'declaration', 'ownership',
'affiliation', 'party_finance', 'election_result', 'other'));
COMMENT ON COLUMN claim.claim_type IS 'Kind of fact. election_result: a party''s result in a region (counterpart = the elected body) or a candidate''s preference votes (person, counterpart = party).';
@@ -0,0 +1,6 @@
-- enrich.embed looks up claims and documents without a chunk on every batch. chunk_uniq is an expression index
-- (COALESCE(...)), which those lookups can't use, so each batch scanned the whole table: after the history
-- backfill (over a million claims) a run took hours and blocked the other enrichers. Built CONCURRENTLY by hand on
-- 2026-11-14; IF NOT EXISTS makes this a no-op there.
CREATE INDEX IF NOT EXISTS chunk_claim_idx ON chunk (claim_id) WHERE claim_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS chunk_document_idx ON chunk (document_id) WHERE document_id IS NOT NULL;
@@ -0,0 +1,7 @@
-- enrich.topics fetches the next batch of untagged claims on every iteration; without an index each batch scanned
-- all claims (over a million after the history backfill) and a run took hours, blocking the enrich queue. The
-- predicate must stay identical to the query in enrich/topics.py. Built CONCURRENTLY by hand on 2026-11-14.
CREATE INDEX IF NOT EXISTS claim_untagged_idx ON claim (id)
WHERE claim_type IN ('media_mention', 'statement', 'donation', 'side_job', 'appearance', 'meeting', 'travel',
'money_flow', 'government_decision', 'wiki_edit', 'career')
AND NOT (data ? 'topics_tagged');
+25 -4
View File
@@ -12,6 +12,7 @@ from typing import Annotated
from fastapi import Depends, FastAPI, Header, HTTPException, Query from fastapi import Depends, FastAPI, Header, HTTPException, Query
from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
from psycopg import Connection from psycopg import Connection
from politrack import queries from politrack import queries
@@ -66,6 +67,23 @@ def healthz(conn: DB) -> dict:
return {"ok": True} return {"ok": True}
BACKUP_SCHEDULE = "30 2 * * *" # the `backup` service in docker-compose.yml logs each dump as a run
@app.get("/api/v1/status/sources", include_in_schema=False)
def source_status(conn: DB) -> JSONResponse:
"""For monitoring (e.g. Uptime Kuma): HTTP 503 as soon as a scheduled source or the nightly backup missed its
last two slots. Lists every task with its last successful run."""
from politrack.core import registry
registry.load_builtin()
schedules = {name: cls.schedule for name, cls in {**registry.ADAPTERS, **registry.ENRICHERS}.items() if cls.schedule}
schedules["backup"] = BACKUP_SCHEDULE
rows = queries.source_freshness(conn, schedules)
stale = [r["adapter"] for r in rows if r["stale"]]
return JSONResponse({"ok": not stale, "stale": stale, "sources": rows}, status_code=503 if stale else 200)
# -- reference data ----------------------------------------------------------------------- # -- reference data -----------------------------------------------------------------------
@app.get("/api/v1/stats", response_model=s.Stats) @app.get("/api/v1/stats", response_model=s.Stats)
@@ -133,8 +151,10 @@ def get_person(slug: str, conn: DB):
@app.get("/api/v1/persons/{slug}/votes", response_model=list[s.PersonVote]) @app.get("/api/v1/persons/{slug}/votes", response_model=list[s.PersonVote])
def get_person_votes(slug: str, conn: DB, topics: str | None = None, limit: Limit = 100, offset: Offset = 0): def get_person_votes(slug: str, conn: DB, topics: str | None = None, include_parts: bool = False, limit: Limit = 100,
return queries.person_votes(conn, slug, topics=csv_list(topics), limit=limit, offset=offset) offset: Offset = 0):
return queries.person_votes(conn, slug, topics=csv_list(topics), include_parts=include_parts, limit=limit,
offset=offset)
@app.get("/api/v1/persons/{slug}/network", response_model=s.Network) @app.get("/api/v1/persons/{slug}/network", response_model=s.Network)
@@ -218,9 +238,10 @@ def get_wiki_edits(conn: DB, limit: Limit = 100, offset: Offset = 0):
@app.get("/api/v1/votes", response_model=list[s.VoteItem]) @app.get("/api/v1/votes", response_model=list[s.VoteItem])
def get_votes(conn: DB, party: str | None = None, topics: str | None = None, roll_call: bool | None = None, def get_votes(conn: DB, party: str | None = None, topics: str | None = None, roll_call: bool | None = None,
since: date | None = None, until: date | None = None, limit: Limit = 50, offset: Offset = 0): since: date | None = None, until: date | None = None, include_parts: bool = False, limit: Limit = 50,
offset: Offset = 0):
return queries.list_votes(conn, party=party, topics=csv_list(topics), roll_call=roll_call, since=since, return queries.list_votes(conn, party=party, topics=csv_list(topics), roll_call=roll_call, since=since,
until=until, limit=limit, offset=offset) until=until, include_parts=include_parts, limit=limit, offset=offset)
@app.get("/api/v1/votes/{vote_id}", response_model=s.VoteDetail) @app.get("/api/v1/votes/{vote_id}", response_model=s.VoteDetail)
+1 -1
View File
@@ -11,7 +11,7 @@ from pydantic import BaseModel, ConfigDict
ClaimType = Literal["role", "vote", "donation", "money_flow", "statement", "appearance", "meeting", "travel", ClaimType = Literal["role", "vote", "donation", "money_flow", "statement", "appearance", "meeting", "travel",
"side_job", "legal_proceeding", "lobbying", "media_mention", "career", "parliamentary_question", "side_job", "legal_proceeding", "lobbying", "media_mention", "career", "parliamentary_question",
"government_decision", "wiki_edit", "declaration", "ownership", "affiliation", "party_finance", "government_decision", "wiki_edit", "declaration", "ownership", "affiliation", "party_finance",
"other"] "election_result", "other"]
PersonStatus = Literal["active", "former", "party_member", "deceased"] PersonStatus = Literal["active", "former", "party_member", "deceased"]
+5 -4
View File
@@ -250,14 +250,15 @@ class Loader:
chamber = self.resolver.org(rec.chamber, create=False) if rec.chamber else None chamber = self.resolver.org(rec.chamber, create=False) if rec.chamber else None
row = self.conn.execute( row = self.conn.execute(
"""INSERT INTO vote_event (event_key, chamber_id, item_ref, item_title, stage, voted_on, result, roll_call, """INSERT INTO vote_event (event_key, chamber_id, item_ref, item_title, stage, voted_on, result, roll_call,
url, protocol_url, raw_text) url, protocol_url, raw_text, part_vote)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (event_key) DO UPDATE SET result = EXCLUDED.result, raw_text = EXCLUDED.raw_text, ON CONFLICT (event_key) DO UPDATE SET result = EXCLUDED.result, raw_text = EXCLUDED.raw_text,
roll_call = EXCLUDED.roll_call, item_title = EXCLUDED.item_title, roll_call = EXCLUDED.roll_call, item_title = EXCLUDED.item_title, part_vote = EXCLUDED.part_vote,
item_ref = EXCLUDED.item_ref, stage = EXCLUDED.stage,
protocol_url = COALESCE(EXCLUDED.protocol_url, vote_event.protocol_url) protocol_url = COALESCE(EXCLUDED.protocol_url, vote_event.protocol_url)
RETURNING id""", RETURNING id""",
(rec.key, chamber, rec.item_ref, rec.title, rec.stage, rec.voted_on, rec.result, rec.roll_call, rec.url, (rec.key, chamber, rec.item_ref, rec.title, rec.stage, rec.voted_on, rec.result, rec.roll_call, rec.url,
rec.protocol_url, rec.raw_text), rec.protocol_url, rec.raw_text, rec.part_vote),
).fetchone() ).fetchone()
self.ctx.stats["vote_events"] += 1 self.ctx.stats["vote_events"] += 1
return row["id"] return row["id"]
+1
View File
@@ -149,6 +149,7 @@ class VoteEvent:
url: str | None = None url: str | None = None
protocol_url: str | None = None protocol_url: str | None = None
raw_text: str | None = None raw_text: str | None = None
part_vote: bool = False # a vote on an amendment or part of a text, not on the whole text
@dataclass @dataclass
+10
View File
@@ -7,6 +7,8 @@ Order (first hit wins, ambiguity never guesses):
Hint strategies for persons (documented in docs/data-model.md): Hint strategies for persons (documented in docs/data-model.md):
surname_in_body + on members of a chamber on a date with that surname (roll-call lists) surname_in_body + on members of a chamber on a date with that surname (roll-call lists)
office + on holder of an office title on a date (Council of Ministers minutes) office + on holder of an office title on a date (Council of Ministers minutes)
birth_year exact name and year of birth (candidate lists); add no_name_match so that a name
alone never matches when the year doesn't
""" """
from __future__ import annotations from __future__ import annotations
@@ -133,6 +135,14 @@ class Resolver:
return suffix[0]["id"] if len(suffix) == 1 else None return suffix[0]["id"] if len(suffix) == 1 else None
if "office" in hints: if "office" in hints:
return self.office_holder(hints["office"], on) return self.office_holder(hints["office"], on)
if "birth_year" in hints and ref.name:
rows = self.conn.execute(
"""SELECT DISTINCT a.entity_id AS id FROM alias a JOIN person p ON p.id = a.entity_id
WHERE a.entity_type = 'person' AND a.alias_norm = %s
AND EXTRACT(YEAR FROM p.birth_date) = %s""",
(normalize(strip_titles(ref.name)), int(hints["birth_year"])),
).fetchall()
return rows[0]["id"] if len(rows) == 1 else None
return None return None
def office_holder(self, office: str, on: date | None) -> int | None: def office_holder(self, office: str, on: date | None) -> int | None:
+4 -3
View File
@@ -63,6 +63,7 @@ def build_chunks(conn: Connection, batch: int = 5000) -> int:
JOIN document d ON d.id = e.document_id JOIN document d ON d.id = e.document_id
JOIN source s ON s.id = d.source_id JOIN source s ON s.id = d.source_id
WHERE c.merged_into IS NULL AND c.review_status = 'published' WHERE c.merged_into IS NULL AND c.review_status = 'published'
AND NOT COALESCE((c.data->>'part_vote')::boolean, FALSE)
AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.claim_id = c.id) AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.claim_id = c.id)
GROUP BY c.id, p.full_name, o.name GROUP BY c.id, p.full_name, o.name
LIMIT %s LIMIT %s
@@ -76,14 +77,14 @@ def build_chunks(conn: Connection, batch: int = 5000) -> int:
conn.execute(UPSERT_CHUNK, (c["id"], None, 0, text.strip())) conn.execute(UPSERT_CHUNK, (c["id"], None, 0, text.strip()))
docs = conn.execute( docs = conn.execute(
""" r"""
SELECT d.id, d.title, d.full_text FROM document d SELECT d.id, d.title, d.full_text FROM document d
WHERE d.full_text IS NOT NULL AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.document_id = d.id) WHERE d.full_text ~ '\S' AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.document_id = d.id)
LIMIT %s LIMIT %s
""", """,
(batch // 10,), (batch // 10,),
).fetchall() ).fetchall()
for d in docs: for d in docs: # blank texts are excluded above: they yield no chunks and would be selected forever
for i, part in enumerate(split_text(d["full_text"])): for i, part in enumerate(split_text(d["full_text"])):
conn.execute(UPSERT_CHUNK, (None, d["id"], i, f"{d['title'] or ''}\n{part}".strip())) conn.execute(UPSERT_CHUNK, (None, d["id"], i, f"{d['title'] or ''}\n{part}".strip()))
conn.commit() conn.commit()
+6 -4
View File
@@ -115,7 +115,8 @@ def list_claims(
) -> str: ) -> str:
"""Claims about a person/party/organisation, newest first. claim_types: role, vote, donation, money_flow, """Claims about a person/party/organisation, newest first. claim_types: role, vote, donation, money_flow,
statement, appearance, meeting, travel, side_job, legal_proceeding, lobbying, media_mention, career, statement, appearance, meeting, travel, side_job, legal_proceeding, lobbying, media_mention, career,
parliamentary_question, government_decision, wiki_edit, declaration, ownership, affiliation, party_finance. parliamentary_question, government_decision, wiki_edit, declaration, ownership, affiliation, party_finance,
election_result.
Dates as YYYY-MM-DD.""" Dates as YYYY-MM-DD."""
with connection() as conn: with connection() as conn:
return _dump(queries.list_claims( return _dump(queries.list_claims(
@@ -170,11 +171,12 @@ def calendar(start: str, end: str, person: str | None = None, party: str | None
@mcp.tool() @mcp.tool()
def person_votes(slug: str, topics: list[str] | None = None, limit: int = 50) -> str: def person_votes(slug: str, topics: list[str] | None = None, include_parts: bool = False, limit: int = 50) -> str:
"""Votes: individual roll-call votes (with dissent flag) and faction votes during the person's mandates. """Votes: individual roll-call votes (with dissent flag) and faction votes during the person's mandates.
`spoke` links a speech by the person on the same item (say vs. vote).""" `spoke` links a speech by the person on the same item (say vs. vote). include_parts adds European
Parliament votes on amendments and parts of a text."""
with connection() as conn: with connection() as conn:
return _dump(queries.person_votes(conn, slug, topics=topics, limit=limit)) return _dump(queries.person_votes(conn, slug, topics=topics, include_parts=include_parts, limit=limit))
@mcp.tool() @mcp.tool()
+10 -2
View File
@@ -1,6 +1,14 @@
"""Read queries shared by the REST API and the MCP server (one module per domain).""" """Read queries shared by the REST API and the MCP server (one module per domain)."""
from politrack.queries.admin import list_sources, list_topics, pending_claims, recent_runs, set_review, stats from politrack.queries.admin import (
list_sources,
list_topics,
pending_claims,
recent_runs,
set_review,
source_freshness,
stats,
)
from politrack.queries.claims import calendar, list_claims, timeline_histogram from politrack.queries.claims import calendar, list_claims, timeline_histogram
from politrack.queries.common import SourceFilter from politrack.queries.common import SourceFilter
from politrack.queries.money import list_money, money_summary, social_feed, wiki_edits from politrack.queries.money import list_money, money_summary, social_feed, wiki_edits
@@ -13,5 +21,5 @@ __all__ = [
"SourceFilter", "calendar", "former_overview", "get_organization", "get_party", "get_person", "list_claims", "SourceFilter", "calendar", "former_overview", "get_organization", "get_party", "get_person", "list_claims",
"list_money", "list_organizations", "list_parties", "list_persons", "list_sources", "list_topics", "list_votes", "list_money", "list_organizations", "list_parties", "list_persons", "list_sources", "list_topics", "list_votes",
"money_summary", "pending_claims", "recent_runs", "person_network", "person_votes", "search", "set_review", "social_feed", "money_summary", "pending_claims", "recent_runs", "person_network", "person_votes", "search", "set_review", "social_feed",
"stats", "timeline_histogram", "vote_detail", "wiki_edits", "source_freshness", "stats", "timeline_histogram", "vote_detail", "wiki_edits",
] ]
+23
View File
@@ -2,6 +2,8 @@
from __future__ import annotations from __future__ import annotations
from datetime import datetime, timedelta, timezone
from psycopg import Connection from psycopg import Connection
@@ -61,6 +63,27 @@ def stats(conn: Connection) -> dict:
).fetchone() ).fetchone()
def source_freshness(conn: Connection, schedules: dict[str, str], now: datetime | None = None) -> list[dict]:
"""Last successful run per scheduled task. Stale = its two latest scheduled slots passed without a
successful run (plus an hour's grace for long runs), so one slow or failed run alone never alerts."""
from croniter import croniter
now = now or datetime.now(timezone.utc)
last_ok = {r["adapter"]: r["at"] for r in conn.execute(
"SELECT adapter, max(COALESCE(finished_at, started_at)) AS at FROM run "
"WHERE status IN ('ok', 'partial') GROUP BY adapter")}
rows = []
for name, schedule in sorted(schedules.items()):
it = croniter(schedule, now)
due = it.get_prev(datetime)
before = it.get_prev(datetime)
deadline = before - timedelta(hours=1)
last = last_ok.get(name)
rows.append({"adapter": name, "schedule": schedule, "last_success": last.isoformat() if last else None,
"stale": last is None or last < deadline, "latest_slot": due.isoformat()})
return rows
def recent_runs(conn: Connection, limit: int = 50) -> list[dict]: def recent_runs(conn: Connection, limit: int = 50) -> list[dict]:
return conn.execute( return conn.execute(
"SELECT adapter, mode, status, started_at, finished_at, stats, left(error, 2000) AS error FROM run " "SELECT adapter, mode, status, started_at, finished_at, stats, left(error, 2000) AS error FROM run "
+4 -3
View File
@@ -6,7 +6,7 @@ from datetime import date
from psycopg import Connection from psycopg import Connection
from politrack.queries.common import VISIBLE, SourceFilter from politrack.queries.common import NOT_PART_VOTE, VISIBLE, SourceFilter
def list_claims( def list_claims(
@@ -62,6 +62,7 @@ def list_claims(
WHERE e.claim_id = c.id AND {source_sql} WHERE e.claim_id = c.id AND {source_sql}
) ev ) ev
WHERE ev.evidence IS NOT NULL AND {VISIBLE} WHERE ev.evidence IS NOT NULL AND {VISIBLE}
AND (%(claim_id)s::bigint IS NOT NULL OR {NOT_PART_VOTE})
AND (%(person)s::text IS NULL OR p.slug = %(person)s OR cpp.slug = %(person)s) AND (%(person)s::text IS NULL OR p.slug = %(person)s OR cpp.slug = %(person)s)
AND (%(party)s::text IS NULL OR o.slug = %(party)s AND (%(party)s::text IS NULL OR o.slug = %(party)s
OR c.counterpart_org_id = (SELECT id FROM organization WHERE slug = %(party)s)) OR c.counterpart_org_id = (SELECT id FROM organization WHERE slug = %(party)s))
@@ -97,7 +98,7 @@ def timeline_histogram(conn: Connection, *, sources: SourceFilter, person: str |
LEFT JOIN organization o ON o.id = c.organization_id LEFT JOIN organization o ON o.id = c.organization_id
JOIN claim_evidence e ON e.claim_id = c.id JOIN document d ON d.id = e.document_id JOIN claim_evidence e ON e.claim_id = c.id JOIN document d ON d.id = e.document_id
JOIN source s ON s.id = d.source_id JOIN source s ON s.id = d.source_id
WHERE {VISIBLE} AND {source_sql} AND COALESCE(c.occurred_at, c.valid_from::timestamptz) IS NOT NULL WHERE {VISIBLE} AND {NOT_PART_VOTE} AND {source_sql} AND COALESCE(c.occurred_at, c.valid_from::timestamptz) IS NOT NULL
AND (%(person)s::text IS NULL OR p.slug = %(person)s OR cpp.slug = %(person)s) AND (%(person)s::text IS NULL OR p.slug = %(person)s OR cpp.slug = %(person)s)
AND (%(party)s::text IS NULL OR o.slug = %(party)s) AND (%(party)s::text IS NULL OR o.slug = %(party)s)
AND c.claim_type <> 'role' AND c.claim_type <> 'role'
@@ -149,7 +150,7 @@ def calendar(conn: Connection, *, sources: SourceFilter, start: date, end: date,
LEFT JOIN organization o ON o.id = c.organization_id LEFT JOIN organization o ON o.id = c.organization_id
JOIN claim_evidence e ON e.claim_id = c.id JOIN document d ON d.id = e.document_id JOIN claim_evidence e ON e.claim_id = c.id JOIN document d ON d.id = e.document_id
JOIN source s ON s.id = d.source_id JOIN source s ON s.id = d.source_id
WHERE {VISIBLE} AND {source_sql} AND c.occurred_at::date BETWEEN %(start)s AND %(end)s WHERE {VISIBLE} AND {NOT_PART_VOTE} AND {source_sql} AND c.occurred_at::date BETWEEN %(start)s AND %(end)s
AND (%(person)s::text IS NULL OR p.slug = %(person)s OR cpp.slug = %(person)s) AND (%(person)s::text IS NULL OR p.slug = %(person)s OR cpp.slug = %(person)s)
AND (%(party)s::text IS NULL OR o.slug = %(party)s) AND (%(party)s::text IS NULL OR o.slug = %(party)s)
GROUP BY 1, 2 ORDER BY 1 GROUP BY 1, 2 ORDER BY 1
+2
View File
@@ -13,6 +13,8 @@ from typing import Any
VISIBLE = ("c.merged_into IS NULL AND (c.review_status = 'published' OR " VISIBLE = ("c.merged_into IS NULL AND (c.review_status = 'published' OR "
"(%(include_pending)s AND c.review_status = 'pending'))") "(%(include_pending)s AND c.review_status = 'pending'))")
# Votes on amendments and parts of a text only appear where someone asks for them: they would bury everything else.
NOT_PART_VOTE = "NOT COALESCE((c.data->>'part_vote')::boolean, FALSE)"
@dataclass @dataclass
+1 -1
View File
@@ -72,7 +72,7 @@ def get_person(conn: Connection, slug: str) -> dict | None:
r["claim_type"]: r["count"] r["claim_type"]: r["count"]
for r in conn.execute( for r in conn.execute(
"SELECT claim_type, count(*) AS count FROM claim WHERE person_id = %s AND merged_into IS NULL " "SELECT claim_type, count(*) AS count FROM claim WHERE person_id = %s AND merged_into IS NULL "
"AND review_status = 'published' GROUP BY claim_type", "AND review_status = 'published' AND NOT COALESCE((data->>'part_vote')::boolean, FALSE) GROUP BY claim_type",
(person["id"],), (person["id"],),
) )
} }
+10 -8
View File
@@ -17,9 +17,9 @@ SPOKE = """
""" """
def person_votes(conn: Connection, slug: str, *, topics: list[str] | None = None, limit: int = 100, def person_votes(conn: Connection, slug: str, *, topics: list[str] | None = None, include_parts: bool = False,
offset: int = 0) -> list[dict]: limit: int = 100, offset: int = 0) -> list[dict]:
"""Votes concerning a person. """Votes concerning a person (include_parts: also votes on amendments and parts of a text).
* individual=true: roll-call votes with the person's own vote; dissent=true when it differs from the * individual=true: roll-call votes with the person's own vote; dissent=true when it differs from the
majority of their faction (vote_position.party_id = their faction that day). majority of their faction (vote_position.party_id = their faction that day).
@@ -41,7 +41,8 @@ def person_votes(conn: Connection, slug: str, *, topics: list[str] | None = None
JOIN organization ch ON ch.id = ve.chamber_id JOIN organization ch ON ch.id = ve.chamber_id
LEFT JOIN organization fo ON fo.id = vp.party_id LEFT JOIN organization fo ON fo.id = vp.party_id
LEFT JOIN vote_position fp ON fp.vote_event_id = ve.id AND fp.organization_id = vp.party_id LEFT JOIN vote_position fp ON fp.vote_event_id = ve.id AND fp.organization_id = vp.party_id
WHERE (%(topics)s::text[] IS NULL OR EXISTS ( WHERE (%(include_parts)s OR NOT ve.part_vote)
AND (%(topics)s::text[] IS NULL OR EXISTS (
SELECT 1 FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id SELECT 1 FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id
WHERE ta.claim_id = vp.claim_id AND t.slug = ANY(%(topics)s))) WHERE ta.claim_id = vp.claim_id AND t.slug = ANY(%(topics)s)))
UNION ALL UNION ALL
@@ -63,13 +64,13 @@ def person_votes(conn: Connection, slug: str, *, topics: list[str] | None = None
ORDER BY voted_on DESC NULLS LAST ORDER BY voted_on DESC NULLS LAST
LIMIT %(limit)s OFFSET %(offset)s LIMIT %(limit)s OFFSET %(offset)s
""", """,
{"slug": slug, "topics": topics, "limit": limit, "offset": offset}, {"slug": slug, "topics": topics, "include_parts": include_parts, "limit": limit, "offset": offset},
).fetchall() ).fetchall()
def list_votes(conn: Connection, *, party: str | None = None, topics: list[str] | None = None, def list_votes(conn: Connection, *, party: str | None = None, topics: list[str] | None = None,
roll_call: bool | None = None, since: date | None = None, until: date | None = None, roll_call: bool | None = None, since: date | None = None, until: date | None = None,
limit: int = 50, offset: int = 0) -> list[dict]: include_parts: bool = False, limit: int = 50, offset: int = 0) -> list[dict]:
return conn.execute( return conn.execute(
""" """
SELECT ve.id, ve.item_title, ve.item_ref, ve.stage, ve.voted_on, ve.result, ve.url, ve.protocol_url, SELECT ve.id, ve.item_title, ve.item_ref, ve.stage, ve.voted_on, ve.result, ve.url, ve.protocol_url,
@@ -83,7 +84,8 @@ def list_votes(conn: Connection, *, party: str | None = None, topics: list[str]
JOIN organization ch ON ch.id = ve.chamber_id JOIN organization ch ON ch.id = ve.chamber_id
LEFT JOIN vote_position vp ON vp.vote_event_id = ve.id AND vp.organization_id IS NOT NULL LEFT JOIN vote_position vp ON vp.vote_event_id = ve.id AND vp.organization_id IS NOT NULL
LEFT JOIN organization o ON o.id = vp.organization_id LEFT JOIN organization o ON o.id = vp.organization_id
WHERE (%(party)s::text IS NULL OR EXISTS ( WHERE (%(include_parts)s OR NOT ve.part_vote)
AND (%(party)s::text IS NULL OR EXISTS (
SELECT 1 FROM vote_position v2 JOIN organization o2 ON o2.id = v2.organization_id SELECT 1 FROM vote_position v2 JOIN organization o2 ON o2.id = v2.organization_id
WHERE v2.vote_event_id = ve.id AND o2.slug = %(party)s)) WHERE v2.vote_event_id = ve.id AND o2.slug = %(party)s))
AND (%(topics)s::text[] IS NULL OR EXISTS ( AND (%(topics)s::text[] IS NULL OR EXISTS (
@@ -97,7 +99,7 @@ def list_votes(conn: Connection, *, party: str | None = None, topics: list[str]
LIMIT %(limit)s OFFSET %(offset)s LIMIT %(limit)s OFFSET %(offset)s
""", """,
{"party": party, "topics": topics, "roll_call": roll_call, "since": since, "until": until, {"party": party, "topics": topics, "roll_call": roll_call, "since": since, "until": until,
"limit": limit, "offset": offset}, "include_parts": include_parts, "limit": limit, "offset": offset},
).fetchall() ).fetchall()
+235
View File
@@ -0,0 +1,235 @@
"""Official election results from the Federal Ministry of the Interior (BMI, Bundeswahlbehörde).
Per election: the final result (XLSX, one row per region with votes per party) and the preference votes of the
candidates (XLSX, per list). Only the federal and state totals of the result are kept; municipal rows stay in the
raw archive. Candidates become claims only if they resolve to a known politician: by exact name plus birth year
where the file has one, otherwise by exact, unambiguous name. Unknown candidates are counted, never stored, and
their occupation is never kept.
Not covered yet: the 2019 state and regional lists (a print layout with side-by-side columns), elections before
2019, Landtag and municipal elections (published by the states).
"""
from __future__ import annotations
import io
import re
from collections.abc import Iterator
from dataclasses import dataclass
from datetime import date
from typing import Any
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Evidence, OrgRef, PersonRef
from politrack.text import strip_titles
BMI = "https://www.bmi.gv.at/412"
SPEC = SourceSpec(
slug="bmi-wahlen", name="Bundesministerium für Inneres – Wahlergebnisse", kind="official",
homepage=f"{BMI}/start.aspx", license="Amtliches Werk (§7 UrhG); Ergebnisse auch als CC BY 4.0 auf data.gv.at",
access="bulk", trust=95, min_interval=3.0,
tos_notes="Final results and preference votes as published by the Bundeswahlbehörde (XLSX). robots.txt sets "
"no rules. Only candidates who resolve to a known politician are stored; occupations are never kept.",
)
@dataclass(frozen=True)
class Election:
key: str
name_de: str
name_en: str
held_on: date
body: str # slug of the elected body
path: str # folder on bmi.gv.at
result: str | None # None: only in a format we can't read (EU 2019: legacy .xls)
preferences: tuple[tuple[str, str], ...] # (layout, file): layouts "bund", "land", "eu"
ELECTIONS = (
Election("nrw2024", "Nationalratswahl 2024", "2024 National Council election", date(2024, 9, 29), "nationalrat",
"nationalratswahlen/nationalratswahl_2024/files", "endgueltiges_ergebnis_beschluss_bundeswahlbehoerde_16102024.xlsx",
(("bund", "nrw2024_aufstellung_der_vorzugsstimmen_der_bundesbewerber_16102024.xlsx"),
("land", "nrw2024_aufstellung_der_vorzugsstimmen_der_landes_undregionalbewerber_16102024.xlsx"))),
Election("nrw2019", "Nationalratswahl 2019", "2019 National Council election", date(2019, 9, 29), "nationalrat",
"nationalratswahlen/nationalratswahl_2019/files", "endgultiges_gesamtergebnis_nrw19_16102019.xlsx",
(("bund", "nrw19_vorzugsstimmen_bundesweit_16102019.xlsx"),)),
Election("euw2024", "Europawahl 2024", "2024 European Parliament election", date(2024, 6, 9),
"europaeisches-parlament", "europawahlen/europawahl_2024/files", "endgueltiges_ergebnis_europawahl_26062024.xlsx",
(("eu", "gesamtaufstellung_erzielte_vorzugsstimmen_eu_wahl_2024_parteien.xlsx"),)),
Election("nrw2017", "Nationalratswahl 2017", "2017 National Council election", date(2017, 10, 15), "nationalrat",
"nationalratswahlen/nationalratswahl_2017/files", "nrw17_endgueltiges_gesamtergebnis.xlsx",
(("bund", "nrw17_vorzugsstimmen_bundesweit.xlsx"),)),
Election("euw2019", "Europawahl 2019", "2019 European Parliament election", date(2019, 5, 26),
"europaeisches-parlament", "europawahlen/europawahl_2019/files/endgueltiges_ergebnis", None,
(("eu", "euw_2019_vorzugsstimmen_gesamthomepage.xlsx"),)),
)
BY_KEY = {e.key: e for e in ELECTIONS}
REGION_CODE = re.compile(r"^G\d0000$") # G00000 Österreich, G10000 Burgenland ... G90000 Wien
PARTY_HEAD = re.compile(r"\(([^()]+)\)\s*$")
LIST_HEAD = re.compile(r"^(Landes|Regional)parteilisten im (?:Landes|Regional)wahlkreis\s+(.+?)\s*$")
def de_int(n: int) -> str:
return f"{n:,}".replace(",", ".")
def de_pct(p: float) -> str:
return f"{p:.2f}".replace(".", ",")
def sheets(body: bytes) -> list[tuple[str, list[tuple]]]:
import openpyxl
wb = openpyxl.load_workbook(io.BytesIO(body), read_only=True, data_only=True)
return [(ws.title, list(ws.iter_rows(values_only=True))) for ws in wb.worksheets]
def label(*cells: Any) -> str:
return " ".join(str(c) for c in cells if c not in (None, "")).replace("\n", "").strip().lower()
def parse_results(rows: list[tuple]) -> list[dict[str, Any]]:
"""Votes per party for Austria and the nine states. The header has party names in the first row, either
with the totals in the same row (2024) or split over two rows ("Stimmen" / "gültige", 2019 and EU 2024)."""
head, sub = rows[0], rows[1] if rows[1] and rows[1][0] is None else ()
labels = [label(head[i], sub[i] if i < len(sub) else None) for i in range(len(head))]
valid_col = next(i for i, text in enumerate(labels) if text.endswith("gültige") and "ungültige" not in text)
parties = [(i, str(name).strip()) for i, name in enumerate(head)
if i > valid_col and isinstance(name, str) and name.strip() not in ("", "%")]
out = []
for row in rows:
code = row[0]
if not (isinstance(code, str) and REGION_CODE.match(code)):
continue
valid = int(row[valid_col] or 0)
for col, party in parties:
votes = int(row[col] or 0)
if not votes:
continue
# the share as published (next column), so the figures match the official result exactly
published = row[col + 1] if col + 1 < len(row) and head[col + 1] == "%" else None
percent = (round(float(published), 2) if isinstance(published, (int, float))
else round(votes * 100 / valid, 2) if valid else None)
out.append({"code": code, "region": str(row[1]).strip(), "electorate": int(row[2] or 0),
"valid": valid, "party": party, "votes": votes, "percent": percent})
return out
def split_name(text: str) -> tuple[str, str]:
"""'Brunner, Dr. Magnus' -> ('Brunner', 'Dr. Magnus'); 'Babler, MSc, Andreas' -> ('Babler', 'Andreas')."""
family, _, rest = str(text).partition(",")
parts = [p.strip() for p in rest.split(",")]
return family.strip(), next((p for p in parts if strip_titles(p)), "")
def parse_preferences(layout: str, workbook: list[tuple[str, list[tuple]]]) -> list[dict[str, Any]]:
"""Candidates with their preference votes. Returns rank, name, birth year (if published), party, list."""
out: list[dict[str, Any]] = []
for sheet, rows in workbook:
party, party_name, list_label = None, None, "Bundesliste"
total_col = None
if layout == "eu":
party, party_name, list_label = sheet.strip(), sheet.strip(), "Bundesliste"
for row in rows:
cells = [c for c in row if c is not None and c != ""]
if not cells:
continue
first = row[0]
if layout == "eu" and any(isinstance(c, str) and re.match(r"Bewerber|Kandidat", c) for c in row):
# header: "LP | Bewerber(in) | BGLD ... | Gesamt" (2024) or "_ | Kandidat(in) | ... | Gesamt" (2019).
# 2024 repeats a re-sorted copy of the table further right: its header must not win.
if total_col is None:
total_col = next((i for i, c in enumerate(row) if isinstance(c, str) and c.strip() == "Gesamt"),
None)
continue
if isinstance(first, str) and len(cells) == 1:
if (m := LIST_HEAD.match(first)):
region = re.sub(r"^\w+\s*-\s*", "", m.group(2)) # "1A - Burgenland Nord" -> "Burgenland Nord"
list_label = f"{m.group(1)}parteiliste {region}"
elif (m := PARTY_HEAD.search(first)) and layout != "eu":
party, party_name, list_label = m.group(1).strip(), first.strip(), "Bundesliste"
continue
rank = str(first).rstrip(".") if first is not None else ""
if not rank.isdigit() or party is None:
continue
if layout == "land":
family, given, year, votes = row[1], row[2], row[4], row[6]
elif layout == "eu":
family, given = split_name(row[1])
year, votes = None, row[total_col] if total_col is not None else None
else:
family, given = split_name(row[1])
year, votes = row[2], row[4]
if not family or not isinstance(votes, (int, float)):
continue
out.append({"rank": int(rank), "name": f"{given} {family}".strip(), "birth_year": int(year)
if isinstance(year, (int, float)) else None, "votes": int(votes), "party": party,
"party_name": party_name, "list": list_label})
return out
@register
class ElectionResults(Adapter):
spec = SPEC
name = "bmi.elections"
description = "Final results of federal and EU elections and candidates' preference votes (BMI)"
schedule = "0 6 1 * *" # the files are final; monthly re-checks cost a conditional request each
queue = "bulk"
def start(self, ctx: RunContext) -> Iterator[Request]:
wanted = set(ctx.params.get("elections") or BY_KEY)
for election in ELECTIONS:
if election.key not in wanted:
continue
base = f"{BMI}/{election.path}"
if election.result:
yield Request(url=f"{base}/{election.result}", kind="result", meta={"election": election.key})
for layout, file in election.preferences:
yield Request(url=f"{base}/{file}", kind="preferences", meta={"election": election.key, "layout": layout})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
election = BY_KEY[response.meta["election"]]
workbook = sheets(response.body)
if response.request.kind == "result":
yield from self.result_claims(ctx, election, response.request.url, workbook[0][1])
else:
yield from self.preference_claims(ctx, election, response.request.url,
parse_preferences(response.meta["layout"], workbook))
def result_claims(self, ctx: RunContext, election: Election, url: str, rows: list[tuple]) -> Iterator[Claim]:
for r in parse_results(rows):
pct = f"{de_pct(r['percent'])} %" if r["percent"] is not None else "?"
yield Claim(
claim_type="election_result", org=OrgRef(short_name=r["party"], kind="party", create=False),
counterpart_org=OrgRef(slug=election.body), occurred_at=election.held_on,
key=("election", election.key, r["code"], r["party"]),
summary_de=f"{r['party']}: {pct} ({de_int(r['votes'])} Stimmen) bei der {election.name_de} – {r['region']}",
summary_en=f"{r['party']}: {pct.replace(',', '.')} ({r['votes']:,} votes) in the {election.name_en} "
f"– {r['region']}",
data={"election": election.key, "party": r["party"], "region": r["region"], "region_code": r["code"],
"votes": r["votes"], "percent": r["percent"], "valid_votes": r["valid"],
"electorate": r["electorate"]},
evidence=[Evidence(url=url, doc_type="dataset_row", title=f"BMI: {election.name_de}, endgültiges Ergebnis",
published_at=election.held_on, locator=f"{r['code']}:{r['party']}",
quote=f"{r['region']} | {r['party']} | {r['votes']} | {r['valid']} gültige")],
)
def preference_claims(self, ctx: RunContext, election: Election, url: str,
candidates: list[dict[str, Any]]) -> Iterator[Claim]:
for c in candidates:
hints = {"birth_year": c["birth_year"], "no_name_match": True} if c["birth_year"] else {}
yield Claim(
claim_type="election_result", person=PersonRef(name=c["name"], hints=hints),
counterpart_org=OrgRef(short_name=c["party"], kind="party", create=False), occurred_at=election.held_on,
key=("preference-votes", election.key, c["list"], c["party"], c["rank"], c["name"]),
summary_de=f"{de_int(c['votes'])} Vorzugsstimmen bei der {election.name_de} "
f"({c['list']} {c['party']}, Platz {c['rank']})",
summary_en=f"{c['votes']:,} preference votes in the {election.name_en} "
f"({c['party']}, {c['list']}, position {c['rank']})",
data={"election": election.key, "preference_votes": c["votes"], "rank": c["rank"], "list": c["list"],
"party": c["party"]},
evidence=[Evidence(url=url, doc_type="dataset_row", title=f"BMI: {election.name_de}, Vorzugsstimmen",
published_at=election.held_on, locator=f"{c['list']}:{c['party']}:{c['rank']}",
quote=f"{c['list']} {c['party']}, Platz {c['rank']}: {c['votes']} Vorzugsstimmen")],
)
+43 -19
View File
@@ -42,7 +42,9 @@ EP_SPEC = SourceSpec(
tos_notes="EP Open Data API v2 (limit 500 requests / 5 min / endpoint).", tos_notes="EP Open Data API v2 (limit 500 requests / 5 min / endpoint).",
) )
EC_EXPORT = "https://ec.europa.eu/transparency-initiative/meetings/data/meetings/dataxlsx" EC_EXPORT = "https://ec.europa.eu/transparency-initiative/meetings/data/meetings/dataxlsx"
EC_TERMS = ("meetingscommissionrepresentatives2429", "meetingscommissionrepresentatives1924") EC_TERMS = ("meetingscommissionrepresentatives2429", "meetingscommissionrepresentatives1924",
"meetingscommissionrepresentatives1419")
EP_PAST_TERMS = (9,) # Austrian MEPs of earlier terms: their votes and declarations, roles with end dates
EP_API = "https://data.europarl.europa.eu/api/v2" EP_API = "https://data.europarl.europa.eu/api/v2"
# Austrian Commissioners in the terms covered by EC_TERMS (fallback if the DB has no Commission roles yet). # Austrian Commissioners in the terms covered by EC_TERMS (fallback if the DB has no Commission roles yet).
AUSTRIAN_COMMISSIONERS = ("Johannes Hahn", "Magnus Brunner") AUSTRIAN_COMMISSIONERS = ("Johannes Hahn", "Magnus Brunner")
@@ -123,12 +125,20 @@ class EuropeanParliament(Adapter):
description = "Current Austrian MEPs (mandates) and their declarations" description = "Current Austrian MEPs (mandates) and their declarations"
schedule = "30 4 * * 2" schedule = "30 4 * * 2"
creates_persons = True creates_persons = True
parser_version = 2 # roles of former MEPs end with their last membership
def start(self, ctx: RunContext) -> Iterator[Request]: def start(self, ctx: RunContext) -> Iterator[Request]:
yield Request(url=f"{EP_API}/meps/show-current", kind="meps", yield Request(url=f"{EP_API}/meps/show-current", kind="meps",
params={"country-of-representation": "AT", "format": "application/ld+json", "limit": 100}) params={"country-of-representation": "AT", "format": "application/ld+json", "limit": 100})
for term in EP_PAST_TERMS:
yield Request(url=f"{EP_API}/meps", kind="meps",
params={"parliamentary-term": term, "country-of-representation": "AT",
"format": "application/ld+json", "limit": 100})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if not response.body.strip(): # the API answers some MEPs' declaration requests with an empty body
ctx.stats["empty_responses"] += 1
return
data = response.json().get("data") or [] data = response.json().get("data") or []
if response.request.kind == "meps": if response.request.kind == "meps":
for mep in data: for mep in data:
@@ -154,19 +164,22 @@ class EuropeanParliament(Adapter):
if mep.get("img"): if mep.get("img"):
yield PersonRecord(ref=ref, full_name=meta["name"], photo_url=mep["img"], yield PersonRecord(ref=ref, full_name=meta["name"], photo_url=mep["img"],
birth_date=date.fromisoformat(mep["bday"][:10]) if mep.get("bday") else None) birth_date=date.fromisoformat(mep["bday"][:10]) if mep.get("bday") else None)
starts = [m.get("memberDuring", {}).get("startDate") for m in mep.get("hasMembership") or [] periods = sorted(((m.get("memberDuring") or {}).get("startDate"), (m.get("memberDuring") or {}).get("endDate"))
if "EU_PARLIAMENT" in str(m.get("membershipClassification", ""))] for m in mep.get("hasMembership") or []
starts = sorted(s for s in starts if s) if "EU_PARLIAMENT" in str(m.get("membershipClassification", ""))
and (m.get("memberDuring") or {}).get("startDate"))
body = OrgRef(slug="europaeisches-parlament") body = OrgRef(slug="europaeisches-parlament")
start = date.fromisoformat(starts[-1]) if starts else None start = date.fromisoformat(periods[-1][0][:10]) if periods else None
# former MEPs: the latest membership has ended; current ones have an open end
end = date.fromisoformat(periods[-1][1][:10]) if periods and periods[-1][1] else None
url = f"https://www.europarl.europa.eu/meps/de/{meta['mep_id']}" url = f"https://www.europarl.europa.eu/meps/de/{meta['mep_id']}"
yield Claim( yield Claim(
claim_type="role", person=ref, org=body, valid_from=start, claim_type="role", person=ref, org=body, valid_from=start, valid_to=end,
summary_de=f"Mitglied des Europäischen Parlaments (Fraktion {meta.get('group') or '?'})", summary_de=f"Mitglied des Europäischen Parlaments (Fraktion {meta.get('group') or '?'})",
summary_en=f"Member of the European Parliament ({meta.get('group') or '?'})", summary_en=f"Member of the European Parliament ({meta.get('group') or '?'})",
data={"ep_group": meta.get("group")}, data={"ep_group": meta.get("group")},
evidence=[Evidence(url=url, doc_type="api_record", title=f"Europäisches Parlament: {meta['name']}")], evidence=[Evidence(url=url, doc_type="api_record", title=f"Europäisches Parlament: {meta['name']}")],
facet=Role(role="mp", level="eu", body=body, start=start), facet=Role(role="mp", level="eu", body=body, start=start, end=end),
) )
def parse_declarations(self, response: Response, docs: list[dict]) -> Iterator[Claim]: def parse_declarations(self, response: Response, docs: list[dict]) -> Iterator[Claim]:
@@ -197,6 +210,7 @@ class EuropeanParliament(Adapter):
LD_JSON = "application/ld+json" LD_JSON = "application/ld+json"
EP_DECISIONS_PAGE = 500 EP_DECISIONS_PAGE = 500
EP_REFERENCE = re.compile(r"^(?:RC-)?[A-C]\d+-\d+/\d{4}$") # A10-0272/2026, RC-B9-0253/2024, C9-0010/2024
EP_OUTCOME = {"ADOPTED": ("accepted", "angenommen", "adopted"), "REJECTED": ("rejected", "abgelehnt", "rejected")} EP_OUTCOME = {"ADOPTED": ("accepted", "angenommen", "adopted"), "REJECTED": ("rejected", "abgelehnt", "rejected")}
EP_VOTERS = (("had_voter_favor", "for", "Stimmte für", "Voted for"), EP_VOTERS = (("had_voter_favor", "for", "Stimmte für", "Voted for"),
("had_voter_against", "against", "Stimmte gegen", "Voted against"), ("had_voter_against", "against", "Stimmte gegen", "Voted against"),
@@ -217,18 +231,19 @@ def ep_person_ids(values: list[str] | None) -> set[str]:
@register @register
class ParliamentVotes(Adapter): class ParliamentVotes(Adapter):
"""Final roll-call votes (whole texts, no amendments) of the Austrian MEPs known to the database. """Roll-call votes of the Austrian MEPs known to the database: on whole texts and on amendments or parts.
Amendment votes are left out on purpose: a sitting has over a hundred of them, which would bury Part votes are flagged (`VoteEvent.part_vote`, `data.part_vote`), because a sitting has over a hundred of
the final votes on a member's page. The official result counts; corrections MEPs file afterwards are them and they would otherwise bury the final votes. The official result counts; corrections MEPs file
kept in `data.intended`, because they don't change the result. afterwards are kept in `data.intended`, because they don't change the result.
""" """
spec = EP_SPEC spec = EP_SPEC
name = "europarl.votes" name = "europarl.votes"
description = "Final roll-call votes of Austrian MEPs in the European Parliament plenary" description = "Roll-call votes of Austrian MEPs in the European Parliament plenary, including amendments"
schedule = "45 5 * * *" schedule = "45 5 * * *"
queue = "scrape" queue = "scrape"
parser_version = 3 # 2: amendment and part votes; 3: references of joint motions and Council documents
def start(self, ctx: RunContext) -> Iterator[Request]: def start(self, ctx: RunContext) -> Iterator[Request]:
ep_ids = [] ep_ids = []
@@ -243,6 +258,9 @@ class ParliamentVotes(Adapter):
meta={"ep_ids": ep_ids}) meta={"ep_ids": ep_ids})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if not response.body.strip(): # sittings without recorded votes come back with an empty body
ctx.stats["empty_responses"] += 1
return
kind = response.request.kind kind = response.request.kind
data = response.json().get("data") or [] data = response.json().get("data") or []
meta = response.meta meta = response.meta
@@ -271,9 +289,10 @@ class ParliamentVotes(Adapter):
austrian = set(meta["ep_ids"]) austrian = set(meta["ep_ids"])
chamber = OrgRef(slug="europaeisches-parlament") chamber = OrgRef(slug="europaeisches-parlament")
for decision in data: for decision in data:
if "ROLLCALL" not in str(decision.get("decision_method")) or "decisionAboutId" in decision: if "ROLLCALL" not in str(decision.get("decision_method")):
ctx.stats["not_final_rollcall"] += 1 ctx.stats["not_rollcall"] += 1
continue continue
part_vote = "decisionAboutId" in decision
voters = {position: ep_person_ids(decision.get(field)) & austrian for field, position, _, _ in EP_VOTERS} voters = {position: ep_person_ids(decision.get(field)) & austrian for field, position, _, _ in EP_VOTERS}
if not any(voters.values()): if not any(voters.values()):
ctx.stats["no_austrian_voters"] += 1 ctx.stats["no_austrian_voters"] += 1
@@ -283,6 +302,9 @@ class ParliamentVotes(Adapter):
items = decision.get("inverse_consists_of") or [None] items = decision.get("inverse_consists_of") or [None]
title = meta["titles"].get(items[0]) or label title = meta["titles"].get(items[0]) or label
stage = ep_label(decision.get("referenceText")) stage = ep_label(decision.get("referenceText"))
label_parts = label.split(" – ", 2) # "A10-0272/2026 – rapporteur – § 13, nach Buchstabe a – Änd. 9"
if part_vote and len(label_parts) == 3:
stage = label_parts[2]
status = str(decision.get("decision_outcome", "")).rsplit("/", 1)[-1] status = str(decision.get("decision_outcome", "")).rsplit("/", 1)[-1]
result, outcome_de, outcome_en = EP_OUTCOME.get(status, ("unknown", "Ergebnis unbekannt", "outcome unknown")) result, outcome_de, outcome_en = EP_OUTCOME.get(status, ("unknown", "Ergebnis unbekannt", "outcome unknown"))
counts = {"yes": decision.get("number_of_votes_favor"), "no": decision.get("number_of_votes_against"), counts = {"yes": decision.get("number_of_votes_favor"), "no": decision.get("number_of_votes_against"),
@@ -295,17 +317,19 @@ class ParliamentVotes(Adapter):
if pv else None) if pv else None)
event_key = fingerprint("ep-vote", decision["id"]) event_key = fingerprint("ep-vote", decision["id"])
yield VoteEvent(key=event_key, chamber=chamber, title=title, voted_on=voted_on, result=result, stage=stage, yield VoteEvent(key=event_key, chamber=chamber, title=title, voted_on=voted_on, result=result, stage=stage,
item_ref=label.split(" – ", 1)[0], roll_call=True, url=url, protocol_url=protocol, item_ref=label_parts[0] if EP_REFERENCE.match(label_parts[0]) else None, roll_call=True,
raw_text=f"{stage}: {tally}") url=url, protocol_url=protocol, raw_text=f"{stage}: {tally}", part_vote=part_vote)
subject = f"{title}, {stage}" if part_vote and title != label else title # a label already names the part
intended = {mep: position for field, position in EP_INTENDED for mep in ep_person_ids(decision.get(field))} intended = {mep: position for field, position in EP_INTENDED for mep in ep_person_ids(decision.get(field))}
for _, position, verb_de, verb_en in EP_VOTERS: for _, position, verb_de, verb_en in EP_VOTERS:
for mep in sorted(voters[position]): for mep in sorted(voters[position]):
yield Claim( yield Claim(
claim_type="vote", person=PersonRef(ids={"ep": mep}), counterpart_org=chamber, occurred_at=voted_on, claim_type="vote", person=PersonRef(ids={"ep": mep}), counterpart_org=chamber, occurred_at=voted_on,
summary_de=f"{verb_de}: {title} (Europäisches Parlament) – {outcome_de}", summary_de=f"{verb_de}: {subject} (Europäisches Parlament) – {outcome_de}",
summary_en=f"{verb_en}: {title} (European Parliament) – {outcome_en}", summary_en=f"{verb_en}: {subject} (European Parliament) – {outcome_en}",
data={"position": position, "result": result, "roll_call": True, "chamber": "EP", **counts, data={"position": position, "result": result, "roll_call": True, "chamber": "EP", **counts,
**({"intended": intended[mep]} if mep in intended else {})}, **({"intended": intended[mep]} if mep in intended else {}),
**({"part_vote": True} if part_vote else {})},
evidence=[Evidence(url=url, doc_type="api_record", title=title, published_at=voted_on, evidence=[Evidence(url=url, doc_type="api_record", title=title, published_at=voted_on,
quote=f"{stage}: {tally}", locator=f"vote:{decision['id']}")], quote=f"{stage}: {tally}", locator=f"vote:{decision['id']}")],
topics=("Europäische Union",), topics=("Europäische Union",),
+5 -1
View File
@@ -48,7 +48,11 @@ class ParlamentPress(Adapter):
def start(self, ctx: RunContext) -> Iterator[Request]: def start(self, ctx: RunContext) -> Iterator[Request]:
last = ctx.state.get("last_date") last = ctx.state.get("last_date")
backfill = int(ctx.params.get("backfill_days", 365)) backfill = int(ctx.params.get("backfill_days", 365))
since = (date.fromisoformat(last) - timedelta(days=2)) if last else date.today() - timedelta(days=backfill) # an explicit backfill_days (history backfill) wins over the cursor of the regular runs
if last and "backfill_days" not in ctx.params:
since = date.fromisoformat(last) - timedelta(days=2)
else:
since = date.today() - timedelta(days=backfill)
yield list_request(110, {"DATERANGE": [since.isoformat(), date.today().isoformat()]}, kind="list", yield list_request(110, {"DATERANGE": [since.isoformat(), date.today().isoformat()]}, kind="list",
meta={"since": since.isoformat()}) meta={"since": since.isoformat()})
+33 -1
View File
@@ -41,6 +41,7 @@ LEGAL_FORM_END = re.compile(r"(?i:gesmbh|gmbh|ges\.\s?m\.\s?b\.\s?h\.?|m\.\s?b\.
r"e\.\s?u\.|\bse|privatstiftung|genossenschaft|sparkasse)(?=[A-ZÄÖÜ\s,]|$)") r"e\.\s?u\.|\bse|privatstiftung|genossenschaft|sparkasse)(?=[A-ZÄÖÜ\s,]|$)")
COMMITTEE_END = re.compile(r"\(\d{2}\.\d{2}\.\d{4}\s*-\s*(\d{2}\.\d{2}\.\d{4})\s*\)\s*$")
STREET_WORD = r"\S*(?:straße|strasse|str\.|gasse|platz|weg|kai|ring|allee|zeile|lände|markt|steig)" STREET_WORD = r"\S*(?:straße|strasse|str\.|gasse|platz|weg|kai|ring|allee|zeile|lände|markt|steig)"
@@ -79,7 +80,7 @@ class ParlamentPersons(Adapter):
description = "Members of Nationalrat and Bundesrat since 1918, mandates, functions, side jobs, speeches" description = "Members of Nationalrat and Bundesrat since 1918, mandates, functions, side jobs, speeches"
schedule = "15 3 * * *" schedule = "15 3 * * *"
creates_persons = True creates_persons = True
parser_version = 3 parser_version = 4
def start(self, ctx: RunContext) -> Iterator[Request]: def start(self, ctx: RunContext) -> Iterator[Request]:
yield list_request(409, {"NRBR": ["NR", "BR"]}, kind="persons") yield list_request(409, {"NRBR": ["NR", "BR"]}, kind="persons")
@@ -92,6 +93,8 @@ class ParlamentPersons(Adapter):
yield from self.parse_person(ctx, response) yield from self.parse_person(ctx, response)
elif kind == "speeches": elif kind == "speeches":
yield from self.parse_speeches(ctx, response) yield from self.parse_speeches(ctx, response)
elif kind == "committees":
yield from self.parse_committees(ctx, response)
# -- persons & mandates ------------------------------------------------------------- # -- persons & mandates -------------------------------------------------------------
@@ -180,6 +183,35 @@ class ParlamentPersons(Adapter):
yield from self.side_jobs(ref, pad, name, content.get("unvtrans") or {}) yield from self.side_jobs(ref, pad, name, content.get("unvtrans") or {})
yield list_request(251, {"PAD_INTERN": [int(pad)]}, kind="speeches", meta={"pad": pad, "name": name}) yield list_request(251, {"PAD_INTERN": [int(pad)]}, kind="speeches", meta={"pad": pad, "name": name})
yield list_request(250, {"PAD_INTERN": [int(pad)]}, kind="committees", meta={"pad": pad, "name": name})
def parse_committees(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
"""Committee memberships (list 250). Affiliations, not roles: a member has dozens of them per period,
and they say nothing about whether the person currently holds a mandate."""
periods = set(ctx.params.get("periods") or get_settings().parlament_periods)
pad, name = response.meta["pad"], response.meta["name"]
for row in list_rows(response.json()):
if row.get("GP") not in periods:
continue
labelled = row.get("Ausschuss") or ""
committee = re.sub(r"\s*\([^()]*\)\s*$", "", labelled).strip()
if not committee:
ctx.stats["committee_without_name"] += 1
continue
function = row.get("Funktion") or "Mitglied"
start = parse_date(row.get("Beginn"))
end = COMMITTEE_END.search(labelled)
chamber = "Bundesrat" if row.get("nrbr") == "BR" else "Nationalrat"
yield Claim(
claim_type="affiliation", person=PersonRef(ids={"parlament": pad}),
key=("parl-committee", pad, row.get("GP"), committee, function, start),
valid_from=start, valid_to=parse_date(end.group(1)) if end else None,
summary_de=f"{name}: {function}, {committee} ({chamber}, {row.get('GP')}. GP)",
summary_en=f"{name}: {function}, {committee} ({chamber}, {row.get('GP')} legislative period)",
data={"committee": committee, "function": function, "gp": row.get("GP"), "chamber": row.get("nrbr")},
evidence=[Evidence(url=f"{BASE}/person/{pad}", doc_type="api_record", title=f"Parlament: {name}",
locator=f"ausschuss:{row.get('GP')}:{committee}:{function}:{start}")],
)
def side_jobs(self, ref: PersonRef, pad: str, name: str, unv: dict) -> Iterator[Claim]: def side_jobs(self, ref: PersonRef, pad: str, name: str, unv: dict) -> Iterator[Claim]:
"""Disclosures under §6 Unv-Transparenz-G: leading positions, paid side jobs, honorary positions.""" """Disclosures under §6 Unv-Transparenz-G: leading positions, paid side jobs, honorary positions."""
+36 -5
View File
@@ -14,6 +14,7 @@ fetch items whose lastmod changed. Runs on the `bulk` queue.
from __future__ import annotations from __future__ import annotations
import re
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
from collections.abc import Iterator from collections.abc import Iterator
from datetime import date from datetime import date
@@ -139,8 +140,28 @@ def parse_award_notice(xml: bytes) -> dict[str, Any] | None:
} }
PLACEHOLDER_NAME = re.compile(r"^[\W_]*$|^(?:n/?a|k\.?\s?a\.?|unbekannt|keine angabe|tbd|xxx+)$", re.IGNORECASE)
FRAMEWORK_BUYER = re.compile(r"^Auftraggeber (?:sind|ist)\b", re.IGNORECASE)
def plausible_date(value: str | None, latest: date) -> date | None:
"""An award date, or None if it can't be one: spreadsheet placeholders (1899-12-30, 1900-01-01) and dates after
the notice was fetched (some publishers put the contract's end date in the conclusion field)."""
d = as_date(value)
return d if d and date(1995, 1, 1) <= d <= latest else None
def buyer_name(name: str) -> tuple[str, str | None]:
"""Framework agreements name every eligible buyer in one sentence ("Auftraggeber sind die Republik Österreich
(Bund), die Bundesbeschaffung GmbH sowie alle weiteren ..."). They are concluded by the federal procurement
agency, so it is the buyer; the sentence is kept as the original wording."""
if FRAMEWORK_BUYER.match(name) and "Bundesbeschaffung" in name:
return "Bundesbeschaffung GmbH", name
return name, None
def org_ref(org: dict[str, Any], kind: str) -> OrgRef | None: def org_ref(org: dict[str, Any], kind: str) -> OrgRef | None:
if not org.get("name"): if not org.get("name") or PLACEHOLDER_NAME.match(org["name"].strip()):
return None return None
company_id = (org.get("company_id") or "").replace(" ", "") company_id = (org.get("company_id") or "").replace(" ", "")
ids = {} ids = {}
@@ -167,7 +188,7 @@ class PublicContracts(Adapter):
description = "Contract awards of Austrian public bodies (eForms and BVergG core data via data.gv.at)" description = "Contract awards of Austrian public bodies (eForms and BVergG core data via data.gv.at)"
schedule = "0 2 * * *" schedule = "0 2 * * *"
queue = "bulk" queue = "bulk"
parser_version = 2 parser_version = 3 # 3: implausible award dates, placeholder contractors, framework-agreement buyers
def start(self, ctx: RunContext) -> Iterator[Request]: def start(self, ctx: RunContext) -> Iterator[Request]:
for query in CATALOG_QUERIES: for query in CATALOG_QUERIES:
@@ -230,22 +251,32 @@ class PublicContracts(Adapter):
if notice is None: if notice is None:
ctx.stats["not_an_award"] += 1 ctx.stats["not_an_award"] += 1
return return
buyer = next((org_ref(b, "public_body") for b in notice["buyers"] if b.get("name")), None) named = next((b for b in notice["buyers"] if b.get("name")), None)
buyer_text = None
if named is not None:
name, buyer_text = buyer_name(named["name"])
named = {**named, "name": name}
buyer = org_ref(named, "public_body") if named else None
if buyer is None: if buyer is None:
return return
latest = response.fetched_at.date()
for contract in notice["contracts"]: for contract in notice["contracts"]:
for winner in contract["winners"]: for winner in contract["winners"]:
supplier = org_ref(winner, "company") supplier = org_ref(winner, "company")
if supplier is None: if supplier is None:
ctx.stats["contractor_unnamed"] += 1
continue continue
amount = contract["amount"] or (notice["total"] if len(notice["contracts"]) == 1 else None) amount = contract["amount"] or (notice["total"] if len(notice["contracts"]) == 1 else None)
awarded = as_date(contract["awarded"]) or as_date(notice["issued"]) awarded = plausible_date(contract["awarded"], latest) or plausible_date(notice["issued"], latest)
if contract["awarded"] and awarded is None:
ctx.stats["implausible_award_date"] += 1
title = contract["title"] or notice["title"] or "Auftrag" title = contract["title"] or notice["title"] or "Auftrag"
yield Claim( yield Claim(
claim_type="money_flow", org=buyer, counterpart_org=supplier, occurred_at=awarded, claim_type="money_flow", org=buyer, counterpart_org=supplier, occurred_at=awarded,
summary_de=f"Auftrag „{title}“ an {supplier.name}" + (f" – {format_eur_de(amount)} €" if amount else ""), summary_de=f"Auftrag „{title}“ an {supplier.name}" + (f" – {format_eur_de(amount)} €" if amount else ""),
summary_en=f"Contract \"{title}\" awarded to {supplier.name}" + (f" – EUR {amount}" if amount else ""), summary_en=f"Contract \"{title}\" awarded to {supplier.name}" + (f" – EUR {amount}" if amount else ""),
data={"notice_id": notice["notice_id"], "cpv": notice["cpv"], "publisher": response.meta.get("publisher")}, data={"notice_id": notice["notice_id"], "cpv": notice["cpv"], "publisher": response.meta.get("publisher"),
**({"buyer_text": buyer_text} if buyer_text else {})},
evidence=[Evidence(url=response.request.url, doc_type="register_entry", title=f"Vergabe: {title}", evidence=[Evidence(url=response.request.url, doc_type="register_entry", title=f"Vergabe: {title}",
published_at=awarded, quote=f"{buyer.name} → {supplier.name}: {title}")], published_at=awarded, quote=f"{buyer.name} → {supplier.name}: {title}")],
facet=Money(kind="contract", amount=amount, payer=buyer, payer_name=buyer.name, payee=supplier, facet=Money(kind="contract", amount=amount, payer=buyer, payer_name=buyer.name, payee=supplier,
+17 -5
View File
@@ -99,16 +99,22 @@ class BlueskyPosts(Adapter):
queue = "social" queue = "social"
def start(self, ctx: RunContext) -> Iterator[Request]: def start(self, ctx: RunContext) -> Iterator[Request]:
pages = int(ctx.params.get("backfill_pages", 0)) # older pages per account (history backfill)
for acc in accounts(ctx, "bluesky", r"bsky\.app/profile/"): for acc in accounts(ctx, "bluesky", r"bsky\.app/profile/"):
m = re.search(r"bsky\.app/profile/([^/?#]+)", acc["url"]) m = re.search(r"bsky\.app/profile/([^/?#]+)", acc["url"])
if m: if m:
yield Request(url="https://public.api.bsky.app/xrpc/app.bsky.feed.getAuthorFeed", yield Request(url="https://public.api.bsky.app/xrpc/app.bsky.feed.getAuthorFeed",
params={"actor": m.group(1), "limit": 50, "filter": "posts_no_replies"}, kind="feed", params={"actor": m.group(1), "limit": 50, "filter": "posts_no_replies"}, kind="feed",
meta={**acc["owner"], "handle": m.group(1)}) meta={**acc["owner"], "handle": m.group(1), **({"pages_left": pages} if pages else {})})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]: def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
handle = response.meta["handle"] handle = response.meta["handle"]
for item in response.json().get("feed", []): data = response.json()
if response.meta.get("pages_left") and data.get("cursor") and data.get("feed"):
yield Request(url=response.request.url, kind="feed",
params={**(response.request.params or {}), "cursor": data["cursor"]},
meta={**response.meta, "pages_left": response.meta["pages_left"] - 1})
for item in data.get("feed", []):
post = item.get("post") or {} post = item.get("post") or {}
if item.get("reason"): # repost of someone else's post if item.get("reason"): # repost of someone else's post
continue continue
@@ -131,12 +137,13 @@ class MastodonPosts(Adapter):
queue = "social" queue = "social"
def start(self, ctx: RunContext) -> Iterator[Request]: def start(self, ctx: RunContext) -> Iterator[Request]:
pages = int(ctx.params.get("backfill_pages", 0)) # older pages per account (history backfill)
for acc in accounts(ctx, "mastodon"): # instances have arbitrary hosts: platform only for acc in accounts(ctx, "mastodon"): # instances have arbitrary hosts: platform only
m = re.match(r"^https?://([^/]+)/@([^/@]+)", acc["url"]) m = re.match(r"^https?://([^/]+)/@([^/@]+)", acc["url"])
if m: if m:
host, user = m.groups() host, user = m.groups()
yield Request(url=f"https://{host}/api/v1/accounts/lookup", params={"acct": user}, kind="lookup", yield Request(url=f"https://{host}/api/v1/accounts/lookup", params={"acct": user}, kind="lookup",
meta={**acc["owner"], "host": host}) meta={**acc["owner"], "host": host, **({"pages_left": pages} if pages else {})})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
meta = response.meta meta = response.meta
@@ -147,7 +154,12 @@ class MastodonPosts(Adapter):
params={"limit": 40, "exclude_replies": "true", "exclude_reblogs": "true"}, params={"limit": 40, "exclude_replies": "true", "exclude_reblogs": "true"},
kind="statuses", meta=meta) kind="statuses", meta=meta)
return return
for status in response.json(): statuses = response.json()
if meta.get("pages_left") and statuses:
yield Request(url=response.request.url, kind="statuses",
params={**(response.request.params or {}), "max_id": statuses[-1]["id"]},
meta={**meta, "pages_left": meta["pages_left"] - 1})
for status in statuses:
if status.get("visibility") not in (None, "public"): if status.get("visibility") not in (None, "public"):
continue continue
yield post_claim(owner=meta, platform="mastodon", url=status.get("url") or status.get("uri"), yield post_claim(owner=meta, platform="mastodon", url=status.get("url") or status.get("uri"),
+97 -1
View File
@@ -9,21 +9,62 @@ One periodic task is generated per registered adapter/enricher from its `schedul
Periodic tasks are deferred by every running worker and de-duplicated by Procrastinate; queueing locks Periodic tasks are deferred by every running worker and de-duplicated by Procrastinate; queueing locks
prevent two runs of the same adapter at once. Workers can run on other machines that reach the DB. prevent two runs of the same adapter at once. Workers can run on other machines that reach the DB.
Procrastinate only remembers in memory when it last deferred a task, and skips a slot that is more than
ten minutes overdue. In November 2026 every daily and weekly source silently missed two days that way,
and jobs of killed workers stayed "doing" forever. The watchdog below makes both self-healing.
""" """
from __future__ import annotations from __future__ import annotations
import asyncio
import logging import logging
import os
import socket
from datetime import datetime, timedelta, timezone
from logging.handlers import RotatingFileHandler
from pathlib import Path
from croniter import croniter
from procrastinate import App, PsycopgConnector from procrastinate import App, PsycopgConnector
from procrastinate.exceptions import AlreadyEnqueued
from procrastinate.jobs import Status
from politrack.config import get_settings from politrack.config import get_settings
from politrack.core import registry from politrack.core import registry
from politrack.core.runner import run from politrack.core.runner import run
from politrack.db import connection
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") LOG_FORMAT = "%(asctime)s %(levelname)s %(name)s: %(message)s"
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
log = logging.getLogger("politrack.watchdog")
def _log_to_file() -> None:
"""Container logs vanish when a container is recreated (every deploy); keep a rotating copy in
$POLITRACK_LOG_DIR/<hostname>.log so failures can still be traced afterwards."""
log_dir = os.environ.get("POLITRACK_LOG_DIR")
if not log_dir:
return
try:
handler = RotatingFileHandler(Path(log_dir) / f"{socket.gethostname()}.log", maxBytes=20_000_000,
backupCount=5, encoding="utf-8")
except OSError as exc: # e.g. the host folder belongs to root: logs stay on stdout only
log.warning("cannot write logs to %s: %s", log_dir, exc)
return
handler.setFormatter(logging.Formatter(LOG_FORMAT))
logging.getLogger().addHandler(handler)
_log_to_file()
app = App(connector=PsycopgConnector(conninfo=get_settings().database_url)) app = App(connector=PsycopgConnector(conninfo=get_settings().database_url))
TASKS: dict = {}
SCHEDULES: dict[str, str] = {}
STALLED_AFTER = 300 # seconds without a worker heartbeat
OVERDUE_GRACE = timedelta(minutes=15) # normal deferral happens within a minute of the slot
ABANDONED_RUN = timedelta(hours=6) # CLI runs have no job; don't close them while they may still run
def _make_task(name: str, queue: str, schedule: str | None) -> None: def _make_task(name: str, queue: str, schedule: str | None) -> None:
@@ -32,10 +73,65 @@ def _make_task(name: str, queue: str, schedule: str | None) -> None:
job.__name__ = f"run_{name.replace('.', '_')}" job.__name__ = f"run_{name.replace('.', '_')}"
task = app.task(name=f"politrack.run.{name}", queue=queue, queueing_lock=name, pass_context=False)(job) task = app.task(name=f"politrack.run.{name}", queue=queue, queueing_lock=name, pass_context=False)(job)
TASKS[name] = task
if schedule: if schedule:
SCHEDULES[name] = schedule
app.periodic(cron=schedule, periodic_id=name)(task) app.periodic(cron=schedule, periodic_id=name)(task)
registry.load_builtin() registry.load_builtin()
for _name, _cls in {**registry.ADAPTERS, **registry.ENRICHERS}.items(): for _name, _cls in {**registry.ADAPTERS, **registry.ENRICHERS}.items():
_make_task(_name, _cls.queue, _cls.schedule) _make_task(_name, _cls.queue, _cls.schedule)
def overdue_tasks(now: datetime) -> list[str]:
"""Scheduled tasks whose latest slot passed without a run, a waiting job or a running job."""
due: list[str] = []
with connection() as conn:
for name, schedule in SCHEDULES.items():
slot = croniter(schedule, now).get_prev(datetime)
if now - slot < OVERDUE_GRACE:
continue
seen = conn.execute(
"""SELECT EXISTS (SELECT 1 FROM run WHERE adapter = %(name)s AND started_at >= %(slot)s)
OR EXISTS (SELECT 1 FROM procrastinate_jobs WHERE task_name = %(task)s
AND status IN ('todo', 'doing')) AS seen""",
{"name": name, "slot": slot, "task": f"politrack.run.{name}"},
).fetchone()["seen"]
if not seen:
due.append(name)
return due
def close_abandoned_runs() -> int:
"""Runs still marked running although no job for them is in progress (the worker was stopped)."""
with connection() as conn:
closed = conn.execute(
"""UPDATE run r SET status = 'error', finished_at = now(),
error = 'abandoned: the worker stopped before the run finished (closed by the watchdog)'
WHERE r.status = 'running' AND r.started_at < now() - %s
AND NOT EXISTS (SELECT 1 FROM procrastinate_jobs j
WHERE j.status = 'doing' AND j.task_name = 'politrack.run.' || r.adapter)""",
(ABANDONED_RUN,),
).rowcount
conn.commit()
return closed
@app.periodic(cron="*/10 * * * *", periodic_id="watchdog")
@app.task(name="politrack.watchdog", queue="social", queueing_lock="watchdog", pass_context=False)
async def watchdog(timestamp: int | None = None) -> dict:
stats = {"stalled_failed": 0, "overdue_deferred": 0}
for job in await app.job_manager.get_stalled_jobs(seconds_since_heartbeat=STALLED_AFTER):
await app.job_manager.finish_job(job, status=Status.FAILED, delete_job=False)
stats["stalled_failed"] += 1
log.warning("failed stalled job %s (%s): its worker is gone", job.id, job.task_name)
stats["abandoned_runs_closed"] = await asyncio.to_thread(close_abandoned_runs)
for name in await asyncio.to_thread(overdue_tasks, datetime.now(timezone.utc)):
try:
await TASKS[name].defer_async()
stats["overdue_deferred"] += 1
log.warning("deferred %s: its last scheduled slot passed without a run", name)
except AlreadyEnqueued:
pass
return stats
+1
View File
@@ -17,6 +17,7 @@ dependencies = [
"python-slugify>=8.0", "python-slugify>=8.0",
"rapidfuzz>=3.9", "rapidfuzz>=3.9",
"procrastinate>=2.14,<4", "procrastinate>=2.14,<4",
"croniter>=2.0",
"typer>=0.12", "typer>=0.12",
"pyyaml>=6.0", "pyyaml>=6.0",
"mcp>=1.9,<2", "mcp>=1.9,<2",
+1 -1
View File
@@ -13,7 +13,7 @@ RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends eatmyda
&& eatmydata apt-get install -y -qq --no-install-recommends \ && eatmydata apt-get install -y -qq --no-install-recommends \
python3 python3-psycopg python3-psycopg-pool python3-pydantic-settings python3-fastapi python3-httpx \ python3 python3-psycopg python3-psycopg-pool python3-pydantic-settings python3-fastapi python3-httpx \
python3-protego python3-feedparser python3-bs4 python3-slugify python3-rapidfuzz python3-typer \ python3-protego python3-feedparser python3-bs4 python3-slugify python3-rapidfuzz python3-typer \
python3-yaml python3-uvicorn python3-pytest python3-openpyxl ca-certificates >/dev/null \ python3-yaml python3-uvicorn python3-pytest python3-openpyxl python3-croniter ca-certificates >/dev/null \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
# Keeps the agent image's user, entrypoint and /workspace as they are. # Keeps the agent image's user, entrypoint and /workspace as they are.
@@ -0,0 +1,262 @@
[
{
"__type__": "Claim",
"claim_type": "election_result",
"counterpart_org": {
"__type__": "OrgRef",
"create": false,
"ids": {},
"kind": "party",
"name": null,
"short_name": "ÖVP",
"slug": null
},
"counterpart_person": null,
"data": {
"election": "nrw2024",
"list": "Bundesliste",
"party": "ÖVP",
"preference_votes": 60402,
"rank": 1
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "dataset_row",
"excerpt": null,
"full_text": null,
"locator": "Bundesliste:ÖVP:1",
"meta": {},
"published_at": "2024-09-29",
"quote": "Bundesliste ÖVP, Platz 1: 60402 Vorzugsstimmen",
"source": null,
"stance": "supports",
"title": "BMI: Nationalratswahl 2024, Vorzugsstimmen",
"url": "https://www.bmi.gv.at/412/nationalratswahlen/nationalratswahl_2024/files/nrw2024_aufstellung_der_vorzugsstimmen_der_bundesbewerber_16102024.xlsx"
}
],
"facet": null,
"key": [
"preference-votes",
"nrw2024",
"Bundesliste",
"ÖVP",
1,
"Karl Nehammer"
],
"occurred_at": "2024-09-29",
"org": null,
"person": {
"__type__": "PersonRef",
"hints": {
"birth_year": 1972,
"no_name_match": true
},
"ids": {},
"name": "Karl Nehammer"
},
"review": false,
"sensitive": false,
"summary_de": "60.402 Vorzugsstimmen bei der Nationalratswahl 2024 (Bundesliste ÖVP, Platz 1)",
"summary_en": "60,402 preference votes in the 2024 National Council election (ÖVP, Bundesliste, position 1)",
"topics": [],
"valid_from": null,
"valid_to": null
},
{
"__type__": "Claim",
"claim_type": "election_result",
"counterpart_org": {
"__type__": "OrgRef",
"create": false,
"ids": {},
"kind": "party",
"name": null,
"short_name": "ÖVP",
"slug": null
},
"counterpart_person": null,
"data": {
"election": "nrw2024",
"list": "Bundesliste",
"party": "ÖVP",
"preference_votes": 7820,
"rank": 2
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "dataset_row",
"excerpt": null,
"full_text": null,
"locator": "Bundesliste:ÖVP:2",
"meta": {},
"published_at": "2024-09-29",
"quote": "Bundesliste ÖVP, Platz 2: 7820 Vorzugsstimmen",
"source": null,
"stance": "supports",
"title": "BMI: Nationalratswahl 2024, Vorzugsstimmen",
"url": "https://www.bmi.gv.at/412/nationalratswahlen/nationalratswahl_2024/files/nrw2024_aufstellung_der_vorzugsstimmen_der_bundesbewerber_16102024.xlsx"
}
],
"facet": null,
"key": [
"preference-votes",
"nrw2024",
"Bundesliste",
"ÖVP",
2,
"Claudia Plakolm"
],
"occurred_at": "2024-09-29",
"org": null,
"person": {
"__type__": "PersonRef",
"hints": {
"birth_year": 1994,
"no_name_match": true
},
"ids": {},
"name": "Claudia Plakolm"
},
"review": false,
"sensitive": false,
"summary_de": "7.820 Vorzugsstimmen bei der Nationalratswahl 2024 (Bundesliste ÖVP, Platz 2)",
"summary_en": "7,820 preference votes in the 2024 National Council election (ÖVP, Bundesliste, position 2)",
"topics": [],
"valid_from": null,
"valid_to": null
},
{
"__type__": "Claim",
"claim_type": "election_result",
"counterpart_org": {
"__type__": "OrgRef",
"create": false,
"ids": {},
"kind": "party",
"name": null,
"short_name": "ÖVP",
"slug": null
},
"counterpart_person": null,
"data": {
"election": "nrw2024",
"list": "Bundesliste",
"party": "ÖVP",
"preference_votes": 2069,
"rank": 3
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "dataset_row",
"excerpt": null,
"full_text": null,
"locator": "Bundesliste:ÖVP:3",
"meta": {},
"published_at": "2024-09-29",
"quote": "Bundesliste ÖVP, Platz 3: 2069 Vorzugsstimmen",
"source": null,
"stance": "supports",
"title": "BMI: Nationalratswahl 2024, Vorzugsstimmen",
"url": "https://www.bmi.gv.at/412/nationalratswahlen/nationalratswahl_2024/files/nrw2024_aufstellung_der_vorzugsstimmen_der_bundesbewerber_16102024.xlsx"
}
],
"facet": null,
"key": [
"preference-votes",
"nrw2024",
"Bundesliste",
"ÖVP",
3,
"Dr. Magnus Brunner"
],
"occurred_at": "2024-09-29",
"org": null,
"person": {
"__type__": "PersonRef",
"hints": {
"birth_year": 1972,
"no_name_match": true
},
"ids": {},
"name": "Dr. Magnus Brunner"
},
"review": false,
"sensitive": false,
"summary_de": "2.069 Vorzugsstimmen bei der Nationalratswahl 2024 (Bundesliste ÖVP, Platz 3)",
"summary_en": "2,069 preference votes in the 2024 National Council election (ÖVP, Bundesliste, position 3)",
"topics": [],
"valid_from": null,
"valid_to": null
},
{
"__type__": "Claim",
"claim_type": "election_result",
"counterpart_org": {
"__type__": "OrgRef",
"create": false,
"ids": {},
"kind": "party",
"name": null,
"short_name": "SPÖ",
"slug": null
},
"counterpart_person": null,
"data": {
"election": "nrw2024",
"list": "Bundesliste",
"party": "SPÖ",
"preference_votes": 46440,
"rank": 1
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "dataset_row",
"excerpt": null,
"full_text": null,
"locator": "Bundesliste:SPÖ:1",
"meta": {},
"published_at": "2024-09-29",
"quote": "Bundesliste SPÖ, Platz 1: 46440 Vorzugsstimmen",
"source": null,
"stance": "supports",
"title": "BMI: Nationalratswahl 2024, Vorzugsstimmen",
"url": "https://www.bmi.gv.at/412/nationalratswahlen/nationalratswahl_2024/files/nrw2024_aufstellung_der_vorzugsstimmen_der_bundesbewerber_16102024.xlsx"
}
],
"facet": null,
"key": [
"preference-votes",
"nrw2024",
"Bundesliste",
"SPÖ",
1,
"Andreas Babler"
],
"occurred_at": "2024-09-29",
"org": null,
"person": {
"__type__": "PersonRef",
"hints": {
"birth_year": 1973,
"no_name_match": true
},
"ids": {},
"name": "Andreas Babler"
},
"review": false,
"sensitive": false,
"summary_de": "46.440 Vorzugsstimmen bei der Nationalratswahl 2024 (Bundesliste SPÖ, Platz 1)",
"summary_en": "46,440 preference votes in the 2024 National Council election (SPÖ, Bundesliste, position 1)",
"topics": [],
"valid_from": null,
"valid_to": null
}
]
@@ -0,0 +1,13 @@
{
"url": "https://www.bmi.gv.at/412/nationalratswahlen/nationalratswahl_2024/files/nrw2024_aufstellung_der_vorzugsstimmen_der_bundesbewerber_16102024.xlsx",
"kind": "preferences",
"method": "GET",
"params": null,
"json": null,
"meta": {
"election": "nrw2024",
"layout": "bund"
},
"content_type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"body_file": "preferences_bund.xlsx"
}
Binary file not shown.
@@ -0,0 +1,250 @@
[
{
"__type__": "Claim",
"claim_type": "election_result",
"counterpart_org": {
"__type__": "OrgRef",
"create": false,
"ids": {},
"kind": "party",
"name": null,
"short_name": "ÖVP",
"slug": null
},
"counterpart_person": null,
"data": {
"election": "euw2024",
"list": "Bundesliste",
"party": "ÖVP",
"preference_votes": 58013,
"rank": 1
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "dataset_row",
"excerpt": null,
"full_text": null,
"locator": "Bundesliste:ÖVP:1",
"meta": {},
"published_at": "2024-06-09",
"quote": "Bundesliste ÖVP, Platz 1: 58013 Vorzugsstimmen",
"source": null,
"stance": "supports",
"title": "BMI: Europawahl 2024, Vorzugsstimmen",
"url": "https://www.bmi.gv.at/412/europawahlen/europawahl_2024/files/gesamtaufstellung_erzielte_vorzugsstimmen_eu_wahl_2024_parteien.xlsx"
}
],
"facet": null,
"key": [
"preference-votes",
"euw2024",
"Bundesliste",
"ÖVP",
1,
"Dr. Reinhold Lopatka"
],
"occurred_at": "2024-06-09",
"org": null,
"person": {
"__type__": "PersonRef",
"hints": {},
"ids": {},
"name": "Dr. Reinhold Lopatka"
},
"review": false,
"sensitive": false,
"summary_de": "58.013 Vorzugsstimmen bei der Europawahl 2024 (Bundesliste ÖVP, Platz 1)",
"summary_en": "58,013 preference votes in the 2024 European Parliament election (ÖVP, Bundesliste, position 1)",
"topics": [],
"valid_from": null,
"valid_to": null
},
{
"__type__": "Claim",
"claim_type": "election_result",
"counterpart_org": {
"__type__": "OrgRef",
"create": false,
"ids": {},
"kind": "party",
"name": null,
"short_name": "ÖVP",
"slug": null
},
"counterpart_person": null,
"data": {
"election": "euw2024",
"list": "Bundesliste",
"party": "ÖVP",
"preference_votes": 38426,
"rank": 2
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "dataset_row",
"excerpt": null,
"full_text": null,
"locator": "Bundesliste:ÖVP:2",
"meta": {},
"published_at": "2024-06-09",
"quote": "Bundesliste ÖVP, Platz 2: 38426 Vorzugsstimmen",
"source": null,
"stance": "supports",
"title": "BMI: Europawahl 2024, Vorzugsstimmen",
"url": "https://www.bmi.gv.at/412/europawahlen/europawahl_2024/files/gesamtaufstellung_erzielte_vorzugsstimmen_eu_wahl_2024_parteien.xlsx"
}
],
"facet": null,
"key": [
"preference-votes",
"euw2024",
"Bundesliste",
"ÖVP",
2,
"Mag. Dr. Angelika Winzig"
],
"occurred_at": "2024-06-09",
"org": null,
"person": {
"__type__": "PersonRef",
"hints": {},
"ids": {},
"name": "Mag. Dr. Angelika Winzig"
},
"review": false,
"sensitive": false,
"summary_de": "38.426 Vorzugsstimmen bei der Europawahl 2024 (Bundesliste ÖVP, Platz 2)",
"summary_en": "38,426 preference votes in the 2024 European Parliament election (ÖVP, Bundesliste, position 2)",
"topics": [],
"valid_from": null,
"valid_to": null
},
{
"__type__": "Claim",
"claim_type": "election_result",
"counterpart_org": {
"__type__": "OrgRef",
"create": false,
"ids": {},
"kind": "party",
"name": null,
"short_name": "SPÖ",
"slug": null
},
"counterpart_person": null,
"data": {
"election": "euw2024",
"list": "Bundesliste",
"party": "SPÖ",
"preference_votes": 61982,
"rank": 1
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "dataset_row",
"excerpt": null,
"full_text": null,
"locator": "Bundesliste:SPÖ:1",
"meta": {},
"published_at": "2024-06-09",
"quote": "Bundesliste SPÖ, Platz 1: 61982 Vorzugsstimmen",
"source": null,
"stance": "supports",
"title": "BMI: Europawahl 2024, Vorzugsstimmen",
"url": "https://www.bmi.gv.at/412/europawahlen/europawahl_2024/files/gesamtaufstellung_erzielte_vorzugsstimmen_eu_wahl_2024_parteien.xlsx"
}
],
"facet": null,
"key": [
"preference-votes",
"euw2024",
"Bundesliste",
"SPÖ",
1,
"Mag. Andreas Schieder"
],
"occurred_at": "2024-06-09",
"org": null,
"person": {
"__type__": "PersonRef",
"hints": {},
"ids": {},
"name": "Mag. Andreas Schieder"
},
"review": false,
"sensitive": false,
"summary_de": "61.982 Vorzugsstimmen bei der Europawahl 2024 (Bundesliste SPÖ, Platz 1)",
"summary_en": "61,982 preference votes in the 2024 European Parliament election (SPÖ, Bundesliste, position 1)",
"topics": [],
"valid_from": null,
"valid_to": null
},
{
"__type__": "Claim",
"claim_type": "election_result",
"counterpart_org": {
"__type__": "OrgRef",
"create": false,
"ids": {},
"kind": "party",
"name": null,
"short_name": "SPÖ",
"slug": null
},
"counterpart_person": null,
"data": {
"election": "euw2024",
"list": "Bundesliste",
"party": "SPÖ",
"preference_votes": 17448,
"rank": 2
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "dataset_row",
"excerpt": null,
"full_text": null,
"locator": "Bundesliste:SPÖ:2",
"meta": {},
"published_at": "2024-06-09",
"quote": "Bundesliste SPÖ, Platz 2: 17448 Vorzugsstimmen",
"source": null,
"stance": "supports",
"title": "BMI: Europawahl 2024, Vorzugsstimmen",
"url": "https://www.bmi.gv.at/412/europawahlen/europawahl_2024/files/gesamtaufstellung_erzielte_vorzugsstimmen_eu_wahl_2024_parteien.xlsx"
}
],
"facet": null,
"key": [
"preference-votes",
"euw2024",
"Bundesliste",
"SPÖ",
2,
"Mag. Evelyn Regner"
],
"occurred_at": "2024-06-09",
"org": null,
"person": {
"__type__": "PersonRef",
"hints": {},
"ids": {},
"name": "Mag. Evelyn Regner"
},
"review": false,
"sensitive": false,
"summary_de": "17.448 Vorzugsstimmen bei der Europawahl 2024 (Bundesliste SPÖ, Platz 2)",
"summary_en": "17,448 preference votes in the 2024 European Parliament election (SPÖ, Bundesliste, position 2)",
"topics": [],
"valid_from": null,
"valid_to": null
}
]
@@ -0,0 +1,13 @@
{
"url": "https://www.bmi.gv.at/412/europawahlen/europawahl_2024/files/gesamtaufstellung_erzielte_vorzugsstimmen_eu_wahl_2024_parteien.xlsx",
"kind": "preferences",
"method": "GET",
"params": null,
"json": null,
"meta": {
"election": "euw2024",
"layout": "eu"
},
"content_type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"body_file": "preferences_eu.xlsx"
}
Binary file not shown.
@@ -0,0 +1,197 @@
[
{
"__type__": "Claim",
"claim_type": "election_result",
"counterpart_org": {
"__type__": "OrgRef",
"create": false,
"ids": {},
"kind": "party",
"name": null,
"short_name": "ÖVP",
"slug": null
},
"counterpart_person": null,
"data": {
"election": "nrw2024",
"list": "Landesparteiliste Burgenland",
"party": "ÖVP",
"preference_votes": 3937,
"rank": 1
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "dataset_row",
"excerpt": null,
"full_text": null,
"locator": "Landesparteiliste Burgenland:ÖVP:1",
"meta": {},
"published_at": "2024-09-29",
"quote": "Landesparteiliste Burgenland ÖVP, Platz 1: 3937 Vorzugsstimmen",
"source": null,
"stance": "supports",
"title": "BMI: Nationalratswahl 2024, Vorzugsstimmen",
"url": "https://www.bmi.gv.at/412/nationalratswahlen/nationalratswahl_2024/files/nrw2024_aufstellung_der_vorzugsstimmen_der_landes_undregionalbewerber_16102024.xlsx"
}
],
"facet": null,
"key": [
"preference-votes",
"nrw2024",
"Landesparteiliste Burgenland",
"ÖVP",
1,
"Christoph Zarits"
],
"occurred_at": "2024-09-29",
"org": null,
"person": {
"__type__": "PersonRef",
"hints": {
"birth_year": 1980,
"no_name_match": true
},
"ids": {},
"name": "Christoph Zarits"
},
"review": false,
"sensitive": false,
"summary_de": "3.937 Vorzugsstimmen bei der Nationalratswahl 2024 (Landesparteiliste Burgenland ÖVP, Platz 1)",
"summary_en": "3,937 preference votes in the 2024 National Council election (ÖVP, Landesparteiliste Burgenland, position 1)",
"topics": [],
"valid_from": null,
"valid_to": null
},
{
"__type__": "Claim",
"claim_type": "election_result",
"counterpart_org": {
"__type__": "OrgRef",
"create": false,
"ids": {},
"kind": "party",
"name": null,
"short_name": "ÖVP",
"slug": null
},
"counterpart_person": null,
"data": {
"election": "nrw2024",
"list": "Landesparteiliste Burgenland",
"party": "ÖVP",
"preference_votes": 2734,
"rank": 2
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "dataset_row",
"excerpt": null,
"full_text": null,
"locator": "Landesparteiliste Burgenland:ÖVP:2",
"meta": {},
"published_at": "2024-09-29",
"quote": "Landesparteiliste Burgenland ÖVP, Platz 2: 2734 Vorzugsstimmen",
"source": null,
"stance": "supports",
"title": "BMI: Nationalratswahl 2024, Vorzugsstimmen",
"url": "https://www.bmi.gv.at/412/nationalratswahlen/nationalratswahl_2024/files/nrw2024_aufstellung_der_vorzugsstimmen_der_landes_undregionalbewerber_16102024.xlsx"
}
],
"facet": null,
"key": [
"preference-votes",
"nrw2024",
"Landesparteiliste Burgenland",
"ÖVP",
2,
"Nikolaus Berlakovich"
],
"occurred_at": "2024-09-29",
"org": null,
"person": {
"__type__": "PersonRef",
"hints": {
"birth_year": 1961,
"no_name_match": true
},
"ids": {},
"name": "Nikolaus Berlakovich"
},
"review": false,
"sensitive": false,
"summary_de": "2.734 Vorzugsstimmen bei der Nationalratswahl 2024 (Landesparteiliste Burgenland ÖVP, Platz 2)",
"summary_en": "2,734 preference votes in the 2024 National Council election (ÖVP, Landesparteiliste Burgenland, position 2)",
"topics": [],
"valid_from": null,
"valid_to": null
},
{
"__type__": "Claim",
"claim_type": "election_result",
"counterpart_org": {
"__type__": "OrgRef",
"create": false,
"ids": {},
"kind": "party",
"name": null,
"short_name": "ÖVP",
"slug": null
},
"counterpart_person": null,
"data": {
"election": "nrw2024",
"list": "Regionalparteiliste Burgenland Nord",
"party": "ÖVP",
"preference_votes": 7361,
"rank": 1
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "dataset_row",
"excerpt": null,
"full_text": null,
"locator": "Regionalparteiliste Burgenland Nord:ÖVP:1",
"meta": {},
"published_at": "2024-09-29",
"quote": "Regionalparteiliste Burgenland Nord ÖVP, Platz 1: 7361 Vorzugsstimmen",
"source": null,
"stance": "supports",
"title": "BMI: Nationalratswahl 2024, Vorzugsstimmen",
"url": "https://www.bmi.gv.at/412/nationalratswahlen/nationalratswahl_2024/files/nrw2024_aufstellung_der_vorzugsstimmen_der_landes_undregionalbewerber_16102024.xlsx"
}
],
"facet": null,
"key": [
"preference-votes",
"nrw2024",
"Regionalparteiliste Burgenland Nord",
"ÖVP",
1,
"Christoph Zarits"
],
"occurred_at": "2024-09-29",
"org": null,
"person": {
"__type__": "PersonRef",
"hints": {
"birth_year": 1980,
"no_name_match": true
},
"ids": {},
"name": "Christoph Zarits"
},
"review": false,
"sensitive": false,
"summary_de": "7.361 Vorzugsstimmen bei der Nationalratswahl 2024 (Regionalparteiliste Burgenland Nord ÖVP, Platz 1)",
"summary_en": "7,361 preference votes in the 2024 National Council election (ÖVP, Regionalparteiliste Burgenland Nord, position 1)",
"topics": [],
"valid_from": null,
"valid_to": null
}
]
@@ -0,0 +1,13 @@
{
"url": "https://www.bmi.gv.at/412/nationalratswahlen/nationalratswahl_2024/files/nrw2024_aufstellung_der_vorzugsstimmen_der_landes_undregionalbewerber_16102024.xlsx",
"kind": "preferences",
"method": "GET",
"params": null,
"json": null,
"meta": {
"election": "nrw2024",
"layout": "land"
},
"content_type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"body_file": "preferences_land.xlsx"
}
Binary file not shown.
@@ -0,0 +1,266 @@
[
{
"__type__": "Claim",
"claim_type": "election_result",
"counterpart_org": {
"__type__": "OrgRef",
"create": true,
"ids": {},
"kind": "other",
"name": null,
"short_name": null,
"slug": "europaeisches-parlament"
},
"counterpart_person": null,
"data": {
"election": "euw2024",
"electorate": 6372205,
"party": "ÖVP",
"percent": 24.52,
"region": "Österreich",
"region_code": "G00000",
"valid_votes": 3523935,
"votes": 864072
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "dataset_row",
"excerpt": null,
"full_text": null,
"locator": "G00000:ÖVP",
"meta": {},
"published_at": "2024-06-09",
"quote": "Österreich | ÖVP | 864072 | 3523935 gültige",
"source": null,
"stance": "supports",
"title": "BMI: Europawahl 2024, endgültiges Ergebnis",
"url": "https://www.bmi.gv.at/412/europawahlen/europawahl_2024/files/endgueltiges_ergebnis_europawahl_26062024.xlsx"
}
],
"facet": null,
"key": [
"election",
"euw2024",
"G00000",
"ÖVP"
],
"occurred_at": "2024-06-09",
"org": {
"__type__": "OrgRef",
"create": false,
"ids": {},
"kind": "party",
"name": null,
"short_name": "ÖVP",
"slug": null
},
"person": null,
"review": false,
"sensitive": false,
"summary_de": "ÖVP: 24,52 % (864.072 Stimmen) bei der Europawahl 2024 – Österreich",
"summary_en": "ÖVP: 24.52 % (864,072 votes) in the 2024 European Parliament election – Österreich",
"topics": [],
"valid_from": null,
"valid_to": null
},
{
"__type__": "Claim",
"claim_type": "election_result",
"counterpart_org": {
"__type__": "OrgRef",
"create": true,
"ids": {},
"kind": "other",
"name": null,
"short_name": null,
"slug": "europaeisches-parlament"
},
"counterpart_person": null,
"data": {
"election": "euw2024",
"electorate": 6372205,
"party": "SPÖ",
"percent": 23.22,
"region": "Österreich",
"region_code": "G00000",
"valid_votes": 3523935,
"votes": 818287
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "dataset_row",
"excerpt": null,
"full_text": null,
"locator": "G00000:SPÖ",
"meta": {},
"published_at": "2024-06-09",
"quote": "Österreich | SPÖ | 818287 | 3523935 gültige",
"source": null,
"stance": "supports",
"title": "BMI: Europawahl 2024, endgültiges Ergebnis",
"url": "https://www.bmi.gv.at/412/europawahlen/europawahl_2024/files/endgueltiges_ergebnis_europawahl_26062024.xlsx"
}
],
"facet": null,
"key": [
"election",
"euw2024",
"G00000",
"SPÖ"
],
"occurred_at": "2024-06-09",
"org": {
"__type__": "OrgRef",
"create": false,
"ids": {},
"kind": "party",
"name": null,
"short_name": "SPÖ",
"slug": null
},
"person": null,
"review": false,
"sensitive": false,
"summary_de": "SPÖ: 23,22 % (818.287 Stimmen) bei der Europawahl 2024 – Österreich",
"summary_en": "SPÖ: 23.22 % (818,287 votes) in the 2024 European Parliament election – Österreich",
"topics": [],
"valid_from": null,
"valid_to": null
},
{
"__type__": "Claim",
"claim_type": "election_result",
"counterpart_org": {
"__type__": "OrgRef",
"create": true,
"ids": {},
"kind": "other",
"name": null,
"short_name": null,
"slug": "europaeisches-parlament"
},
"counterpart_person": null,
"data": {
"election": "euw2024",
"electorate": 235492,
"party": "ÖVP",
"percent": 27.91,
"region": "Burgenland",
"region_code": "G10000",
"valid_votes": 134907,
"votes": 37654
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "dataset_row",
"excerpt": null,
"full_text": null,
"locator": "G10000:ÖVP",
"meta": {},
"published_at": "2024-06-09",
"quote": "Burgenland | ÖVP | 37654 | 134907 gültige",
"source": null,
"stance": "supports",
"title": "BMI: Europawahl 2024, endgültiges Ergebnis",
"url": "https://www.bmi.gv.at/412/europawahlen/europawahl_2024/files/endgueltiges_ergebnis_europawahl_26062024.xlsx"
}
],
"facet": null,
"key": [
"election",
"euw2024",
"G10000",
"ÖVP"
],
"occurred_at": "2024-06-09",
"org": {
"__type__": "OrgRef",
"create": false,
"ids": {},
"kind": "party",
"name": null,
"short_name": "ÖVP",
"slug": null
},
"person": null,
"review": false,
"sensitive": false,
"summary_de": "ÖVP: 27,91 % (37.654 Stimmen) bei der Europawahl 2024 – Burgenland",
"summary_en": "ÖVP: 27.91 % (37,654 votes) in the 2024 European Parliament election – Burgenland",
"topics": [],
"valid_from": null,
"valid_to": null
},
{
"__type__": "Claim",
"claim_type": "election_result",
"counterpart_org": {
"__type__": "OrgRef",
"create": true,
"ids": {},
"kind": "other",
"name": null,
"short_name": null,
"slug": "europaeisches-parlament"
},
"counterpart_person": null,
"data": {
"election": "euw2024",
"electorate": 235492,
"party": "SPÖ",
"percent": 29.84,
"region": "Burgenland",
"region_code": "G10000",
"valid_votes": 134907,
"votes": 40253
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "dataset_row",
"excerpt": null,
"full_text": null,
"locator": "G10000:SPÖ",
"meta": {},
"published_at": "2024-06-09",
"quote": "Burgenland | SPÖ | 40253 | 134907 gültige",
"source": null,
"stance": "supports",
"title": "BMI: Europawahl 2024, endgültiges Ergebnis",
"url": "https://www.bmi.gv.at/412/europawahlen/europawahl_2024/files/endgueltiges_ergebnis_europawahl_26062024.xlsx"
}
],
"facet": null,
"key": [
"election",
"euw2024",
"G10000",
"SPÖ"
],
"occurred_at": "2024-06-09",
"org": {
"__type__": "OrgRef",
"create": false,
"ids": {},
"kind": "party",
"name": null,
"short_name": "SPÖ",
"slug": null
},
"person": null,
"review": false,
"sensitive": false,
"summary_de": "SPÖ: 29,84 % (40.253 Stimmen) bei der Europawahl 2024 – Burgenland",
"summary_en": "SPÖ: 29.84 % (40,253 votes) in the 2024 European Parliament election – Burgenland",
"topics": [],
"valid_from": null,
"valid_to": null
}
]
@@ -0,0 +1,12 @@
{
"url": "https://www.bmi.gv.at/412/europawahlen/europawahl_2024/files/endgueltiges_ergebnis_europawahl_26062024.xlsx",
"kind": "result",
"method": "GET",
"params": null,
"json": null,
"meta": {
"election": "euw2024"
},
"content_type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"body_file": "result_euw2024.xlsx"
}
Binary file not shown.
@@ -0,0 +1,398 @@
[
{
"__type__": "Claim",
"claim_type": "election_result",
"counterpart_org": {
"__type__": "OrgRef",
"create": true,
"ids": {},
"kind": "other",
"name": null,
"short_name": null,
"slug": "nationalrat"
},
"counterpart_person": null,
"data": {
"election": "nrw2024",
"electorate": 6346059,
"party": "ÖVP",
"percent": 26.26,
"region": "Österreich",
"region_code": "G00000",
"valid_votes": 4882888,
"votes": 1282734
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "dataset_row",
"excerpt": null,
"full_text": null,
"locator": "G00000:ÖVP",
"meta": {},
"published_at": "2024-09-29",
"quote": "Österreich | ÖVP | 1282734 | 4882888 gültige",
"source": null,
"stance": "supports",
"title": "BMI: Nationalratswahl 2024, endgültiges Ergebnis",
"url": "https://www.bmi.gv.at/412/nationalratswahlen/nationalratswahl_2024/files/endgueltiges_ergebnis_beschluss_bundeswahlbehoerde_16102024.xlsx"
}
],
"facet": null,
"key": [
"election",
"nrw2024",
"G00000",
"ÖVP"
],
"occurred_at": "2024-09-29",
"org": {
"__type__": "OrgRef",
"create": false,
"ids": {},
"kind": "party",
"name": null,
"short_name": "ÖVP",
"slug": null
},
"person": null,
"review": false,
"sensitive": false,
"summary_de": "ÖVP: 26,26 % (1.282.734 Stimmen) bei der Nationalratswahl 2024 – Österreich",
"summary_en": "ÖVP: 26.26 % (1,282,734 votes) in the 2024 National Council election – Österreich",
"topics": [],
"valid_from": null,
"valid_to": null
},
{
"__type__": "Claim",
"claim_type": "election_result",
"counterpart_org": {
"__type__": "OrgRef",
"create": true,
"ids": {},
"kind": "other",
"name": null,
"short_name": null,
"slug": "nationalrat"
},
"counterpart_person": null,
"data": {
"election": "nrw2024",
"electorate": 6346059,
"party": "SPÖ",
"percent": 21.13,
"region": "Österreich",
"region_code": "G00000",
"valid_votes": 4882888,
"votes": 1032233
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "dataset_row",
"excerpt": null,
"full_text": null,
"locator": "G00000:SPÖ",
"meta": {},
"published_at": "2024-09-29",
"quote": "Österreich | SPÖ | 1032233 | 4882888 gültige",
"source": null,
"stance": "supports",
"title": "BMI: Nationalratswahl 2024, endgültiges Ergebnis",
"url": "https://www.bmi.gv.at/412/nationalratswahlen/nationalratswahl_2024/files/endgueltiges_ergebnis_beschluss_bundeswahlbehoerde_16102024.xlsx"
}
],
"facet": null,
"key": [
"election",
"nrw2024",
"G00000",
"SPÖ"
],
"occurred_at": "2024-09-29",
"org": {
"__type__": "OrgRef",
"create": false,
"ids": {},
"kind": "party",
"name": null,
"short_name": "SPÖ",
"slug": null
},
"person": null,
"review": false,
"sensitive": false,
"summary_de": "SPÖ: 21,13 % (1.032.233 Stimmen) bei der Nationalratswahl 2024 – Österreich",
"summary_en": "SPÖ: 21.13 % (1,032,233 votes) in the 2024 National Council election – Österreich",
"topics": [],
"valid_from": null,
"valid_to": null
},
{
"__type__": "Claim",
"claim_type": "election_result",
"counterpart_org": {
"__type__": "OrgRef",
"create": true,
"ids": {},
"kind": "other",
"name": null,
"short_name": null,
"slug": "nationalrat"
},
"counterpart_person": null,
"data": {
"election": "nrw2024",
"electorate": 6346059,
"party": "FPÖ",
"percent": 28.84,
"region": "Österreich",
"region_code": "G00000",
"valid_votes": 4882888,
"votes": 1408512
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "dataset_row",
"excerpt": null,
"full_text": null,
"locator": "G00000:FPÖ",
"meta": {},
"published_at": "2024-09-29",
"quote": "Österreich | FPÖ | 1408512 | 4882888 gültige",
"source": null,
"stance": "supports",
"title": "BMI: Nationalratswahl 2024, endgültiges Ergebnis",
"url": "https://www.bmi.gv.at/412/nationalratswahlen/nationalratswahl_2024/files/endgueltiges_ergebnis_beschluss_bundeswahlbehoerde_16102024.xlsx"
}
],
"facet": null,
"key": [
"election",
"nrw2024",
"G00000",
"FPÖ"
],
"occurred_at": "2024-09-29",
"org": {
"__type__": "OrgRef",
"create": false,
"ids": {},
"kind": "party",
"name": null,
"short_name": "FPÖ",
"slug": null
},
"person": null,
"review": false,
"sensitive": false,
"summary_de": "FPÖ: 28,84 % (1.408.512 Stimmen) bei der Nationalratswahl 2024 – Österreich",
"summary_en": "FPÖ: 28.84 % (1,408,512 votes) in the 2024 National Council election – Österreich",
"topics": [],
"valid_from": null,
"valid_to": null
},
{
"__type__": "Claim",
"claim_type": "election_result",
"counterpart_org": {
"__type__": "OrgRef",
"create": true,
"ids": {},
"kind": "other",
"name": null,
"short_name": null,
"slug": "nationalrat"
},
"counterpart_person": null,
"data": {
"election": "nrw2024",
"electorate": 233738,
"party": "ÖVP",
"percent": 28.55,
"region": "Burgenland",
"region_code": "G10000",
"valid_votes": 190099,
"votes": 54287
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "dataset_row",
"excerpt": null,
"full_text": null,
"locator": "G10000:ÖVP",
"meta": {},
"published_at": "2024-09-29",
"quote": "Burgenland | ÖVP | 54287 | 190099 gültige",
"source": null,
"stance": "supports",
"title": "BMI: Nationalratswahl 2024, endgültiges Ergebnis",
"url": "https://www.bmi.gv.at/412/nationalratswahlen/nationalratswahl_2024/files/endgueltiges_ergebnis_beschluss_bundeswahlbehoerde_16102024.xlsx"
}
],
"facet": null,
"key": [
"election",
"nrw2024",
"G10000",
"ÖVP"
],
"occurred_at": "2024-09-29",
"org": {
"__type__": "OrgRef",
"create": false,
"ids": {},
"kind": "party",
"name": null,
"short_name": "ÖVP",
"slug": null
},
"person": null,
"review": false,
"sensitive": false,
"summary_de": "ÖVP: 28,55 % (54.287 Stimmen) bei der Nationalratswahl 2024 – Burgenland",
"summary_en": "ÖVP: 28.55 % (54,287 votes) in the 2024 National Council election – Burgenland",
"topics": [],
"valid_from": null,
"valid_to": null
},
{
"__type__": "Claim",
"claim_type": "election_result",
"counterpart_org": {
"__type__": "OrgRef",
"create": true,
"ids": {},
"kind": "other",
"name": null,
"short_name": null,
"slug": "nationalrat"
},
"counterpart_person": null,
"data": {
"election": "nrw2024",
"electorate": 233738,
"party": "SPÖ",
"percent": 27.02,
"region": "Burgenland",
"region_code": "G10000",
"valid_votes": 190099,
"votes": 51374
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "dataset_row",
"excerpt": null,
"full_text": null,
"locator": "G10000:SPÖ",
"meta": {},
"published_at": "2024-09-29",
"quote": "Burgenland | SPÖ | 51374 | 190099 gültige",
"source": null,
"stance": "supports",
"title": "BMI: Nationalratswahl 2024, endgültiges Ergebnis",
"url": "https://www.bmi.gv.at/412/nationalratswahlen/nationalratswahl_2024/files/endgueltiges_ergebnis_beschluss_bundeswahlbehoerde_16102024.xlsx"
}
],
"facet": null,
"key": [
"election",
"nrw2024",
"G10000",
"SPÖ"
],
"occurred_at": "2024-09-29",
"org": {
"__type__": "OrgRef",
"create": false,
"ids": {},
"kind": "party",
"name": null,
"short_name": "SPÖ",
"slug": null
},
"person": null,
"review": false,
"sensitive": false,
"summary_de": "SPÖ: 27,02 % (51.374 Stimmen) bei der Nationalratswahl 2024 – Burgenland",
"summary_en": "SPÖ: 27.02 % (51,374 votes) in the 2024 National Council election – Burgenland",
"topics": [],
"valid_from": null,
"valid_to": null
},
{
"__type__": "Claim",
"claim_type": "election_result",
"counterpart_org": {
"__type__": "OrgRef",
"create": true,
"ids": {},
"kind": "other",
"name": null,
"short_name": null,
"slug": "nationalrat"
},
"counterpart_person": null,
"data": {
"election": "nrw2024",
"electorate": 233738,
"party": "FPÖ",
"percent": 28.78,
"region": "Burgenland",
"region_code": "G10000",
"valid_votes": 190099,
"votes": 54718
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "dataset_row",
"excerpt": null,
"full_text": null,
"locator": "G10000:FPÖ",
"meta": {},
"published_at": "2024-09-29",
"quote": "Burgenland | FPÖ | 54718 | 190099 gültige",
"source": null,
"stance": "supports",
"title": "BMI: Nationalratswahl 2024, endgültiges Ergebnis",
"url": "https://www.bmi.gv.at/412/nationalratswahlen/nationalratswahl_2024/files/endgueltiges_ergebnis_beschluss_bundeswahlbehoerde_16102024.xlsx"
}
],
"facet": null,
"key": [
"election",
"nrw2024",
"G10000",
"FPÖ"
],
"occurred_at": "2024-09-29",
"org": {
"__type__": "OrgRef",
"create": false,
"ids": {},
"kind": "party",
"name": null,
"short_name": "FPÖ",
"slug": null
},
"person": null,
"review": false,
"sensitive": false,
"summary_de": "FPÖ: 28,78 % (54.718 Stimmen) bei der Nationalratswahl 2024 – Burgenland",
"summary_en": "FPÖ: 28.78 % (54,718 votes) in the 2024 National Council election – Burgenland",
"topics": [],
"valid_from": null,
"valid_to": null
}
]
@@ -0,0 +1,12 @@
{
"url": "https://www.bmi.gv.at/412/nationalratswahlen/nationalratswahl_2024/files/endgueltiges_ergebnis_beschluss_bundeswahlbehoerde_16102024.xlsx",
"kind": "result",
"method": "GET",
"params": null,
"json": null,
"meta": {
"election": "nrw2024"
},
"content_type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"body_file": "result_nrw2024.xlsx"
}
Binary file not shown.
@@ -12,6 +12,7 @@
}, },
"item_ref": "A10-0259/2026", "item_ref": "A10-0259/2026",
"key": "3ab02bb907351dfa059987800a98ee3b1e2a77a9", "key": "3ab02bb907351dfa059987800a98ee3b1e2a77a9",
"part_vote": false,
"protocol_url": "https://www.europarl.europa.eu/doceo/document/PV-10-2026-10-21-RCV_DE.html", "protocol_url": "https://www.europarl.europa.eu/doceo/document/PV-10-2026-10-21-RCV_DE.html",
"raw_text": "Entschließungsantrag (gesamter Text): 504 Ja, 63 Nein, 99 Enthaltungen", "raw_text": "Entschließungsantrag (gesamter Text): 504 Ja, 63 Nein, 99 Enthaltungen",
"result": "accepted", "result": "accepted",
@@ -219,5 +220,95 @@
], ],
"valid_from": null, "valid_from": null,
"valid_to": null "valid_to": null
},
{
"__type__": "VoteEvent",
"chamber": {
"__type__": "OrgRef",
"create": true,
"ids": {},
"kind": "other",
"name": null,
"short_name": null,
"slug": "europaeisches-parlament"
},
"item_ref": "A10-0272/2026",
"key": "6f5c1cd4005ecf97b7b450c671850a66d9fd5943",
"part_vote": true,
"protocol_url": "https://www.europarl.europa.eu/doceo/document/PV-10-2026-10-21-RCV_DE.html",
"raw_text": "nach § 3 – Änd. 70: 177 Ja, 455 Nein, 41 Enthaltungen",
"result": "rejected",
"roll_call": true,
"stage": "nach § 3 – Änd. 70",
"title": "A10-0272/2026 – Nils Ušakovs, Michalis Hadjipantela – nach § 3 – Änd. 70",
"url": "https://data.europarl.europa.eu/eli/dl/event/MTG-PL-2026-10-21-DEC-198977",
"voted_on": "2026-10-21"
},
{
"__type__": "Claim",
"claim_type": "vote",
"counterpart_org": {
"__type__": "OrgRef",
"create": true,
"ids": {},
"kind": "other",
"name": null,
"short_name": null,
"slug": "europaeisches-parlament"
},
"counterpart_person": null,
"data": {
"abstained": 41,
"chamber": "EP",
"no": 455,
"part_vote": true,
"position": "for",
"result": "rejected",
"roll_call": true,
"yes": 177
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "api_record",
"excerpt": null,
"full_text": null,
"locator": "vote:eli/dl/event/MTG-PL-2026-10-21-DEC-198977",
"meta": {},
"published_at": "2026-10-21",
"quote": "nach § 3 – Änd. 70: 177 Ja, 455 Nein, 41 Enthaltungen",
"source": null,
"stance": "supports",
"title": "A10-0272/2026 – Nils Ušakovs, Michalis Hadjipantela – nach § 3 – Änd. 70",
"url": "https://data.europarl.europa.eu/eli/dl/event/MTG-PL-2026-10-21-DEC-198977"
}
],
"facet": {
"__type__": "VotePosition",
"event_key": "6f5c1cd4005ecf97b7b450c671850a66d9fd5943",
"party": null,
"position": "for"
},
"key": null,
"occurred_at": "2026-10-21",
"org": null,
"person": {
"__type__": "PersonRef",
"hints": {},
"ids": {
"ep": "257104"
},
"name": null
},
"review": false,
"sensitive": false,
"summary_de": "Stimmte für: A10-0272/2026 – Nils Ušakovs, Michalis Hadjipantela – nach § 3 – Änd. 70 (Europäisches Parlament) – abgelehnt",
"summary_en": "Voted for: A10-0272/2026 – Nils Ušakovs, Michalis Hadjipantela – nach § 3 – Änd. 70 (European Parliament) – rejected",
"topics": [
"Europäische Union"
],
"valid_from": null,
"valid_to": null
} }
] ]
@@ -12,6 +12,7 @@
}, },
"item_ref": "416/A", "item_ref": "416/A",
"key": "931917bd2feea2b95c21f3e15a86267702bce32f", "key": "931917bd2feea2b95c21f3e15a86267702bce32f",
"part_vote": false,
"protocol_url": "https://www.parlament.gv.at/dokument/XXVIII/NRSITZ/44/fnameorig_1764795.html#160", "protocol_url": "https://www.parlament.gv.at/dokument/XXVIII/NRSITZ/44/fnameorig_1764795.html#160",
"raw_text": "44. Sitzung des Nationalrates: Gesetzesvorschlag in zweiter Lesung angenommen Dafür: ÖVP, SPÖ, NEOS, GRÜNE, dagegen: FPÖ", "raw_text": "44. Sitzung des Nationalrates: Gesetzesvorschlag in zweiter Lesung angenommen Dafür: ÖVP, SPÖ, NEOS, GRÜNE, dagegen: FPÖ",
"result": "accepted", "result": "accepted",
@@ -459,6 +460,7 @@
}, },
"item_ref": "416/A", "item_ref": "416/A",
"key": "106b0163e733dbe54ccf3a7c148a2041ee607b6b", "key": "106b0163e733dbe54ccf3a7c148a2041ee607b6b",
"part_vote": false,
"protocol_url": "https://www.parlament.gv.at/dokument/XXVIII/NRSITZ/44/fnameorig_1764795.html#160", "protocol_url": "https://www.parlament.gv.at/dokument/XXVIII/NRSITZ/44/fnameorig_1764795.html#160",
"raw_text": "44. Sitzung des Nationalrates: Gesetzesvorschlag in dritter Lesung angenommen Dafür: ÖVP, SPÖ, NEOS, GRÜNE, dagegen: FPÖ", "raw_text": "44. Sitzung des Nationalrates: Gesetzesvorschlag in dritter Lesung angenommen Dafür: ÖVP, SPÖ, NEOS, GRÜNE, dagegen: FPÖ",
"result": "accepted", "result": "accepted",
@@ -906,6 +908,7 @@
}, },
"item_ref": "416/A", "item_ref": "416/A",
"key": "bc19d7af2b29acce57bfa6ad139b44c43aacc98a", "key": "bc19d7af2b29acce57bfa6ad139b44c43aacc98a",
"part_vote": false,
"protocol_url": "https://www.parlament.gv.at/dokument/BR/BRSITZ/982/fnameorig_1768518.html#45", "protocol_url": "https://www.parlament.gv.at/dokument/BR/BRSITZ/982/fnameorig_1768518.html#45",
"raw_text": "982. Sitzung: Antrag, keinen Einspruch zu erheben, angenommen Dafür: ÖVP, SPÖ, GRÜNE, NEOS, dagegen: FPÖ", "raw_text": "982. Sitzung: Antrag, keinen Einspruch zu erheben, angenommen Dafür: ÖVP, SPÖ, GRÜNE, NEOS, dagegen: FPÖ",
"result": "accepted", "result": "accepted",
@@ -0,0 +1,167 @@
[
{
"__type__": "Claim",
"claim_type": "affiliation",
"counterpart_org": null,
"counterpart_person": null,
"data": {
"chamber": "NR",
"committee": "Ständiger Unterausschuss des Budgetausschusses",
"function": "Ersatzmitglied",
"gp": "XXVIII"
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "api_record",
"excerpt": null,
"full_text": null,
"locator": "ausschuss:XXVIII:Ständiger Unterausschuss des Budgetausschusses:Ersatzmitglied:2024-10-24",
"meta": {},
"published_at": null,
"quote": null,
"source": null,
"stance": "supports",
"title": "Parlament: Hannes Amesbauer",
"url": "https://www.parlament.gv.at/person/51879"
}
],
"facet": null,
"key": [
"parl-committee",
"51879",
"XXVIII",
"Ständiger Unterausschuss des Budgetausschusses",
"Ersatzmitglied",
"2024-10-24"
],
"occurred_at": null,
"org": null,
"person": {
"__type__": "PersonRef",
"hints": {},
"ids": {
"parlament": "51879"
},
"name": null
},
"review": false,
"sensitive": false,
"summary_de": "Hannes Amesbauer: Ersatzmitglied, Ständiger Unterausschuss des Budgetausschusses (Nationalrat, XXVIII. GP)",
"summary_en": "Hannes Amesbauer: Ersatzmitglied, Ständiger Unterausschuss des Budgetausschusses (Nationalrat, XXVIII legislative period)",
"topics": [],
"valid_from": "2024-10-24",
"valid_to": "2024-12-18"
},
{
"__type__": "Claim",
"claim_type": "affiliation",
"counterpart_org": null,
"counterpart_person": null,
"data": {
"chamber": "NR",
"committee": "Ständiger Unterausschuss des Hauptausschusses",
"function": "Ersatzmitglied",
"gp": "XXVIII"
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "api_record",
"excerpt": null,
"full_text": null,
"locator": "ausschuss:XXVIII:Ständiger Unterausschuss des Hauptausschusses:Ersatzmitglied:2024-10-24",
"meta": {},
"published_at": null,
"quote": null,
"source": null,
"stance": "supports",
"title": "Parlament: Hannes Amesbauer",
"url": "https://www.parlament.gv.at/person/51879"
}
],
"facet": null,
"key": [
"parl-committee",
"51879",
"XXVIII",
"Ständiger Unterausschuss des Hauptausschusses",
"Ersatzmitglied",
"2024-10-24"
],
"occurred_at": null,
"org": null,
"person": {
"__type__": "PersonRef",
"hints": {},
"ids": {
"parlament": "51879"
},
"name": null
},
"review": false,
"sensitive": false,
"summary_de": "Hannes Amesbauer: Ersatzmitglied, Ständiger Unterausschuss des Hauptausschusses (Nationalrat, XXVIII. GP)",
"summary_en": "Hannes Amesbauer: Ersatzmitglied, Ständiger Unterausschuss des Hauptausschusses (Nationalrat, XXVIII legislative period)",
"topics": [],
"valid_from": "2024-10-24",
"valid_to": "2024-12-18"
},
{
"__type__": "Claim",
"claim_type": "affiliation",
"counterpart_org": null,
"counterpart_person": null,
"data": {
"chamber": "NR",
"committee": "Ständiger Unterausschuss des Budgetausschusses",
"function": "Obfrau",
"gp": "XXVIII"
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "api_record",
"excerpt": null,
"full_text": null,
"locator": "ausschuss:XXVIII:Ständiger Unterausschuss des Budgetausschusses:Obfrau:2024-10-24",
"meta": {},
"published_at": null,
"quote": null,
"source": null,
"stance": "supports",
"title": "Parlament: Hannes Amesbauer",
"url": "https://www.parlament.gv.at/person/51879"
}
],
"facet": null,
"key": [
"parl-committee",
"51879",
"XXVIII",
"Ständiger Unterausschuss des Budgetausschusses",
"Obfrau",
"2024-10-24"
],
"occurred_at": null,
"org": null,
"person": {
"__type__": "PersonRef",
"hints": {},
"ids": {
"parlament": "51879"
},
"name": null
},
"review": false,
"sensitive": false,
"summary_de": "Hannes Amesbauer: Obfrau, Ständiger Unterausschuss des Budgetausschusses (Nationalrat, XXVIII. GP)",
"summary_en": "Hannes Amesbauer: Obfrau, Ständiger Unterausschuss des Budgetausschusses (Nationalrat, XXVIII legislative period)",
"topics": [],
"valid_from": "2024-10-24",
"valid_to": null
}
]
+206
View File
@@ -0,0 +1,206 @@
{
"pages": 1,
"count": 4,
"header": [
{
"css_class": "datum-col",
"feldId": 92,
"hidden": false,
"rnr": 1,
"listeId": 250,
"label": "GP",
"sortable": "1",
"feld_name": "GP",
"class": "datum-col"
},
{
"feldId": 93,
"hidden": true,
"rnr": 2,
"listeId": 250,
"label": "Sort"
},
{
"css_class": "col-20",
"feldId": 94,
"hidden": false,
"rnr": 3,
"listeId": 250,
"label": "Funktion",
"feld_name": "FUNKTION",
"class": "col-20"
},
{
"css_class": "fraction-col",
"link_rnr": "6",
"feldId": 95,
"hidden": false,
"rnr": 4,
"listeId": 250,
"label": "Ausschuss",
"class": "fraction-col"
},
{
"feldId": 96,
"hidden": true,
"rnr": 5,
"listeId": 250,
"label": "nrbr"
},
{
"feldId": 97,
"hidden": true,
"rnr": 6,
"listeId": 250,
"label": "URL"
},
{
"feldId": 98,
"hidden": true,
"rnr": 7,
"listeId": 250,
"label": "PAD_INTERN",
"feld_name": "PAD_INTERN"
},
{
"feldId": 969,
"hidden": true,
"rnr": 8,
"mediaType": "fromDate",
"listeId": 250,
"label": "fromDate"
},
{
"feldId": 564,
"hidden": true,
"rnr": 9,
"listeId": 250,
"label": "Ausschuss",
"feld_name": "AUSSCHUSS"
},
{
"feldId": 565,
"hidden": true,
"rnr": 10,
"listeId": 250,
"label": "Gremium",
"feld_name": "GREMIUM"
},
{
"feldId": 584,
"hidden": true,
"rnr": 11,
"listeId": 250,
"label": "gp_text_full",
"feld_name": "GP_TEXT_FULL"
},
{
"feldId": 1645,
"hidden": true,
"ist_range": "1",
"rnr": 12,
"mediaType": "datetime",
"listeId": 250,
"label": "Beginn",
"feld_name": "BEGINNDATUM_ISO"
},
{
"feldId": 1665,
"hidden": false,
"sort_rnr": 14,
"label": "Funktionsbeginn",
"sortable": "1",
"css_class": "datum-col",
"rnr": 13,
"listeId": 250,
"feld_name": "FUNKTIONSBEGINN",
"class": "datum-col"
},
{
"feldId": 1666,
"hidden": true,
"rnr": 14,
"listeId": 250,
"label": "Beginn4Sort",
"feld_name": "BEGINN4SORT"
},
{
"feldId": 1685,
"hidden": true,
"rnr": 15,
"listeId": 250,
"label": "Personenregister",
"feld_name": "PERSONENREGISTER"
}
],
"rows": [
[
"XXVIII",
"2Z797589750013220241024STÄNDIGER UNTERAUSSCHUSS DES BUDGETAUSSCHUSSES79758975",
"Ersatzmitglied",
"Ständiger Unterausschuss des Budgetausschusses (24.10.2024 - 18.12.2024)",
"NR",
"/PAKT/VHG/XXVIII/SA-BU/SA-BU_00001_00908/index.shtml",
"51879",
"Thu Oct 24 02:00:00 CEST 2024",
"Ständiger Unterausschuss des Budgetausschusses",
"Nationalrat",
"ab 24.10.2024: XXVIII. Gesetzgebungsperiode des NR",
"2024-10-24T00:00:00",
"24.10.2024",
"20241024",
null
],
[
"XXVIII",
"2Z797589750013220241024STÄNDIGER UNTERAUSSCHUSS DES HAUPTAUSSCHUSSES79758975",
"Ersatzmitglied",
"Ständiger Unterausschuss des Hauptausschusses (24.10.2024 - 18.12.2024)",
"NR",
"/PAKT/VHG/XXVIII/SA-HA/SA-HA_00001_00912/index.shtml",
"51879",
"Thu Oct 24 02:00:00 CEST 2024",
"Ständiger Unterausschuss des Hauptausschusses",
"Nationalrat",
"ab 24.10.2024: XXVIII. Gesetzgebungsperiode des NR",
"2024-10-24T00:00:00",
"24.10.2024",
"20241024",
null
],
[
"XXVI",
"2Z798288900013120171221AUSSCHUSS FÜR KONSUMENTENSCHUTZ79828778",
"Mitglied",
"Ausschuss für Konsumentenschutz (21.12.2017 - 25.09.2018)",
"NR",
"/PAKT/VHG/XXVI/A-KO/A-KO_00001_00842/index.shtml",
"51879",
"Thu Dec 21 01:00:00 CET 2017",
"Ausschuss für Konsumentenschutz",
"Nationalrat",
"09.11.2017 - 22.10.2019: XXVI. Gesetzgebungsperiode des NR",
"2017-12-21T00:00:00",
"21.12.2017",
"20171221",
null
],
[
"XXVIII",
"2Z797589750013220241024STÄNDIGER UNTERAUSSCHUSS DES BUDGETAUSSCHUSSES79758975",
"Obfrau",
"Ständiger Unterausschuss des Budgetausschusses (24.10.2024 - )",
"NR",
"/PAKT/VHG/XXVIII/SA-BU/SA-BU_00001_00908/index.shtml",
"51879",
"Thu Oct 24 02:00:00 CEST 2024",
"Ständiger Unterausschuss des Budgetausschusses",
"Nationalrat",
"ab 24.10.2024: XXVIII. Gesetzgebungsperiode des NR",
"2024-10-24T00:00:00",
"24.10.2024",
"20241024",
null
]
]
}
@@ -0,0 +1,23 @@
{
"url": "https://www.parlament.gv.at/Filter/api/filter/data/250?js=eval&showAll=true",
"kind": "committees",
"method": "POST",
"params": null,
"json": {
"PAD_INTERN": [
51879
]
},
"meta": {
"pad": "51879",
"name": "Hannes Amesbauer"
},
"content_type": "application/json",
"body_file": "committees.json",
"run_params": {
"periods": [
"XXVII",
"XXVIII"
]
}
}
@@ -677,5 +677,24 @@
"params": null, "params": null,
"skip_unchanged": true, "skip_unchanged": true,
"url": "https://www.parlament.gv.at/Filter/api/filter/data/251?js=eval&showAll=true" "url": "https://www.parlament.gv.at/Filter/api/filter/data/251?js=eval&showAll=true"
},
{
"__type__": "Request",
"conditional": true,
"headers": null,
"json": {
"PAD_INTERN": [
1977
]
},
"kind": "committees",
"meta": {
"name": "Tanja Graf",
"pad": "1977"
},
"method": "POST",
"params": null,
"skip_unchanged": true,
"url": "https://www.parlament.gv.at/Filter/api/filter/data/250?js=eval&showAll=true"
} }
] ]
+59
View File
@@ -158,3 +158,62 @@ def test_parlament_calendar_titles_are_html_decoded():
params={"since_year": 2026}) params={"since_year": 2026})
assert [e.title for e in events] == ["84. Sitzung des Nationalrates"] assert [e.title for e in events] == ["84. Sitzung des Nationalrates"]
assert events[0].description == "Plenarsitzung NR" assert events[0].description == "Plenarsitzung NR"
def test_procurement_rejects_implausible_award_dates():
from datetime import date
from politrack.sources.procurement import plausible_date
today = date(2026, 11, 13)
assert plausible_date("2025-02-17", today) == date(2025, 2, 17)
assert plausible_date("1900-12-30", today) is None # spreadsheet placeholder
assert plausible_date("2028-09-30", today) is None # the contract's end, not its award
assert plausible_date(None, today) is None
def test_procurement_placeholder_contractor_and_framework_buyer():
from politrack.sources.procurement import buyer_name, org_ref
assert org_ref({"name": "-"}, "company") is None
assert org_ref({"name": " n/a "}, "company") is None
assert org_ref({"name": "Swietelsky AG"}, "company").name == "Swietelsky AG"
sentence = ("Auftraggeber sind die Republik Österreich (Bund), die Bundesbeschaffung GmbH sowie alle weiteren "
"Auftraggeber gemäß der den Ausschreibungsunterlagen beiliegenden Liste")
assert buyer_name(sentence) == ("Bundesbeschaffung GmbH", sentence)
assert buyer_name("ASFINAG Alpenstraßen GmbH") == ("ASFINAG Alpenstraßen GmbH", None)
def test_source_freshness_alerts_after_two_missed_slots():
from datetime import datetime, timezone
from politrack.queries.admin import source_freshness
class Conn:
def __init__(self, rows):
self.rows = rows
def execute(self, *_):
return self.rows
now = datetime(2026, 11, 13, 12, 0, tzinfo=timezone.utc)
daily = "15 3 * * *"
rows = [{"adapter": "fresh", "at": datetime(2026, 11, 13, 3, 40, tzinfo=timezone.utc)},
{"adapter": "one_missed", "at": datetime(2026, 11, 12, 3, 40, tzinfo=timezone.utc)},
{"adapter": "two_missed", "at": datetime(2026, 11, 11, 3, 40, tzinfo=timezone.utc)}]
result = {r["adapter"]: r["stale"] for r in source_freshness(
Conn(rows), {"fresh": daily, "one_missed": daily, "two_missed": daily, "never": daily}, now)}
assert result == {"fresh": False, "one_missed": False, "two_missed": True, "never": True}
def test_election_preferences_eu_layouts():
from politrack.sources.elections import parse_preferences, split_name
assert split_name("Babler, MSc, Andreas") == ("Babler", "Andreas")
assert split_name("Brunner, Dr. Magnus") == ("Brunner", "Dr. Magnus")
eu2019 = [("ÖVP", [(None, "Kandidat(in)", "BGLD", "Gesamt"), (1, "Karas, Dr. Othmar ", 3012, 103035)])]
eu2024 = [("ÖVP", [("LP", "Bewerber(in)", "BGLD", "Gesamt", None, "Darstellung", None, None),
(None, None, None, None, None, "(LP)", "Bewerber(in)", "Gesamt"),
(1, "Lopatka, Dr. Reinhold", 2268, 58013, None, 3, "Bernhuber, Alexander", 44641)])]
assert [(c["name"], c["votes"]) for c in parse_preferences("eu", eu2019)] == [("Dr. Othmar Karas", 103035)]
assert [(c["name"], c["votes"]) for c in parse_preferences("eu", eu2024)] == [("Dr. Reinhold Lopatka", 58013)]
+33
View File
@@ -0,0 +1,33 @@
#!/bin/sh
# Nightly database backup, run by the `backup` service in docker-compose.yml.
# Writes politrack-YYYY-MM-DD.dump (pg_dump custom format) to /backups at 02:30 UTC, deletes dumps older than
# $BACKUP_KEEP_DAYS days, and records each attempt as a run of "backup", so /api/v1/status/sources alerts
# when backups stop. Restore: pg_restore -h <host> -U politrack -d politrack --clean <file>
set -u
KEEP="${BACKUP_KEEP_DAYS:-14}"
export PGHOST=db PGUSER=politrack PGDATABASE=politrack
record() { # record <status> <json stats>
psql -q -c "INSERT INTO run (adapter, mode, status, finished_at, stats) VALUES ('backup', 'backup', '$1', now(), '$2'::jsonb)" \
|| echo "could not record the backup run"
}
while true; do
now=$(date -u +%s)
next=$(date -u -d "today 02:30" +%s)
[ "$next" -le "$now" ] && next=$(date -u -d "tomorrow 02:30" +%s)
echo "next backup at $(date -u -d "@$next")"
sleep $((next - now))
file="/backups/politrack-$(date -u +%F).dump"
if pg_dump -Fc -f "$file.tmp" && mv "$file.tmp" "$file"; then
size=$(stat -c %s "$file")
echo "backup ok: $file ($size bytes)"
record ok "{\"file\": \"$(basename "$file")\", \"bytes\": $size}"
else
rm -f "$file.tmp"
echo "backup FAILED"
record error '{}'
fi
find /backups -name 'politrack-*.dump' -mtime "+$KEEP" -delete
done
+26
View File
@@ -9,9 +9,12 @@ x-backend: &backend
environment: environment:
POLITRACK_DATABASE_URL: postgresql://politrack:${POSTGRES_PASSWORD}@db:5432/politrack POLITRACK_DATABASE_URL: postgresql://politrack:${POSTGRES_PASSWORD}@db:5432/politrack
POLITRACK_RAW_DIR: /data/raw POLITRACK_RAW_DIR: /data/raw
POLITRACK_LOG_DIR: /data/logs
volumes: volumes:
# Raw-response archive: lets `politrack reparse` rebuild data after parser fixes without refetching. # Raw-response archive: lets `politrack reparse` rebuild data after parser fixes without refetching.
- ${DATA_DIR:-./data}/raw:/data/raw - ${DATA_DIR:-./data}/raw:/data/raw
# Worker logs (<service>.log, rotating), kept across redeploys.
- ${DATA_DIR:-./data}/logs:/data/logs
depends_on: depends_on:
db: db:
condition: service_healthy condition: service_healthy
@@ -62,16 +65,34 @@ services:
# another machine pointed at this database. # another machine pointed at this database.
worker-scrape: worker-scrape:
<<: *worker <<: *worker
hostname: worker-scrape # names the log file
command: ["procrastinate", "--app=politrack.worker.app", "worker", "--queues=scrape,social", "--concurrency=2"] command: ["procrastinate", "--app=politrack.worker.app", "worker", "--queues=scrape,social", "--concurrency=2"]
worker-bulk: worker-bulk:
<<: *worker <<: *worker
hostname: worker-bulk
command: ["procrastinate", "--app=politrack.worker.app", "worker", "--queues=bulk", "--concurrency=1"] command: ["procrastinate", "--app=politrack.worker.app", "worker", "--queues=bulk", "--concurrency=1"]
worker-enrich: worker-enrich:
<<: *worker <<: *worker
hostname: worker-enrich
command: ["procrastinate", "--app=politrack.worker.app", "worker", "--queues=enrich", "--concurrency=1"] command: ["procrastinate", "--app=politrack.worker.app", "worker", "--queues=enrich", "--concurrency=1"]
# Nightly pg_dump at 02:30 UTC into ${BACKUP_DIR}, kept for ${BACKUP_KEEP_DAYS} days (see backup.sh).
backup:
image: pgvector/pgvector:pg16
restart: unless-stopped
environment:
PGPASSWORD: ${POSTGRES_PASSWORD}
BACKUP_KEEP_DAYS: ${BACKUP_KEEP_DAYS:-14}
volumes:
- ./backup.sh:/backup.sh:ro
- ${BACKUP_DIR:-./data/backups}:/backups
entrypoint: ["sh", "/backup.sh"]
depends_on:
db:
condition: service_healthy
mcp: mcp:
<<: *worker <<: *worker
command: ["politrack", "mcp"] command: ["politrack", "mcp"]
@@ -84,6 +105,11 @@ services:
restart: unless-stopped restart: unless-stopped
environment: environment:
API_INTERNAL_URL: http://api:8000 API_INTERNAL_URL: http://api:8000
POLITRACK_USER_AGENT: ${POLITRACK_USER_AGENT:-}
POLITRACK_LEGAL_NAME: ${POLITRACK_LEGAL_NAME:-}
POLITRACK_LEGAL_ADDRESS: ${POLITRACK_LEGAL_ADDRESS:-}
POLITRACK_LEGAL_EMAIL: ${POLITRACK_LEGAL_EMAIL:-}
POLITRACK_CORRECTIONS_EMAIL: ${POLITRACK_CORRECTIONS_EMAIL:-}
ports: ports:
- "${WEB_PORT:-3000}:3000" - "${WEB_PORT:-3000}:3000"
depends_on: depends_on:
+3 -1
View File
@@ -71,7 +71,9 @@ a permalink shows the same to everyone.
from the home page's "Entdecken" cards. from the home page's "Entdecken" cards.
4. Page modules may only export Next.js page fields (`default`, `generateMetadata`, …); keep helpers and 4. Page modules may only export Next.js page fields (`default`, `generateMetadata`, …); keep helpers and
constants module-local or in `src/components`. constants module-local or in `src/components`.
5. Check it at 390 px and in both themes (design-system.md §7). 5. Check it at 390 px and in both themes (design-system.md §7). `node web/scripts/mobile-audit.mjs` emulates a
phone, reports pages that scroll sideways (and which elements cause it) and saves screenshots; the
script's header shows how to call it. All pages passed on 2026-11-12.
## Showing a new claim type ## Showing a new claim type
+5 -1
View File
@@ -69,8 +69,12 @@ counterpart (`counterpart_person_id`, `counterpart_org_id`), time (`occurred_at`
| `government_decision` | – | Council of Ministers minutes | | `government_decision` | – | Council of Ministers minutes |
| `wiki_edit` | – | Wikipedia edits from institutional IP ranges (subject = network owner) | | `wiki_edit` | – | Wikipedia edits from institutional IP ranges (subject = network owner) |
| `declaration` | – | EP declarations of interests | | `declaration` | – | EP declarations of interests |
| `ownership`, `affiliation` | – | party reports (holdings, affiliated organisations) | | `ownership`, `affiliation` | – | party reports (holdings, affiliated organisations), committee memberships |
| `party_finance` | – | party annual reports (totals in `data`) | | `party_finance` | – | party annual reports (totals in `data`) |
| `election_result` | – | BMI: a party's votes in Austria or a state (counterpart = elected body), a candidate's preference votes (counterpart = party) |
EP votes on amendments and parts of a text: `vote_event.part_vote` and `claim.data.part_vote` are true; general
lists, counts and search chunks leave them out (`NOT_PART_VOTE` in `queries/common.py`).
**Status:** `reported` (one source) → `corroborated` (≥ 2 distinct sources support/mention it) or **Status:** `reported` (one source) → `corroborated` (≥ 2 distinct sources support/mention it) or
`disputed` (a source contradicts). Computed by the Loader on every new piece of evidence and by `disputed` (a source contradicts). Computed by the Loader on every new piece of evidence and by
+15 -9
View File
@@ -23,9 +23,9 @@
docker compose build docker compose build
docker compose up -d docker compose up -d
``` ```
Docker creates `${DATA_DIR}/raw` as root, but the backend runs as a non-root user, and every fetch Docker creates `${DATA_DIR}/raw` and `${DATA_DIR}/logs` as root, but the backend runs as a non-root user, and
then fails with "Permission denied". Hand it over once after the first `up`: every fetch then fails with "Permission denied". Hand them over once after the first `up`:
`chown -R "$(docker run --rm --entrypoint id politrack-backend -u):$(docker run --rm --entrypoint id politrack-backend -g)" "$DATA_DIR/raw"`. `chown -R "$(docker run --rm --entrypoint id politrack-backend -u):$(docker run --rm --entrypoint id politrack-backend -g)" "$DATA_DIR/raw" "$DATA_DIR/logs"`.
`migrate` runs once (schema, seed data, source registry, job-queue tables), then `api`, `web`, `mcp` `migrate` runs once (schema, seed data, source registry, job-queue tables), then `api`, `web`, `mcp`
and three workers start: and three workers start:
@@ -76,11 +76,11 @@ bge-m3 runs fine on CPU for this data volume. With an NVIDIA GPU, add a device r
## Backups ## Backups
* Database: a nightly `pg_dump`, e.g. with the **User Scripts** plugin: * Database: the `backup` service dumps it every night at 02:30 UTC to `BACKUP_DIR` (default `./data/backups`;
```bash better a different share, e.g. `/mnt/user/backups/politrack`) and keeps `BACKUP_KEEP_DAYS` (14) days. Each dump is
docker exec politrack-db-1 pg_dump -U politrack -Fc politrack > /mnt/user/backups/politrack/$(date +%F).dump recorded as a run of `backup`, so the status endpoint below alerts when backups stop. Restore with
find /mnt/user/backups/politrack -mtime +30 -delete `pg_restore -h localhost -p 5433 -U politrack -d politrack --clean <file>`. For off-site copies, sync `BACKUP_DIR`
``` elsewhere.
* Raw archive (`${DATA_DIR}/raw`): optional to back up – it saves refetching after parser fixes, and it can * Raw archive (`${DATA_DIR}/raw`): optional to back up – it saves refetching after parser fixes, and it can
be pruned (`find raw -mtime +365 -delete`); missing blobs are simply refetched. be pruned (`find raw -mtime +365 -delete`); missing blobs are simply refetched.
@@ -96,7 +96,13 @@ bge-m3 runs fine on CPU for this data volume. With an NVIDIA GPU, add a device r
* `GET /api/v1/admin/runs` with `Authorization: Bearer $POLITRACK_ADMIN_TOKEN`. * `GET /api/v1/admin/runs` with `Authorization: Bearer $POLITRACK_ADMIN_TOKEN`.
* Fetch log: `SELECT adapter, status, count(*) FROM fetch_log WHERE fetched_at > now() - interval '1 day' GROUP BY 1, 2;` * Fetch log: `SELECT adapter, status, count(*) FROM fetch_log WHERE fetched_at > now() - interval '1 day' GROUP BY 1, 2;`
* Job queue: `SELECT task_name, status, count(*) FROM procrastinate_jobs GROUP BY 1, 2;` * Job queue: `SELECT task_name, status, count(*) FROM procrastinate_jobs GROUP BY 1, 2;`
* Uptime Kuma can poll `http://api:8000/healthz`. * `GET /api/v1/status/sources` returns HTTP 503 as soon as a scheduled source (or the backup) missed its last two
slots, and lists every task with its last successful run: point an HTTP monitor (Uptime Kuma) at it.
`/healthz` only checks that the API reaches the database.
* The `watchdog` task (every 10 minutes) fails jobs of dead workers, closes abandoned runs and starts any scheduled
source whose last slot was missed (Procrastinate skips slots that are more than ten minutes overdue).
* Worker logs: `${DATA_DIR}/logs/worker-*.log` (rotating), kept across redeploys; `docker compose logs` only covers
the current containers.
## Troubleshooting ## Troubleshooting
+14 -5
View File
@@ -32,7 +32,14 @@ Next.js frontend (de/en), Docker Compose for Unraid.
- [x] Social: Bluesky, Mastodon, public Telegram channels, YouTube (key), X via official API (token); - [x] Social: Bluesky, Mastodon, public Telegram channels, YouTube (key), X via official API (token);
party/institution channels; curated `social_accounts.yaml` party/institution channels; curated `social_accounts.yaml`
- [x] X deletion sync (`social.x_deletions`): posts gone from X keep a marked record without their text - [x] X deletion sync (`social.x_deletions`): posts gone from X keep a marked record without their text
- [x] EP final roll-call votes of Austrian MEPs (`europarl.votes`) - [x] EP roll-call votes of Austrian MEPs, including amendments and part votes (`europarl.votes`)
- [x] Committee memberships of the configured legislative periods (list 250, `parlament.persons`)
- [x] Election results and preference votes: NR 2024, NR 2019, NR 2017, EU 2024, EU 2019 preference votes (`bmi.elections`)
- [x] Former Austrian MEPs of the 2019-2024 term; EC meetings of the 2014-2019 Commission; Bluesky/Mastodon history paging
- [x] Operations: watchdog for missed schedules and stalled jobs, nightly backups, `/api/v1/status/sources` for
monitoring, worker logs kept across redeploys; procurement data cleaned (award dates, placeholder contractors,
framework-agreement buyers)
- [x] Draft legal pages (`/impressum`, `/datenschutz`), corrections contact, portrait proxy (`/api/img`)
- [x] Pages: organisations, money explorer, social feed, Wikipedia edits, person network ("follow the money"), - [x] Pages: organisations, money explorer, social feed, Wikipedia edits, person network ("follow the money"),
say-vs-vote on roll calls say-vs-vote on roll calls
@@ -45,9 +52,10 @@ Next.js frontend (de/en), Docker Compose for Unraid.
## Next ## Next
**Data** **Data**
- [ ] Full speech texts and interjections from the protocols; committee memberships (list 250) - [ ] Full speech texts and interjections from the protocols
- [ ] EU Council participant lists, ministries' press releases, Federal President's schedule - [ ] EU Council participant lists, ministries' press releases, Federal President's schedule
- [ ] Election results and candidate lists; AMA/TAM subsidy recipients (after checking bulk access and terms) - [ ] Election results before 2019, NR 2019 state lists, Landtag and municipal elections
- [ ] AMA/TAM subsidy recipients (after checking bulk access and terms)
- [ ] Cabinet staff and appointments to state-owned companies (from answers to parliamentary questions) - [ ] Cabinet staff and appointments to state-owned companies (from answers to parliamentary questions)
- [ ] One adapter per Landtag (OGD where available); municipal councils and mayors - [ ] One adapter per Landtag (OGD where available); municipal councils and mayors
- [ ] Lobbying register – if the BMJ ever offers stable URLs or an export - [ ] Lobbying register – if the BMJ ever offers stable URLs or an export
@@ -59,8 +67,8 @@ Next.js frontend (de/en), Docker Compose for Unraid.
- [ ] Wayback Machine archiving of evidence URLs (`document.archive_url`) - [ ] Wayback Machine archiving of evidence URLs (`document.archive_url`)
**Product** **Product**
- [ ] Before going public: Impressum/Offenlegung (§ 5 ECG, § 25 MedienG), privacy policy, a corrections contact - [ ] Before going public: have `/impressum` and `/datenschutz` legally reviewed, set `POLITRACK_LEGAL_*` in `.env`,
- [ ] Image proxy for portraits and logos (no visitor IPs to Wikimedia/parliament) and document the balancing test (see legal.md)
- [ ] Compare two people or parties; topic pages (`/themen/{slug}`); Open Graph share images; CSV export - [ ] Compare two people or parties; topic pages (`/themen/{slug}`); Open Graph share images; CSV export
- [ ] Review queue for editors in the web UI (today `politrack review`) - [ ] Review queue for editors in the web UI (today `politrack review`)
- [ ] Community link submissions and a corrections/right-of-reply form - [ ] Community link submissions and a corrections/right-of-reply form
@@ -68,6 +76,7 @@ Next.js frontend (de/en), Docker Compose for Unraid.
- [ ] Optional accounts via reverse-proxy SSO using `app_user` - [ ] Optional accounts via reverse-proxy SSO using `app_user`
- [ ] Partnerships: MeineAbgeordneten, Parteispenden.at, APA (OTS full-text licence) - [ ] Partnerships: MeineAbgeordneten, Parteispenden.at, APA (OTS full-text licence)
- [ ] Map view of trips and appearances - [ ] Map view of trips and appearances
- [ ] Semantic search with embeddings (`POLITRACK_OLLAMA_URL`): parked by the owner on 2026-11-09, full-text search only for now
## Not yet verified (v2) ## Not yet verified (v2)
+3 -2
View File
@@ -31,6 +31,7 @@ Status: ✅ implemented (adapter name in `code`) · 🟡 planned · ⛔ delibera
| Source | Data | Access | Licence / terms | Status | | Source | Data | Access | Licence / terms | Status |
|---|---|---|---|---| |---|---|---|---|---|
| Parlament Österreich | MPs since 1918, mandates, functions, **side jobs & income category** (§6 Unv-Transparenz-G), social-media links, plenary speeches, **faction votes** per item and reading | JSON API (`/Filter/api/filter/data/409,101,251`, `?json=TRUE` detail pages) | [CC BY 4.0](https://www.parlament.gv.at/recherchieren/open-data/daten-und-lizenz/) | ✅ `parlament.persons`, `parlament.items` | | Parlament Österreich | MPs since 1918, mandates, functions, **side jobs & income category** (§6 Unv-Transparenz-G), social-media links, plenary speeches, **faction votes** per item and reading | JSON API (`/Filter/api/filter/data/409,101,251`, `?json=TRUE` detail pages) | [CC BY 4.0](https://www.parlament.gv.at/recherchieren/open-data/daten-und-lizenz/) | ✅ `parlament.persons`, `parlament.items` |
| – committee memberships | who sat on which committee, with function and dates | JSON API (`/Filter/api/filter/data/250`) | CC BY 4.0 | ✅ `parlament.persons` |
| – stenographic protocols | **Roll-call votes**: every member's yes/no from the "Mit „Ja“/„Nein“ stimmten …" lists | HTML fragments | CC BY 4.0 | ✅ `parlament.items` | | – stenographic protocols | **Roll-call votes**: every member's yes/no from the "Mit „Ja“/„Nein“ stimmten …" lists | HTML fragments | CC BY 4.0 | ✅ `parlament.items` |
| – immunity waivers | prosecutors' requests to lift immunity: authority, offence, outcome | items + committee report | CC BY 4.0 | ✅ `parlament.items` | | – immunity waivers | prosecutors' requests to lift immunity: authority, offence, outcome | items + committee report | CC BY 4.0 | ✅ `parlament.items` |
| – written questions and answers | who asked which minister what | item list (J / AB) | CC BY 4.0 | ✅ `parlament.items` | | – written questions and answers | who asked which minister what | item list (J / AB) | CC BY 4.0 | ✅ `parlament.items` |
@@ -45,7 +46,7 @@ Status: ✅ implemented (adapter name in `code`) · 🟡 planned · ⛔ delibera
| Public procurement – data.gv.at "Vergabe-Kerndaten" | **contract awards**: buyer, winner, amount, CPV (eForms) | OGD hub catalogue → per-publisher XML | open data (CC BY 4.0) | ✅ `vergabe.awards` | | Public procurement – data.gv.at "Vergabe-Kerndaten" | **contract awards**: buyer, winner, amount, CPV (eForms) | OGD hub catalogue → per-publisher XML | open data (CC BY 4.0) | ✅ `vergabe.awards` |
| RTR / KommAustria Medientransparenz | **government advertising** and **media subsidies** per payer → medium | open-data API | free re-use, attribution | ✅ `rtr.medientransparenz` | | RTR / KommAustria Medientransparenz | **government advertising** and **media subsidies** per payer → medium | open-data API | free re-use, attribution | ✅ `rtr.medientransparenz` |
| European Parliament | Austrian MEPs, roles, **declarations of interests** | open-data/website XML | CC BY 4.0 | ✅ `europarl.meps` | | European Parliament | Austrian MEPs, roles, **declarations of interests** | open-data/website XML | CC BY 4.0 | ✅ `europarl.meps` |
| – plenary roll calls | **final votes** of Austrian MEPs (whole texts, no amendments), with totals and later corrections | Open Data API v2 (`/meetings/{id}/decisions`) | CC BY 4.0 | ✅ `europarl.votes` | | – plenary roll calls | **votes of Austrian MEPs** on whole texts and on amendments or parts (flagged `part_vote`), with totals and later corrections | Open Data API v2 (`/meetings/{id}/decisions`) | CC BY 4.0 | ✅ `europarl.votes` |
| European Commission – meetings | meetings of Commissioners/cabinets with **interest representatives** (Austrians and Austrian organisations) | XLSX export | EU re-use policy | ✅ `eu.commission_meetings` | | European Commission – meetings | meetings of Commissioners/cabinets with **interest representatives** (Austrians and Austrian organisations) | XLSX export | EU re-use policy | ✅ `eu.commission_meetings` |
| Wikipedia (de/en) | anonymous edits from **parliament/government IP ranges** (`config/ip_ranges.yaml`) | MediaWiki API `usercontribs` | CC BY-SA (edit metadata) | ✅ `wikipedia.institution_edits` | | Wikipedia (de/en) | anonymous edits from **parliament/government IP ranges** (`config/ip_ranges.yaml`) | MediaWiki API `usercontribs` | CC BY-SA (edit metadata) | ✅ `wikipedia.institution_edits` |
| Google Political Ads Transparency | Austrian political ads: advertiser, spend bracket, impressions | public bundle (zip, CSV) | Google's transparency report terms | ✅ `ads.google` | | Google Political Ads Transparency | Austrian political ads: advertiser, spend bracket, impressions | public bundle (zip, CSV) | Google's transparency report terms | ✅ `ads.google` |
@@ -56,7 +57,7 @@ Status: ✅ implemented (adapter name in `code`) · 🟡 planned · ⛔ delibera
| Municipalities | mayors, councils | fragmented | – | 🟡 via Wikidata | | Municipalities | mayors, councils | fragmented | – | 🟡 via Wikidata |
| EU Council participant lists | which minister attended which Council | PDF per meeting | EU re-use policy | 🟡 not implemented yet | | EU Council participant lists | which minister attended which Council | PDF per meeting | EU re-use policy | 🟡 not implemented yet |
| AMA farm subsidies, EU state-aid register (TAM) | subsidy recipients | search interfaces | – | 🟡 not implemented yet (bulk access and terms still to be checked) | | AMA farm subsidies, EU state-aid register (TAM) | subsidy recipients | search interfaces | – | 🟡 not implemented yet (bulk access and terms still to be checked) |
| Election results, candidate lists | results per municipality, candidates | BMI / data.gv.at, official gazettes | open data / official | 🟡 not implemented yet | | Election results, preference votes | party results for Austria and the states; candidates' preference votes (only candidates who resolve to a known politician) | BMI XLSX (Bundeswahlbehörde) | official works; CC BY 4.0 on data.gv.at | ✅ `bmi.elections` (NR 2024, NR 2019, EU 2024); 🟡 earlier elections, NR 2019 state lists, Landtag and municipal elections |
| Insolvency notices (Ediktsdatei) | companies tied to politicians | Justiz web application | terms to be checked before automating | 🟡 | | Insolvency notices (Ediktsdatei) | companies tied to politicians | Justiz web application | terms to be checked before automating | 🟡 |
| Firmenbuch, Grundbuch, beneficial owners (WiEReG) | company roles, property | paid clearing houses; WiEReG public access restricted since the 2022 CJEU ruling | – | ⛔ cost and legal limits | | Firmenbuch, Grundbuch, beneficial owners (WiEReG) | company roles, property | paid clearing houses; WiEReG public access restricted since the 2022 CJEU ruling | – | ⛔ cost and legal limits |
| Criminal record (*Strafregister*) | – | not public | – | ⛔ does not exist publicly | | Criminal record (*Strafregister*) | – | not public | – | ⛔ does not exist publicly |
+2
View File
@@ -16,6 +16,8 @@ ENV NODE_ENV=production NEXT_TELEMETRY_DISABLED=1 PORT=3000 HOSTNAME=0.0.0.0
COPY --from=build /app/.next/standalone ./ COPY --from=build /app/.next/standalone ./
COPY --from=build /app/.next/static ./.next/static COPY --from=build /app/.next/static ./.next/static
COPY --from=build /app/public ./public COPY --from=build /app/public ./public
# Next.js caches fetched data here at runtime (portrait proxy).
RUN mkdir -p .next/cache && chown -R node:node .next/cache
USER node USER node
EXPOSE 3000 EXPOSE 3000
CMD ["node", "server.js"] CMD ["node", "server.js"]
+45 -3
View File
@@ -16,7 +16,44 @@
"organizations": "Organisationen", "organizations": "Organisationen",
"money": "Geld", "money": "Geld",
"social": "Social Media", "social": "Social Media",
"wikipedia": "Wikipedia-Bearbeitungen" "wikipedia": "Wikipedia-Bearbeitungen",
"imprint": "Impressum",
"privacy": "Datenschutz"
},
"Imprint": {
"title": "Impressum",
"intro": "Offenlegung gemäß § 5 ECG und §§ 24, 25 MedienG.",
"notConfigured": "Die Angaben zum Medieninhaber sind noch nicht eingetragen. Vor einer Veröffentlichung müssen sie vollständig sein.",
"missing": "[noch nicht eingetragen]",
"ownerHeading": "Medieninhaber und Herausgeber",
"email": "E-Mail",
"purposeHeading": "Unternehmensgegenstand",
"purpose": "Keiner. Politrack ist ein privates, nicht gewinnorientiertes Projekt ohne Werbung.",
"directionHeading": "Grundlegende Richtung",
"direction": "Information über die öffentliche Tätigkeit österreichischer Politikerinnen, Politiker und Parteien auf Grundlage öffentlich zugänglicher Quellen, unabhängig von Parteien und Interessengruppen.",
"correctionsHeading": "Korrekturen und Gegendarstellung",
"corrections": "Eine Angabe ist falsch oder unvollständig? Schreib uns bitte die Nummer der Angabe und eine Quelle. Widersprochene Angaben markieren wir als umstritten, falsche korrigieren wir.",
"liabilityHeading": "Haftung",
"liability": "Alle Angaben stammen aus den jeweils verlinkten Quellen; maßgeblich sind diese. Für die Inhalte verlinkter Seiten sind deren Betreiber verantwortlich."
},
"Privacy": {
"title": "Datenschutz",
"intro": "Welche Daten Politrack verarbeitet und welche Rechte du hast.",
"draft": "Entwurf: Dieser Text wurde noch nicht rechtlich geprüft.",
"missing": "[noch nicht eingetragen]",
"controllerHeading": "Verantwortlicher",
"visitorsHeading": "Wenn du diese Website besuchst",
"visitors": "Politrack verwendet keine Werbung, kein Tracking und keine Skripte, Schriften oder Einbettungen von Dritten. Beim Abruf fallen technisch notwendige Zugriffsdaten an (IP-Adresse, Zeitpunkt, aufgerufene Seite), die nur so lange gespeichert werden, wie es für den sicheren Betrieb nötig ist.",
"cookiesHeading": "Cookies",
"cookies": "Wir speichern nur deine eigenen Einstellungen in Cookies: Quellen- und Themenfilter sowie das Farbschema. Sie dienen keiner Nachverfolgung.",
"imagesHeading": "Porträts",
"images": "Porträts laden wir über unseren eigenen Server, sodass deine IP-Adresse nicht an das Parlament oder andere Quellen übermittelt wird. Links zu externen Quellen öffnest du selbst; dort gelten die Bestimmungen der jeweiligen Betreiber.",
"subjectsHeading": "Daten über Politikerinnen und Politiker",
"subjects": "Politrack verarbeitet personenbezogene Daten ausschließlich zur öffentlichen Rolle von Personen: Funktionen, Abstimmungen, öffentliche Aussagen und Auftritte, gemeldete Nebentätigkeiten, Parteispenden und öffentlich bekannte Verfahren. Private Spender werden nur so genannt, wie es die amtliche Veröffentlichung vorsieht.",
"basisHeading": "Rechtsgrundlage",
"basis": "Berechtigtes Interesse und öffentliches Interesse an politischer Transparenz (Art. 6 Abs. 1 lit. f DSGVO) sowie die Freiheit der Meinungsäußerung und Information (Art. 85 DSGVO).",
"rightsHeading": "Deine Rechte",
"rights": "Du hast das Recht auf Auskunft, Berichtigung, Löschung, Einschränkung und Widerspruch. Wende dich dazu an die oben genannte Adresse. Außerdem kannst du dich bei der Österreichischen Datenschutzbehörde beschweren."
}, },
"Home": { "Home": {
"headline": "Politik in Österreich – nachvollziehbar.", "headline": "Politik in Österreich – nachvollziehbar.",
@@ -120,7 +157,9 @@
"travel": "Reisen & Treffen", "travel": "Reisen & Treffen",
"social": "Social Media" "social": "Social Media"
}, },
"factionVoteNote": "Im Nationalrat wird meist per Aufstehen abgestimmt; offiziell protokolliert ist dann nur das Stimmverhalten der Fraktion. Bei namentlichen Abstimmungen zeigen wir die eigene Stimme – und ob sie von der Fraktion abweicht.", "showPartVotes": "Auch Änderungsanträge und Teilabstimmungen anzeigen",
"hidePartVotes": "Nur Abstimmungen über ganze Texte anzeigen",
"factionVoteNote":"Im Nationalrat wird meist per Aufstehen abgestimmt; offiziell protokolliert ist dann nur das Stimmverhalten der Fraktion. Bei namentlichen Abstimmungen zeigen wir die eigene Stimme – und ob sie von der Fraktion abweicht.",
"roleCorroborated": "mehrfach belegt", "roleCorroborated": "mehrfach belegt",
"unverified": "laut Wikidata – nicht amtlich bestätigt", "unverified": "laut Wikidata – nicht amtlich bestätigt",
"accountUnverified": "Laut Wikidata – nicht amtlich bestätigt", "accountUnverified": "Laut Wikidata – nicht amtlich bestätigt",
@@ -186,6 +225,7 @@
}, },
"Votes": { "Votes": {
"title": "Abstimmungen", "title": "Abstimmungen",
"includePartVotes": "Auch Änderungsanträge und Teilabstimmungen (EU-Parlament)",
"accepted": "angenommen", "accepted": "angenommen",
"rejected": "abgelehnt", "rejected": "abgelehnt",
"for": "dafür", "for": "dafür",
@@ -226,6 +266,7 @@
"contradicts": "widerspricht", "contradicts": "widerspricht",
"none": "Keine Angaben mit deinen Quellen- und Themenfiltern.", "none": "Keine Angaben mit deinen Quellen- und Themenfiltern.",
"type": { "type": {
"election_result": "Wahlergebnis",
"role": "Funktion", "role": "Funktion",
"vote": "Abstimmung", "vote": "Abstimmung",
"donation": "Spende", "donation": "Spende",
@@ -336,7 +377,8 @@
"kind": "Art", "kind": "Art",
"notCollected": "Wir umgehen keine Bezahlschranken, Logins oder technischen Sperren und halten uns an robots.txt und Nutzungsbedingungen. Von Medien speichern wir nur Überschrift, Teaser und Link.", "notCollected": "Wir umgehen keine Bezahlschranken, Logins oder technischen Sperren und halten uns an robots.txt und Nutzungsbedingungen. Von Medien speichern wir nur Überschrift, Teaser und Link.",
"data": "Die Betreiber:innen dieser Instanz stellen die Daten auch maschinenlesbar bereit: als REST-API mit OpenAPI-Dokumentation, als MCP-Server für KI-Agenten und als Lesezugriff auf die Datenbank. Jede Angabe kommt dort mit denselben Belegen wie hier.", "data": "Die Betreiber:innen dieser Instanz stellen die Daten auch maschinenlesbar bereit: als REST-API mit OpenAPI-Dokumentation, als MCP-Server für KI-Agenten und als Lesezugriff auf die Datenbank. Jede Angabe kommt dort mit denselben Belegen wie hier.",
"corrections": "Fehler gefunden? Wir korrigieren falsche Angaben, markieren widersprochene als umstritten und löschen nichts stillschweigend. Bitte nenne die Nummer der Angabe (siehe Permalink) und eine Quelle für die Korrektur." "corrections": "Fehler gefunden? Wir korrigieren falsche Angaben, markieren widersprochene als umstritten und löschen nichts stillschweigend. Bitte nenne die Nummer der Angabe (siehe Permalink) und eine Quelle für die Korrektur.",
"correctionsContact": "Kontakt für Korrekturen"
}, },
"Common": { "Common": {
"loadError": "Die Daten konnten nicht geladen werden. Bitte versuche es gleich noch einmal.", "loadError": "Die Daten konnten nicht geladen werden. Bitte versuche es gleich noch einmal.",
+45 -3
View File
@@ -16,7 +16,9 @@
"organizations": "Organisations", "organizations": "Organisations",
"money": "Money", "money": "Money",
"social": "Social media", "social": "Social media",
"wikipedia": "Wikipedia edits" "wikipedia": "Wikipedia edits",
"imprint": "Legal notice",
"privacy": "Privacy"
}, },
"Home": { "Home": {
"headline": "Austrian politics – traceable.", "headline": "Austrian politics – traceable.",
@@ -120,7 +122,9 @@
"travel": "Travel & meetings", "travel": "Travel & meetings",
"social": "Social media" "social": "Social media"
}, },
"factionVoteNote": "The National Council mostly votes by standing up; then only each faction's vote is recorded. For roll-call votes we show the person's own vote – and whether it differs from their faction.", "showPartVotes": "Also show amendments and part votes",
"hidePartVotes": "Show votes on whole texts only",
"factionVoteNote":"The National Council mostly votes by standing up; then only each faction's vote is recorded. For roll-call votes we show the person's own vote – and whether it differs from their faction.",
"roleCorroborated": "corroborated", "roleCorroborated": "corroborated",
"unverified": "per Wikidata – not officially confirmed", "unverified": "per Wikidata – not officially confirmed",
"accountUnverified": "According to Wikidata – not officially confirmed", "accountUnverified": "According to Wikidata – not officially confirmed",
@@ -186,6 +190,7 @@
}, },
"Votes": { "Votes": {
"title": "Votes", "title": "Votes",
"includePartVotes": "Include amendments and part votes (European Parliament)",
"accepted": "accepted", "accepted": "accepted",
"rejected": "rejected", "rejected": "rejected",
"for": "for", "for": "for",
@@ -226,6 +231,7 @@
"contradicts": "contradicts", "contradicts": "contradicts",
"none": "No claims match your source and topic filters.", "none": "No claims match your source and topic filters.",
"type": { "type": {
"election_result": "Election result",
"role": "Office", "role": "Office",
"vote": "Vote", "vote": "Vote",
"donation": "Donation", "donation": "Donation",
@@ -336,7 +342,8 @@
"kind": "Kind", "kind": "Kind",
"notCollected": "We don't get around paywalls, logins or technical barriers, and we respect robots.txt and terms of use. From media we store only the headline, teaser and link.", "notCollected": "We don't get around paywalls, logins or technical barriers, and we respect robots.txt and terms of use. From media we store only the headline, teaser and link.",
"data": "The operators of this instance also provide the data in machine-readable form: a REST API with OpenAPI documentation, an MCP server for AI agents and read access to the database. Every fact comes with the same sources as here.", "data": "The operators of this instance also provide the data in machine-readable form: a REST API with OpenAPI documentation, an MCP server for AI agents and read access to the database. Every fact comes with the same sources as here.",
"corrections": "Found a mistake? We correct wrong facts, mark contradicted ones as disputed and never delete anything silently. Please quote the fact's number (see its permalink) and a source for the correction." "corrections": "Found a mistake? We correct wrong facts, mark contradicted ones as disputed and never delete anything silently. Please quote the fact's number (see its permalink) and a source for the correction.",
"correctionsContact": "Contact for corrections"
}, },
"Common": { "Common": {
"loadError": "The data could not be loaded. Please try again in a moment.", "loadError": "The data could not be loaded. Please try again in a moment.",
@@ -586,6 +593,41 @@
"about": "About Politrack" "about": "About Politrack"
} }
}, },
"Imprint": {
"title": "Legal notice",
"intro": "Disclosure under § 5 ECG and §§ 24, 25 MedienG (Austrian law).",
"notConfigured": "The publisher's details have not been entered yet. They must be complete before publication.",
"missing": "[not entered yet]",
"ownerHeading": "Media owner and publisher",
"email": "Email",
"purposeHeading": "Business purpose",
"purpose": "None. Politrack is a private, non-profit project without advertising.",
"directionHeading": "Editorial line",
"direction": "Information on the public activity of Austrian politicians and parties, based on publicly available sources and independent of parties and interest groups.",
"correctionsHeading": "Corrections and right of reply",
"corrections": "Is a fact wrong or incomplete? Please send us its number and a source. We mark contradicted facts as disputed and correct wrong ones.",
"liabilityHeading": "Liability",
"liability": "All facts come from the sources linked with them, which are authoritative. Linked sites are the responsibility of their operators."
},
"Privacy": {
"title": "Privacy",
"intro": "What data Politrack processes and what rights you have.",
"draft": "Draft: this text has not been legally reviewed yet.",
"missing": "[not entered yet]",
"controllerHeading": "Controller",
"visitorsHeading": "When you visit this website",
"visitors": "Politrack uses no advertising, no tracking and no third-party scripts, fonts or embeds. Serving a page produces technically necessary access data (IP address, time, page requested), kept only as long as needed to run the site securely.",
"cookiesHeading": "Cookies",
"cookies": "Cookies only store your own settings: source and topic filters and the colour scheme. They are not used for tracking.",
"imagesHeading": "Portraits",
"images": "Portraits are loaded through our own server, so your IP address is not passed to the parliament or other sources. External source links are yours to open; the operators' own terms apply there.",
"subjectsHeading": "Data about politicians",
"subjects": "Politrack processes personal data only about people's public role: offices, votes, public statements and appearances, declared side jobs, party donations and publicly known proceedings. Private donors are named only as the official publication names them.",
"basisHeading": "Legal basis",
"basis": "Legitimate and public interest in political transparency (Art. 6(1)(f) GDPR) and freedom of expression and information (Art. 85 GDPR).",
"rightsHeading": "Your rights",
"rights": "You have the right to access, rectification, erasure, restriction and objection. Please contact the address above. You can also complain to the Austrian Data Protection Authority."
},
"Footer": { "Footer": {
"tagline": "Austrian politics, sourced. An independent project that collects public information on politicians and parties – every fact with its source.", "tagline": "Austrian politics, sourced. An independent project that collects public information on politicians and parties – every fact with its source.",
"principleEvidence": "Every fact with its source", "principleEvidence": "Every fact with its source",
+72
View File
@@ -0,0 +1,72 @@
// Phone-width audit: emulate a 390x844 phone through the DevTools protocol (a plain headless window can't go
// below ~500 px and fakes overflow), then for each page report horizontal overflow, the elements causing it and
// small tap targets, and save a full-page screenshot. Needs Node 22+ and any Chromium browser (Chrome, Edge).
// Usage: node web/scripts/mobile-audit.mjs <browser executable> <outdir> <baseUrl> <path>...
// Example: node web/scripts/mobile-audit.mjs "/usr/bin/chromium" /tmp/audit http://localhost:3000 / /personen /geld
// From Git Bash, prefix MSYS_NO_PATHCONV=1, or it rewrites "/personen" into a Windows path.
import {spawn} from 'node:child_process';
import {mkdirSync, writeFileSync} from 'node:fs';
const [edge, outDir, base, ...paths] = process.argv.slice(2);
mkdirSync(outDir, {recursive: true});
const port = 9333;
const proc = spawn(edge, ['--headless=new', `--remote-debugging-port=${port}`, '--disable-gpu', '--no-first-run',
`--user-data-dir=${outDir}/profile`, 'about:blank'], {stdio: 'ignore'});
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
let target;
for (let i = 0; i < 40 && !target; i++) {
try { target = (await (await fetch(`http://127.0.0.1:${port}/json/list`)).json()).find((t) => t.type === 'page'); }
catch { await sleep(250); }
}
const ws = new WebSocket(target.webSocketDebuggerUrl);
await new Promise((r) => ws.addEventListener('open', r, {once: true}));
let seq = 0; const pending = new Map();
ws.addEventListener('message', (ev) => {
const msg = JSON.parse(ev.data);
if (msg.id && pending.has(msg.id)) { pending.get(msg.id)(msg); pending.delete(msg.id); }
});
const send = (method, params = {}) => new Promise((resolve) => {
const id = ++seq; pending.set(id, resolve); ws.send(JSON.stringify({id, method, params}));
});
const evaluate = async (expression) => (await send('Runtime.evaluate', {expression, returnByValue: true})).result?.result?.value;
await send('Page.enable');
await send('Emulation.setDeviceMetricsOverride', {width: 390, height: 844, deviceScaleFactor: 1, mobile: true});
await send('Emulation.setTouchEmulationEnabled', {enabled: true});
await send('Network.enable');
await send('Network.setUserAgentOverride', {userAgent: 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 Mobile/15E148'});
const report = [];
for (const path of paths) {
await send('Page.navigate', {url: base + path});
await sleep(4000);
const info = await evaluate(`(() => {
const vw = document.documentElement.clientWidth;
const offenders = [];
for (const el of document.querySelectorAll('body *')) {
const r = el.getBoundingClientRect();
if (r.width > 0 && r.right > vw + 1) {
const style = getComputedStyle(el);
let p = el.parentElement, clipped = false;
while (p) { const s = getComputedStyle(p); if (/(auto|scroll|hidden|clip)/.test(s.overflowX)) { clipped = true; break; } p = p.parentElement; }
if (!clipped) offenders.push({tag: el.tagName.toLowerCase(), cls: String(el.className).slice(0, 60), right: Math.round(r.right), text: (el.textContent || '').trim().slice(0, 50)});
}
}
// keep only the outermost offenders
return {vw, scrollWidth: document.documentElement.scrollWidth, height: document.documentElement.scrollHeight,
offenders: offenders.slice(0, 12), tapTargetsTooSmall: [...document.querySelectorAll('a,button,input,select,summary')]
.filter((e) => { const r = e.getBoundingClientRect(); return r.width > 0 && r.height > 0 && (r.height < 24 || r.width < 24); }).length};
})()`);
const name = path.replace(/[^a-z0-9]+/gi, '_').replace(/^_|_$/g, '') || 'home';
const height = Math.min(info?.height ?? 2000, 4000);
await send('Emulation.setDeviceMetricsOverride', {width: 390, height, deviceScaleFactor: 1, mobile: true});
await sleep(500);
const shot = await send('Page.captureScreenshot', {format: 'png'});
writeFileSync(`${outDir}/${name}.png`, Buffer.from(shot.result.data, 'base64'));
await send('Emulation.setDeviceMetricsOverride', {width: 390, height: 844, deviceScaleFactor: 1, mobile: true});
report.push({path, ...info});
}
console.log(JSON.stringify(report, null, 1));
ws.close(); proc.kill();
process.exit(0);
+7 -3
View File
@@ -10,7 +10,7 @@ const PAGE = 50;
type Props = { type Props = {
params: Promise<{locale: string}>; params: Promise<{locale: string}>;
searchParams: Promise<{party?: string; page?: string; roll_call?: string}>; searchParams: Promise<{party?: string; page?: string; roll_call?: string; parts?: string}>;
}; };
export async function generateMetadata({params}: Props): Promise<Metadata> { export async function generateMetadata({params}: Props): Promise<Metadata> {
@@ -25,16 +25,20 @@ export default async function VotesPage({params, searchParams}: Props) {
const sp = await searchParams; const sp = await searchParams;
const t = await getTranslations('Votes'); const t = await getTranslations('Votes');
const page = Math.max(1, Number(sp.page) || 1); const page = Math.max(1, Number(sp.page) || 1);
const votes = await api.votes({party: sp.party, roll_call: sp.roll_call ? 'true' : undefined, limit: PAGE, offset: (page - 1) * PAGE}); const votes = await api.votes({party: sp.party, roll_call: sp.roll_call ? 'true' : undefined,
include_parts: sp.parts ? 'true' : undefined, limit: PAGE, offset: (page - 1) * PAGE});
return ( return (
<> <>
<PageHeader title={t('title')} lede={t('intro')} meta={sp.party ? t('filteredParty', {party: sp.party}) : undefined} /> <PageHeader title={t('title')} lede={t('intro')} meta={sp.party ? t('filteredParty', {party: sp.party}) : undefined} />
<FilterBar id="votes" active={countActive(sp.roll_call)} resetHref={sp.party ? `/abstimmungen?party=${sp.party}` : '/abstimmungen'} <FilterBar id="votes" active={countActive(sp.roll_call, sp.parts)} resetHref={sp.party ? `/abstimmungen?party=${sp.party}` : '/abstimmungen'}
hidden={{party: sp.party}}> hidden={{party: sp.party}}>
<label className="field field--check"> <label className="field field--check">
<input type="checkbox" name="roll_call" value="1" defaultChecked={Boolean(sp.roll_call)} /> {t('onlyRollCall')} <input type="checkbox" name="roll_call" value="1" defaultChecked={Boolean(sp.roll_call)} /> {t('onlyRollCall')}
</label> </label>
<label className="field field--check">
<input type="checkbox" name="parts" value="1" defaultChecked={Boolean(sp.parts)} /> {t('includePartVotes')}
</label>
</FilterBar> </FilterBar>
<VoteLegend /> <VoteLegend />
<VoteList votes={votes} /> <VoteList votes={votes} />
+42
View File
@@ -0,0 +1,42 @@
import type {Metadata} from 'next';
import {getTranslations, setRequestLocale} from 'next-intl/server';
import {Notice} from '@/components/Notice';
import {PageHeader} from '@/components/PageHeader';
import {legalDetails} from '@/lib/legal';
export const dynamic = 'force-dynamic'; // the operator's details are read from the environment at request time
const SECTIONS = ['visitors', 'cookies', 'images', 'subjects', 'basis', 'rights'] as const;
export async function generateMetadata({params}: {params: Promise<{locale: string}>}): Promise<Metadata> {
const {locale} = await params;
const t = await getTranslations({locale, namespace: 'Privacy'});
return {title: t('title')};
}
export default async function PrivacyPage({params}: {params: Promise<{locale: string}>}) {
const {locale} = await params;
setRequestLocale(locale);
const t = await getTranslations('Privacy');
const {name, address, email} = legalDetails();
return (
<article>
<PageHeader title={t('title')} lede={t('intro')} />
<div className="prose">
<Notice tone="caution">{t('draft')}</Notice>
<section>
<h2>{t('controllerHeading')}</h2>
<p>{name ?? t('missing')}<br />{address ?? t('missing')}</p>
<p>{email ? <a href={`mailto:${email}`}>{email}</a> : t('missing')}</p>
</section>
{SECTIONS.map((key) => (
<section key={key}>
<h2>{t(`${key}Heading`)}</h2>
<p>{t(key)}</p>
</section>
))}
</div>
</article>
);
}
+52
View File
@@ -0,0 +1,52 @@
import type {Metadata} from 'next';
import {getTranslations, setRequestLocale} from 'next-intl/server';
import {Notice} from '@/components/Notice';
import {PageHeader} from '@/components/PageHeader';
import {legalDetails} from '@/lib/legal';
export const dynamic = 'force-dynamic'; // the operator's details are read from the environment at request time
export async function generateMetadata({params}: {params: Promise<{locale: string}>}): Promise<Metadata> {
const {locale} = await params;
const t = await getTranslations({locale, namespace: 'Imprint'});
return {title: t('title')};
}
export default async function ImprintPage({params}: {params: Promise<{locale: string}>}) {
const {locale} = await params;
setRequestLocale(locale);
const t = await getTranslations('Imprint');
const {name, address, email, correctionsEmail} = legalDetails();
const missing = t('missing');
return (
<article>
<PageHeader title={t('title')} lede={t('intro')} />
<div className="prose">
{(!name || !address || !email) && <Notice tone="caution">{t('notConfigured')}</Notice>}
<section>
<h2>{t('ownerHeading')}</h2>
<p>{name ?? missing}<br />{address ?? missing}</p>
<p>{t('email')}: {email ? <a href={`mailto:${email}`}>{email}</a> : missing}</p>
</section>
<section>
<h2>{t('purposeHeading')}</h2>
<p>{t('purpose')}</p>
</section>
<section>
<h2>{t('directionHeading')}</h2>
<p>{t('direction')}</p>
</section>
<section id="korrekturen">
<h2>{t('correctionsHeading')}</h2>
<p>{t('corrections')}</p>
<p>{correctionsEmail ? <a href={`mailto:${correctionsEmail}`}>{correctionsEmail}</a> : missing}</p>
</section>
<section>
<h2>{t('liabilityHeading')}</h2>
<p>{t('liability')}</p>
</section>
</div>
</article>
);
}
+2 -1
View File
@@ -5,6 +5,7 @@ import {SourceMeter, StatusBadge} from '@/components/badges';
import {ExternalLink} from '@/components/ExternalLink'; import {ExternalLink} from '@/components/ExternalLink';
import {Notice} from '@/components/Notice'; import {Notice} from '@/components/Notice';
import {PageHeader} from '@/components/PageHeader'; import {PageHeader} from '@/components/PageHeader';
import {Link} from '@/i18n/navigation';
import {api, type LegalStage} from '@/lib/api'; import {api, type LegalStage} from '@/lib/api';
const LEGAL_STAGES: LegalStage[] = ['allegation_reported', 'investigation', 'immunity_waiver_requested', 'immunity_waived', const LEGAL_STAGES: LegalStage[] = ['allegation_reported', 'investigation', 'immunity_waiver_requested', 'immunity_waived',
@@ -125,7 +126,7 @@ export default async function MethodPage({params}: {params: Promise<{locale: str
</section> </section>
<section id={ANCHORS.corrections}> <section id={ANCHORS.corrections}>
<h2>{t('section.corrections')}</h2> <h2>{t('section.corrections')}</h2>
<p>{t('corrections')}</p> <p>{t('corrections')} <Link href="/impressum#korrekturen">{t('correctionsContact')}</Link></p>
</section> </section>
</div> </div>
</article> </article>
@@ -42,7 +42,7 @@ export default async function PartyPage({params}: Props) {
const party = await loadParty(slug); const party = await loadParty(slug);
const [votes, claims, posts] = await Promise.all([ const [votes, claims, posts] = await Promise.all([
api.votes({party: slug, limit: 10}), api.votes({party: slug, limit: 10}),
api.claims({party: slug, types: ['donation', 'media_mention', 'legal_proceeding', 'money_flow', 'lobbying'], limit: 20}), api.claims({party: slug, types: ['donation', 'media_mention', 'legal_proceeding', 'money_flow', 'lobbying', 'election_result'], limit: 20}),
api.social({party: slug, limit: 6}) api.social({party: slug, limit: 6})
]); ]);
@@ -23,7 +23,7 @@ import {isCurrent, roleLabel} from '@/lib/roles';
// Claim types per tab (also used for the tab counts); tabs with their own component are listed in CUSTOM_TABS. // Claim types per tab (also used for the tab counts); tabs with their own component are listed in CUSTOM_TABS.
const TABS = { const TABS = {
overview: ['statement', 'vote', 'donation', 'side_job', 'media_mention', 'meeting', 'travel', 'legal_proceeding', overview: ['statement', 'vote', 'donation', 'side_job', 'media_mention', 'meeting', 'travel', 'legal_proceeding',
'career', 'government_decision', 'declaration'], 'career', 'government_decision', 'declaration', 'election_result'],
timeline: [], timeline: [],
votes: ['vote'], votes: ['vote'],
travel: ['travel', 'meeting', 'appearance', 'government_decision'], travel: ['travel', 'meeting', 'appearance', 'government_decision'],
@@ -48,7 +48,7 @@ const GLANCE: [string, ClaimType[], Tab][] = [
type Props = { type Props = {
params: Promise<{locale: string; slug: string}>; params: Promise<{locale: string; slug: string}>;
searchParams: Promise<{tab?: string; since?: string; until?: string}>; searchParams: Promise<{tab?: string; since?: string; until?: string; parts?: string}>;
}; };
async function loadPerson(slug: string): Promise<Person> { async function loadPerson(slug: string): Promise<Person> {
@@ -70,7 +70,7 @@ export async function generateMetadata({params}: Props): Promise<Metadata> {
export default async function PersonPage({params, searchParams}: Props) { export default async function PersonPage({params, searchParams}: Props) {
const {locale, slug} = await params; const {locale, slug} = await params;
setRequestLocale(locale); setRequestLocale(locale);
const {tab: rawTab, since, until} = await searchParams; const {tab: rawTab, since, until, parts} = await searchParams;
const tab: Tab = rawTab && rawTab in TABS ? (rawTab as Tab) : 'overview'; const tab: Tab = rawTab && rawTab in TABS ? (rawTab as Tab) : 'overview';
const t = await getTranslations('Person'); const t = await getTranslations('Person');
const tp = await getTranslations('Persons'); const tp = await getTranslations('Persons');
@@ -217,7 +217,10 @@ export default async function PersonPage({params, searchParams}: Props) {
{tab === 'votes' && ( {tab === 'votes' && (
<section> <section>
<Notice tone="info">{t('factionVoteNote')}</Notice> <Notice tone="info">{t('factionVoteNote')}</Notice>
<VoteList votes={await api.personVotes(slug)} /> <p className="small">
<Link href={parts ? href('votes') : `${href('votes')}&parts=1`}>{t(parts ? 'hidePartVotes' : 'showPartVotes')}</Link>
</p>
<VoteList votes={await api.personVotes(slug, Boolean(parts))} />
</section> </section>
)} )}
+47
View File
@@ -0,0 +1,47 @@
// Image proxy for portraits, so that visitors' IP addresses don't reach the publishers.
// Only raster images from the hosts below: anything else would turn this into an open proxy.
import type {NextRequest} from 'next/server';
const ALLOWED_HOSTS = new Set(['www.parlament.gv.at', 'www.europarl.europa.eu']);
const IMAGE_TYPES = new Set(['image/jpeg', 'image/png', 'image/webp', 'image/gif']);
const MAX_BYTES = 5 * 1024 * 1024;
const MAX_REDIRECTS = 3;
const WEEK = 7 * 24 * 60 * 60;
function allowedUrl(raw: string | null): URL | null {
try {
const url = new URL(raw ?? '');
return url.protocol === 'https:' && ALLOWED_HOSTS.has(url.hostname) ? url : null;
} catch {
return null;
}
}
export async function GET(request: NextRequest): Promise<Response> {
let url = allowedUrl(request.nextUrl.searchParams.get('u'));
if (!url) return new Response(null, {status: 400});
const headers = {'User-Agent': process.env.POLITRACK_USER_AGENT ?? 'PolitrackBot/0.1'};
for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
let upstream: Response;
try {
upstream = await fetch(url, {headers, redirect: 'manual', signal: AbortSignal.timeout(10_000), next: {revalidate: WEEK}});
} catch {
return new Response(null, {status: 502});
}
if (upstream.status >= 300 && upstream.status < 400) {
url = allowedUrl(new URL(upstream.headers.get('location') ?? '', url).href);
if (!url) return new Response(null, {status: 502});
continue;
}
if (!upstream.ok) return new Response(null, {status: 404});
const type = (upstream.headers.get('content-type') ?? '').split(';')[0].trim().toLowerCase();
if (!IMAGE_TYPES.has(type)) return new Response(null, {status: 415});
const body = await upstream.arrayBuffer();
if (body.byteLength > MAX_BYTES) return new Response(null, {status: 413});
return new Response(body, {
headers: {'Content-Type': type, 'Cache-Control': `public, max-age=${WEEK}`, 'X-Content-Type-Options': 'nosniff'}
});
}
return new Response(null, {status: 502});
}
+3 -3
View File
@@ -1,13 +1,13 @@
import {initials} from '@/lib/format'; import {initials} from '@/lib/format';
/** /**
* Portrait or initials. Portraits come from Wikimedia/parliament: loaded lazily and without a referrer. * Portrait or initials. Portraits go through our own image proxy (app/api/img), so that visitors' IP addresses
* Decorative (alt=""), because the name is always printed next to it. * don't reach third parties. Decorative (alt=""), because the name is always printed next to it.
*/ */
export function Avatar({name, src, size = 'md'}: {name: string; src?: string | null; size?: 'md' | 'lg'}) { export function Avatar({name, src, size = 'md'}: {name: string; src?: string | null; size?: 'md' | 'lg'}) {
const className = size === 'lg' ? 'avatar avatar--lg' : 'avatar'; const className = size === 'lg' ? 'avatar avatar--lg' : 'avatar';
if (src) { if (src) {
return <img src={src} alt="" className={className} loading="lazy" decoding="async" referrerPolicy="no-referrer" />; return <img src={`/api/img?u=${encodeURIComponent(src)}`} alt="" className={className} loading="lazy" decoding="async" />;
} }
return <span className={className} aria-hidden="true">{initials(name)}</span>; return <span className={className} aria-hidden="true">{initials(name)}</span>;
} }
+2 -1
View File
@@ -26,7 +26,8 @@ const TYPE_ICONS: Partial<Record<ClaimType, IconName>> = {
wiki_edit: 'pencil', wiki_edit: 'pencil',
declaration: 'document', declaration: 'document',
ownership: 'link', ownership: 'link',
affiliation: 'link' affiliation: 'link',
election_result: 'vote'
}; };
export async function TypeLabel({type}: {type: ClaimType}) { export async function TypeLabel({type}: {type: ClaimType}) {
+4 -2
View File
@@ -16,7 +16,9 @@ const ITEMS = {
former: {href: '/ehemalige', key: 'former', icon: 'history'}, former: {href: '/ehemalige', key: 'former', icon: 'history'},
method: {href: '/methodik', key: 'method', icon: 'book'}, method: {href: '/methodik', key: 'method', icon: 'book'},
settings: {href: '/einstellungen', key: 'settings', icon: 'sliders'}, settings: {href: '/einstellungen', key: 'settings', icon: 'sliders'},
search: {href: '/suche', key: 'search', icon: 'search'} search: {href: '/suche', key: 'search', icon: 'search'},
imprint: {href: '/impressum', key: 'imprint', icon: 'document'},
privacy: {href: '/datenschutz', key: 'privacy', icon: 'shield'}
} satisfies Record<string, NavItem>; } satisfies Record<string, NavItem>;
export const PRIMARY_NAV: NavItem[] = [ITEMS.persons, ITEMS.parties, ITEMS.votes, ITEMS.money, ITEMS.organizations, ITEMS.calendar]; export const PRIMARY_NAV: NavItem[] = [ITEMS.persons, ITEMS.parties, ITEMS.votes, ITEMS.money, ITEMS.organizations, ITEMS.calendar];
@@ -26,5 +28,5 @@ export const NAV_GROUPS: {key: string; items: NavItem[]}[] = [
{key: 'people', items: [ITEMS.persons, ITEMS.parties, ITEMS.former]}, {key: 'people', items: [ITEMS.persons, ITEMS.parties, ITEMS.former]},
{key: 'decisions', items: [ITEMS.votes, ITEMS.calendar, ITEMS.social]}, {key: 'decisions', items: [ITEMS.votes, ITEMS.calendar, ITEMS.social]},
{key: 'influence', items: [ITEMS.money, ITEMS.organizations, ITEMS.wikipedia]}, {key: 'influence', items: [ITEMS.money, ITEMS.organizations, ITEMS.wikipedia]},
{key: 'about', items: [ITEMS.method, ITEMS.settings, ITEMS.search]} {key: 'about', items: [ITEMS.method, ITEMS.settings, ITEMS.search, ITEMS.imprint, ITEMS.privacy]}
]; ];
+1 -1
View File
@@ -22,7 +22,7 @@ export type CalendarEvent = {
export type Claim = { export type Claim = {
id: number; id: number;
claim_type: 'role' | 'vote' | 'donation' | 'money_flow' | 'statement' | 'appearance' | 'meeting' | 'travel' | 'side_job' | 'legal_proceeding' | 'lobbying' | 'media_mention' | 'career' | 'parliamentary_question' | 'government_decision' | 'wiki_edit' | 'declaration' | 'ownership' | 'affiliation' | 'party_finance' | 'other'; claim_type: 'role' | 'vote' | 'donation' | 'money_flow' | 'statement' | 'appearance' | 'meeting' | 'travel' | 'side_job' | 'legal_proceeding' | 'lobbying' | 'media_mention' | 'career' | 'parliamentary_question' | 'government_decision' | 'wiki_edit' | 'declaration' | 'ownership' | 'affiliation' | 'party_finance' | 'election_result' | 'other';
summary_de: string; summary_de: string;
summary_en: string | null; summary_en: string | null;
occurred_at: string | null; occurred_at: string | null;
+2 -2
View File
@@ -74,9 +74,9 @@ export const api = {
persons: (params: Params) => get<T.PersonListItem[]>('/api/v1/persons', params), persons: (params: Params) => get<T.PersonListItem[]>('/api/v1/persons', params),
person: (slug: string) => get<T.PersonDetail>(`/api/v1/persons/${enc(slug)}`), person: (slug: string) => get<T.PersonDetail>(`/api/v1/persons/${enc(slug)}`),
personNetwork: (slug: string) => get<T.Network>(`/api/v1/persons/${enc(slug)}/network`), personNetwork: (slug: string) => get<T.Network>(`/api/v1/persons/${enc(slug)}/network`),
personVotes: async (slug: string) => { personVotes: async (slug: string, includeParts = false) => {
const prefs = await getPrefs(); const prefs = await getPrefs();
return get<T.PersonVote[]>(`/api/v1/persons/${enc(slug)}/votes`, {topics: prefs.topics}); return get<T.PersonVote[]>(`/api/v1/persons/${enc(slug)}/votes`, {topics: prefs.topics, include_parts: includeParts || undefined});
}, },
former: (params: Params) => get<T.FormerPerson[]>('/api/v1/former', params), former: (params: Params) => get<T.FormerPerson[]>('/api/v1/former', params),
+11
View File
@@ -0,0 +1,11 @@
// The operator's details for the legal pages. They come from the environment (.env on the server), so the
// operator's name and address stay out of this public repository. Read at request time, not at build time.
export function legalDetails() {
const email = process.env.POLITRACK_LEGAL_EMAIL || null;
return {
name: process.env.POLITRACK_LEGAL_NAME || null,
address: process.env.POLITRACK_LEGAL_ADDRESS || null,
email,
correctionsEmail: process.env.POLITRACK_CORRECTIONS_EMAIL || email
};
}