Politrack: track Austrian politicians from public sources
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
583ccba31c
187 files changed
+39890
No files matched your search
@@ -0,0 +1,144 @@
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
|
||||
from politrack.enrich.embed import split_text
|
||||
from politrack.enrich.linker import detect_context, tokens
|
||||
from politrack.enrich.topics import keyword_pattern
|
||||
from politrack.sources.parlament.common import parse_date, parse_vote_text
|
||||
from politrack.sources.rechnungshof import (
|
||||
clean_name,
|
||||
decode,
|
||||
find_columns,
|
||||
parse_amount,
|
||||
parse_de_date,
|
||||
read_rows,
|
||||
)
|
||||
from politrack.sources.wikidata import classify_level
|
||||
from politrack.text import fingerprint, normalize, strip_titles
|
||||
|
||||
|
||||
def test_vote_text_simple():
|
||||
v = parse_vote_text("137. Sitzung: Gesetzesvorschlag in dritter Lesung angenommen Dafür: ÖVP, SPÖ, GRÜNE, "
|
||||
"dagegen: FPÖ, NEOS")
|
||||
assert v["result"] == "accepted"
|
||||
assert v["for"] == ["ÖVP", "SPÖ", "GRÜNE"]
|
||||
assert v["against"] == ["FPÖ", "NEOS"]
|
||||
assert v["stage"] == "137. Sitzung: Gesetzesvorschlag in dritter Lesung"
|
||||
|
||||
|
||||
def test_vote_text_rejected_motion():
|
||||
v = parse_vote_text("50. Sitzung: Entschließungsantrag abgelehnt Dafür: FPÖ, dagegen: ÖVP, SPÖ, NEOS, GRÜNE")
|
||||
assert v["result"] == "rejected"
|
||||
assert v["for"] == ["FPÖ"]
|
||||
|
||||
|
||||
def test_vote_text_split_vote_is_not_attributed():
|
||||
v = parse_vote_text("12. Sitzung: angenommen Dafür: V, S, N (teilweise G), dagegen: F")
|
||||
assert v["split"] is True
|
||||
assert v["for"] == [] and v["against"] == []
|
||||
|
||||
|
||||
def test_vote_text_empty_against():
|
||||
v = parse_vote_text("997. Sitzung: Antrag, keinen Einspruch zu erheben, angenommen Dafür: ÖVP, SPÖ, dagegen: -")
|
||||
assert v["against"] == []
|
||||
assert v["stage"].endswith("zu erheben")
|
||||
|
||||
|
||||
def test_vote_text_without_result():
|
||||
assert parse_vote_text("Zuweisung an den Ausschuss") is None
|
||||
|
||||
|
||||
def test_parse_date_formats():
|
||||
assert parse_date("16.12.2021") == date(2021, 12, 16)
|
||||
assert parse_date("2026-10-14T12:00:00") == date(2026, 10, 14)
|
||||
assert parse_date("") is None
|
||||
|
||||
|
||||
def test_rechnungshof_header_after_title_row():
|
||||
rows = [["Zu veröffentlichen", "", ""],
|
||||
["Partei", "Meldedatum_an_RH", "Spendeneingangsdatum", "Name_der_Spenderin_des_Spenders",
|
||||
"PLZ_der_Spenderin_des_Spenders", "Betrag", "Waehrung", "Empfaengerin_Empfaenger_der_Spende", "Anmerkungen"]]
|
||||
start, idx = find_columns(rows)
|
||||
assert start == 2
|
||||
assert idx["date"] == 2 # Spendeneingangsdatum wins over Meldedatum
|
||||
assert idx["donor"] == 3
|
||||
assert idx["recipient"] == 7
|
||||
assert 4 not in idx.values() # postcode is never mapped
|
||||
|
||||
|
||||
def test_rechnungshof_headerless_file_skips_address():
|
||||
text = "08.07.19;300.000,00;Dr. Hans Peter Haselsteiner;1220 Wien Donau-City-Straße 9;NEOS\n"
|
||||
idx, rows = read_rows(text)
|
||||
assert idx["date"] == 0 and idx["amount"] == 1 and idx["donor"] == 2 and idx["party"] == 4
|
||||
assert 3 not in idx.values()
|
||||
assert rows[0][2] == "Dr. Hans Peter Haselsteiner"
|
||||
|
||||
|
||||
def test_rechnungshof_clean_name_drops_addresses():
|
||||
assert clean_name("2522 Oberwaltersdorf Hofeldstraße 4;Team Stronach für Österreich") == "Team Stronach für Österreich"
|
||||
assert clean_name("1220 Wien Donau-City-Straße 9") == ""
|
||||
assert clean_name("NEOS – Das Neue Österreich") == "NEOS – Das Neue Österreich"
|
||||
|
||||
|
||||
def test_rechnungshof_decoding():
|
||||
assert decode("Österreich".encode("cp1252")) == "Österreich"
|
||||
assert decode("Österreich".encode("mac_roman")) == "Österreich"
|
||||
assert decode("Österreich".encode("utf-8")) == "Österreich"
|
||||
|
||||
|
||||
def test_amount_and_date():
|
||||
assert parse_amount("1.971.000,00") == Decimal("1971000.00")
|
||||
assert parse_amount("790") == Decimal("790")
|
||||
assert parse_amount("") is None
|
||||
assert parse_de_date("26.08.24") == date(2024, 8, 26)
|
||||
|
||||
|
||||
def test_text_helpers():
|
||||
assert normalize(" Österreichische Volkspartei ") == "osterreichische volkspartei"
|
||||
assert strip_titles("Mag. Dr. Selma Yildirim, BA") == "Selma Yildirim"
|
||||
assert fingerprint("a", 1) == fingerprint("a", 1) != fingerprint("a", 2)
|
||||
|
||||
|
||||
def test_linker_helpers():
|
||||
assert tokens("Karl Nehammer (ÖVP) im ZIB 2-Interview") == ["karl", "nehammer", "ovp", "im", "zib", "2-interview"]
|
||||
assert detect_context("Babler in der ZIB 2") == "tv"
|
||||
assert detect_context("Gastkommentar: Warum wir mehr brauchen") == "op_ed"
|
||||
assert detect_context("Budget beschlossen") is None
|
||||
|
||||
|
||||
def test_keyword_pattern_short_words_are_whole_words():
|
||||
pattern = keyword_pattern(["EU", "Klima"])
|
||||
assert pattern.search("Die EU beschließt")
|
||||
assert not pattern.search("Euro-Rettung")
|
||||
assert pattern.search("Klimaschutzgesetz")
|
||||
|
||||
|
||||
def test_classify_level():
|
||||
assert classify_level("Mitglied des Tiroler Landtages") == "state"
|
||||
assert classify_level("Bürgermeister von Graz") == "municipal"
|
||||
assert classify_level("Abgeordneter zum Nationalrat") == "federal"
|
||||
assert classify_level("Mitglied des Europäischen Parlaments") == "eu"
|
||||
|
||||
|
||||
def test_split_text():
|
||||
chunks = split_text("\n".join(["x" * 500] * 6), size=1200)
|
||||
assert all(len(c) <= 1800 for c in chunks)
|
||||
assert sum(len(c.replace("\n", "")) for c in chunks) == 3000
|
||||
|
||||
|
||||
def test_employer_without_address():
|
||||
from politrack.sources.parlament.persons import employer
|
||||
|
||||
assert employer("Hypo Tirol Bank AG, Meraner Straße 8, 6020 Innsbruck") == ("Hypo Tirol Bank AG", None)
|
||||
# parliament's JSON glues name, address and register number together
|
||||
assert employer("25 PersonaldienstleistungsgesmbHFranz Josef Kai 95020 Salzburg FN306527a") == (
|
||||
"25 PersonaldienstleistungsgesmbH", "306527a")
|
||||
assert employer("Salzburger Parkgaragen Gesellschaft m.b.H") == ("Salzburger Parkgaragen Gesellschaft m.b.H", None)
|
||||
assert employer("Verein Musterhilfe Hauptstraße 12 1010 Wien")[0] == "Verein Musterhilfe"
|
||||
|
||||
|
||||
def test_firmenbuch_number():
|
||||
from politrack.text import firmenbuch_number
|
||||
|
||||
assert firmenbuch_number("FN 306527 a") == firmenbuch_number("FN306527A") == firmenbuch_number("306527a") == "306527a"
|
||||
assert firmenbuch_number("ATU12345678") is None
|
||||
Reference in new issue
Block a user