diff --git a/backend/politrack/sources/landtage.py b/backend/politrack/sources/landtage.py index 95c5cc9..a96fd87 100644 --- a/backend/politrack/sources/landtage.py +++ b/backend/politrack/sources/landtage.py @@ -12,9 +12,11 @@ which states are covered, and the person_status view then trusts the official li vbg vorarlberg.at/web/landtag/abgeordnete-alle sbg salzburg.gv.at, one page per Landtag club bgld bgld-landtag.at/organe/abgeordnete (club sections; the postal addresses printed there are not kept) - wien wien.gv.at politician search (allowed in robots.txt), Landtag = Gemeinderat -Kärnten is missing on purpose: ktn.gv.at disallows all crawlers in robots.txt, so its members stay Wikidata-only. +Kärnten and Wien are missing on purpose, so their members stay Wikidata-only: ktn.gv.at disallows all crawlers in +robots.txt, and wien.gv.at disallows /advuew/ (the politician search) for every crawler except its own search +appliance. Every request sets skip_unchanged=False: enrich.landtage judges coverage by which role claims a run +touched, so an unchanged page must still be parsed and loaded. Only names, party, district/region and mandate dates are kept (public role); no contact details or addresses. """ @@ -24,7 +26,7 @@ import re from collections.abc import Iterator from datetime import date from typing import Any -from urllib.parse import parse_qs, urljoin, urlsplit +from urllib.parse import urljoin from bs4 import BeautifulSoup @@ -44,9 +46,6 @@ LANDTAGE = { "vbg": ("Vorarlberg", "Vorarlberger Landtag", 36, "https://vorarlberg.at/web/landtag/abgeordnete-alle"), "sbg": ("Salzburg", "Salzburger Landtag", 36, "https://www.salzburg.gv.at/pol/landtag/lt-klubs"), "bgld": ("Burgenland", "Burgenländischer Landtag", 36, "https://www.bgld-landtag.at/organe/abgeordnete/"), - "wien": ("Wien", "Wiener Landtag", 100, - "https://www.wien.gv.at/advuew/internet/AdvPrSrv.asp?Layout=polstelle_at&Type=K" - "&stellecd=1997031808394662&Hlayout=politikersuche&AUSSEN"), } SBG_CLUBS = {"lt-oevp": "ÖVP", "lt-fpoe": "FPÖ", "lt-spoe": "SPÖ", "lt-kpoeplus": "KPÖ", "lt-diegruenen": "GRÜNE"} PARTY_ALIASES = {"KEIN KLUB": None, "VP TIROL": "ÖVP", "VP": "ÖVP", "VOLKSPARTEI": "ÖVP", "GRÜNE": "GRÜNE", "DIE GRÜNEN": "GRÜNE", @@ -218,18 +217,6 @@ def parse_bgld(html: str, base: str) -> list[dict[str, Any]]: return out -def parse_wien(body: bytes, base: str) -> list[dict[str, Any]]: - soup = BeautifulSoup(body, "html.parser") # the page declares its (legacy) charset itself - out = [] - for a in soup.find_all("a", href=re.compile(r"Layout=politiker_at")): - cell = a.find_parent("td") - after = [t for t in (cell.stripped_strings if cell else []) if t not in a.stripped_strings] - party = after[-1].split()[-1] if after else None # "SPÖ" or "Klubvorsitzende NEOS" - person = parse_qs(urlsplit(a["href"]).query).get("personcd", [None])[0] - out.append(member(surname_first(a.get_text(" ", strip=True)), person, party, url=urljoin(base, a["href"]))) - return out - - # -- adapters ------------------------------------------------------------------------------------------------- def landtag_spec(key: str) -> SourceSpec: @@ -248,7 +235,7 @@ class LandtagMembers(Adapter): creates_persons = True def start(self, ctx: RunContext) -> Iterator[Request]: - yield Request(url=LANDTAGE[self.key][3], kind="list") + yield Request(url=LANDTAGE[self.key][3], kind="list", skip_unchanged=False) def members(self, ctx: RunContext, response: Response) -> Iterator[Any]: raise NotImplementedError @@ -294,9 +281,9 @@ def _noe(self, ctx, response): profiles, pages = parse_noe_list(response.text, response.url) if not response.meta.get("page"): for page in pages: - yield Request(url=page, kind="list", meta={"page": True}) + yield Request(url=page, kind="list", meta={"page": True}, skip_unchanged=False) for url in profiles: - yield Request(url=url, kind="profile") + yield Request(url=url, kind="profile", skip_unchanged=False) else: yield parse_noe_profile(response.text, response.request.url) @@ -305,7 +292,7 @@ def _sbg(self, ctx, response): if response.request.kind == "list": for club in SBG_CLUBS: yield Request(url=f"https://www.salzburg.gv.at/pol/landtag/lt-klubs/{club}", kind="club", - meta={"party": SBG_CLUBS[club]}) + meta={"party": SBG_CLUBS[club]}, skip_unchanged=False) else: yield from parse_sbg_club(response.text, response.url, response.meta["party"]) @@ -317,7 +304,6 @@ _register("tirol", lambda self, ctx, r: iter(parse_tirol(r.text, r.url))) _register("vbg", lambda self, ctx, r: iter(parse_vbg(r.text, r.url))) _register("sbg", _sbg) _register("bgld", lambda self, ctx, r: iter(parse_bgld(r.text, r.url))) -_register("wien", lambda self, ctx, r: iter(parse_wien(r.body, r.url))) @register diff --git a/backend/tests/test_landtage.py b/backend/tests/test_landtage.py index b4da95d..d1c3b65 100644 --- a/backend/tests/test_landtage.py +++ b/backend/tests/test_landtage.py @@ -48,7 +48,7 @@ def test_stmk_tirol_vbg(): ("Manuela Auer", "auer-manuela", "SPÖ")] -def test_sbg_bgld_wien(): +def test_sbg_bgld(): sbg = """
""" assert [(m["name"], m["id"], m["party"]) for m in L.parse_sbg_club(sbg, "https://x", "ÖVP")] == [ @@ -60,9 +60,9 @@ def test_sbg_bgld_wien(): assert [(m["name"], m["party"]) for m in members] == [ ("Gerhard Bachmann", "SPÖ"), ("Astrid Eisenkopf", "SPÖ"), ("Claudia Schlager", "SPÖ")] assert all(m["district"] is None for m in members) # postal addresses are never kept - wien = """|
- Abrahamczik Nina, Maga SPÖ | +43 1 4000 81925 |
| Arapovic Selma, Dipl.-Ing in
- Klubvorsitzende NEOS | +43 1 |