diff --git a/backend/politrack/sources/eu.py b/backend/politrack/sources/eu.py index 6589c5b..a56ff5d 100644 --- a/backend/politrack/sources/eu.py +++ b/backend/politrack/sources/eu.py @@ -129,6 +129,9 @@ class EuropeanParliament(Adapter): params={"country-of-representation": "AT", "format": "application/ld+json", "limit": 100}) def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: + if not response.body.strip(): # the API answers some MEPs' declaration requests with an empty body + ctx.stats["empty_responses"] += 1 + return data = response.json().get("data") or [] if response.request.kind == "meps": for mep in data: @@ -243,6 +246,9 @@ class ParliamentVotes(Adapter): meta={"ep_ids": ep_ids}) def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: + if not response.body.strip(): # sittings without recorded votes come back with an empty body + ctx.stats["empty_responses"] += 1 + return kind = response.request.kind data = response.json().get("data") or [] meta = response.meta diff --git a/backend/politrack/sources/parlament/persons.py b/backend/politrack/sources/parlament/persons.py index 4b8e9b8..e8f3efa 100644 --- a/backend/politrack/sources/parlament/persons.py +++ b/backend/politrack/sources/parlament/persons.py @@ -41,6 +41,7 @@ LEGAL_FORM_END = re.compile(r"(?i:gesmbh|gmbh|ges\.\s?m\.\s?b\.\s?h\.?|m\.\s?b\. r"e\.\s?u\.|\bse|privatstiftung|genossenschaft|sparkasse)(?=[A-ZÄÖÜ\s,]|$)") +COMMITTEE_END = re.compile(r"\(\d{2}\.\d{2}\.\d{4}\s*-\s*(\d{2}\.\d{2}\.\d{4})\s*\)\s*$") STREET_WORD = r"\S*(?:straße|strasse|str\.|gasse|platz|weg|kai|ring|allee|zeile|lände|markt|steig)" @@ -79,7 +80,7 @@ class ParlamentPersons(Adapter): description = "Members of Nationalrat and Bundesrat since 1918, mandates, functions, side jobs, speeches" schedule = "15 3 * * *" creates_persons = True - parser_version = 3 + parser_version = 4 def start(self, ctx: RunContext) -> Iterator[Request]: yield list_request(409, {"NRBR": ["NR", "BR"]}, kind="persons") @@ -92,6 +93,8 @@ class ParlamentPersons(Adapter): yield from self.parse_person(ctx, response) elif kind == "speeches": yield from self.parse_speeches(ctx, response) + elif kind == "committees": + yield from self.parse_committees(ctx, response) # -- persons & mandates ------------------------------------------------------------- @@ -180,6 +183,35 @@ class ParlamentPersons(Adapter): yield from self.side_jobs(ref, pad, name, content.get("unvtrans") or {}) yield list_request(251, {"PAD_INTERN": [int(pad)]}, kind="speeches", meta={"pad": pad, "name": name}) + yield list_request(250, {"PAD_INTERN": [int(pad)]}, kind="committees", meta={"pad": pad, "name": name}) + + def parse_committees(self, ctx: RunContext, response: Response) -> Iterator[Claim]: + """Committee memberships (list 250). Affiliations, not roles: a member has dozens of them per period, + and they say nothing about whether the person currently holds a mandate.""" + periods = set(ctx.params.get("periods") or get_settings().parlament_periods) + pad, name = response.meta["pad"], response.meta["name"] + for row in list_rows(response.json()): + if row.get("GP") not in periods: + continue + labelled = row.get("Ausschuss") or "" + committee = re.sub(r"\s*\([^()]*\)\s*$", "", labelled).strip() + if not committee: + ctx.stats["committee_without_name"] += 1 + continue + function = row.get("Funktion") or "Mitglied" + start = parse_date(row.get("Beginn")) + end = COMMITTEE_END.search(labelled) + chamber = "Bundesrat" if row.get("nrbr") == "BR" else "Nationalrat" + yield Claim( + claim_type="affiliation", person=PersonRef(ids={"parlament": pad}), + key=("parl-committee", pad, row.get("GP"), committee, function, start), + valid_from=start, valid_to=parse_date(end.group(1)) if end else None, + summary_de=f"{name}: {function}, {committee} ({chamber}, {row.get('GP')}. GP)", + summary_en=f"{name}: {function}, {committee} ({chamber}, {row.get('GP')} legislative period)", + data={"committee": committee, "function": function, "gp": row.get("GP"), "chamber": row.get("nrbr")}, + evidence=[Evidence(url=f"{BASE}/person/{pad}", doc_type="api_record", title=f"Parlament: {name}", + locator=f"ausschuss:{row.get('GP')}:{committee}:{function}:{start}")], + ) def side_jobs(self, ref: PersonRef, pad: str, name: str, unv: dict) -> Iterator[Claim]: """Disclosures under §6 Unv-Transparenz-G: leading positions, paid side jobs, honorary positions.""" diff --git a/backend/tests/fixtures/parlament.persons/committees.expected.json b/backend/tests/fixtures/parlament.persons/committees.expected.json new file mode 100644 index 0000000..da8ca3f --- /dev/null +++ b/backend/tests/fixtures/parlament.persons/committees.expected.json @@ -0,0 +1,167 @@ +[ + { + "__type__": "Claim", + "claim_type": "affiliation", + "counterpart_org": null, + "counterpart_person": null, + "data": { + "chamber": "NR", + "committee": "Ständiger Unterausschuss des Budgetausschusses", + "function": "Ersatzmitglied", + "gp": "XXVIII" + }, + "evidence": [ + { + "__type__": "Evidence", + "confidence": 1.0, + "doc_type": "api_record", + "excerpt": null, + "full_text": null, + "locator": "ausschuss:XXVIII:Ständiger Unterausschuss des Budgetausschusses:Ersatzmitglied:2024-10-24", + "meta": {}, + "published_at": null, + "quote": null, + "source": null, + "stance": "supports", + "title": "Parlament: Hannes Amesbauer", + "url": "https://www.parlament.gv.at/person/51879" + } + ], + "facet": null, + "key": [ + "parl-committee", + "51879", + "XXVIII", + "Ständiger Unterausschuss des Budgetausschusses", + "Ersatzmitglied", + "2024-10-24" + ], + "occurred_at": null, + "org": null, + "person": { + "__type__": "PersonRef", + "hints": {}, + "ids": { + "parlament": "51879" + }, + "name": null + }, + "review": false, + "sensitive": false, + "summary_de": "Hannes Amesbauer: Ersatzmitglied, Ständiger Unterausschuss des Budgetausschusses (Nationalrat, XXVIII. GP)", + "summary_en": "Hannes Amesbauer: Ersatzmitglied, Ständiger Unterausschuss des Budgetausschusses (Nationalrat, XXVIII legislative period)", + "topics": [], + "valid_from": "2024-10-24", + "valid_to": "2024-12-18" + }, + { + "__type__": "Claim", + "claim_type": "affiliation", + "counterpart_org": null, + "counterpart_person": null, + "data": { + "chamber": "NR", + "committee": "Ständiger Unterausschuss des Hauptausschusses", + "function": "Ersatzmitglied", + "gp": "XXVIII" + }, + "evidence": [ + { + "__type__": "Evidence", + "confidence": 1.0, + "doc_type": "api_record", + "excerpt": null, + "full_text": null, + "locator": "ausschuss:XXVIII:Ständiger Unterausschuss des Hauptausschusses:Ersatzmitglied:2024-10-24", + "meta": {}, + "published_at": null, + "quote": null, + "source": null, + "stance": "supports", + "title": "Parlament: Hannes Amesbauer", + "url": "https://www.parlament.gv.at/person/51879" + } + ], + "facet": null, + "key": [ + "parl-committee", + "51879", + "XXVIII", + "Ständiger Unterausschuss des Hauptausschusses", + "Ersatzmitglied", + "2024-10-24" + ], + "occurred_at": null, + "org": null, + "person": { + "__type__": "PersonRef", + "hints": {}, + "ids": { + "parlament": "51879" + }, + "name": null + }, + "review": false, + "sensitive": false, + "summary_de": "Hannes Amesbauer: Ersatzmitglied, Ständiger Unterausschuss des Hauptausschusses (Nationalrat, XXVIII. GP)", + "summary_en": "Hannes Amesbauer: Ersatzmitglied, Ständiger Unterausschuss des Hauptausschusses (Nationalrat, XXVIII legislative period)", + "topics": [], + "valid_from": "2024-10-24", + "valid_to": "2024-12-18" + }, + { + "__type__": "Claim", + "claim_type": "affiliation", + "counterpart_org": null, + "counterpart_person": null, + "data": { + "chamber": "NR", + "committee": "Ständiger Unterausschuss des Budgetausschusses", + "function": "Obfrau", + "gp": "XXVIII" + }, + "evidence": [ + { + "__type__": "Evidence", + "confidence": 1.0, + "doc_type": "api_record", + "excerpt": null, + "full_text": null, + "locator": "ausschuss:XXVIII:Ständiger Unterausschuss des Budgetausschusses:Obfrau:2024-10-24", + "meta": {}, + "published_at": null, + "quote": null, + "source": null, + "stance": "supports", + "title": "Parlament: Hannes Amesbauer", + "url": "https://www.parlament.gv.at/person/51879" + } + ], + "facet": null, + "key": [ + "parl-committee", + "51879", + "XXVIII", + "Ständiger Unterausschuss des Budgetausschusses", + "Obfrau", + "2024-10-24" + ], + "occurred_at": null, + "org": null, + "person": { + "__type__": "PersonRef", + "hints": {}, + "ids": { + "parlament": "51879" + }, + "name": null + }, + "review": false, + "sensitive": false, + "summary_de": "Hannes Amesbauer: Obfrau, Ständiger Unterausschuss des Budgetausschusses (Nationalrat, XXVIII. GP)", + "summary_en": "Hannes Amesbauer: Obfrau, Ständiger Unterausschuss des Budgetausschusses (Nationalrat, XXVIII legislative period)", + "topics": [], + "valid_from": "2024-10-24", + "valid_to": null + } +] diff --git a/backend/tests/fixtures/parlament.persons/committees.json b/backend/tests/fixtures/parlament.persons/committees.json new file mode 100644 index 0000000..05427ed --- /dev/null +++ b/backend/tests/fixtures/parlament.persons/committees.json @@ -0,0 +1,206 @@ +{ + "pages": 1, + "count": 4, + "header": [ + { + "css_class": "datum-col", + "feldId": 92, + "hidden": false, + "rnr": 1, + "listeId": 250, + "label": "GP", + "sortable": "1", + "feld_name": "GP", + "class": "datum-col" + }, + { + "feldId": 93, + "hidden": true, + "rnr": 2, + "listeId": 250, + "label": "Sort" + }, + { + "css_class": "col-20", + "feldId": 94, + "hidden": false, + "rnr": 3, + "listeId": 250, + "label": "Funktion", + "feld_name": "FUNKTION", + "class": "col-20" + }, + { + "css_class": "fraction-col", + "link_rnr": "6", + "feldId": 95, + "hidden": false, + "rnr": 4, + "listeId": 250, + "label": "Ausschuss", + "class": "fraction-col" + }, + { + "feldId": 96, + "hidden": true, + "rnr": 5, + "listeId": 250, + "label": "nrbr" + }, + { + "feldId": 97, + "hidden": true, + "rnr": 6, + "listeId": 250, + "label": "URL" + }, + { + "feldId": 98, + "hidden": true, + "rnr": 7, + "listeId": 250, + "label": "PAD_INTERN", + "feld_name": "PAD_INTERN" + }, + { + "feldId": 969, + "hidden": true, + "rnr": 8, + "mediaType": "fromDate", + "listeId": 250, + "label": "fromDate" + }, + { + "feldId": 564, + "hidden": true, + "rnr": 9, + "listeId": 250, + "label": "Ausschuss", + "feld_name": "AUSSCHUSS" + }, + { + "feldId": 565, + "hidden": true, + "rnr": 10, + "listeId": 250, + "label": "Gremium", + "feld_name": "GREMIUM" + }, + { + "feldId": 584, + "hidden": true, + "rnr": 11, + "listeId": 250, + "label": "gp_text_full", + "feld_name": "GP_TEXT_FULL" + }, + { + "feldId": 1645, + "hidden": true, + "ist_range": "1", + "rnr": 12, + "mediaType": "datetime", + "listeId": 250, + "label": "Beginn", + "feld_name": "BEGINNDATUM_ISO" + }, + { + "feldId": 1665, + "hidden": false, + "sort_rnr": 14, + "label": "Funktionsbeginn", + "sortable": "1", + "css_class": "datum-col", + "rnr": 13, + "listeId": 250, + "feld_name": "FUNKTIONSBEGINN", + "class": "datum-col" + }, + { + "feldId": 1666, + "hidden": true, + "rnr": 14, + "listeId": 250, + "label": "Beginn4Sort", + "feld_name": "BEGINN4SORT" + }, + { + "feldId": 1685, + "hidden": true, + "rnr": 15, + "listeId": 250, + "label": "Personenregister", + "feld_name": "PERSONENREGISTER" + } + ], + "rows": [ + [ + "XXVIII", + "2Z797589750013220241024STÄNDIGER UNTERAUSSCHUSS DES BUDGETAUSSCHUSSES79758975", + "Ersatzmitglied", + "Ständiger Unterausschuss des Budgetausschusses (24.10.2024 - 18.12.2024)", + "NR", + "/PAKT/VHG/XXVIII/SA-BU/SA-BU_00001_00908/index.shtml", + "51879", + "Thu Oct 24 02:00:00 CEST 2024", + "Ständiger Unterausschuss des Budgetausschusses", + "Nationalrat", + "ab 24.10.2024: XXVIII. Gesetzgebungsperiode des NR", + "2024-10-24T00:00:00", + "24.10.2024", + "20241024", + null + ], + [ + "XXVIII", + "2Z797589750013220241024STÄNDIGER UNTERAUSSCHUSS DES HAUPTAUSSCHUSSES79758975", + "Ersatzmitglied", + "Ständiger Unterausschuss des Hauptausschusses (24.10.2024 - 18.12.2024)", + "NR", + "/PAKT/VHG/XXVIII/SA-HA/SA-HA_00001_00912/index.shtml", + "51879", + "Thu Oct 24 02:00:00 CEST 2024", + "Ständiger Unterausschuss des Hauptausschusses", + "Nationalrat", + "ab 24.10.2024: XXVIII. Gesetzgebungsperiode des NR", + "2024-10-24T00:00:00", + "24.10.2024", + "20241024", + null + ], + [ + "XXVI", + "2Z798288900013120171221AUSSCHUSS FÜR KONSUMENTENSCHUTZ79828778", + "Mitglied", + "Ausschuss für Konsumentenschutz (21.12.2017 - 25.09.2018)", + "NR", + "/PAKT/VHG/XXVI/A-KO/A-KO_00001_00842/index.shtml", + "51879", + "Thu Dec 21 01:00:00 CET 2017", + "Ausschuss für Konsumentenschutz", + "Nationalrat", + "09.11.2017 - 22.10.2019: XXVI. Gesetzgebungsperiode des NR", + "2017-12-21T00:00:00", + "21.12.2017", + "20171221", + null + ], + [ + "XXVIII", + "2Z797589750013220241024STÄNDIGER UNTERAUSSCHUSS DES BUDGETAUSSCHUSSES79758975", + "Obfrau", + "Ständiger Unterausschuss des Budgetausschusses (24.10.2024 - )", + "NR", + "/PAKT/VHG/XXVIII/SA-BU/SA-BU_00001_00908/index.shtml", + "51879", + "Thu Oct 24 02:00:00 CEST 2024", + "Ständiger Unterausschuss des Budgetausschusses", + "Nationalrat", + "ab 24.10.2024: XXVIII. Gesetzgebungsperiode des NR", + "2024-10-24T00:00:00", + "24.10.2024", + "20241024", + null + ] + ] +} diff --git a/backend/tests/fixtures/parlament.persons/committees.request.json b/backend/tests/fixtures/parlament.persons/committees.request.json new file mode 100644 index 0000000..b91d3d2 --- /dev/null +++ b/backend/tests/fixtures/parlament.persons/committees.request.json @@ -0,0 +1,23 @@ +{ + "url": "https://www.parlament.gv.at/Filter/api/filter/data/250?js=eval&showAll=true", + "kind": "committees", + "method": "POST", + "params": null, + "json": { + "PAD_INTERN": [ + 51879 + ] + }, + "meta": { + "pad": "51879", + "name": "Hannes Amesbauer" + }, + "content_type": "application/json", + "body_file": "committees.json", + "run_params": { + "periods": [ + "XXVII", + "XXVIII" + ] + } +} diff --git a/backend/tests/fixtures/parlament.persons/person_detail.expected.json b/backend/tests/fixtures/parlament.persons/person_detail.expected.json index ae526f5..14203ad 100644 --- a/backend/tests/fixtures/parlament.persons/person_detail.expected.json +++ b/backend/tests/fixtures/parlament.persons/person_detail.expected.json @@ -677,5 +677,24 @@ "params": null, "skip_unchanged": true, "url": "https://www.parlament.gv.at/Filter/api/filter/data/251?js=eval&showAll=true" + }, + { + "__type__": "Request", + "conditional": true, + "headers": null, + "json": { + "PAD_INTERN": [ + 1977 + ] + }, + "kind": "committees", + "meta": { + "name": "Tanja Graf", + "pad": "1977" + }, + "method": "POST", + "params": null, + "skip_unchanged": true, + "url": "https://www.parlament.gv.at/Filter/api/filter/data/250?js=eval&showAll=true" } ] diff --git a/docs/roadmap.md b/docs/roadmap.md index 5eb3ef9..eefedc3 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -33,6 +33,7 @@ Next.js frontend (de/en), Docker Compose for Unraid. party/institution channels; curated `social_accounts.yaml` - [x] X deletion sync (`social.x_deletions`): posts gone from X keep a marked record without their text - [x] EP final roll-call votes of Austrian MEPs (`europarl.votes`) +- [x] Committee memberships of the configured legislative periods (list 250, `parlament.persons`) - [x] Draft legal pages (`/impressum`, `/datenschutz`), corrections contact, portrait proxy (`/api/img`) - [x] Pages: organisations, money explorer, social feed, Wikipedia edits, person network ("follow the money"), say-vs-vote on roll calls @@ -46,7 +47,7 @@ Next.js frontend (de/en), Docker Compose for Unraid. ## Next **Data** -- [ ] Full speech texts and interjections from the protocols; committee memberships (list 250) +- [ ] Full speech texts and interjections from the protocols - [ ] EU Council participant lists, ministries' press releases, Federal President's schedule - [ ] Election results and candidate lists; AMA/TAM subsidy recipients (after checking bulk access and terms) - [ ] Cabinet staff and appointments to state-owned companies (from answers to parliamentary questions) diff --git a/docs/sources.md b/docs/sources.md index d55620a..37a2b01 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -31,6 +31,7 @@ Status: ✅ implemented (adapter name in `code`) · 🟡 planned · ⛔ delibera | Source | Data | Access | Licence / terms | Status | |---|---|---|---|---| | Parlament Österreich | MPs since 1918, mandates, functions, **side jobs & income category** (§6 Unv-Transparenz-G), social-media links, plenary speeches, **faction votes** per item and reading | JSON API (`/Filter/api/filter/data/409,101,251`, `?json=TRUE` detail pages) | [CC BY 4.0](https://www.parlament.gv.at/recherchieren/open-data/daten-und-lizenz/) | ✅ `parlament.persons`, `parlament.items` | +| – committee memberships | who sat on which committee, with function and dates | JSON API (`/Filter/api/filter/data/250`) | CC BY 4.0 | ✅ `parlament.persons` | | – stenographic protocols | **Roll-call votes**: every member's yes/no from the "Mit „Ja“/„Nein“ stimmten …" lists | HTML fragments | CC BY 4.0 | ✅ `parlament.items` | | – immunity waivers | prosecutors' requests to lift immunity: authority, offence, outcome | items + committee report | CC BY 4.0 | ✅ `parlament.items` | | – written questions and answers | who asked which minister what | item list (J / AB) | CC BY 4.0 | ✅ `parlament.items` |