Add committee memberships; tolerate empty EP declaration responses

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-09 02:19:25 +01:00
1 parent b4e26bb296
commit 9c58bb3d8f
@@ -129,6 +129,9 @@ class EuropeanParliament(Adapter):
                      params={"country-of-representation": "AT", "format": "application/ld+json", "limit": 100})

    def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
+        if not response.body.strip():  # the API answers some MEPs' declaration requests with an empty body
+            ctx.stats["empty_responses"] += 1
+            return
        data = response.json().get("data") or []
        if response.request.kind == "meps":
            for mep in data:
@@ -243,6 +246,9 @@ class ParliamentVotes(Adapter):
                          meta={"ep_ids": ep_ids})

    def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
+        if not response.body.strip():  # sittings without recorded votes come back with an empty body
+            ctx.stats["empty_responses"] += 1
+            return
        kind = response.request.kind
        data = response.json().get("data") or []
        meta = response.meta
@@ -41,6 +41,7 @@ LEGAL_FORM_END = re.compile(r"(?i:gesmbh|gmbh|ges\.\s?m\.\s?b\.\s?h\.?|m\.\s?b\.
                            r"e\.\s?u\.|\bse|privatstiftung|genossenschaft|sparkasse)(?=[A-ZÄÖÜ\s,]|$)")


+COMMITTEE_END = re.compile(r"\(\d{2}\.\d{2}\.\d{4}\s*-\s*(\d{2}\.\d{2}\.\d{4})\s*\)\s*$")
 STREET_WORD = r"\S*(?:straße|strasse|str\.|gasse|platz|weg|kai|ring|allee|zeile|lände|markt|steig)"


@@ -79,7 +80,7 @@ class ParlamentPersons(Adapter):
    description = "Members of Nationalrat and Bundesrat since 1918, mandates, functions, side jobs, speeches"
    schedule = "15 3 * * *"
    creates_persons = True
-    parser_version = 3
+    parser_version = 4

    def start(self, ctx: RunContext) -> Iterator[Request]:
        yield list_request(409, {"NRBR": ["NR", "BR"]}, kind="persons")
@@ -92,6 +93,8 @@ class ParlamentPersons(Adapter):
            yield from self.parse_person(ctx, response)
        elif kind == "speeches":
            yield from self.parse_speeches(ctx, response)
+        elif kind == "committees":
+            yield from self.parse_committees(ctx, response)

    # -- persons & mandates -------------------------------------------------------------

@@ -180,6 +183,35 @@ class ParlamentPersons(Adapter):

        yield from self.side_jobs(ref, pad, name, content.get("unvtrans") or {})
        yield list_request(251, {"PAD_INTERN": [int(pad)]}, kind="speeches", meta={"pad": pad, "name": name})
+        yield list_request(250, {"PAD_INTERN": [int(pad)]}, kind="committees", meta={"pad": pad, "name": name})
+
+    def parse_committees(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
+        """Committee memberships (list 250). Affiliations, not roles: a member has dozens of them per period,
+        and they say nothing about whether the person currently holds a mandate."""
+        periods = set(ctx.params.get("periods") or get_settings().parlament_periods)
+        pad, name = response.meta["pad"], response.meta["name"]
+        for row in list_rows(response.json()):
+            if row.get("GP") not in periods:
+                continue
+            labelled = row.get("Ausschuss") or ""
+            committee = re.sub(r"\s*\([^()]*\)\s*$", "", labelled).strip()
+            if not committee:
+                ctx.stats["committee_without_name"] += 1
+                continue
+            function = row.get("Funktion") or "Mitglied"
+            start = parse_date(row.get("Beginn"))
+            end = COMMITTEE_END.search(labelled)
+            chamber = "Bundesrat" if row.get("nrbr") == "BR" else "Nationalrat"
+            yield Claim(
+                claim_type="affiliation", person=PersonRef(ids={"parlament": pad}),
+                key=("parl-committee", pad, row.get("GP"), committee, function, start),
+                valid_from=start, valid_to=parse_date(end.group(1)) if end else None,
+                summary_de=f"{name}: {function}, {committee} ({chamber}, {row.get('GP')}. GP)",
+                summary_en=f"{name}: {function}, {committee} ({chamber}, {row.get('GP')} legislative period)",
+                data={"committee": committee, "function": function, "gp": row.get("GP"), "chamber": row.get("nrbr")},
+                evidence=[Evidence(url=f"{BASE}/person/{pad}", doc_type="api_record", title=f"Parlament: {name}",
+                                   locator=f"ausschuss:{row.get('GP')}:{committee}:{function}:{start}")],
+            )

    def side_jobs(self, ref: PersonRef, pad: str, name: str, unv: dict) -> Iterator[Claim]:
        """Disclosures under §6 Unv-Transparenz-G: leading positions, paid side jobs, honorary positions."""
@@ -0,0 +1,167 @@
+[
+ {
+  "__type__": "Claim",
+  "claim_type": "affiliation",
+  "counterpart_org": null,
+  "counterpart_person": null,
+  "data": {
+   "chamber": "NR",
+   "committee": "Ständiger Unterausschuss des Budgetausschusses",
+   "function": "Ersatzmitglied",
+   "gp": "XXVIII"
+  },
+  "evidence": [
+   {
+    "__type__": "Evidence",
+    "confidence": 1.0,
+    "doc_type": "api_record",
+    "excerpt": null,
+    "full_text": null,
+    "locator": "ausschuss:XXVIII:Ständiger Unterausschuss des Budgetausschusses:Ersatzmitglied:2024-10-24",
+    "meta": {},
+    "published_at": null,
+    "quote": null,
+    "source": null,
+    "stance": "supports",
+    "title": "Parlament: Hannes Amesbauer",
+    "url": "https://www.parlament.gv.at/person/51879"
+   }
+  ],
+  "facet": null,
+  "key": [
+   "parl-committee",
+   "51879",
+   "XXVIII",
+   "Ständiger Unterausschuss des Budgetausschusses",
+   "Ersatzmitglied",
+   "2024-10-24"
+  ],
+  "occurred_at": null,
+  "org": null,
+  "person": {
+   "__type__": "PersonRef",
+   "hints": {},
+   "ids": {
+    "parlament": "51879"
+   },
+   "name": null
+  },
+  "review": false,
+  "sensitive": false,
+  "summary_de": "Hannes Amesbauer: Ersatzmitglied, Ständiger Unterausschuss des Budgetausschusses (Nationalrat, XXVIII. GP)",
+  "summary_en": "Hannes Amesbauer: Ersatzmitglied, Ständiger Unterausschuss des Budgetausschusses (Nationalrat, XXVIII legislative period)",
+  "topics": [],
+  "valid_from": "2024-10-24",
+  "valid_to": "2024-12-18"
+ },
+ {
+  "__type__": "Claim",
+  "claim_type": "affiliation",
+  "counterpart_org": null,
+  "counterpart_person": null,
+  "data": {
+   "chamber": "NR",
+   "committee": "Ständiger Unterausschuss des Hauptausschusses",
+   "function": "Ersatzmitglied",
+   "gp": "XXVIII"
+  },
+  "evidence": [
+   {
+    "__type__": "Evidence",
+    "confidence": 1.0,
+    "doc_type": "api_record",
+    "excerpt": null,
+    "full_text": null,
+    "locator": "ausschuss:XXVIII:Ständiger Unterausschuss des Hauptausschusses:Ersatzmitglied:2024-10-24",
+    "meta": {},
+    "published_at": null,
+    "quote": null,
+    "source": null,
+    "stance": "supports",
+    "title": "Parlament: Hannes Amesbauer",
+    "url": "https://www.parlament.gv.at/person/51879"
+   }
+  ],
+  "facet": null,
+  "key": [
+   "parl-committee",
+   "51879",
+   "XXVIII",
+   "Ständiger Unterausschuss des Hauptausschusses",
+   "Ersatzmitglied",
+   "2024-10-24"
+  ],
+  "occurred_at": null,
+  "org": null,
+  "person": {
+   "__type__": "PersonRef",
+   "hints": {},
+   "ids": {
+    "parlament": "51879"
+   },
+   "name": null
+  },
+  "review": false,
+  "sensitive": false,
+  "summary_de": "Hannes Amesbauer: Ersatzmitglied, Ständiger Unterausschuss des Hauptausschusses (Nationalrat, XXVIII. GP)",
+  "summary_en": "Hannes Amesbauer: Ersatzmitglied, Ständiger Unterausschuss des Hauptausschusses (Nationalrat, XXVIII legislative period)",
+  "topics": [],
+  "valid_from": "2024-10-24",
+  "valid_to": "2024-12-18"
+ },
+ {
+  "__type__": "Claim",
+  "claim_type": "affiliation",
+  "counterpart_org": null,
+  "counterpart_person": null,
+  "data": {
+   "chamber": "NR",
+   "committee": "Ständiger Unterausschuss des Budgetausschusses",
+   "function": "Obfrau",
+   "gp": "XXVIII"
+  },
+  "evidence": [
+   {
+    "__type__": "Evidence",
+    "confidence": 1.0,
+    "doc_type": "api_record",
+    "excerpt": null,
+    "full_text": null,
+    "locator": "ausschuss:XXVIII:Ständiger Unterausschuss des Budgetausschusses:Obfrau:2024-10-24",
+    "meta": {},
+    "published_at": null,
+    "quote": null,
+    "source": null,
+    "stance": "supports",
+    "title": "Parlament: Hannes Amesbauer",
+    "url": "https://www.parlament.gv.at/person/51879"
+   }
+  ],
+  "facet": null,
+  "key": [
+   "parl-committee",
+   "51879",
+   "XXVIII",
+   "Ständiger Unterausschuss des Budgetausschusses",
+   "Obfrau",
+   "2024-10-24"
+  ],
+  "occurred_at": null,
+  "org": null,
+  "person": {
+   "__type__": "PersonRef",
+   "hints": {},
+   "ids": {
+    "parlament": "51879"
+   },
+   "name": null
+  },
+  "review": false,
+  "sensitive": false,
+  "summary_de": "Hannes Amesbauer: Obfrau, Ständiger Unterausschuss des Budgetausschusses (Nationalrat, XXVIII. GP)",
+  "summary_en": "Hannes Amesbauer: Obfrau, Ständiger Unterausschuss des Budgetausschusses (Nationalrat, XXVIII legislative period)",
+  "topics": [],
+  "valid_from": "2024-10-24",
+  "valid_to": null
+ }
+]
@@ -0,0 +1,206 @@
+{
+ "pages": 1,
+ "count": 4,
+ "header": [
+  {
+   "css_class": "datum-col",
+   "feldId": 92,
+   "hidden": false,
+   "rnr": 1,
+   "listeId": 250,
+   "label": "GP",
+   "sortable": "1",
+   "feld_name": "GP",
+   "class": "datum-col"
+  },
+  {
+   "feldId": 93,
+   "hidden": true,
+   "rnr": 2,
+   "listeId": 250,
+   "label": "Sort"
+  },
+  {
+   "css_class": "col-20",
+   "feldId": 94,
+   "hidden": false,
+   "rnr": 3,
+   "listeId": 250,
+   "label": "Funktion",
+   "feld_name": "FUNKTION",
+   "class": "col-20"
+  },
+  {
+   "css_class": "fraction-col",
+   "link_rnr": "6",
+   "feldId": 95,
+   "hidden": false,
+   "rnr": 4,
+   "listeId": 250,
+   "label": "Ausschuss",
+   "class": "fraction-col"
+  },
+  {
+   "feldId": 96,
+   "hidden": true,
+   "rnr": 5,
+   "listeId": 250,
+   "label": "nrbr"
+  },
+  {
+   "feldId": 97,
+   "hidden": true,
+   "rnr": 6,
+   "listeId": 250,
+   "label": "URL"
+  },
+  {
+   "feldId": 98,
+   "hidden": true,
+   "rnr": 7,
+   "listeId": 250,
+   "label": "PAD_INTERN",
+   "feld_name": "PAD_INTERN"
+  },
+  {
+   "feldId": 969,
+   "hidden": true,
+   "rnr": 8,
+   "mediaType": "fromDate",
+   "listeId": 250,
+   "label": "fromDate"
+  },
+  {
+   "feldId": 564,
+   "hidden": true,
+   "rnr": 9,
+   "listeId": 250,
+   "label": "Ausschuss",
+   "feld_name": "AUSSCHUSS"
+  },
+  {
+   "feldId": 565,
+   "hidden": true,
+   "rnr": 10,
+   "listeId": 250,
+   "label": "Gremium",
+   "feld_name": "GREMIUM"
+  },
+  {
+   "feldId": 584,
+   "hidden": true,
+   "rnr": 11,
+   "listeId": 250,
+   "label": "gp_text_full",
+   "feld_name": "GP_TEXT_FULL"
+  },
+  {
+   "feldId": 1645,
+   "hidden": true,
+   "ist_range": "1",
+   "rnr": 12,
+   "mediaType": "datetime",
+   "listeId": 250,
+   "label": "Beginn",
+   "feld_name": "BEGINNDATUM_ISO"
+  },
+  {
+   "feldId": 1665,
+   "hidden": false,
+   "sort_rnr": 14,
+   "label": "Funktionsbeginn",
+   "sortable": "1",
+   "css_class": "datum-col",
+   "rnr": 13,
+   "listeId": 250,
+   "feld_name": "FUNKTIONSBEGINN",
+   "class": "datum-col"
+  },
+  {
+   "feldId": 1666,
+   "hidden": true,
+   "rnr": 14,
+   "listeId": 250,
+   "label": "Beginn4Sort",
+   "feld_name": "BEGINN4SORT"
+  },
+  {
+   "feldId": 1685,
+   "hidden": true,
+   "rnr": 15,
+   "listeId": 250,
+   "label": "Personenregister",
+   "feld_name": "PERSONENREGISTER"
+  }
+ ],
+ "rows": [
+  [
+   "XXVIII",
+   "2Z797589750013220241024STÄNDIGER UNTERAUSSCHUSS DES BUDGETAUSSCHUSSES79758975",
+   "Ersatzmitglied",
+   "Ständiger Unterausschuss des Budgetausschusses (24.10.2024 - 18.12.2024)",
+   "NR",
+   "/PAKT/VHG/XXVIII/SA-BU/SA-BU_00001_00908/index.shtml",
+   "51879",
+   "Thu Oct 24 02:00:00 CEST 2024",
+   "Ständiger Unterausschuss des Budgetausschusses",
+   "Nationalrat",
+   "ab 24.10.2024: XXVIII. Gesetzgebungsperiode des NR",
+   "2024-10-24T00:00:00",
+   "24.10.2024",
+   "20241024",
+   null
+  ],
+  [
+   "XXVIII",
+   "2Z797589750013220241024STÄNDIGER UNTERAUSSCHUSS DES HAUPTAUSSCHUSSES79758975",
+   "Ersatzmitglied",
+   "Ständiger Unterausschuss des Hauptausschusses (24.10.2024 - 18.12.2024)",
+   "NR",
+   "/PAKT/VHG/XXVIII/SA-HA/SA-HA_00001_00912/index.shtml",
+   "51879",
+   "Thu Oct 24 02:00:00 CEST 2024",
+   "Ständiger Unterausschuss des Hauptausschusses",
+   "Nationalrat",
+   "ab 24.10.2024: XXVIII. Gesetzgebungsperiode des NR",
+   "2024-10-24T00:00:00",
+   "24.10.2024",
+   "20241024",
+   null
+  ],
+  [
+   "XXVI",
+   "2Z798288900013120171221AUSSCHUSS FÜR KONSUMENTENSCHUTZ79828778",
+   "Mitglied",
+   "Ausschuss für Konsumentenschutz (21.12.2017 - 25.09.2018)",
+   "NR",
+   "/PAKT/VHG/XXVI/A-KO/A-KO_00001_00842/index.shtml",
+   "51879",
+   "Thu Dec 21 01:00:00 CET 2017",
+   "Ausschuss für Konsumentenschutz",
+   "Nationalrat",
+   "09.11.2017 - 22.10.2019: XXVI. Gesetzgebungsperiode des NR",
+   "2017-12-21T00:00:00",
+   "21.12.2017",
+   "20171221",
+   null
+  ],
+  [
+   "XXVIII",
+   "2Z797589750013220241024STÄNDIGER UNTERAUSSCHUSS DES BUDGETAUSSCHUSSES79758975",
+   "Obfrau",
+   "Ständiger Unterausschuss des Budgetausschusses (24.10.2024 - )",
+   "NR",
+   "/PAKT/VHG/XXVIII/SA-BU/SA-BU_00001_00908/index.shtml",
+   "51879",
+   "Thu Oct 24 02:00:00 CEST 2024",
+   "Ständiger Unterausschuss des Budgetausschusses",
+   "Nationalrat",
+   "ab 24.10.2024: XXVIII. Gesetzgebungsperiode des NR",
+   "2024-10-24T00:00:00",
+   "24.10.2024",
+   "20241024",
+   null
+  ]
+ ]
+}
@@ -0,0 +1,23 @@
+{
+  "url": "https://www.parlament.gv.at/Filter/api/filter/data/250?js=eval&showAll=true",
+  "kind": "committees",
+  "method": "POST",
+  "params": null,
+  "json": {
+    "PAD_INTERN": [
+      51879
+    ]
+  },
+  "meta": {
+    "pad": "51879",
+    "name": "Hannes Amesbauer"
+  },
+  "content_type": "application/json",
+  "body_file": "committees.json",
+  "run_params": {
+    "periods": [
+      "XXVII",
+      "XXVIII"
+    ]
+  }
+}
@@ -677,5 +677,24 @@
  "params": null,
  "skip_unchanged": true,
  "url": "https://www.parlament.gv.at/Filter/api/filter/data/251?js=eval&showAll=true"
+ },
+ {
+  "__type__": "Request",
+  "conditional": true,
+  "headers": null,
+  "json": {
+   "PAD_INTERN": [
+    1977
+   ]
+  },
+  "kind": "committees",
+  "meta": {
+   "name": "Tanja Graf",
+   "pad": "1977"
+  },
+  "method": "POST",
+  "params": null,
+  "skip_unchanged": true,
+  "url": "https://www.parlament.gv.at/Filter/api/filter/data/250?js=eval&showAll=true"
 }
 ]
@@ -33,6 +33,7 @@ Next.js frontend (de/en), Docker Compose for Unraid.
      party/institution channels; curated `social_accounts.yaml`
 - [x] X deletion sync (`social.x_deletions`): posts gone from X keep a marked record without their text
 - [x] EP final roll-call votes of Austrian MEPs (`europarl.votes`)
+- [x] Committee memberships of the configured legislative periods (list 250, `parlament.persons`)
 - [x] Draft legal pages (`/impressum`, `/datenschutz`), corrections contact, portrait proxy (`/api/img`)
 - [x] Pages: organisations, money explorer, social feed, Wikipedia edits, person network ("follow the money"),
      say-vs-vote on roll calls
@@ -46,7 +47,7 @@ Next.js frontend (de/en), Docker Compose for Unraid.
 ## Next

 **Data**
- [ ] Full speech texts and interjections from the protocols; committee memberships (list 250)
+- [ ] Full speech texts and interjections from the protocols
 - [ ] EU Council participant lists, ministries' press releases, Federal President's schedule
 - [ ] Election results and candidate lists; AMA/TAM subsidy recipients (after checking bulk access and terms)
 - [ ] Cabinet staff and appointments to state-owned companies (from answers to parliamentary questions)
@@ -31,6 +31,7 @@ Status: ✅ implemented (adapter name in `code`) · 🟡 planned · ⛔ delibera
 | Source | Data | Access | Licence / terms | Status |
 |---|---|---|---|---|
 | Parlament Österreich | MPs since 1918, mandates, functions, **side jobs & income category** (§6 Unv-Transparenz-G), social-media links, plenary speeches, **faction votes** per item and reading | JSON API (`/Filter/api/filter/data/409,101,251`, `?json=TRUE` detail pages) | [CC BY 4.0](https://www.parlament.gv.at/recherchieren/open-data/daten-und-lizenz/) | ✅ `parlament.persons`, `parlament.items` |
+| – committee memberships | who sat on which committee, with function and dates | JSON API (`/Filter/api/filter/data/250`) | CC BY 4.0 | ✅ `parlament.persons` |
 | – stenographic protocols | **Roll-call votes**: every member's yes/no from the "Mit „Ja“/„Nein“ stimmten …" lists | HTML fragments | CC BY 4.0 | ✅ `parlament.items` |
 | – immunity waivers | prosecutors' requests to lift immunity: authority, offence, outcome | items + committee report | CC BY 4.0 | ✅ `parlament.items` |
 | – written questions and answers | who asked which minister what | item list (J / AB) | CC BY 4.0 | ✅ `parlament.items` |