Add committee memberships; tolerate empty EP declaration responses

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
simonandClaude Opus 5.5 committed 2026-11-09 02:19:25 +01:00
1 parent b4e26bb296
commit 9c58bb3d8f
8 files changed
+457 -2

No files matched your search

+6
View File
@@ -129,6 +129,9 @@ class EuropeanParliament(Adapter):
params={"country-of-representation": "AT", "format": "application/ld+json", "limit": 100})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if not response.body.strip(): # the API answers some MEPs' declaration requests with an empty body
ctx.stats["empty_responses"] += 1
return
data = response.json().get("data") or []
if response.request.kind == "meps":
for mep in data:
@@ -243,6 +246,9 @@ class ParliamentVotes(Adapter):
meta={"ep_ids": ep_ids})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if not response.body.strip(): # sittings without recorded votes come back with an empty body
ctx.stats["empty_responses"] += 1
return
kind = response.request.kind
data = response.json().get("data") or []
meta = response.meta
+33 -1
View File
@@ -41,6 +41,7 @@ LEGAL_FORM_END = re.compile(r"(?i:gesmbh|gmbh|ges\.\s?m\.\s?b\.\s?h\.?|m\.\s?b\.
r"e\.\s?u\.|\bse|privatstiftung|genossenschaft|sparkasse)(?=[A-ZÄÖÜ\s,]|$)")
COMMITTEE_END = re.compile(r"\(\d{2}\.\d{2}\.\d{4}\s*-\s*(\d{2}\.\d{2}\.\d{4})\s*\)\s*$")
STREET_WORD = r"\S*(?:straße|strasse|str\.|gasse|platz|weg|kai|ring|allee|zeile|lände|markt|steig)"
@@ -79,7 +80,7 @@ class ParlamentPersons(Adapter):
description = "Members of Nationalrat and Bundesrat since 1918, mandates, functions, side jobs, speeches"
schedule = "15 3 * * *"
creates_persons = True
parser_version = 3
parser_version = 4
def start(self, ctx: RunContext) -> Iterator[Request]:
yield list_request(409, {"NRBR": ["NR", "BR"]}, kind="persons")
@@ -92,6 +93,8 @@ class ParlamentPersons(Adapter):
yield from self.parse_person(ctx, response)
elif kind == "speeches":
yield from self.parse_speeches(ctx, response)
elif kind == "committees":
yield from self.parse_committees(ctx, response)
# -- persons & mandates -------------------------------------------------------------
@@ -180,6 +183,35 @@ class ParlamentPersons(Adapter):
yield from self.side_jobs(ref, pad, name, content.get("unvtrans") or {})
yield list_request(251, {"PAD_INTERN": [int(pad)]}, kind="speeches", meta={"pad": pad, "name": name})
yield list_request(250, {"PAD_INTERN": [int(pad)]}, kind="committees", meta={"pad": pad, "name": name})
def parse_committees(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
"""Committee memberships (list 250). Affiliations, not roles: a member has dozens of them per period,
and they say nothing about whether the person currently holds a mandate."""
periods = set(ctx.params.get("periods") or get_settings().parlament_periods)
pad, name = response.meta["pad"], response.meta["name"]
for row in list_rows(response.json()):
if row.get("GP") not in periods:
continue
labelled = row.get("Ausschuss") or ""
committee = re.sub(r"\s*\([^()]*\)\s*$", "", labelled).strip()
if not committee:
ctx.stats["committee_without_name"] += 1
continue
function = row.get("Funktion") or "Mitglied"
start = parse_date(row.get("Beginn"))
end = COMMITTEE_END.search(labelled)
chamber = "Bundesrat" if row.get("nrbr") == "BR" else "Nationalrat"
yield Claim(
claim_type="affiliation", person=PersonRef(ids={"parlament": pad}),
key=("parl-committee", pad, row.get("GP"), committee, function, start),
valid_from=start, valid_to=parse_date(end.group(1)) if end else None,
summary_de=f"{name}: {function}, {committee} ({chamber}, {row.get('GP')}. GP)",
summary_en=f"{name}: {function}, {committee} ({chamber}, {row.get('GP')} legislative period)",
data={"committee": committee, "function": function, "gp": row.get("GP"), "chamber": row.get("nrbr")},
evidence=[Evidence(url=f"{BASE}/person/{pad}", doc_type="api_record", title=f"Parlament: {name}",
locator=f"ausschuss:{row.get('GP')}:{committee}:{function}:{start}")],
)
def side_jobs(self, ref: PersonRef, pad: str, name: str, unv: dict) -> Iterator[Claim]:
"""Disclosures under §6 Unv-Transparenz-G: leading positions, paid side jobs, honorary positions."""