Backfill: former MEPs, older elections, EC 2014-2019, social history paging

- europarl.meps also loads the Austrian MEPs of the 2019-2024 term; roles end
  with the last membership, so former MEPs don't show as current
- bmi.elections: National Council 2017 and EU 2019 preference votes
- eu.commission_meetings: the 2014-2019 Commission
- social.bluesky / social.mastodon: -p backfill_pages=N pages older posts

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
simonandClaude Opus 5.5 committed 2026-11-13 00:34:24 +01:00
1 parent 4bd4e5b2db
commit 79639e5319
3 files changed
+49 -16

No files matched your search

+17 -7
View File
@@ -42,7 +42,9 @@ EP_SPEC = SourceSpec(
tos_notes="EP Open Data API v2 (limit 500 requests / 5 min / endpoint).",
)
EC_EXPORT = "https://ec.europa.eu/transparency-initiative/meetings/data/meetings/dataxlsx"
EC_TERMS = ("meetingscommissionrepresentatives2429", "meetingscommissionrepresentatives1924")
EC_TERMS = ("meetingscommissionrepresentatives2429", "meetingscommissionrepresentatives1924",
"meetingscommissionrepresentatives1419")
EP_PAST_TERMS = (9,) # Austrian MEPs of earlier terms: their votes and declarations, roles with end dates
EP_API = "https://data.europarl.europa.eu/api/v2"
# Austrian Commissioners in the terms covered by EC_TERMS (fallback if the DB has no Commission roles yet).
AUSTRIAN_COMMISSIONERS = ("Johannes Hahn", "Magnus Brunner")
@@ -123,10 +125,15 @@ class EuropeanParliament(Adapter):
description = "Current Austrian MEPs (mandates) and their declarations"
schedule = "30 4 * * 2"
creates_persons = True
parser_version = 2 # roles of former MEPs end with their last membership
def start(self, ctx: RunContext) -> Iterator[Request]:
yield Request(url=f"{EP_API}/meps/show-current", kind="meps",
params={"country-of-representation": "AT", "format": "application/ld+json", "limit": 100})
for term in EP_PAST_TERMS:
yield Request(url=f"{EP_API}/meps", kind="meps",
params={"parliamentary-term": term, "country-of-representation": "AT",
"format": "application/ld+json", "limit": 100})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if not response.body.strip(): # the API answers some MEPs' declaration requests with an empty body
@@ -157,19 +164,22 @@ class EuropeanParliament(Adapter):
if mep.get("img"):
yield PersonRecord(ref=ref, full_name=meta["name"], photo_url=mep["img"],
birth_date=date.fromisoformat(mep["bday"][:10]) if mep.get("bday") else None)
starts = [m.get("memberDuring", {}).get("startDate") for m in mep.get("hasMembership") or []
if "EU_PARLIAMENT" in str(m.get("membershipClassification", ""))]
starts = sorted(s for s in starts if s)
periods = sorted(((m.get("memberDuring") or {}).get("startDate"), (m.get("memberDuring") or {}).get("endDate"))
for m in mep.get("hasMembership") or []
if "EU_PARLIAMENT" in str(m.get("membershipClassification", ""))
and (m.get("memberDuring") or {}).get("startDate"))
body = OrgRef(slug="europaeisches-parlament")
start = date.fromisoformat(starts[-1]) if starts else None
start = date.fromisoformat(periods[-1][0][:10]) if periods else None
# former MEPs: the latest membership has ended; current ones have an open end
end = date.fromisoformat(periods[-1][1][:10]) if periods and periods[-1][1] else None
url = f"https://www.europarl.europa.eu/meps/de/{meta['mep_id']}"
yield Claim(
claim_type="role", person=ref, org=body, valid_from=start,
claim_type="role", person=ref, org=body, valid_from=start, valid_to=end,
summary_de=f"Mitglied des Europäischen Parlaments (Fraktion {meta.get('group') or '?'})",
summary_en=f"Member of the European Parliament ({meta.get('group') or '?'})",
data={"ep_group": meta.get("group")},
evidence=[Evidence(url=url, doc_type="api_record", title=f"Europäisches Parlament: {meta['name']}")],
facet=Role(role="mp", level="eu", body=body, start=start),
facet=Role(role="mp", level="eu", body=body, start=start, end=end),
)
def parse_declarations(self, response: Response, docs: list[dict]) -> Iterator[Claim]: