Backfill: former MEPs, older elections, EC 2014-2019, social history paging
- europarl.meps also loads the Austrian MEPs of the 2019-2024 term; roles end with the last membership, so former MEPs don't show as current - bmi.elections: National Council 2017 and EU 2019 preference votes - eu.commission_meetings: the 2014-2019 Commission - social.bluesky / social.mastodon: -p backfill_pages=N pages older posts Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4bd4e5b2db
commit
79639e5319
3 files changed
+49
-16
No files matched your search
@@ -42,7 +42,9 @@ EP_SPEC = SourceSpec(
|
||||
tos_notes="EP Open Data API v2 (limit 500 requests / 5 min / endpoint).",
|
||||
)
|
||||
EC_EXPORT = "https://ec.europa.eu/transparency-initiative/meetings/data/meetings/dataxlsx"
|
||||
EC_TERMS = ("meetingscommissionrepresentatives2429", "meetingscommissionrepresentatives1924")
|
||||
EC_TERMS = ("meetingscommissionrepresentatives2429", "meetingscommissionrepresentatives1924",
|
||||
"meetingscommissionrepresentatives1419")
|
||||
EP_PAST_TERMS = (9,) # Austrian MEPs of earlier terms: their votes and declarations, roles with end dates
|
||||
EP_API = "https://data.europarl.europa.eu/api/v2"
|
||||
# Austrian Commissioners in the terms covered by EC_TERMS (fallback if the DB has no Commission roles yet).
|
||||
AUSTRIAN_COMMISSIONERS = ("Johannes Hahn", "Magnus Brunner")
|
||||
@@ -123,10 +125,15 @@ class EuropeanParliament(Adapter):
|
||||
description = "Current Austrian MEPs (mandates) and their declarations"
|
||||
schedule = "30 4 * * 2"
|
||||
creates_persons = True
|
||||
parser_version = 2 # roles of former MEPs end with their last membership
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
yield Request(url=f"{EP_API}/meps/show-current", kind="meps",
|
||||
params={"country-of-representation": "AT", "format": "application/ld+json", "limit": 100})
|
||||
for term in EP_PAST_TERMS:
|
||||
yield Request(url=f"{EP_API}/meps", kind="meps",
|
||||
params={"parliamentary-term": term, "country-of-representation": "AT",
|
||||
"format": "application/ld+json", "limit": 100})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
if not response.body.strip(): # the API answers some MEPs' declaration requests with an empty body
|
||||
@@ -157,19 +164,22 @@ class EuropeanParliament(Adapter):
|
||||
if mep.get("img"):
|
||||
yield PersonRecord(ref=ref, full_name=meta["name"], photo_url=mep["img"],
|
||||
birth_date=date.fromisoformat(mep["bday"][:10]) if mep.get("bday") else None)
|
||||
starts = [m.get("memberDuring", {}).get("startDate") for m in mep.get("hasMembership") or []
|
||||
if "EU_PARLIAMENT" in str(m.get("membershipClassification", ""))]
|
||||
starts = sorted(s for s in starts if s)
|
||||
periods = sorted(((m.get("memberDuring") or {}).get("startDate"), (m.get("memberDuring") or {}).get("endDate"))
|
||||
for m in mep.get("hasMembership") or []
|
||||
if "EU_PARLIAMENT" in str(m.get("membershipClassification", ""))
|
||||
and (m.get("memberDuring") or {}).get("startDate"))
|
||||
body = OrgRef(slug="europaeisches-parlament")
|
||||
start = date.fromisoformat(starts[-1]) if starts else None
|
||||
start = date.fromisoformat(periods[-1][0][:10]) if periods else None
|
||||
# former MEPs: the latest membership has ended; current ones have an open end
|
||||
end = date.fromisoformat(periods[-1][1][:10]) if periods and periods[-1][1] else None
|
||||
url = f"https://www.europarl.europa.eu/meps/de/{meta['mep_id']}"
|
||||
yield Claim(
|
||||
claim_type="role", person=ref, org=body, valid_from=start,
|
||||
claim_type="role", person=ref, org=body, valid_from=start, valid_to=end,
|
||||
summary_de=f"Mitglied des Europäischen Parlaments (Fraktion {meta.get('group') or '?'})",
|
||||
summary_en=f"Member of the European Parliament ({meta.get('group') or '?'})",
|
||||
data={"ep_group": meta.get("group")},
|
||||
evidence=[Evidence(url=url, doc_type="api_record", title=f"Europäisches Parlament: {meta['name']}")],
|
||||
facet=Role(role="mp", level="eu", body=body, start=start),
|
||||
facet=Role(role="mp", level="eu", body=body, start=start, end=end),
|
||||
)
|
||||
|
||||
def parse_declarations(self, response: Response, docs: list[dict]) -> Iterator[Claim]:
|
||||
|
||||
Reference in new issue
Block a user