Backfill: former MEPs, older elections, EC 2014-2019, social history paging

- europarl.meps also loads the Austrian MEPs of the 2019-2024 term; roles end
  with the last membership, so former MEPs don't show as current
- bmi.elections: National Council 2017 and EU 2019 preference votes
- eu.commission_meetings: the 2014-2019 Commission
- social.bluesky / social.mastodon: -p backfill_pages=N pages older posts

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
simonandClaude Opus 5.5 committed 2026-11-13 00:34:24 +01:00
1 parent 4bd4e5b2db
commit 79639e5319
3 files changed
+49 -16

No files matched your search

+17 -5
View File
@@ -99,16 +99,22 @@ class BlueskyPosts(Adapter):
queue = "social"
def start(self, ctx: RunContext) -> Iterator[Request]:
pages = int(ctx.params.get("backfill_pages", 0)) # older pages per account (history backfill)
for acc in accounts(ctx, "bluesky", r"bsky\.app/profile/"):
m = re.search(r"bsky\.app/profile/([^/?#]+)", acc["url"])
if m:
yield Request(url="https://public.api.bsky.app/xrpc/app.bsky.feed.getAuthorFeed",
params={"actor": m.group(1), "limit": 50, "filter": "posts_no_replies"}, kind="feed",
meta={**acc["owner"], "handle": m.group(1)})
meta={**acc["owner"], "handle": m.group(1), **({"pages_left": pages} if pages else {})})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
handle = response.meta["handle"]
for item in response.json().get("feed", []):
data = response.json()
if response.meta.get("pages_left") and data.get("cursor") and data.get("feed"):
yield Request(url=response.request.url, kind="feed",
params={**(response.request.params or {}), "cursor": data["cursor"]},
meta={**response.meta, "pages_left": response.meta["pages_left"] - 1})
for item in data.get("feed", []):
post = item.get("post") or {}
if item.get("reason"): # repost of someone else's post
continue
@@ -131,12 +137,13 @@ class MastodonPosts(Adapter):
queue = "social"
def start(self, ctx: RunContext) -> Iterator[Request]:
pages = int(ctx.params.get("backfill_pages", 0)) # older pages per account (history backfill)
for acc in accounts(ctx, "mastodon"): # instances have arbitrary hosts: platform only
m = re.match(r"^https?://([^/]+)/@([^/@]+)", acc["url"])
if m:
host, user = m.groups()
yield Request(url=f"https://{host}/api/v1/accounts/lookup", params={"acct": user}, kind="lookup",
meta={**acc["owner"], "host": host})
meta={**acc["owner"], "host": host, **({"pages_left": pages} if pages else {})})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
meta = response.meta
@@ -147,7 +154,12 @@ class MastodonPosts(Adapter):
params={"limit": 40, "exclude_replies": "true", "exclude_reblogs": "true"},
kind="statuses", meta=meta)
return
for status in response.json():
statuses = response.json()
if meta.get("pages_left") and statuses:
yield Request(url=response.request.url, kind="statuses",
params={**(response.request.params or {}), "max_id": statuses[-1]["id"]},
meta={**meta, "pages_left": meta["pages_left"] - 1})
for status in statuses:
if status.get("visibility") not in (None, "public"):
continue
yield post_claim(owner=meta, platform="mastodon", url=status.get("url") or status.get("uri"),