Backfill: former MEPs, older elections, EC 2014-2019, social history paging
- europarl.meps also loads the Austrian MEPs of the 2019-2024 term; roles end with the last membership, so former MEPs don't show as current - bmi.elections: National Council 2017 and EU 2019 preference votes - eu.commission_meetings: the 2014-2019 Commission - social.bluesky / social.mastodon: -p backfill_pages=N pages older posts Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4bd4e5b2db
commit
79639e5319
3 files changed
+49
-16
No files matched your search
@@ -99,16 +99,22 @@ class BlueskyPosts(Adapter):
|
||||
queue = "social"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
pages = int(ctx.params.get("backfill_pages", 0)) # older pages per account (history backfill)
|
||||
for acc in accounts(ctx, "bluesky", r"bsky\.app/profile/"):
|
||||
m = re.search(r"bsky\.app/profile/([^/?#]+)", acc["url"])
|
||||
if m:
|
||||
yield Request(url="https://public.api.bsky.app/xrpc/app.bsky.feed.getAuthorFeed",
|
||||
params={"actor": m.group(1), "limit": 50, "filter": "posts_no_replies"}, kind="feed",
|
||||
meta={**acc["owner"], "handle": m.group(1)})
|
||||
meta={**acc["owner"], "handle": m.group(1), **({"pages_left": pages} if pages else {})})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
handle = response.meta["handle"]
|
||||
for item in response.json().get("feed", []):
|
||||
data = response.json()
|
||||
if response.meta.get("pages_left") and data.get("cursor") and data.get("feed"):
|
||||
yield Request(url=response.request.url, kind="feed",
|
||||
params={**(response.request.params or {}), "cursor": data["cursor"]},
|
||||
meta={**response.meta, "pages_left": response.meta["pages_left"] - 1})
|
||||
for item in data.get("feed", []):
|
||||
post = item.get("post") or {}
|
||||
if item.get("reason"): # repost of someone else's post
|
||||
continue
|
||||
@@ -131,12 +137,13 @@ class MastodonPosts(Adapter):
|
||||
queue = "social"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
pages = int(ctx.params.get("backfill_pages", 0)) # older pages per account (history backfill)
|
||||
for acc in accounts(ctx, "mastodon"): # instances have arbitrary hosts: platform only
|
||||
m = re.match(r"^https?://([^/]+)/@([^/@]+)", acc["url"])
|
||||
if m:
|
||||
host, user = m.groups()
|
||||
yield Request(url=f"https://{host}/api/v1/accounts/lookup", params={"acct": user}, kind="lookup",
|
||||
meta={**acc["owner"], "host": host})
|
||||
meta={**acc["owner"], "host": host, **({"pages_left": pages} if pages else {})})
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
meta = response.meta
|
||||
@@ -147,7 +154,12 @@ class MastodonPosts(Adapter):
|
||||
params={"limit": 40, "exclude_replies": "true", "exclude_reblogs": "true"},
|
||||
kind="statuses", meta=meta)
|
||||
return
|
||||
for status in response.json():
|
||||
statuses = response.json()
|
||||
if meta.get("pages_left") and statuses:
|
||||
yield Request(url=response.request.url, kind="statuses",
|
||||
params={**(response.request.params or {}), "max_id": statuses[-1]["id"]},
|
||||
meta={**meta, "pages_left": meta["pages_left"] - 1})
|
||||
for status in statuses:
|
||||
if status.get("visibility") not in (None, "public"):
|
||||
continue
|
||||
yield post_claim(owner=meta, platform="mastodon", url=status.get("url") or status.get("uri"),
|
||||
|
||||
Reference in new issue
Block a user