Files
simonandClaude Opus 5.5 79639e5319 Backfill: former MEPs, older elections, EC 2014-2019, social history paging
- europarl.meps also loads the Austrian MEPs of the 2019-2024 term; roles end
  with the last membership, so former MEPs don't show as current
- bmi.elections: National Council 2017 and EU 2019 preference votes
- eu.commission_meetings: the 2014-2019 Commission
- social.bluesky / social.mastodon: -p backfill_pages=N pages older posts

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 00:34:24 +01:00

351 lines
19 KiB
Python

"""Public posts of politicians on open platforms: Bluesky, Mastodon, Telegram (public channels), YouTube,
and X via the official (paid) API when a token is configured.
Accounts come from the `social_account` table: parliament profiles (verified), Wikidata
P2002/P12361/P4033/P3789/P2397 (unverified until confirmed) and the curated config/social_accounts.yaml
(enricher `config.social_accounts`). Accounts belong to a person or to an organisation (party channels).
Not collected (see docs/sources.md): X without the official API (ToS forbid scraping and proxies such as
Nitter), Facebook/Instagram (ToS), WhatsApp channels (no public read access, ToS forbid automated
collection), Signal (private by design, no public channels), private Telegram groups.
"""
from __future__ import annotations
import re
from collections.abc import Iterator
from typing import Any
from politrack.config import get_settings
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import ArchiveCopy, Claim, Evidence, OrgRef, PersonRef, PostRemoved, Statement
from politrack.text import excerpt, html_to_text
def social_spec(slug: str, name: str, homepage: str, notes: str, robots_exempt: tuple[str, ...] = ()) -> SourceSpec:
return SourceSpec(slug=slug, name=name, kind="social", homepage=homepage, license="Per-post copyright of the author",
access="api", trust=40, store_fulltext=True, keep_raw=True, min_interval=2.0,
robots_exempt=robots_exempt, tos_notes=notes)
BLUESKY = social_spec("bluesky", "Bluesky", "https://bsky.app",
"Public AT Protocol AppView API, no auth for public reads. Posts by the politicians themselves.")
MASTODON = social_spec("mastodon", "Mastodon / Fediverse", "https://joinmastodon.org",
"Public REST API of the account's instance (no auth for public statuses). Respect instance rules.")
TELEGRAM = social_spec("telegram", "Telegram (öffentliche Kanäle)", "https://telegram.org",
"Public web preview of *public broadcast channels* (t.me/s/<channel>); no robots.txt restrictions. "
"Private groups/chats are never accessed. Low frequency, latest posts only.")
YOUTUBE = social_spec("youtube", "YouTube", "https://www.youtube.com",
"YouTube Data API v3 (free quota, API key required via POLITRACK_YOUTUBE_API_KEY).")
X_API = "https://api.x.com/2/"
X = SourceSpec(slug="x", name="X (Twitter)", kind="social", homepage="https://x.com",
license="Per-post copyright of the author; X Developer Agreement", access="api", trust=40,
store_fulltext=False, keep_raw=False, min_interval=2.0, robots_exempt=(X_API,),
tos_notes="Official X API v2 only (paid tier, POLITRACK_X_BEARER_TOKEN). Scraping x.com or using "
"mirrors/proxies (Nitter etc.) violates the ToS and is not done. The Developer Agreement requires "
"removing posts that were deleted on X: no raw archive or full text is kept, and "
"social.x_deletions removes the stored excerpt of such posts, keeping only a marked record.")
PLATFORM_LABELS = {"bluesky": "Bluesky", "mastodon": "Mastodon", "telegram": "Telegram", "youtube": "YouTube",
"x": "X"}
def accounts(ctx: RunContext, platform: str, url_pattern: str | None = None) -> list[dict[str, Any]]:
"""Accounts of persons or organisations on one platform; each row carries an `owner` dict for Request.meta.
`url_pattern` also picks up accounts stored as plain websites, so it must match only that platform's
own host (never a generic shape like "https://host/@user", which TikTok and others share).
"""
if ctx.conn is None:
return []
rows = ctx.conn.execute(
"""SELECT person_id, organization_id, url, handle, verified FROM social_account
WHERE (platform = %s OR (%s::text IS NOT NULL AND url ~* %s))
AND (person_id IS NOT NULL OR organization_id IS NOT NULL)
ORDER BY verified DESC, id""",
(platform, url_pattern, url_pattern),
).fetchall()
for row in rows:
row["owner"] = {"person_id": str(row["person_id"]) if row["person_id"] else None,
"org_id": None if row["person_id"] else str(row["organization_id"]),
"verified": row["verified"]}
return rows
def post_claim(*, owner: dict[str, Any], platform: str, url: str, text: str, created: Any,
title: str | None = None, data: dict | None = None) -> Claim:
"""A public post as a `statement` claim of the account owner (person, or organisation for party channels)."""
label = PLATFORM_LABELS[platform]
return Claim(
claim_type="statement", occurred_at=created,
person=PersonRef(ids={"politrack": owner["person_id"]}) if owner.get("person_id") else None,
org=OrgRef(ids={"politrack_org": owner["org_id"]}, create=False) if owner.get("org_id") else None,
summary_de=f"Auf {label}: „{excerpt(title or text, 200)}“",
summary_en=f"On {label}: \"{excerpt(title or text, 200)}\"",
data={"platform": platform, "account_verified": owner.get("verified", False), **(data or {})},
evidence=[Evidence(url=url, source=platform, doc_type="social_post", title=excerpt(title or text, 120),
published_at=created, full_text=text, excerpt=text, quote=excerpt(text, 500))],
facet=Statement(context="social_post", url=url, said_at=created, platform=platform, title=title,
excerpt=text),
)
@register
class BlueskyPosts(Adapter):
spec = BLUESKY
name = "social.bluesky"
description = "Public Bluesky posts of politicians"
schedule = "7 * * * *"
queue = "social"
def start(self, ctx: RunContext) -> Iterator[Request]:
pages = int(ctx.params.get("backfill_pages", 0)) # older pages per account (history backfill)
for acc in accounts(ctx, "bluesky", r"bsky\.app/profile/"):
m = re.search(r"bsky\.app/profile/([^/?#]+)", acc["url"])
if m:
yield Request(url="https://public.api.bsky.app/xrpc/app.bsky.feed.getAuthorFeed",
params={"actor": m.group(1), "limit": 50, "filter": "posts_no_replies"}, kind="feed",
meta={**acc["owner"], "handle": m.group(1), **({"pages_left": pages} if pages else {})})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
handle = response.meta["handle"]
data = response.json()
if response.meta.get("pages_left") and data.get("cursor") and data.get("feed"):
yield Request(url=response.request.url, kind="feed",
params={**(response.request.params or {}), "cursor": data["cursor"]},
meta={**response.meta, "pages_left": response.meta["pages_left"] - 1})
for item in data.get("feed", []):
post = item.get("post") or {}
if item.get("reason"): # repost of someone else's post
continue
record = post.get("record") or {}
text = record.get("text") or ""
author = (post.get("author") or {}).get("handle") or handle
rkey = post.get("uri", "").rsplit("/", 1)[-1]
yield post_claim(owner=response.meta, platform="bluesky",
url=f"https://bsky.app/profile/{author}/post/{rkey}", text=text,
created=record.get("createdAt"),
data={"likes": post.get("likeCount"), "reposts": post.get("repostCount")})
@register
class MastodonPosts(Adapter):
spec = MASTODON
name = "social.mastodon"
description = "Public Mastodon/Fediverse posts of politicians"
schedule = "17 * * * *"
queue = "social"
def start(self, ctx: RunContext) -> Iterator[Request]:
pages = int(ctx.params.get("backfill_pages", 0)) # older pages per account (history backfill)
for acc in accounts(ctx, "mastodon"): # instances have arbitrary hosts: platform only
m = re.match(r"^https?://([^/]+)/@([^/@]+)", acc["url"])
if m:
host, user = m.groups()
yield Request(url=f"https://{host}/api/v1/accounts/lookup", params={"acct": user}, kind="lookup",
meta={**acc["owner"], "host": host, **({"pages_left": pages} if pages else {})})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
meta = response.meta
if response.request.kind == "lookup":
account_id = response.json().get("id")
if account_id:
yield Request(url=f"https://{meta['host']}/api/v1/accounts/{account_id}/statuses",
params={"limit": 40, "exclude_replies": "true", "exclude_reblogs": "true"},
kind="statuses", meta=meta)
return
statuses = response.json()
if meta.get("pages_left") and statuses:
yield Request(url=response.request.url, kind="statuses",
params={**(response.request.params or {}), "max_id": statuses[-1]["id"]},
meta={**meta, "pages_left": meta["pages_left"] - 1})
for status in statuses:
if status.get("visibility") not in (None, "public"):
continue
yield post_claim(owner=meta, platform="mastodon", url=status.get("url") or status.get("uri"),
text=html_to_text(status.get("content")), created=status.get("created_at"),
data={"favourites": status.get("favourites_count"), "reblogs": status.get("reblogs_count")})
@register
class TelegramChannels(Adapter):
spec = TELEGRAM
name = "social.telegram"
description = "Public Telegram broadcast channels of politicians (web preview, latest posts)"
schedule = "27 */2 * * *"
queue = "social"
def start(self, ctx: RunContext) -> Iterator[Request]:
for acc in accounts(ctx, "telegram", r"(t\.me|telegram\.me)/"):
m = re.search(r"(?:t\.me|telegram\.me)/(?:s/)?([A-Za-z0-9_]{4,})", acc["url"])
if m:
yield Request(url=f"https://t.me/s/{m.group(1)}", kind="channel",
meta={**acc["owner"], "channel": m.group(1)})
def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
soup = response.soup
subscribers = soup.select_one(".tgme_channel_info_counter .counter_value")
for msg in soup.select("div.tgme_widget_message[data-post]"):
text_el = msg.select_one(".tgme_widget_message_text")
time_el = msg.select_one(".tgme_widget_message_date time")
text = text_el.get_text("\n", strip=True) if text_el else ""
if not text:
continue # media-only posts
forwarded = msg.select_one(".tgme_widget_message_forwarded_from")
views = msg.select_one(".tgme_widget_message_views")
yield post_claim(
owner=response.meta, platform="telegram", url=f"https://t.me/{msg['data-post']}",
text=text, created=time_el.get("datetime") if time_el else None,
data={"views": views.get_text(strip=True) if views else None,
"forwarded_from": forwarded.get_text(" ", strip=True) if forwarded else None,
"channel_subscribers": subscribers.get_text(strip=True) if subscribers else None},
)
@register
class YouTubeVideos(Adapter):
spec = YOUTUBE
name = "social.youtube"
description = "Latest YouTube uploads of politicians' channels (requires POLITRACK_YOUTUBE_API_KEY)"
schedule = "37 */6 * * *"
queue = "social"
def start(self, ctx: RunContext) -> Iterator[Request]:
key = get_settings().youtube_api_key
if not key:
ctx.stats["skipped_no_api_key"] = 1
return
api = "https://www.googleapis.com/youtube/v3/channels"
for acc in accounts(ctx, "youtube", r"youtube\.com/"):
meta = acc["owner"]
if m := re.search(r"youtube\.com/channel/(UC[\w-]+)", acc["url"]):
params = {"id": m.group(1)}
elif m := re.search(r"youtube\.com/@([\w.-]+)", acc["url"]):
params = {"forHandle": f"@{m.group(1)}"}
elif m := re.search(r"youtube\.com/(?:user/|c/)?([\w.-]+)/?$", acc["url"]):
params = {"forUsername": m.group(1)}
else:
continue
yield Request(url=api, params={**params, "part": "contentDetails", "key": key}, kind="channel", meta=meta)
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
data = response.json()
meta = response.meta
if response.request.kind == "channel":
for item in data.get("items", []):
uploads = item["contentDetails"]["relatedPlaylists"]["uploads"]
yield Request(url="https://www.googleapis.com/youtube/v3/playlistItems", kind="uploads", meta=meta,
params={"playlistId": uploads, "part": "snippet", "maxResults": 25,
"key": get_settings().youtube_api_key})
return
for item in data.get("items", []):
snippet = item.get("snippet") or {}
video = (snippet.get("resourceId") or {}).get("videoId")
if not video:
continue
yield post_claim(owner=meta, platform="youtube",
url=f"https://www.youtube.com/watch?v={video}", title=snippet.get("title"),
text=f"{snippet.get('title', '')}\n{snippet.get('description', '')}".strip(),
created=snippet.get("publishedAt"))
@register
class XPosts(Adapter):
spec = X
name = "social.x"
description = "Posts of politicians on X via the official API v2 (requires POLITRACK_X_BEARER_TOKEN)"
schedule = "47 */6 * * *"
queue = "social"
parser_version = 2 # data.removed_from_platform: a post seen in the timeline again is no longer removed
def start(self, ctx: RunContext) -> Iterator[Request]:
token = get_settings().x_bearer_token
if not token:
ctx.stats["skipped_no_api_key"] = 1
return
for acc in accounts(ctx, "twitter", r"//(www\.)?(twitter|x)\.com/"):
m = re.search(r"(?:twitter|x)\.com/(?:#!/)?@?(\w{1,15})/?$", acc["url"])
if m:
yield Request(url=f"{X_API}users/by/username/{m.group(1)}", kind="user", headers=self.auth(),
meta=acc["owner"])
@staticmethod
def auth() -> dict[str, str]:
return {"Authorization": f"Bearer {get_settings().x_bearer_token}"}
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
data = response.json()
meta = response.meta
if response.request.kind == "user":
user = data.get("data") or {}
if user.get("id"):
yield Request(url=f"{X_API}users/{user['id']}/tweets", kind="tweets", headers=self.auth(),
params={"max_results": 50, "exclude": "replies,retweets",
"tweet.fields": "created_at,public_metrics"},
meta={**meta, "username": user.get("username")})
return
for tweet in data.get("data") or []:
metrics = tweet.get("public_metrics") or {}
yield post_claim(owner=meta, platform="x",
url=f"https://x.com/{meta['username']}/status/{tweet['id']}", text=tweet.get("text", ""),
created=tweet.get("created_at"),
data={"likes": metrics.get("like_count"), "reposts": metrics.get("retweet_count"),
"removed_from_platform": False})
X_LOOKUP_BATCH = 100 # maximum ids per request on GET /2/tweets
WAYBACK_API = "https://archive.org/wayback/available"
@register
class XDeletionSync(Adapter):
"""Finds stored X posts that no longer exist there, so their text can be removed (Developer Agreement).
Only "resource-not-found" counts: posts that are merely protected or whose author is suspended come back
with other error types and stay untouched. Runs four times a day because deletions must be honoured promptly.
"""
spec = X
name = "social.x_deletions"
description = "Marks stored X posts that are gone from X and removes their text (requires POLITRACK_X_BEARER_TOKEN)"
schedule = "17 */6 * * *"
queue = "social"
def start(self, ctx: RunContext) -> Iterator[Request]:
if not get_settings().x_bearer_token:
ctx.stats["skipped_no_api_key"] = 1
return
if ctx.conn is None:
return
rows = ctx.conn.execute(
"""SELECT st.url FROM statement st JOIN claim c ON c.id = st.claim_id
WHERE st.platform = 'x' AND st.context = 'social_post'
AND NOT COALESCE((c.data->>'removed_from_platform')::boolean, FALSE)
ORDER BY st.said_at DESC NULLS LAST"""
).fetchall()
urls = {m.group(1): row["url"] for row in rows if (m := re.search(r"/status/(\d+)", row["url"]))}
ids = list(urls)
for i in range(0, len(ids), X_LOOKUP_BATCH):
batch = ids[i:i + X_LOOKUP_BATCH]
yield Request(url=f"{X_API}tweets", kind="lookup", headers=XPosts.auth(), params={"ids": ",".join(batch)},
meta={"urls": {post_id: urls[post_id] for post_id in batch}},
conditional=False, skip_unchanged=False)
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if response.request.kind == "wayback":
closest = (response.json().get("archived_snapshots") or {}).get("closest") or {}
if closest.get("available") and str(closest.get("status")) == "200" and closest.get("url"):
yield ArchiveCopy(url=response.meta["url"],
archive_url=closest["url"].replace("http://", "https://", 1))
return
urls = response.meta["urls"]
for error in response.json().get("errors") or []:
if error.get("resource_type") != "tweet" or not str(error.get("type", "")).endswith("/resource-not-found"):
continue
url = urls.get(str(error.get("resource_id")))
if url:
yield PostRemoved(url=url, platform="x", platform_label=PLATFORM_LABELS["x"],
detected_at=response.fetched_at)
yield Request(url=WAYBACK_API, kind="wayback", params={"url": url}, meta={"url": url},
conditional=False, skip_unchanged=False)