"""Public posts of politicians on open platforms: Bluesky, Mastodon, Telegram (public channels), YouTube, and X via the official (paid) API when a token is configured. Accounts come from the `social_account` table: parliament profiles (verified), Wikidata P2002/P12361/P4033/P3789/P2397 (unverified until confirmed) and the curated config/social_accounts.yaml (enricher `config.social_accounts`). Accounts belong to a person or to an organisation (party channels). Not collected (see docs/sources.md): X without the official API (ToS forbid scraping and proxies such as Nitter), Facebook/Instagram (ToS), WhatsApp channels (no public read access, ToS forbid automated collection), Signal (private by design, no public channels), private Telegram groups. """ from __future__ import annotations import re from collections.abc import Iterator from typing import Any from politrack.config import get_settings from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register from politrack.core.records import Claim, Evidence, OrgRef, PersonRef, Statement from politrack.text import excerpt, html_to_text def social_spec(slug: str, name: str, homepage: str, notes: str, robots_exempt: tuple[str, ...] = ()) -> SourceSpec: return SourceSpec(slug=slug, name=name, kind="social", homepage=homepage, license="Per-post copyright of the author", access="api", trust=40, store_fulltext=True, keep_raw=True, min_interval=2.0, robots_exempt=robots_exempt, tos_notes=notes) BLUESKY = social_spec("bluesky", "Bluesky", "https://bsky.app", "Public AT Protocol AppView API, no auth for public reads. Posts by the politicians themselves.") MASTODON = social_spec("mastodon", "Mastodon / Fediverse", "https://joinmastodon.org", "Public REST API of the account's instance (no auth for public statuses). Respect instance rules.") TELEGRAM = social_spec("telegram", "Telegram (öffentliche Kanäle)", "https://telegram.org", "Public web preview of *public broadcast channels* (t.me/s/); no robots.txt restrictions. " "Private groups/chats are never accessed. Low frequency, latest posts only.") YOUTUBE = social_spec("youtube", "YouTube", "https://www.youtube.com", "YouTube Data API v3 (free quota, API key required via POLITRACK_YOUTUBE_API_KEY).") X_API = "https://api.x.com/2/" X = SourceSpec(slug="x", name="X (Twitter)", kind="social", homepage="https://x.com", license="Per-post copyright of the author; X Developer Agreement", access="api", trust=40, store_fulltext=False, keep_raw=False, min_interval=2.0, robots_exempt=(X_API,), tos_notes="Official X API v2 only (paid tier, POLITRACK_X_BEARER_TOKEN). Scraping x.com or using " "mirrors/proxies (Nitter etc.) violates the ToS and is not done. The Developer Agreement requires " "removing posts that were deleted on X; no raw archive or full text is kept, but the " "deletion sync is still open (docs/roadmap.md).") PLATFORM_LABELS = {"bluesky": "Bluesky", "mastodon": "Mastodon", "telegram": "Telegram", "youtube": "YouTube", "x": "X"} def accounts(ctx: RunContext, platform: str, url_pattern: str | None = None) -> list[dict[str, Any]]: """Accounts of persons or organisations on one platform; each row carries an `owner` dict for Request.meta. `url_pattern` also picks up accounts stored as plain websites, so it must match only that platform's own host (never a generic shape like "https://host/@user", which TikTok and others share). """ if ctx.conn is None: return [] rows = ctx.conn.execute( """SELECT person_id, organization_id, url, handle, verified FROM social_account WHERE (platform = %s OR (%s::text IS NOT NULL AND url ~* %s)) AND (person_id IS NOT NULL OR organization_id IS NOT NULL) ORDER BY verified DESC, id""", (platform, url_pattern, url_pattern), ).fetchall() for row in rows: row["owner"] = {"person_id": str(row["person_id"]) if row["person_id"] else None, "org_id": None if row["person_id"] else str(row["organization_id"]), "verified": row["verified"]} return rows def post_claim(*, owner: dict[str, Any], platform: str, url: str, text: str, created: Any, title: str | None = None, data: dict | None = None) -> Claim: """A public post as a `statement` claim of the account owner (person, or organisation for party channels).""" label = PLATFORM_LABELS[platform] return Claim( claim_type="statement", occurred_at=created, person=PersonRef(ids={"politrack": owner["person_id"]}) if owner.get("person_id") else None, org=OrgRef(ids={"politrack_org": owner["org_id"]}, create=False) if owner.get("org_id") else None, summary_de=f"Auf {label}: „{excerpt(title or text, 200)}“", summary_en=f"On {label}: \"{excerpt(title or text, 200)}\"", data={"platform": platform, "account_verified": owner.get("verified", False), **(data or {})}, evidence=[Evidence(url=url, source=platform, doc_type="social_post", title=excerpt(title or text, 120), published_at=created, full_text=text, excerpt=text, quote=excerpt(text, 500))], facet=Statement(context="social_post", url=url, said_at=created, platform=platform, title=title, excerpt=text), ) @register class BlueskyPosts(Adapter): spec = BLUESKY name = "social.bluesky" description = "Public Bluesky posts of politicians" schedule = "7 * * * *" queue = "social" def start(self, ctx: RunContext) -> Iterator[Request]: for acc in accounts(ctx, "bluesky", r"bsky\.app/profile/"): m = re.search(r"bsky\.app/profile/([^/?#]+)", acc["url"]) if m: yield Request(url="https://public.api.bsky.app/xrpc/app.bsky.feed.getAuthorFeed", params={"actor": m.group(1), "limit": 50, "filter": "posts_no_replies"}, kind="feed", meta={**acc["owner"], "handle": m.group(1)}) def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]: handle = response.meta["handle"] for item in response.json().get("feed", []): post = item.get("post") or {} if item.get("reason"): # repost of someone else's post continue record = post.get("record") or {} text = record.get("text") or "" author = (post.get("author") or {}).get("handle") or handle rkey = post.get("uri", "").rsplit("/", 1)[-1] yield post_claim(owner=response.meta, platform="bluesky", url=f"https://bsky.app/profile/{author}/post/{rkey}", text=text, created=record.get("createdAt"), data={"likes": post.get("likeCount"), "reposts": post.get("repostCount")}) @register class MastodonPosts(Adapter): spec = MASTODON name = "social.mastodon" description = "Public Mastodon/Fediverse posts of politicians" schedule = "17 * * * *" queue = "social" def start(self, ctx: RunContext) -> Iterator[Request]: for acc in accounts(ctx, "mastodon"): # instances have arbitrary hosts: platform only m = re.match(r"^https?://([^/]+)/@([^/@]+)", acc["url"]) if m: host, user = m.groups() yield Request(url=f"https://{host}/api/v1/accounts/lookup", params={"acct": user}, kind="lookup", meta={**acc["owner"], "host": host}) def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: meta = response.meta if response.request.kind == "lookup": account_id = response.json().get("id") if account_id: yield Request(url=f"https://{meta['host']}/api/v1/accounts/{account_id}/statuses", params={"limit": 40, "exclude_replies": "true", "exclude_reblogs": "true"}, kind="statuses", meta=meta) return for status in response.json(): if status.get("visibility") not in (None, "public"): continue yield post_claim(owner=meta, platform="mastodon", url=status.get("url") or status.get("uri"), text=html_to_text(status.get("content")), created=status.get("created_at"), data={"favourites": status.get("favourites_count"), "reblogs": status.get("reblogs_count")}) @register class TelegramChannels(Adapter): spec = TELEGRAM name = "social.telegram" description = "Public Telegram broadcast channels of politicians (web preview, latest posts)" schedule = "27 */2 * * *" queue = "social" def start(self, ctx: RunContext) -> Iterator[Request]: for acc in accounts(ctx, "telegram", r"(t\.me|telegram\.me)/"): m = re.search(r"(?:t\.me|telegram\.me)/(?:s/)?([A-Za-z0-9_]{4,})", acc["url"]) if m: yield Request(url=f"https://t.me/s/{m.group(1)}", kind="channel", meta={**acc["owner"], "channel": m.group(1)}) def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]: soup = response.soup subscribers = soup.select_one(".tgme_channel_info_counter .counter_value") for msg in soup.select("div.tgme_widget_message[data-post]"): text_el = msg.select_one(".tgme_widget_message_text") time_el = msg.select_one(".tgme_widget_message_date time") text = text_el.get_text("\n", strip=True) if text_el else "" if not text: continue # media-only posts forwarded = msg.select_one(".tgme_widget_message_forwarded_from") views = msg.select_one(".tgme_widget_message_views") yield post_claim( owner=response.meta, platform="telegram", url=f"https://t.me/{msg['data-post']}", text=text, created=time_el.get("datetime") if time_el else None, data={"views": views.get_text(strip=True) if views else None, "forwarded_from": forwarded.get_text(" ", strip=True) if forwarded else None, "channel_subscribers": subscribers.get_text(strip=True) if subscribers else None}, ) @register class YouTubeVideos(Adapter): spec = YOUTUBE name = "social.youtube" description = "Latest YouTube uploads of politicians' channels (requires POLITRACK_YOUTUBE_API_KEY)" schedule = "37 */6 * * *" queue = "social" def start(self, ctx: RunContext) -> Iterator[Request]: key = get_settings().youtube_api_key if not key: ctx.stats["skipped_no_api_key"] = 1 return api = "https://www.googleapis.com/youtube/v3/channels" for acc in accounts(ctx, "youtube", r"youtube\.com/"): meta = acc["owner"] if m := re.search(r"youtube\.com/channel/(UC[\w-]+)", acc["url"]): params = {"id": m.group(1)} elif m := re.search(r"youtube\.com/@([\w.-]+)", acc["url"]): params = {"forHandle": f"@{m.group(1)}"} elif m := re.search(r"youtube\.com/(?:user/|c/)?([\w.-]+)/?$", acc["url"]): params = {"forUsername": m.group(1)} else: continue yield Request(url=api, params={**params, "part": "contentDetails", "key": key}, kind="channel", meta=meta) def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: data = response.json() meta = response.meta if response.request.kind == "channel": for item in data.get("items", []): uploads = item["contentDetails"]["relatedPlaylists"]["uploads"] yield Request(url="https://www.googleapis.com/youtube/v3/playlistItems", kind="uploads", meta=meta, params={"playlistId": uploads, "part": "snippet", "maxResults": 25, "key": get_settings().youtube_api_key}) return for item in data.get("items", []): snippet = item.get("snippet") or {} video = (snippet.get("resourceId") or {}).get("videoId") if not video: continue yield post_claim(owner=meta, platform="youtube", url=f"https://www.youtube.com/watch?v={video}", title=snippet.get("title"), text=f"{snippet.get('title', '')}\n{snippet.get('description', '')}".strip(), created=snippet.get("publishedAt")) @register class XPosts(Adapter): spec = X name = "social.x" description = "Posts of politicians on X via the official API v2 (requires POLITRACK_X_BEARER_TOKEN)" schedule = "47 */6 * * *" queue = "social" def start(self, ctx: RunContext) -> Iterator[Request]: token = get_settings().x_bearer_token if not token: ctx.stats["skipped_no_api_key"] = 1 return for acc in accounts(ctx, "twitter", r"//(www\.)?(twitter|x)\.com/"): m = re.search(r"(?:twitter|x)\.com/(?:#!/)?@?(\w{1,15})/?$", acc["url"]) if m: yield Request(url=f"{X_API}users/by/username/{m.group(1)}", kind="user", headers=self.auth(), meta=acc["owner"]) @staticmethod def auth() -> dict[str, str]: return {"Authorization": f"Bearer {get_settings().x_bearer_token}"} def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: data = response.json() meta = response.meta if response.request.kind == "user": user = data.get("data") or {} if user.get("id"): yield Request(url=f"{X_API}users/{user['id']}/tweets", kind="tweets", headers=self.auth(), params={"max_results": 50, "exclude": "replies,retweets", "tweet.fields": "created_at,public_metrics"}, meta={**meta, "username": user.get("username")}) return for tweet in data.get("data") or []: metrics = tweet.get("public_metrics") or {} yield post_claim(owner=meta, platform="x", url=f"https://x.com/{meta['username']}/status/{tweet['id']}", text=tweet.get("text", ""), created=tweet.get("created_at"), data={"likes": metrics.get("like_count"), "reposts": metrics.get("retweet_count")})