politics-tracking/backend/politrack/sources/social.py

"""Public posts of politicians on open platforms: Bluesky, Mastodon, Telegram (public channels), YouTube,
and X via the official (paid) API when a token is configured.

Accounts come from the `social_account` table: parliament profiles (verified), Wikidata
P2002/P12361/P4033/P3789/P2397 (unverified until confirmed) and the curated config/social_accounts.yaml
(enricher `config.social_accounts`). Accounts belong to a person or to an organisation (party channels).

Not collected (see docs/sources.md): X without the official API (ToS forbid scraping and proxies such as
Nitter), Facebook/Instagram (ToS), WhatsApp channels (no public read access, ToS forbid automated
collection), Signal (private by design, no public channels), private Telegram groups.
"""

from __future__ import annotations

import re
from collections.abc import Iterator
from typing import Any

from politrack.config import get_settings
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Evidence, OrgRef, PersonRef, Statement
from politrack.text import excerpt, html_to_text


def social_spec(slug: str, name: str, homepage: str, notes: str, robots_exempt: tuple[str, ...] = ()) -> SourceSpec:
    return SourceSpec(slug=slug, name=name, kind="social", homepage=homepage, license="Per-post copyright of the author",
                      access="api", trust=40, store_fulltext=True, keep_raw=True, min_interval=2.0,
                      robots_exempt=robots_exempt, tos_notes=notes)


BLUESKY = social_spec("bluesky", "Bluesky", "https://bsky.app",
                      "Public AT Protocol AppView API, no auth for public reads. Posts by the politicians themselves.")
MASTODON = social_spec("mastodon", "Mastodon / Fediverse", "https://joinmastodon.org",
                       "Public REST API of the account's instance (no auth for public statuses). Respect instance rules.")
TELEGRAM = social_spec("telegram", "Telegram (öffentliche Kanäle)", "https://telegram.org",
                       "Public web preview of *public broadcast channels* (t.me/s/<channel>); no robots.txt restrictions. "
                       "Private groups/chats are never accessed. Low frequency, latest posts only.")
YOUTUBE = social_spec("youtube", "YouTube", "https://www.youtube.com",
                      "YouTube Data API v3 (free quota, API key required via POLITRACK_YOUTUBE_API_KEY).")
X_API = "https://api.x.com/2/"
X = SourceSpec(slug="x", name="X (Twitter)", kind="social", homepage="https://x.com",
               license="Per-post copyright of the author; X Developer Agreement", access="api", trust=40,
               store_fulltext=False, keep_raw=False, min_interval=2.0, robots_exempt=(X_API,),
               tos_notes="Official X API v2 only (paid tier, POLITRACK_X_BEARER_TOKEN). Scraping x.com or using "
                         "mirrors/proxies (Nitter etc.) violates the ToS and is not done. The Developer Agreement requires "
                         "removing posts that were deleted on X; no raw archive or full text is kept, but the "
                         "deletion sync is still open (docs/roadmap.md).")

PLATFORM_LABELS = {"bluesky": "Bluesky", "mastodon": "Mastodon", "telegram": "Telegram", "youtube": "YouTube",
                   "x": "X"}


def accounts(ctx: RunContext, platform: str, url_pattern: str | None = None) -> list[dict[str, Any]]:
    """Accounts of persons or organisations on one platform; each row carries an `owner` dict for Request.meta.

    `url_pattern` also picks up accounts stored as plain websites, so it must match only that platform's
    own host (never a generic shape like "https://host/@user", which TikTok and others share).
    """
    if ctx.conn is None:
        return []
    rows = ctx.conn.execute(
        """SELECT person_id, organization_id, url, handle, verified FROM social_account
           WHERE (platform = %s OR (%s::text IS NOT NULL AND url ~* %s))
             AND (person_id IS NOT NULL OR organization_id IS NOT NULL)
           ORDER BY verified DESC, id""",
        (platform, url_pattern, url_pattern),
    ).fetchall()
    for row in rows:
        row["owner"] = {"person_id": str(row["person_id"]) if row["person_id"] else None,
                        "org_id": None if row["person_id"] else str(row["organization_id"]),
                        "verified": row["verified"]}
    return rows


def post_claim(*, owner: dict[str, Any], platform: str, url: str, text: str, created: Any,
               title: str | None = None, data: dict | None = None) -> Claim:
    """A public post as a `statement` claim of the account owner (person, or organisation for party channels)."""
    label = PLATFORM_LABELS[platform]
    return Claim(
        claim_type="statement", occurred_at=created,
        person=PersonRef(ids={"politrack": owner["person_id"]}) if owner.get("person_id") else None,
        org=OrgRef(ids={"politrack_org": owner["org_id"]}, create=False) if owner.get("org_id") else None,
        summary_de=f"Auf {label}: „{excerpt(title or text, 200)}“",
        summary_en=f"On {label}: \"{excerpt(title or text, 200)}\"",
        data={"platform": platform, "account_verified": owner.get("verified", False), **(data or {})},
        evidence=[Evidence(url=url, source=platform, doc_type="social_post", title=excerpt(title or text, 120),
                           published_at=created, full_text=text, excerpt=text, quote=excerpt(text, 500))],
        facet=Statement(context="social_post", url=url, said_at=created, platform=platform, title=title,
                        excerpt=text),
    )


@register
class BlueskyPosts(Adapter):
    spec = BLUESKY
    name = "social.bluesky"
    description = "Public Bluesky posts of politicians"
    schedule = "7 * * * *"
    queue = "social"

    def start(self, ctx: RunContext) -> Iterator[Request]:
        for acc in accounts(ctx, "bluesky", r"bsky\.app/profile/"):
            m = re.search(r"bsky\.app/profile/([^/?#]+)", acc["url"])
            if m:
                yield Request(url="https://public.api.bsky.app/xrpc/app.bsky.feed.getAuthorFeed",
                              params={"actor": m.group(1), "limit": 50, "filter": "posts_no_replies"}, kind="feed",
                              meta={**acc["owner"], "handle": m.group(1)})

    def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
        handle = response.meta["handle"]
        for item in response.json().get("feed", []):
            post = item.get("post") or {}
            if item.get("reason"):  # repost of someone else's post
                continue
            record = post.get("record") or {}
            text = record.get("text") or ""
            author = (post.get("author") or {}).get("handle") or handle
            rkey = post.get("uri", "").rsplit("/", 1)[-1]
            yield post_claim(owner=response.meta, platform="bluesky",
                             url=f"https://bsky.app/profile/{author}/post/{rkey}", text=text,
                             created=record.get("createdAt"),
                             data={"likes": post.get("likeCount"), "reposts": post.get("repostCount")})


@register
class MastodonPosts(Adapter):
    spec = MASTODON
    name = "social.mastodon"
    description = "Public Mastodon/Fediverse posts of politicians"
    schedule = "17 * * * *"
    queue = "social"

    def start(self, ctx: RunContext) -> Iterator[Request]:
        for acc in accounts(ctx, "mastodon"):  # instances have arbitrary hosts: platform only
            m = re.match(r"^https?://([^/]+)/@([^/@]+)", acc["url"])
            if m:
                host, user = m.groups()
                yield Request(url=f"https://{host}/api/v1/accounts/lookup", params={"acct": user}, kind="lookup",
                              meta={**acc["owner"], "host": host})

    def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
        meta = response.meta
        if response.request.kind == "lookup":
            account_id = response.json().get("id")
            if account_id:
                yield Request(url=f"https://{meta['host']}/api/v1/accounts/{account_id}/statuses",
                              params={"limit": 40, "exclude_replies": "true", "exclude_reblogs": "true"},
                              kind="statuses", meta=meta)
            return
        for status in response.json():
            if status.get("visibility") not in (None, "public"):
                continue
            yield post_claim(owner=meta, platform="mastodon", url=status.get("url") or status.get("uri"),
                             text=html_to_text(status.get("content")), created=status.get("created_at"),
                             data={"favourites": status.get("favourites_count"), "reblogs": status.get("reblogs_count")})


@register
class TelegramChannels(Adapter):
    spec = TELEGRAM
    name = "social.telegram"
    description = "Public Telegram broadcast channels of politicians (web preview, latest posts)"
    schedule = "27 */2 * * *"
    queue = "social"

    def start(self, ctx: RunContext) -> Iterator[Request]:
        for acc in accounts(ctx, "telegram", r"(t\.me|telegram\.me)/"):
            m = re.search(r"(?:t\.me|telegram\.me)/(?:s/)?([A-Za-z0-9_]{4,})", acc["url"])
            if m:
                yield Request(url=f"https://t.me/s/{m.group(1)}", kind="channel",
                              meta={**acc["owner"], "channel": m.group(1)})

    def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
        soup = response.soup
        subscribers = soup.select_one(".tgme_channel_info_counter .counter_value")
        for msg in soup.select("div.tgme_widget_message[data-post]"):
            text_el = msg.select_one(".tgme_widget_message_text")
            time_el = msg.select_one(".tgme_widget_message_date time")
            text = text_el.get_text("\n", strip=True) if text_el else ""
            if not text:
                continue  # media-only posts
            forwarded = msg.select_one(".tgme_widget_message_forwarded_from")
            views = msg.select_one(".tgme_widget_message_views")
            yield post_claim(
                owner=response.meta, platform="telegram", url=f"https://t.me/{msg['data-post']}",
                text=text, created=time_el.get("datetime") if time_el else None,
                data={"views": views.get_text(strip=True) if views else None,
                      "forwarded_from": forwarded.get_text(" ", strip=True) if forwarded else None,
                      "channel_subscribers": subscribers.get_text(strip=True) if subscribers else None},
            )


@register
class YouTubeVideos(Adapter):
    spec = YOUTUBE
    name = "social.youtube"
    description = "Latest YouTube uploads of politicians' channels (requires POLITRACK_YOUTUBE_API_KEY)"
    schedule = "37 */6 * * *"
    queue = "social"

    def start(self, ctx: RunContext) -> Iterator[Request]:
        key = get_settings().youtube_api_key
        if not key:
            ctx.stats["skipped_no_api_key"] = 1
            return
        api = "https://www.googleapis.com/youtube/v3/channels"
        for acc in accounts(ctx, "youtube", r"youtube\.com/"):
            meta = acc["owner"]
            if m := re.search(r"youtube\.com/channel/(UC[\w-]+)", acc["url"]):
                params = {"id": m.group(1)}
            elif m := re.search(r"youtube\.com/@([\w.-]+)", acc["url"]):
                params = {"forHandle": f"@{m.group(1)}"}
            elif m := re.search(r"youtube\.com/(?:user/|c/)?([\w.-]+)/?$", acc["url"]):
                params = {"forUsername": m.group(1)}
            else:
                continue
            yield Request(url=api, params={**params, "part": "contentDetails", "key": key}, kind="channel", meta=meta)

    def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
        data = response.json()
        meta = response.meta
        if response.request.kind == "channel":
            for item in data.get("items", []):
                uploads = item["contentDetails"]["relatedPlaylists"]["uploads"]
                yield Request(url="https://www.googleapis.com/youtube/v3/playlistItems", kind="uploads", meta=meta,
                              params={"playlistId": uploads, "part": "snippet", "maxResults": 25,
                                      "key": get_settings().youtube_api_key})
            return
        for item in data.get("items", []):
            snippet = item.get("snippet") or {}
            video = (snippet.get("resourceId") or {}).get("videoId")
            if not video:
                continue
            yield post_claim(owner=meta, platform="youtube",
                             url=f"https://www.youtube.com/watch?v={video}", title=snippet.get("title"),
                             text=f"{snippet.get('title', '')}\n{snippet.get('description', '')}".strip(),
                             created=snippet.get("publishedAt"))


@register
class XPosts(Adapter):
    spec = X
    name = "social.x"
    description = "Posts of politicians on X via the official API v2 (requires POLITRACK_X_BEARER_TOKEN)"
    schedule = "47 */6 * * *"
    queue = "social"

    def start(self, ctx: RunContext) -> Iterator[Request]:
        token = get_settings().x_bearer_token
        if not token:
            ctx.stats["skipped_no_api_key"] = 1
            return
        for acc in accounts(ctx, "twitter", r"//(www\.)?(twitter|x)\.com/"):
            m = re.search(r"(?:twitter|x)\.com/(?:#!/)?@?(\w{1,15})/?$", acc["url"])
            if m:
                yield Request(url=f"{X_API}users/by/username/{m.group(1)}", kind="user", headers=self.auth(),
                              meta=acc["owner"])

    @staticmethod
    def auth() -> dict[str, str]:
        return {"Authorization": f"Bearer {get_settings().x_bearer_token}"}

    def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
        data = response.json()
        meta = response.meta
        if response.request.kind == "user":
            user = data.get("data") or {}
            if user.get("id"):
                yield Request(url=f"{X_API}users/{user['id']}/tweets", kind="tweets", headers=self.auth(),
                              params={"max_results": 50, "exclude": "replies,retweets",
                                      "tweet.fields": "created_at,public_metrics"},
                              meta={**meta, "username": user.get("username")})
            return
        for tweet in data.get("data") or []:
            metrics = tweet.get("public_metrics") or {}
            yield post_claim(owner=meta, platform="x",
                             url=f"https://x.com/{meta['username']}/status/{tweet['id']}", text=tweet.get("text", ""),
                             created=tweet.get("created_at"),
                             data={"likes": metrics.get("like_count"), "reposts": metrics.get("retweet_count")})