Mark posts that disappear from X instead of erasing them; pin mcp below 2

social.x_deletions checks stored posts through the official API. A post X
reports as not found keeps a marked record (author, link, dates, Internet
Archive link if a copy exists) while every copy of its text is removed, as
the Developer Agreement requires. A post that shows up in the timeline again
is restored by the normal upsert.

mcp 2.x removed FastMCP, which crash-looped the MCP server on first deploy.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
simonandClaude Opus 5.5 committed 2026-11-09 00:43:36 +01:00
1 parent 0e21730b9f
commit 693c32a5e9
18 files changed
+265 -16

No files matched your search

+63 -4
View File
@@ -18,7 +18,7 @@ from typing import Any
from politrack.config import get_settings
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Evidence, OrgRef, PersonRef, Statement
from politrack.core.records import ArchiveCopy, Claim, Evidence, OrgRef, PersonRef, PostRemoved, Statement
from politrack.text import excerpt, html_to_text
@@ -43,8 +43,8 @@ X = SourceSpec(slug="x", name="X (Twitter)", kind="social", homepage="https://x.
store_fulltext=False, keep_raw=False, min_interval=2.0, robots_exempt=(X_API,),
tos_notes="Official X API v2 only (paid tier, POLITRACK_X_BEARER_TOKEN). Scraping x.com or using "
"mirrors/proxies (Nitter etc.) violates the ToS and is not done. The Developer Agreement requires "
"removing posts that were deleted on X; no raw archive or full text is kept, but the "
"deletion sync is still open (docs/roadmap.md).")
"removing posts that were deleted on X: no raw archive or full text is kept, and "
"social.x_deletions removes the stored excerpt of such posts, keeping only a marked record.")
PLATFORM_LABELS = {"bluesky": "Bluesky", "mastodon": "Mastodon", "telegram": "Telegram", "youtube": "YouTube",
"x": "X"}
@@ -244,6 +244,7 @@ class XPosts(Adapter):
description = "Posts of politicians on X via the official API v2 (requires POLITRACK_X_BEARER_TOKEN)"
schedule = "47 */6 * * *"
queue = "social"
parser_version = 2 # data.removed_from_platform: a post seen in the timeline again is no longer removed
def start(self, ctx: RunContext) -> Iterator[Request]:
token = get_settings().x_bearer_token
@@ -276,4 +277,62 @@ class XPosts(Adapter):
yield post_claim(owner=meta, platform="x",
url=f"https://x.com/{meta['username']}/status/{tweet['id']}", text=tweet.get("text", ""),
created=tweet.get("created_at"),
data={"likes": metrics.get("like_count"), "reposts": metrics.get("retweet_count")})
data={"likes": metrics.get("like_count"), "reposts": metrics.get("retweet_count"),
"removed_from_platform": False})
X_LOOKUP_BATCH = 100 # maximum ids per request on GET /2/tweets
WAYBACK_API = "https://archive.org/wayback/available"
@register
class XDeletionSync(Adapter):
"""Finds stored X posts that no longer exist there, so their text can be removed (Developer Agreement).
Only "resource-not-found" counts: posts that are merely protected or whose author is suspended come back
with other error types and stay untouched. Runs four times a day because deletions must be honoured promptly.
"""
spec = X
name = "social.x_deletions"
description = "Marks stored X posts that are gone from X and removes their text (requires POLITRACK_X_BEARER_TOKEN)"
schedule = "17 */6 * * *"
queue = "social"
def start(self, ctx: RunContext) -> Iterator[Request]:
if not get_settings().x_bearer_token:
ctx.stats["skipped_no_api_key"] = 1
return
if ctx.conn is None:
return
rows = ctx.conn.execute(
"""SELECT st.url FROM statement st JOIN claim c ON c.id = st.claim_id
WHERE st.platform = 'x' AND st.context = 'social_post'
AND NOT COALESCE((c.data->>'removed_from_platform')::boolean, FALSE)
ORDER BY st.said_at DESC NULLS LAST"""
).fetchall()
urls = {m.group(1): row["url"] for row in rows if (m := re.search(r"/status/(\d+)", row["url"]))}
ids = list(urls)
for i in range(0, len(ids), X_LOOKUP_BATCH):
batch = ids[i:i + X_LOOKUP_BATCH]
yield Request(url=f"{X_API}tweets", kind="lookup", headers=XPosts.auth(), params={"ids": ",".join(batch)},
meta={"urls": {post_id: urls[post_id] for post_id in batch}},
conditional=False, skip_unchanged=False)
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
if response.request.kind == "wayback":
closest = (response.json().get("archived_snapshots") or {}).get("closest") or {}
if closest.get("available") and str(closest.get("status")) == "200" and closest.get("url"):
yield ArchiveCopy(url=response.meta["url"],
archive_url=closest["url"].replace("http://", "https://", 1))
return
urls = response.meta["urls"]
for error in response.json().get("errors") or []:
if error.get("resource_type") != "tweet" or not str(error.get("type", "")).endswith("/resource-not-found"):
continue
url = urls.get(str(error.get("resource_id")))
if url:
yield PostRemoved(url=url, platform="x", platform_label=PLATFORM_LABELS["x"],
detected_at=response.fetched_at)
yield Request(url=WAYBACK_API, kind="wayback", params={"url": url}, meta={"url": url},
conditional=False, skip_unchanged=False)