Mark posts that disappear from X instead of erasing them; pin mcp below 2
social.x_deletions checks stored posts through the official API. A post X reports as not found keeps a marked record (author, link, dates, Internet Archive link if a copy exists) while every copy of its text is removed, as the Developer Agreement requires. A post that shows up in the timeline again is restored by the normal upsert. mcp 2.x removed FastMCP, which crash-looped the MCP server on first deploy. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0e21730b9f
commit
693c32a5e9
18 files changed
+265
-16
No files matched your search
@@ -18,7 +18,7 @@ from typing import Any
|
||||
|
||||
from politrack.config import get_settings
|
||||
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
||||
from politrack.core.records import Claim, Evidence, OrgRef, PersonRef, Statement
|
||||
from politrack.core.records import ArchiveCopy, Claim, Evidence, OrgRef, PersonRef, PostRemoved, Statement
|
||||
from politrack.text import excerpt, html_to_text
|
||||
|
||||
|
||||
@@ -43,8 +43,8 @@ X = SourceSpec(slug="x", name="X (Twitter)", kind="social", homepage="https://x.
|
||||
store_fulltext=False, keep_raw=False, min_interval=2.0, robots_exempt=(X_API,),
|
||||
tos_notes="Official X API v2 only (paid tier, POLITRACK_X_BEARER_TOKEN). Scraping x.com or using "
|
||||
"mirrors/proxies (Nitter etc.) violates the ToS and is not done. The Developer Agreement requires "
|
||||
"removing posts that were deleted on X; no raw archive or full text is kept, but the "
|
||||
"deletion sync is still open (docs/roadmap.md).")
|
||||
"removing posts that were deleted on X: no raw archive or full text is kept, and "
|
||||
"social.x_deletions removes the stored excerpt of such posts, keeping only a marked record.")
|
||||
|
||||
PLATFORM_LABELS = {"bluesky": "Bluesky", "mastodon": "Mastodon", "telegram": "Telegram", "youtube": "YouTube",
|
||||
"x": "X"}
|
||||
@@ -244,6 +244,7 @@ class XPosts(Adapter):
|
||||
description = "Posts of politicians on X via the official API v2 (requires POLITRACK_X_BEARER_TOKEN)"
|
||||
schedule = "47 */6 * * *"
|
||||
queue = "social"
|
||||
parser_version = 2 # data.removed_from_platform: a post seen in the timeline again is no longer removed
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
token = get_settings().x_bearer_token
|
||||
@@ -276,4 +277,62 @@ class XPosts(Adapter):
|
||||
yield post_claim(owner=meta, platform="x",
|
||||
url=f"https://x.com/{meta['username']}/status/{tweet['id']}", text=tweet.get("text", ""),
|
||||
created=tweet.get("created_at"),
|
||||
data={"likes": metrics.get("like_count"), "reposts": metrics.get("retweet_count")})
|
||||
data={"likes": metrics.get("like_count"), "reposts": metrics.get("retweet_count"),
|
||||
"removed_from_platform": False})
|
||||
|
||||
|
||||
X_LOOKUP_BATCH = 100 # maximum ids per request on GET /2/tweets
|
||||
WAYBACK_API = "https://archive.org/wayback/available"
|
||||
|
||||
|
||||
@register
|
||||
class XDeletionSync(Adapter):
|
||||
"""Finds stored X posts that no longer exist there, so their text can be removed (Developer Agreement).
|
||||
|
||||
Only "resource-not-found" counts: posts that are merely protected or whose author is suspended come back
|
||||
with other error types and stay untouched. Runs four times a day because deletions must be honoured promptly.
|
||||
"""
|
||||
|
||||
spec = X
|
||||
name = "social.x_deletions"
|
||||
description = "Marks stored X posts that are gone from X and removes their text (requires POLITRACK_X_BEARER_TOKEN)"
|
||||
schedule = "17 */6 * * *"
|
||||
queue = "social"
|
||||
|
||||
def start(self, ctx: RunContext) -> Iterator[Request]:
|
||||
if not get_settings().x_bearer_token:
|
||||
ctx.stats["skipped_no_api_key"] = 1
|
||||
return
|
||||
if ctx.conn is None:
|
||||
return
|
||||
rows = ctx.conn.execute(
|
||||
"""SELECT st.url FROM statement st JOIN claim c ON c.id = st.claim_id
|
||||
WHERE st.platform = 'x' AND st.context = 'social_post'
|
||||
AND NOT COALESCE((c.data->>'removed_from_platform')::boolean, FALSE)
|
||||
ORDER BY st.said_at DESC NULLS LAST"""
|
||||
).fetchall()
|
||||
urls = {m.group(1): row["url"] for row in rows if (m := re.search(r"/status/(\d+)", row["url"]))}
|
||||
ids = list(urls)
|
||||
for i in range(0, len(ids), X_LOOKUP_BATCH):
|
||||
batch = ids[i:i + X_LOOKUP_BATCH]
|
||||
yield Request(url=f"{X_API}tweets", kind="lookup", headers=XPosts.auth(), params={"ids": ",".join(batch)},
|
||||
meta={"urls": {post_id: urls[post_id] for post_id in batch}},
|
||||
conditional=False, skip_unchanged=False)
|
||||
|
||||
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
||||
if response.request.kind == "wayback":
|
||||
closest = (response.json().get("archived_snapshots") or {}).get("closest") or {}
|
||||
if closest.get("available") and str(closest.get("status")) == "200" and closest.get("url"):
|
||||
yield ArchiveCopy(url=response.meta["url"],
|
||||
archive_url=closest["url"].replace("http://", "https://", 1))
|
||||
return
|
||||
urls = response.meta["urls"]
|
||||
for error in response.json().get("errors") or []:
|
||||
if error.get("resource_type") != "tweet" or not str(error.get("type", "")).endswith("/resource-not-found"):
|
||||
continue
|
||||
url = urls.get(str(error.get("resource_id")))
|
||||
if url:
|
||||
yield PostRemoved(url=url, platform="x", platform_label=PLATFORM_LABELS["x"],
|
||||
detected_at=response.fetched_at)
|
||||
yield Request(url=WAYBACK_API, kind="wayback", params={"url": url}, meta={"url": url},
|
||||
conditional=False, skip_unchanged=False)
|
||||
Reference in new issue
Block a user