Mark posts that disappear from X instead of erasing them; pin mcp below 2

social.x_deletions checks stored posts through the official API. A post X
reports as not found keeps a marked record (author, link, dates, Internet
Archive link if a copy exists) while every copy of its text is removed, as
the Developer Agreement requires. A post that shows up in the timeline again
is restored by the normal upsert.

mcp 2.x removed FastMCP, which crash-looped the MCP server on first deploy.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/politrack/core/loader.py

@@ -20,6 +20,7 @@ from psycopg.types.json import Jsonb
 
 from politrack.core.adapter import RunContext
 from politrack.core.records import (
+    ArchiveCopy,
     Claim,
     Document,
     Event,
@@ -31,6 +32,7 @@ from politrack.core.records import (
     Participation,
     PersonRecord,
     PersonRef,
+    PostRemoved,
     Role,
     SocialAccount,
     Statement,
@@ -76,6 +78,10 @@ class Loader:
             return self.vote_event(record)
         if isinstance(record, Event):
             return self.event(record)
+        if isinstance(record, PostRemoved):
+            return self.post_removed(record)
+        if isinstance(record, ArchiveCopy):
+            return self.archive_copy(record)
         raise TypeError(f"unknown record type {type(record).__name__}")
 
     # -- documents --------------------------------------------------------------------
@@ -109,6 +115,48 @@ class Loader:
         self.ctx.stats["documents"] += 1
         return row["id"]
 
+    def archive_copy(self, rec: ArchiveCopy) -> None:
+        self.ctx.stats["archive_copies"] += self.conn.execute(
+            "UPDATE document SET archive_url = %s WHERE url = %s AND archive_url IS NULL", (rec.archive_url, rec.url)
+        ).rowcount
+
+    def post_removed(self, rec: PostRemoved) -> None:
+        """Keep the record that a post existed and disappeared, without any of the platform's text.
+
+        The text is copied to several places when a post is loaded; all of them are cleared here. If the
+        post comes back in a later timeline fetch, the normal upsert restores the text and the flag.
+        """
+        claims = self.conn.execute(
+            "SELECT claim_id FROM statement WHERE url = %s AND platform = %s AND context = 'social_post'",
+            (rec.url, rec.platform),
+        ).fetchall()
+        if not claims:
+            self.ctx.stats["removed_post_unknown"] += 1
+            return
+        day = rec.detected_at.date()
+        marker = {"removed_from_platform": True, "removed_detected_at": rec.detected_at.isoformat()}
+        for row in claims:
+            claim_id = row["claim_id"]
+            self.conn.execute("UPDATE statement SET excerpt = NULL, title = NULL WHERE claim_id = %s", (claim_id,))
+            self.conn.execute("UPDATE claim_evidence SET quote = NULL WHERE claim_id = %s", (claim_id,))
+            self.conn.execute("DELETE FROM chunk WHERE claim_id = %s", (claim_id,))
+            self.conn.execute(
+                """UPDATE claim SET summary_de = %s, summary_en = %s, updated_at = now(),
+                       data = (data - 'likes' - 'reposts') || %s
+                   WHERE id = %s""",
+                (f"Beitrag auf {rec.platform_label}, dort nicht mehr verfügbar (festgestellt am {day:%d.%m.%Y})",
+                 f"Post on {rec.platform_label}, no longer available there (detected on {day:%Y-%m-%d})",
+                 Jsonb(marker), claim_id),
+            )
+        self.conn.execute(
+            "DELETE FROM chunk WHERE document_id IN (SELECT id FROM document WHERE url = %s)", (rec.url,)
+        )
+        self.conn.execute(
+            """UPDATE document SET title = %s, excerpt = NULL, full_text = NULL, meta = meta || %s WHERE url = %s""",
+            (f"Beitrag auf {rec.platform_label} (nicht mehr verfügbar)", Jsonb(marker), rec.url),
+        )
+        self.ctx.stats["posts_removed"] += 1
+
     # -- entities ---------------------------------------------------------------------
 
     def person(self, rec: PersonRecord) -> int | None:

backend/politrack/core/records.py

@@ -247,7 +247,29 @@ class Claim:
     facet: Facet | None = None
 
 
-Record = Union[PersonRecord, OrgRecord, SocialAccount, VoteEvent, Event, Claim, Document]
+@dataclass
+class PostRemoved:
+    """A stored post is no longer available on its platform.
+
+    The Loader removes the platform's text everywhere and keeps a marked record (who, when, link), so the
+    fact that the post existed and disappeared stays on the record. See docs/legal.md.
+    """
+
+    url: str
+    platform: str
+    platform_label: str
+    detected_at: datetime
+
+
+@dataclass
+class ArchiveCopy:
+    """A public archive (Internet Archive) holds a copy of this document."""
+
+    url: str
+    archive_url: str
+
+
+Record = Union[PersonRecord, OrgRecord, SocialAccount, VoteEvent, Event, Claim, Document, PostRemoved, ArchiveCopy]
 
 
 def to_jsonable(obj: Any) -> Any:

backend/politrack/sources/social.py

@@ -18,7 +18,7 @@ from typing import Any
 
 from politrack.config import get_settings
 from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
-from politrack.core.records import Claim, Evidence, OrgRef, PersonRef, Statement
+from politrack.core.records import ArchiveCopy, Claim, Evidence, OrgRef, PersonRef, PostRemoved, Statement
 from politrack.text import excerpt, html_to_text
 
 
@@ -43,8 +43,8 @@ X = SourceSpec(slug="x", name="X (Twitter)", kind="social", homepage="https://x.
                store_fulltext=False, keep_raw=False, min_interval=2.0, robots_exempt=(X_API,),
                tos_notes="Official X API v2 only (paid tier, POLITRACK_X_BEARER_TOKEN). Scraping x.com or using "
                          "mirrors/proxies (Nitter etc.) violates the ToS and is not done. The Developer Agreement requires "
-                         "removing posts that were deleted on X; no raw archive or full text is kept, but the "
-                         "deletion sync is still open (docs/roadmap.md).")
+                         "removing posts that were deleted on X: no raw archive or full text is kept, and "
+                         "social.x_deletions removes the stored excerpt of such posts, keeping only a marked record.")
 
 PLATFORM_LABELS = {"bluesky": "Bluesky", "mastodon": "Mastodon", "telegram": "Telegram", "youtube": "YouTube",
                    "x": "X"}
@@ -244,6 +244,7 @@ class XPosts(Adapter):
     description = "Posts of politicians on X via the official API v2 (requires POLITRACK_X_BEARER_TOKEN)"
     schedule = "47 */6 * * *"
     queue = "social"
+    parser_version = 2  # data.removed_from_platform: a post seen in the timeline again is no longer removed
 
     def start(self, ctx: RunContext) -> Iterator[Request]:
         token = get_settings().x_bearer_token
@@ -276,4 +277,62 @@ class XPosts(Adapter):
             yield post_claim(owner=meta, platform="x",
                              url=f"https://x.com/{meta['username']}/status/{tweet['id']}", text=tweet.get("text", ""),
                              created=tweet.get("created_at"),
-                             data={"likes": metrics.get("like_count"), "reposts": metrics.get("retweet_count")})
+                             data={"likes": metrics.get("like_count"), "reposts": metrics.get("retweet_count"),
+                                   "removed_from_platform": False})
+
+
+X_LOOKUP_BATCH = 100  # maximum ids per request on GET /2/tweets
+WAYBACK_API = "https://archive.org/wayback/available"
+
+
+@register
+class XDeletionSync(Adapter):
+    """Finds stored X posts that no longer exist there, so their text can be removed (Developer Agreement).
+
+    Only "resource-not-found" counts: posts that are merely protected or whose author is suspended come back
+    with other error types and stay untouched. Runs four times a day because deletions must be honoured promptly.
+    """
+
+    spec = X
+    name = "social.x_deletions"
+    description = "Marks stored X posts that are gone from X and removes their text (requires POLITRACK_X_BEARER_TOKEN)"
+    schedule = "17 */6 * * *"
+    queue = "social"
+
+    def start(self, ctx: RunContext) -> Iterator[Request]:
+        if not get_settings().x_bearer_token:
+            ctx.stats["skipped_no_api_key"] = 1
+            return
+        if ctx.conn is None:
+            return
+        rows = ctx.conn.execute(
+            """SELECT st.url FROM statement st JOIN claim c ON c.id = st.claim_id
+               WHERE st.platform = 'x' AND st.context = 'social_post'
+                 AND NOT COALESCE((c.data->>'removed_from_platform')::boolean, FALSE)
+               ORDER BY st.said_at DESC NULLS LAST"""
+        ).fetchall()
+        urls = {m.group(1): row["url"] for row in rows if (m := re.search(r"/status/(\d+)", row["url"]))}
+        ids = list(urls)
+        for i in range(0, len(ids), X_LOOKUP_BATCH):
+            batch = ids[i:i + X_LOOKUP_BATCH]
+            yield Request(url=f"{X_API}tweets", kind="lookup", headers=XPosts.auth(), params={"ids": ",".join(batch)},
+                          meta={"urls": {post_id: urls[post_id] for post_id in batch}},
+                          conditional=False, skip_unchanged=False)
+
+    def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
+        if response.request.kind == "wayback":
+            closest = (response.json().get("archived_snapshots") or {}).get("closest") or {}
+            if closest.get("available") and str(closest.get("status")) == "200" and closest.get("url"):
+                yield ArchiveCopy(url=response.meta["url"],
+                                  archive_url=closest["url"].replace("http://", "https://", 1))
+            return
+        urls = response.meta["urls"]
+        for error in response.json().get("errors") or []:
+            if error.get("resource_type") != "tweet" or not str(error.get("type", "")).endswith("/resource-not-found"):
+                continue
+            url = urls.get(str(error.get("resource_id")))
+            if url:
+                yield PostRemoved(url=url, platform="x", platform_label=PLATFORM_LABELS["x"],
+                                  detected_at=response.fetched_at)
+                yield Request(url=WAYBACK_API, kind="wayback", params={"url": url}, meta={"url": url},
+                              conditional=False, skip_unchanged=False)

backend/pyproject.toml

@@ -19,7 +19,7 @@ dependencies = [
     "procrastinate>=2.14,<4",
     "typer>=0.12",
     "pyyaml>=6.0",
-    "mcp>=1.9",
+    "mcp>=1.9,<2",
 ]
 
 # Third-party adapters register themselves here, e.g. in another package's pyproject:

backend/tests/fixtures/social.x_deletions/lookup.expected.json

@@ -0,0 +1,25 @@
+[
+ {
+  "__type__": "PostRemoved",
+  "detected_at": "2026-01-01T00:00:00+00:00",
+  "platform": "x",
+  "platform_label": "X",
+  "url": "https://x.com/example/status/222"
+ },
+ {
+  "__type__": "Request",
+  "conditional": false,
+  "headers": null,
+  "json": null,
+  "kind": "wayback",
+  "meta": {
+   "url": "https://x.com/example/status/222"
+  },
+  "method": "GET",
+  "params": {
+   "url": "https://x.com/example/status/222"
+  },
+  "skip_unchanged": false,
+  "url": "https://archive.org/wayback/available"
+ }
+]

backend/tests/fixtures/social.x_deletions/lookup.json

@@ -0,0 +1,25 @@
+{
+  "data": [
+    {"id": "111", "text": "Still online"}
+  ],
+  "errors": [
+    {
+      "value": "222",
+      "detail": "Could not find tweet with ids: [222].",
+      "title": "Not Found Error",
+      "resource_type": "tweet",
+      "parameter": "ids",
+      "resource_id": "222",
+      "type": "https://api.twitter.com/2/problems/resource-not-found"
+    },
+    {
+      "value": "333",
+      "detail": "Sorry, you are not authorized to see the Tweet with ids: [333].",
+      "title": "Authorization Error",
+      "resource_type": "tweet",
+      "parameter": "ids",
+      "resource_id": "333",
+      "type": "https://api.twitter.com/2/problems/not-authorized-for-resource"
+    }
+  ]
+}

backend/tests/fixtures/social.x_deletions/lookup.request.json

@@ -0,0 +1,18 @@
+{
+  "url": "https://api.x.com/2/tweets",
+  "kind": "lookup",
+  "method": "GET",
+  "params": {
+    "ids": "111,222,333"
+  },
+  "json": null,
+  "meta": {
+    "urls": {
+      "111": "https://x.com/example/status/111",
+      "222": "https://x.com/example/status/222",
+      "333": "https://x.com/example/status/333"
+    }
+  },
+  "content_type": "application/json; charset=utf-8",
+  "body_file": "lookup.json"
+}

backend/tests/fixtures/social.x_deletions/wayback.expected.json

@@ -0,0 +1,7 @@
+[
+ {
+  "__type__": "ArchiveCopy",
+  "archive_url": "https://web.archive.org/web/20260101000000/https://x.com/example/status/222",
+  "url": "https://x.com/example/status/222"
+ }
+]

backend/tests/fixtures/social.x_deletions/wayback.json

@@ -0,0 +1,11 @@
+{
+  "url": "https://x.com/example/status/222",
+  "archived_snapshots": {
+    "closest": {
+      "status": "200",
+      "available": true,
+      "url": "http://web.archive.org/web/20260101000000/https://x.com/example/status/222",
+      "timestamp": "20260101000000"
+    }
+  }
+}

backend/tests/fixtures/social.x_deletions/wayback.request.json

@@ -0,0 +1,14 @@
+{
+  "url": "https://archive.org/wayback/available",
+  "kind": "wayback",
+  "method": "GET",
+  "params": {
+    "url": "https://x.com/example/status/222"
+  },
+  "json": null,
+  "meta": {
+    "url": "https://x.com/example/status/222"
+  },
+  "content_type": "application/json",
+  "body_file": "wayback.json"
+}