Mark posts that disappear from X instead of erasing them; pin mcp below 2

social.x_deletions checks stored posts through the official API. A post X
reports as not found keeps a marked record (author, link, dates, Internet
Archive link if a copy exists) while every copy of its text is removed, as
the Developer Agreement requires. A post that shows up in the timeline again
is restored by the normal upsert.

mcp 2.x removed FastMCP, which crash-looped the MCP server on first deploy.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

AGENTS.md

@@ -36,8 +36,8 @@ next-intl, German by default, English at `/en`. Deployed with Docker Compose on
 3. **Fairness.** Neutral wording. Legal matters carry the procedural stage and "Es gilt die
    Unschuldsvermutung". Automatically detected legal claims stay in review (`review_status = pending`).
    Connections are not accusations. Personal data only in the person's public role.
-4. **Not public yet.** The instance is private as of October 2026. The X deletion sync (roadmap) is
-   deferred until launch, but must exist before the instance goes public or an X token is configured.
+4. **Not public yet.** The instance is private (LAN only) as of November 2026. Posts that disappear from
+   X are marked, not erased, but their text must go (`social.x_deletions`, docs/legal.md).
 5. **Commits:** only when the owner asks, as `Simon Diesenreiter <simon@disi.dev>`, not the global
    (work) address: `git -c user.email=simon@disi.dev commit …`, unless the repo-local config sets it.
 

backend/politrack/core/loader.py

@@ -20,6 +20,7 @@ from psycopg.types.json import Jsonb
 
 from politrack.core.adapter import RunContext
 from politrack.core.records import (
+    ArchiveCopy,
     Claim,
     Document,
     Event,
@@ -31,6 +32,7 @@ from politrack.core.records import (
     Participation,
     PersonRecord,
     PersonRef,
+    PostRemoved,
     Role,
     SocialAccount,
     Statement,
@@ -76,6 +78,10 @@ class Loader:
             return self.vote_event(record)
         if isinstance(record, Event):
             return self.event(record)
+        if isinstance(record, PostRemoved):
+            return self.post_removed(record)
+        if isinstance(record, ArchiveCopy):
+            return self.archive_copy(record)
         raise TypeError(f"unknown record type {type(record).__name__}")
 
     # -- documents --------------------------------------------------------------------
@@ -109,6 +115,48 @@ class Loader:
         self.ctx.stats["documents"] += 1
         return row["id"]
 
+    def archive_copy(self, rec: ArchiveCopy) -> None:
+        self.ctx.stats["archive_copies"] += self.conn.execute(
+            "UPDATE document SET archive_url = %s WHERE url = %s AND archive_url IS NULL", (rec.archive_url, rec.url)
+        ).rowcount
+
+    def post_removed(self, rec: PostRemoved) -> None:
+        """Keep the record that a post existed and disappeared, without any of the platform's text.
+
+        The text is copied to several places when a post is loaded; all of them are cleared here. If the
+        post comes back in a later timeline fetch, the normal upsert restores the text and the flag.
+        """
+        claims = self.conn.execute(
+            "SELECT claim_id FROM statement WHERE url = %s AND platform = %s AND context = 'social_post'",
+            (rec.url, rec.platform),
+        ).fetchall()
+        if not claims:
+            self.ctx.stats["removed_post_unknown"] += 1
+            return
+        day = rec.detected_at.date()
+        marker = {"removed_from_platform": True, "removed_detected_at": rec.detected_at.isoformat()}
+        for row in claims:
+            claim_id = row["claim_id"]
+            self.conn.execute("UPDATE statement SET excerpt = NULL, title = NULL WHERE claim_id = %s", (claim_id,))
+            self.conn.execute("UPDATE claim_evidence SET quote = NULL WHERE claim_id = %s", (claim_id,))
+            self.conn.execute("DELETE FROM chunk WHERE claim_id = %s", (claim_id,))
+            self.conn.execute(
+                """UPDATE claim SET summary_de = %s, summary_en = %s, updated_at = now(),
+                       data = (data - 'likes' - 'reposts') || %s
+                   WHERE id = %s""",
+                (f"Beitrag auf {rec.platform_label}, dort nicht mehr verfügbar (festgestellt am {day:%d.%m.%Y})",
+                 f"Post on {rec.platform_label}, no longer available there (detected on {day:%Y-%m-%d})",
+                 Jsonb(marker), claim_id),
+            )
+        self.conn.execute(
+            "DELETE FROM chunk WHERE document_id IN (SELECT id FROM document WHERE url = %s)", (rec.url,)
+        )
+        self.conn.execute(
+            """UPDATE document SET title = %s, excerpt = NULL, full_text = NULL, meta = meta || %s WHERE url = %s""",
+            (f"Beitrag auf {rec.platform_label} (nicht mehr verfügbar)", Jsonb(marker), rec.url),
+        )
+        self.ctx.stats["posts_removed"] += 1
+
     # -- entities ---------------------------------------------------------------------
 
     def person(self, rec: PersonRecord) -> int | None:

backend/politrack/core/records.py

@@ -247,7 +247,29 @@ class Claim:
     facet: Facet | None = None
 
 
-Record = Union[PersonRecord, OrgRecord, SocialAccount, VoteEvent, Event, Claim, Document]
+@dataclass
+class PostRemoved:
+    """A stored post is no longer available on its platform.
+
+    The Loader removes the platform's text everywhere and keeps a marked record (who, when, link), so the
+    fact that the post existed and disappeared stays on the record. See docs/legal.md.
+    """
+
+    url: str
+    platform: str
+    platform_label: str
+    detected_at: datetime
+
+
+@dataclass
+class ArchiveCopy:
+    """A public archive (Internet Archive) holds a copy of this document."""
+
+    url: str
+    archive_url: str
+
+
+Record = Union[PersonRecord, OrgRecord, SocialAccount, VoteEvent, Event, Claim, Document, PostRemoved, ArchiveCopy]
 
 
 def to_jsonable(obj: Any) -> Any:

backend/politrack/sources/social.py

@@ -18,7 +18,7 @@ from typing import Any
 
 from politrack.config import get_settings
 from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
-from politrack.core.records import Claim, Evidence, OrgRef, PersonRef, Statement
+from politrack.core.records import ArchiveCopy, Claim, Evidence, OrgRef, PersonRef, PostRemoved, Statement
 from politrack.text import excerpt, html_to_text
 
 
@@ -43,8 +43,8 @@ X = SourceSpec(slug="x", name="X (Twitter)", kind="social", homepage="https://x.
                store_fulltext=False, keep_raw=False, min_interval=2.0, robots_exempt=(X_API,),
                tos_notes="Official X API v2 only (paid tier, POLITRACK_X_BEARER_TOKEN). Scraping x.com or using "
                          "mirrors/proxies (Nitter etc.) violates the ToS and is not done. The Developer Agreement requires "
-                         "removing posts that were deleted on X; no raw archive or full text is kept, but the "
-                         "deletion sync is still open (docs/roadmap.md).")
+                         "removing posts that were deleted on X: no raw archive or full text is kept, and "
+                         "social.x_deletions removes the stored excerpt of such posts, keeping only a marked record.")
 
 PLATFORM_LABELS = {"bluesky": "Bluesky", "mastodon": "Mastodon", "telegram": "Telegram", "youtube": "YouTube",
                    "x": "X"}
@@ -244,6 +244,7 @@ class XPosts(Adapter):
     description = "Posts of politicians on X via the official API v2 (requires POLITRACK_X_BEARER_TOKEN)"
     schedule = "47 */6 * * *"
     queue = "social"
+    parser_version = 2  # data.removed_from_platform: a post seen in the timeline again is no longer removed
 
     def start(self, ctx: RunContext) -> Iterator[Request]:
         token = get_settings().x_bearer_token
@@ -276,4 +277,62 @@ class XPosts(Adapter):
             yield post_claim(owner=meta, platform="x",
                              url=f"https://x.com/{meta['username']}/status/{tweet['id']}", text=tweet.get("text", ""),
                              created=tweet.get("created_at"),
-                             data={"likes": metrics.get("like_count"), "reposts": metrics.get("retweet_count")})
+                             data={"likes": metrics.get("like_count"), "reposts": metrics.get("retweet_count"),
+                                   "removed_from_platform": False})
+
+
+X_LOOKUP_BATCH = 100  # maximum ids per request on GET /2/tweets
+WAYBACK_API = "https://archive.org/wayback/available"
+
+
+@register
+class XDeletionSync(Adapter):
+    """Finds stored X posts that no longer exist there, so their text can be removed (Developer Agreement).
+
+    Only "resource-not-found" counts: posts that are merely protected or whose author is suspended come back
+    with other error types and stay untouched. Runs four times a day because deletions must be honoured promptly.
+    """
+
+    spec = X
+    name = "social.x_deletions"
+    description = "Marks stored X posts that are gone from X and removes their text (requires POLITRACK_X_BEARER_TOKEN)"
+    schedule = "17 */6 * * *"
+    queue = "social"
+
+    def start(self, ctx: RunContext) -> Iterator[Request]:
+        if not get_settings().x_bearer_token:
+            ctx.stats["skipped_no_api_key"] = 1
+            return
+        if ctx.conn is None:
+            return
+        rows = ctx.conn.execute(
+            """SELECT st.url FROM statement st JOIN claim c ON c.id = st.claim_id
+               WHERE st.platform = 'x' AND st.context = 'social_post'
+                 AND NOT COALESCE((c.data->>'removed_from_platform')::boolean, FALSE)
+               ORDER BY st.said_at DESC NULLS LAST"""
+        ).fetchall()
+        urls = {m.group(1): row["url"] for row in rows if (m := re.search(r"/status/(\d+)", row["url"]))}
+        ids = list(urls)
+        for i in range(0, len(ids), X_LOOKUP_BATCH):
+            batch = ids[i:i + X_LOOKUP_BATCH]
+            yield Request(url=f"{X_API}tweets", kind="lookup", headers=XPosts.auth(), params={"ids": ",".join(batch)},
+                          meta={"urls": {post_id: urls[post_id] for post_id in batch}},
+                          conditional=False, skip_unchanged=False)
+
+    def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
+        if response.request.kind == "wayback":
+            closest = (response.json().get("archived_snapshots") or {}).get("closest") or {}
+            if closest.get("available") and str(closest.get("status")) == "200" and closest.get("url"):
+                yield ArchiveCopy(url=response.meta["url"],
+                                  archive_url=closest["url"].replace("http://", "https://", 1))
+            return
+        urls = response.meta["urls"]
+        for error in response.json().get("errors") or []:
+            if error.get("resource_type") != "tweet" or not str(error.get("type", "")).endswith("/resource-not-found"):
+                continue
+            url = urls.get(str(error.get("resource_id")))
+            if url:
+                yield PostRemoved(url=url, platform="x", platform_label=PLATFORM_LABELS["x"],
+                                  detected_at=response.fetched_at)
+                yield Request(url=WAYBACK_API, kind="wayback", params={"url": url}, meta={"url": url},
+                              conditional=False, skip_unchanged=False)

backend/pyproject.toml

@@ -19,7 +19,7 @@ dependencies = [
     "procrastinate>=2.14,<4",
     "typer>=0.12",
     "pyyaml>=6.0",
-    "mcp>=1.9",
+    "mcp>=1.9,<2",
 ]
 
 # Third-party adapters register themselves here, e.g. in another package's pyproject:

backend/tests/fixtures/social.x_deletions/lookup.expected.json

@@ -0,0 +1,25 @@
+[
+ {
+  "__type__": "PostRemoved",
+  "detected_at": "2026-01-01T00:00:00+00:00",
+  "platform": "x",
+  "platform_label": "X",
+  "url": "https://x.com/example/status/222"
+ },
+ {
+  "__type__": "Request",
+  "conditional": false,
+  "headers": null,
+  "json": null,
+  "kind": "wayback",
+  "meta": {
+   "url": "https://x.com/example/status/222"
+  },
+  "method": "GET",
+  "params": {
+   "url": "https://x.com/example/status/222"
+  },
+  "skip_unchanged": false,
+  "url": "https://archive.org/wayback/available"
+ }
+]

backend/tests/fixtures/social.x_deletions/lookup.json

@@ -0,0 +1,25 @@
+{
+  "data": [
+    {"id": "111", "text": "Still online"}
+  ],
+  "errors": [
+    {
+      "value": "222",
+      "detail": "Could not find tweet with ids: [222].",
+      "title": "Not Found Error",
+      "resource_type": "tweet",
+      "parameter": "ids",
+      "resource_id": "222",
+      "type": "https://api.twitter.com/2/problems/resource-not-found"
+    },
+    {
+      "value": "333",
+      "detail": "Sorry, you are not authorized to see the Tweet with ids: [333].",
+      "title": "Authorization Error",
+      "resource_type": "tweet",
+      "parameter": "ids",
+      "resource_id": "333",
+      "type": "https://api.twitter.com/2/problems/not-authorized-for-resource"
+    }
+  ]
+}

backend/tests/fixtures/social.x_deletions/lookup.request.json

@@ -0,0 +1,18 @@
+{
+  "url": "https://api.x.com/2/tweets",
+  "kind": "lookup",
+  "method": "GET",
+  "params": {
+    "ids": "111,222,333"
+  },
+  "json": null,
+  "meta": {
+    "urls": {
+      "111": "https://x.com/example/status/111",
+      "222": "https://x.com/example/status/222",
+      "333": "https://x.com/example/status/333"
+    }
+  },
+  "content_type": "application/json; charset=utf-8",
+  "body_file": "lookup.json"
+}

backend/tests/fixtures/social.x_deletions/wayback.expected.json

@@ -0,0 +1,7 @@
+[
+ {
+  "__type__": "ArchiveCopy",
+  "archive_url": "https://web.archive.org/web/20260101000000/https://x.com/example/status/222",
+  "url": "https://x.com/example/status/222"
+ }
+]

backend/tests/fixtures/social.x_deletions/wayback.json

@@ -0,0 +1,11 @@
+{
+  "url": "https://x.com/example/status/222",
+  "archived_snapshots": {
+    "closest": {
+      "status": "200",
+      "available": true,
+      "url": "http://web.archive.org/web/20260101000000/https://x.com/example/status/222",
+      "timestamp": "20260101000000"
+    }
+  }
+}

backend/tests/fixtures/social.x_deletions/wayback.request.json

@@ -0,0 +1,14 @@
+{
+  "url": "https://archive.org/wayback/available",
+  "kind": "wayback",
+  "method": "GET",
+  "params": {
+    "url": "https://x.com/example/status/222"
+  },
+  "json": null,
+  "meta": {
+    "url": "https://x.com/example/status/222"
+  },
+  "content_type": "application/json",
+  "body_file": "wayback.json"
+}

docs/architecture/02-comparison-and-decisions.md

@@ -81,8 +81,9 @@ no public channels and groups are private by design.
 **Context.** Many Austrian politicians are most active on X; proxies such as Nitter work by violating
 X's terms. **Decision.** `social.x` uses the official API v2 with the operator's bearer token and does
 nothing without one. It keeps no raw archive and no full text (`keep_raw=False`, `store_fulltext=False`).
-The Developer Agreement requires removing posts deleted on X; until a deletion sync exists the adapter is
-documented as not production-ready.
+The Developer Agreement requires removing posts deleted on X. For accountability we mark instead of erase:
+`social.x_deletions` clears every copy of the text and keeps a record that the post existed and disappeared
+(a new `PostRemoved` record, so the Loader stays the only writer).
 
 ### ADR-23: Organisation-owned accounts
 **Context.** Party and institution channels (MFG on Telegram, NEOS and the Federal President on Bluesky)

docs/legal.md

@@ -11,7 +11,7 @@
 * **Database right** (§76c ff. UrhG): systematically extracting substantial parts of someone else's curated database (for example MeineAbgeordneten) is not allowed without permission, so we ask for partnerships instead.
 * **ToS**: we never scrape platforms whose terms prohibit it (X, Meta, most newspapers). We use official APIs or leave the source out. Each source's assessment is recorded in its `SourceSpec.tos_notes` (shown on the method page and in `/api/v1/sources`).
 * **Raw archive**: responses are archived only where the publisher's terms allow it (`SourceSpec.keep_raw`) and never for TDM-reserved sites. The archive serves re-parsing and auditability, not republication.
-* **X API**: the Developer Agreement requires removing content of posts that were deleted on X. `social.x` keeps no raw archive or full text, but the deletion sync is still open – don't enable it in production before it exists (see roadmap).
+* **X API**: the Developer Agreement requires removing content of posts that were deleted on X. `social.x` keeps no raw archive or full text, and `social.x_deletions` checks stored posts four times a day. For accountability, a post that disappears is **marked, not erased**: author, link, dates and a neutral "no longer available" note stay, plus an Internet Archive link if a copy exists, while every copy of X's text is removed. Keeping the text itself would need X's written permission (Twitter cut off Politwoops over this in 2015 and restored access only after an agreement). Check the current terms before enabling `social.x`.
 * **Messaging platforms**: only public Telegram broadcast channels through Telegram's public web preview. No WhatsApp (no public read access, terms forbid automated collection), no Signal, no private groups.
 
 ## 2. Personal data (GDPR / DSG)

docs/roadmap.md

@@ -31,6 +31,7 @@ Next.js frontend (de/en), Docker Compose for Unraid.
 - [x] Political ads: Google (bundle), Meta Ad Library (opt-in token)
 - [x] Social: Bluesky, Mastodon, public Telegram channels, YouTube (key), X via official API (token);
       party/institution channels; curated `social_accounts.yaml`
+- [x] X deletion sync (`social.x_deletions`): posts gone from X keep a marked record without their text
 - [x] Pages: organisations, money explorer, social feed, Wikipedia edits, person network ("follow the money"),
       say-vs-vote on roll calls
 
@@ -43,7 +44,6 @@ Next.js frontend (de/en), Docker Compose for Unraid.
 ## Next
 
 **Data**
-- [ ] X deletion sync (Developer Agreement: remove posts deleted on X) – required before enabling `social.x` in production
 - [ ] Full speech texts and interjections from the protocols; committee memberships (list 250)
 - [ ] HowTheyVote.eu / EP roll-call votes of Austrian MEPs
 - [ ] EU Council participant lists, ministries' press releases, Federal President's schedule

docs/sources.md

@@ -105,7 +105,7 @@ Accounts come from parliament profiles (verified), Wikidata (unverified until co
 | Mastodon / Fediverse | public REST API of the account's instance | ✅ `social.mastodon` |
 | Telegram – public broadcast channels | public web preview `t.me/s/<channel>` (no login, no robots.txt restriction); latest posts | ✅ `social.telegram` |
 | YouTube | Data API v3 (free quota, `POLITRACK_YOUTUBE_API_KEY`) | ✅ `social.youtube` (opt-in) |
-| X (Twitter) | **official API v2 only** (paid read access, `POLITRACK_X_BEARER_TOKEN`). No raw archive or full text kept; posts deleted on X must be removed (deletion sync still open, see roadmap) | ✅ `social.x` (opt-in) |
+| X (Twitter) | **official API v2 only** (paid read access, `POLITRACK_X_BEARER_TOKEN`). No raw archive or full text kept; posts gone from X keep a marked record without their text (`social.x_deletions`) | ✅ `social.x` (opt-in) |
 | Facebook / Instagram | ToS forbid scraping; Meta Content Library is for researchers only | ⛔ posts · ✅ political ads via the Ad Library API |
 | TikTok | Research API (academic only) | ⛔ |
 

web/messages/de.json

@@ -542,6 +542,8 @@
     "intro": "Öffentliche Beiträge von Politiker:innen und Parteien auf Bluesky, Mastodon, öffentlichen Telegram-Kanälen und YouTube.",
     "none": "Keine Beiträge.",
     "original": "Original ansehen",
+    "removed": "Dieser Beitrag ist auf der Plattform nicht mehr verfügbar (festgestellt am {date}). Seinen Text zeigen wir daher nicht mehr.",
+    "archivedCopy": "Kopie im Internet Archive",
     "unverified": "Konto nicht bestätigt",
     "unverifiedHint": "Das Konto stammt aus Wikidata und wurde von keiner offiziellen Seite bestätigt.",
     "platform": "Plattform",

web/messages/en.json

@@ -542,6 +542,8 @@
     "intro": "Public posts by politicians and parties on Bluesky, Mastodon, public Telegram channels and YouTube.",
     "none": "No posts.",
     "original": "View original",
+    "removed": "This post is no longer available on the platform (detected on {date}), so we no longer show its text.",
+    "archivedCopy": "Copy in the Internet Archive",
     "unverified": "account unconfirmed",
     "unverifiedHint": "The account comes from Wikidata and was not confirmed by an official page.",
     "platform": "Platform",

web/src/components/SocialFeed.tsx

@@ -16,7 +16,10 @@ export async function SocialFeed({posts, showAuthor = true}: {posts: SocialPost[
   if (!posts.length) return <EmptyState icon="message">{t('none')}</EmptyState>;
   return (
     <ul className="feed-grid">
-      {posts.map((post) => (
+      {posts.map((post) => {
+        const removed = post.data.removed_from_platform === true;
+        const archived = post.evidence.find((ev) => ev.archive_url)?.archive_url;
+        return (
         <li key={post.id}>
           <article className="post">
             <div className="post-head">
@@ -30,10 +33,22 @@ export async function SocialFeed({posts, showAuthor = true}: {posts: SocialPost[
             </div>
             {post.title && <p className="post-title">{post.title}</p>}
             {post.excerpt && <p className="post-text">{post.excerpt}</p>}
-            <p className="post-foot"><ExternalLink href={post.url}>{t('original')}</ExternalLink></p>
+            {removed && (
+              <p className="post-text muted">
+                {t('removed', {date: formatDate(String(post.data.removed_detected_at), locale)})}
+              </p>
+            )}
+            {(!removed || archived) && (
+              <p className="post-foot">
+                {removed
+                  ? <ExternalLink href={archived as string}>{t('archivedCopy')}</ExternalLink>
+                  : <ExternalLink href={post.url}>{t('original')}</ExternalLink>}
+              </p>
+            )}
           </article>
         </li>
-      ))}
+        );
+      })}
     </ul>
   );
 }