Mark posts that disappear from X instead of erasing them; pin mcp below 2
social.x_deletions checks stored posts through the official API. A post X reports as not found keeps a marked record (author, link, dates, Internet Archive link if a copy exists) while every copy of its text is removed, as the Developer Agreement requires. A post that shows up in the timeline again is restored by the normal upsert. mcp 2.x removed FastMCP, which crash-looped the MCP server on first deploy. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0e21730b9f
commit
693c32a5e9
18 files changed
+265
-16
No files matched your search
@@ -20,6 +20,7 @@ from psycopg.types.json import Jsonb
|
||||
|
||||
from politrack.core.adapter import RunContext
|
||||
from politrack.core.records import (
|
||||
ArchiveCopy,
|
||||
Claim,
|
||||
Document,
|
||||
Event,
|
||||
@@ -31,6 +32,7 @@ from politrack.core.records import (
|
||||
Participation,
|
||||
PersonRecord,
|
||||
PersonRef,
|
||||
PostRemoved,
|
||||
Role,
|
||||
SocialAccount,
|
||||
Statement,
|
||||
@@ -76,6 +78,10 @@ class Loader:
|
||||
return self.vote_event(record)
|
||||
if isinstance(record, Event):
|
||||
return self.event(record)
|
||||
if isinstance(record, PostRemoved):
|
||||
return self.post_removed(record)
|
||||
if isinstance(record, ArchiveCopy):
|
||||
return self.archive_copy(record)
|
||||
raise TypeError(f"unknown record type {type(record).__name__}")
|
||||
|
||||
# -- documents --------------------------------------------------------------------
|
||||
@@ -109,6 +115,48 @@ class Loader:
|
||||
self.ctx.stats["documents"] += 1
|
||||
return row["id"]
|
||||
|
||||
def archive_copy(self, rec: ArchiveCopy) -> None:
|
||||
self.ctx.stats["archive_copies"] += self.conn.execute(
|
||||
"UPDATE document SET archive_url = %s WHERE url = %s AND archive_url IS NULL", (rec.archive_url, rec.url)
|
||||
).rowcount
|
||||
|
||||
def post_removed(self, rec: PostRemoved) -> None:
|
||||
"""Keep the record that a post existed and disappeared, without any of the platform's text.
|
||||
|
||||
The text is copied to several places when a post is loaded; all of them are cleared here. If the
|
||||
post comes back in a later timeline fetch, the normal upsert restores the text and the flag.
|
||||
"""
|
||||
claims = self.conn.execute(
|
||||
"SELECT claim_id FROM statement WHERE url = %s AND platform = %s AND context = 'social_post'",
|
||||
(rec.url, rec.platform),
|
||||
).fetchall()
|
||||
if not claims:
|
||||
self.ctx.stats["removed_post_unknown"] += 1
|
||||
return
|
||||
day = rec.detected_at.date()
|
||||
marker = {"removed_from_platform": True, "removed_detected_at": rec.detected_at.isoformat()}
|
||||
for row in claims:
|
||||
claim_id = row["claim_id"]
|
||||
self.conn.execute("UPDATE statement SET excerpt = NULL, title = NULL WHERE claim_id = %s", (claim_id,))
|
||||
self.conn.execute("UPDATE claim_evidence SET quote = NULL WHERE claim_id = %s", (claim_id,))
|
||||
self.conn.execute("DELETE FROM chunk WHERE claim_id = %s", (claim_id,))
|
||||
self.conn.execute(
|
||||
"""UPDATE claim SET summary_de = %s, summary_en = %s, updated_at = now(),
|
||||
data = (data - 'likes' - 'reposts') || %s
|
||||
WHERE id = %s""",
|
||||
(f"Beitrag auf {rec.platform_label}, dort nicht mehr verfügbar (festgestellt am {day:%d.%m.%Y})",
|
||||
f"Post on {rec.platform_label}, no longer available there (detected on {day:%Y-%m-%d})",
|
||||
Jsonb(marker), claim_id),
|
||||
)
|
||||
self.conn.execute(
|
||||
"DELETE FROM chunk WHERE document_id IN (SELECT id FROM document WHERE url = %s)", (rec.url,)
|
||||
)
|
||||
self.conn.execute(
|
||||
"""UPDATE document SET title = %s, excerpt = NULL, full_text = NULL, meta = meta || %s WHERE url = %s""",
|
||||
(f"Beitrag auf {rec.platform_label} (nicht mehr verfügbar)", Jsonb(marker), rec.url),
|
||||
)
|
||||
self.ctx.stats["posts_removed"] += 1
|
||||
|
||||
# -- entities ---------------------------------------------------------------------
|
||||
|
||||
def person(self, rec: PersonRecord) -> int | None:
|
||||
|
||||
Reference in new issue
Block a user