"""Corroboration: find claims from different sources that state the same fact and merge them. Merging keeps the oldest claim, copies the duplicate's evidence onto it and marks the duplicate with `merged_into`. Afterwards `refresh_claim_status` marks the survivor "corroborated" when at least two *independent sources* support it. Matchers: * roles: same person, equivalent body/office, overlapping or near-identical periods (e.g. Wikidata "Abgeordneter zum Nationalrat" == Parliament mandate) * stories: news articles about the same event (similar headlines, shared people, within 72 hours) are grouped; mentions of a person in one story are merged * events: trips/meetings of the same person to the same place/counterpart within 3 days """ from collections import defaultdict from datetime import timedelta from psycopg import Connection from rapidfuzz import fuzz from politrack.core import Enricher, RunContext, register from politrack.core.spec import DERIVED from politrack.enrich.events import counterpart_anchor from politrack.text import normalize REFRESH_STATUS = """ UPDATE claim SET status = CASE WHEN status = 'retracted' THEN status WHEN EXISTS (SELECT 1 FROM claim_evidence e WHERE e.claim_id = claim.id AND e.stance = 'contradicts') THEN 'disputed' WHEN (SELECT count(DISTINCT d.source_id) FROM claim_evidence e JOIN document d ON d.id = e.document_id WHERE e.claim_id = claim.id AND e.stance IN ('supports', 'mentions')) >= 2 THEN 'corroborated' ELSE 'reported' END WHERE id = %s """ def merge_claims(conn: Connection, keep_id: int, dup_id: int) -> None: if keep_id == dup_id: return conn.execute( """ INSERT INTO claim_evidence (claim_id, document_id, stance, quote, locator, extracted_by, confidence) SELECT %s, document_id, stance, quote, locator, extracted_by, confidence FROM claim_evidence WHERE claim_id = %s ON CONFLICT (claim_id, document_id, locator) DO NOTHING """, (keep_id, dup_id), ) conn.execute( """ INSERT INTO topic_assignment (topic_id, claim_id, confidence, method) SELECT topic_id, %s, confidence, method FROM topic_assignment WHERE claim_id = %s ON CONFLICT DO NOTHING """, (keep_id, dup_id), ) conn.execute("UPDATE claim SET merged_into = %s, updated_at = now() WHERE id = %s", (keep_id, dup_id)) conn.execute("UPDATE claim SET merged_into = %s WHERE merged_into = %s", (keep_id, dup_id)) conn.execute(REFRESH_STATUS, (keep_id,)) # -- roles --------------------------------------------------------------------- BODY_KEYWORDS = { "nationalrat": ("nationalrat",), "bundesrat": ("bundesrat", "bundesrates"), } def _same_office(a: dict, b: dict) -> bool: """a: authoritative role (Parliament), b: other source (Wikidata).""" if a["role"] == "mp" and b["role"] == "mp": return a["body_slug"] == b["body_slug"] label = normalize(b["role"]) if a["role"] == "mp" and a["body_slug"] in BODY_KEYWORDS: return any(k in label for k in BODY_KEYWORDS[a["body_slug"]]) and "prasident" not in label return fuzz.token_set_ratio(normalize(a["role"]), label) >= 80 def _periods_match(a: dict, b: dict) -> bool: a_start, b_start = a["start_date"], b["start_date"] a_end, b_end = a["end_date"], b["end_date"] if a_start and b_start and abs((a_start - b_start).days) <= 60: return True # Wikidata often only records one term for several consecutive mandates: accept containment. if a_start and b_start and b_start <= a_start and (b_end is None or a_end is None or a_end <= b_end + timedelta(days=60)): return True return False def merge_roles(conn: Connection) -> int: rows = conn.execute( """ SELECT r.claim_id, r.person_id, r.role, r.start_date, r.end_date, c.created_by, o.slug AS body_slug FROM role r JOIN claim c ON c.id = r.claim_id LEFT JOIN organization o ON o.id = r.organization_id WHERE c.merged_into IS NULL AND r.level <> 'party' AND (c.created_by LIKE 'parlament%%' OR c.created_by LIKE 'wikidata%%' OR c.created_by LIKE 'europarl%%') ORDER BY r.person_id, r.start_date NULLS LAST """ ).fetchall() by_person: dict[int, dict[str, list[dict]]] = defaultdict(lambda: defaultdict(list)) for r in rows: family = "wikidata" if r["created_by"].startswith("wikidata") else "parlament" by_person[r["person_id"]][family].append(r) merged = 0 for roles in by_person.values(): for other in roles.get("wikidata", []): match = next((a for a in roles.get("parlament", []) if _same_office(a, other) and _periods_match(a, other)), None) if match: merge_claims(conn, match["claim_id"], other["claim_id"]) merged += 1 conn.commit() return merged # -- news stories ------------------------------------------------------------- def cluster_stories(conn: Connection, days: int = 14) -> int: """Group recent news documents about the same thing into stories.""" docs = conn.execute( """ SELECT d.id, d.title, d.published_at, d.story_id, d.source_id, d.meta->'people' AS people FROM document d JOIN source s ON s.id = d.source_id WHERE (s.kind = 'media' OR s.slug = 'gdelt') AND d.published_at > now() - make_interval(days => %s) AND d.title IS NOT NULL ORDER BY d.published_at """, (days,), ).fetchall() stories: list[dict] = [] # {id, docs: [...]} for doc in docs: title = normalize(doc["title"]) people = set(doc["people"] or []) best, best_score = None, 0 for story in stories[-400:]: lead = story["docs"][0] if doc["published_at"] - lead["published_at"] > timedelta(hours=72): continue score = fuzz.token_set_ratio(title, lead["norm"]) shared = people & lead["people"] if score >= 82 or (shared and score >= 60): if score > best_score: best, best_score = story, score entry = {"id": doc["id"], "norm": title, "people": people, "published_at": doc["published_at"], "source_id": doc["source_id"], "story_id": doc["story_id"], "title": doc["title"]} if best: best["docs"].append(entry) else: stories.append({"docs": [entry]}) assigned = 0 for story in stories: if len(story["docs"]) < 2: continue existing = next((d["story_id"] for d in story["docs"] if d["story_id"]), None) lead = story["docs"][0] if existing is None: existing = conn.execute( "INSERT INTO story (title, first_seen) VALUES (%s, %s) RETURNING id", (lead["title"], lead["published_at"]) ).fetchone()["id"] ids = [d["id"] for d in story["docs"]] conn.execute("UPDATE document SET story_id = %s WHERE id = ANY(%s)", (existing, ids)) conn.execute( """ UPDATE story SET last_seen = sub.last_seen, document_count = sub.n, source_count = sub.sources FROM (SELECT max(published_at) AS last_seen, count(*) AS n, count(DISTINCT source_id) AS sources FROM document WHERE story_id = %s) sub WHERE story.id = %s """, (existing, existing), ) assigned += len(ids) conn.commit() return assigned def merge_story_mentions(conn: Connection) -> int: """One media_mention claim per (person or party, story); other outlets become additional evidence.""" rows = conn.execute( """ SELECT c.id, c.person_id, c.organization_id, d.story_id, c.occurred_at FROM claim c JOIN claim_evidence e ON e.claim_id = c.id JOIN document d ON d.id = e.document_id WHERE c.claim_type = 'media_mention' AND c.merged_into IS NULL AND d.story_id IS NOT NULL ORDER BY c.occurred_at NULLS LAST, c.id """ ).fetchall() keep: dict[tuple, int] = {} merged = 0 for r in rows: key = (r["person_id"], r["organization_id"], r["story_id"]) if key in keep: if keep[key] != r["id"]: merge_claims(conn, keep[key], r["id"]) merged += 1 else: keep[key] = r["id"] conn.commit() return merged # -- events -------------------------------------------------------------------- def merge_events(conn: Connection) -> int: rows = conn.execute( """ SELECT c.id, c.person_id, c.claim_type, c.occurred_at, c.data->>'country' AS country, c.data->>'counterpart' AS counterpart FROM claim c WHERE c.claim_type IN ('travel', 'meeting') AND c.merged_into IS NULL ORDER BY c.person_id, c.claim_type, c.occurred_at """ ).fetchall() merged = 0 previous: dict[tuple, dict] = {} for r in rows: anchor = r["country"] if r["claim_type"] == "travel" else counterpart_anchor(r["counterpart"] or "") key = (r["person_id"], r["claim_type"], anchor) prev = previous.get(key) if prev and r["occurred_at"] and prev["occurred_at"] and r["occurred_at"] - prev["occurred_at"] <= timedelta(days=3): merge_claims(conn, prev["id"], r["id"]) merged += 1 continue previous[key] = r conn.commit() return merged @register class Corroborator(Enricher): spec = DERIVED name = "enrich.corroborate" description = "Merge near-duplicate claims from different sources; cluster news stories" schedule = "9-59/10 * * * *" def run(self, ctx: RunContext) -> None: conn = ctx.conn ctx.stats["roles_merged"] = merge_roles(conn) ctx.stats["story_documents"] = cluster_stories(conn) ctx.stats["story_mentions_merged"] = merge_story_mentions(conn) ctx.stats["events_merged"] = merge_events(conn) return None