politics-tracking/backend/politrack/enrich/corroborate.py

"""Corroboration: find claims from different sources that state the same fact and merge them.

Merging keeps the oldest claim, copies the duplicate's evidence onto it and marks
the duplicate with `merged_into`. Afterwards `refresh_claim_status` marks the
survivor "corroborated" when at least two *independent sources* support it.

Matchers:
  * roles:    same person, equivalent body/office, overlapping or near-identical periods
              (e.g. Wikidata "Abgeordneter zum Nationalrat" == Parliament mandate)
  * stories:  news articles about the same event (similar headlines, shared people,
              within 72 hours) are grouped; mentions of a person in one story are merged
  * events:   trips/meetings of the same person to the same place/counterpart within 3 days
"""

from collections import defaultdict
from datetime import timedelta

from psycopg import Connection
from rapidfuzz import fuzz

from politrack.core import Enricher, RunContext, register
from politrack.core.spec import DERIVED
from politrack.enrich.events import counterpart_anchor
from politrack.text import normalize

REFRESH_STATUS = """
UPDATE claim SET status = CASE
    WHEN status = 'retracted' THEN status
    WHEN EXISTS (SELECT 1 FROM claim_evidence e WHERE e.claim_id = claim.id AND e.stance = 'contradicts') THEN 'disputed'
    WHEN (SELECT count(DISTINCT d.source_id) FROM claim_evidence e JOIN document d ON d.id = e.document_id
          WHERE e.claim_id = claim.id AND e.stance IN ('supports', 'mentions')) >= 2 THEN 'corroborated'
    ELSE 'reported' END
WHERE id = %s
"""


def merge_claims(conn: Connection, keep_id: int, dup_id: int) -> None:
    if keep_id == dup_id:
        return
    conn.execute(
        """
        INSERT INTO claim_evidence (claim_id, document_id, stance, quote, locator, extracted_by, confidence)
        SELECT %s, document_id, stance, quote, locator, extracted_by, confidence
        FROM claim_evidence WHERE claim_id = %s
        ON CONFLICT (claim_id, document_id, locator) DO NOTHING
        """,
        (keep_id, dup_id),
    )
    conn.execute(
        """
        INSERT INTO topic_assignment (topic_id, claim_id, confidence, method)
        SELECT topic_id, %s, confidence, method FROM topic_assignment WHERE claim_id = %s
        ON CONFLICT DO NOTHING
        """,
        (keep_id, dup_id),
    )
    conn.execute("UPDATE claim SET merged_into = %s, updated_at = now() WHERE id = %s", (keep_id, dup_id))
    conn.execute("UPDATE claim SET merged_into = %s WHERE merged_into = %s", (keep_id, dup_id))
    conn.execute(REFRESH_STATUS, (keep_id,))


# -- roles ---------------------------------------------------------------------

BODY_KEYWORDS = {
    "nationalrat": ("nationalrat",),
    "bundesrat": ("bundesrat", "bundesrates"),
}


def _same_office(a: dict, b: dict) -> bool:
    """a: authoritative role (Parliament), b: other source (Wikidata)."""
    if a["role"] == "mp" and b["role"] == "mp":
        return a["body_slug"] == b["body_slug"]
    label = normalize(b["role"])
    if a["role"] == "mp" and a["body_slug"] in BODY_KEYWORDS:
        return any(k in label for k in BODY_KEYWORDS[a["body_slug"]]) and "prasident" not in label
    return fuzz.token_set_ratio(normalize(a["role"]), label) >= 80


def _periods_match(a: dict, b: dict) -> bool:
    a_start, b_start = a["start_date"], b["start_date"]
    a_end, b_end = a["end_date"], b["end_date"]
    if a_start and b_start and abs((a_start - b_start).days) <= 60:
        return True
    # Wikidata often only records one term for several consecutive mandates: accept containment.
    if a_start and b_start and b_start <= a_start and (b_end is None or a_end is None or a_end <= b_end + timedelta(days=60)):
        return True
    return False


def merge_roles(conn: Connection) -> int:
    rows = conn.execute(
        """
        SELECT r.claim_id, r.person_id, r.role, r.start_date, r.end_date, c.created_by, o.slug AS body_slug
        FROM role r JOIN claim c ON c.id = r.claim_id LEFT JOIN organization o ON o.id = r.organization_id
        WHERE c.merged_into IS NULL AND r.level <> 'party'
          AND (c.created_by LIKE 'parlament%%' OR c.created_by LIKE 'wikidata%%' OR c.created_by LIKE 'europarl%%')
        ORDER BY r.person_id, r.start_date NULLS LAST
        """
    ).fetchall()
    by_person: dict[int, dict[str, list[dict]]] = defaultdict(lambda: defaultdict(list))
    for r in rows:
        family = "wikidata" if r["created_by"].startswith("wikidata") else "parlament"
        by_person[r["person_id"]][family].append(r)
    merged = 0
    for roles in by_person.values():
        for other in roles.get("wikidata", []):
            match = next((a for a in roles.get("parlament", []) if _same_office(a, other) and _periods_match(a, other)),
                         None)
            if match:
                merge_claims(conn, match["claim_id"], other["claim_id"])
                merged += 1
    conn.commit()
    return merged


# -- news stories -------------------------------------------------------------

def cluster_stories(conn: Connection, days: int = 14) -> int:
    """Group recent news documents about the same thing into stories."""
    docs = conn.execute(
        """
        SELECT d.id, d.title, d.published_at, d.story_id, d.source_id, d.meta->'people' AS people
        FROM document d JOIN source s ON s.id = d.source_id
        WHERE (s.kind = 'media' OR s.slug = 'gdelt') AND d.published_at > now() - make_interval(days => %s)
          AND d.title IS NOT NULL
        ORDER BY d.published_at
        """,
        (days,),
    ).fetchall()
    stories: list[dict] = []  # {id, docs: [...]}
    for doc in docs:
        title = normalize(doc["title"])
        people = set(doc["people"] or [])
        best, best_score = None, 0
        for story in stories[-400:]:
            lead = story["docs"][0]
            if doc["published_at"] - lead["published_at"] > timedelta(hours=72):
                continue
            score = fuzz.token_set_ratio(title, lead["norm"])
            shared = people & lead["people"]
            if score >= 82 or (shared and score >= 60):
                if score > best_score:
                    best, best_score = story, score
        entry = {"id": doc["id"], "norm": title, "people": people, "published_at": doc["published_at"],
                 "source_id": doc["source_id"], "story_id": doc["story_id"], "title": doc["title"]}
        if best:
            best["docs"].append(entry)
        else:
            stories.append({"docs": [entry]})

    assigned = 0
    for story in stories:
        if len(story["docs"]) < 2:
            continue
        existing = next((d["story_id"] for d in story["docs"] if d["story_id"]), None)
        lead = story["docs"][0]
        if existing is None:
            existing = conn.execute(
                "INSERT INTO story (title, first_seen) VALUES (%s, %s) RETURNING id", (lead["title"], lead["published_at"])
            ).fetchone()["id"]
        ids = [d["id"] for d in story["docs"]]
        conn.execute("UPDATE document SET story_id = %s WHERE id = ANY(%s)", (existing, ids))
        conn.execute(
            """
            UPDATE story SET last_seen = sub.last_seen, document_count = sub.n, source_count = sub.sources
            FROM (SELECT max(published_at) AS last_seen, count(*) AS n, count(DISTINCT source_id) AS sources
                  FROM document WHERE story_id = %s) sub
            WHERE story.id = %s
            """,
            (existing, existing),
        )
        assigned += len(ids)
    conn.commit()
    return assigned


def merge_story_mentions(conn: Connection) -> int:
    """One media_mention claim per (person or party, story); other outlets become additional evidence."""
    rows = conn.execute(
        """
        SELECT c.id, c.person_id, c.organization_id, d.story_id, c.occurred_at
        FROM claim c
        JOIN claim_evidence e ON e.claim_id = c.id
        JOIN document d ON d.id = e.document_id
        WHERE c.claim_type = 'media_mention' AND c.merged_into IS NULL AND d.story_id IS NOT NULL
        ORDER BY c.occurred_at NULLS LAST, c.id
        """
    ).fetchall()
    keep: dict[tuple, int] = {}
    merged = 0
    for r in rows:
        key = (r["person_id"], r["organization_id"], r["story_id"])
        if key in keep:
            if keep[key] != r["id"]:
                merge_claims(conn, keep[key], r["id"])
                merged += 1
        else:
            keep[key] = r["id"]
    conn.commit()
    return merged


# -- events --------------------------------------------------------------------

def merge_events(conn: Connection) -> int:
    rows = conn.execute(
        """
        SELECT c.id, c.person_id, c.claim_type, c.occurred_at, c.data->>'country' AS country,
               c.data->>'counterpart' AS counterpart
        FROM claim c
        WHERE c.claim_type IN ('travel', 'meeting') AND c.merged_into IS NULL
        ORDER BY c.person_id, c.claim_type, c.occurred_at
        """
    ).fetchall()
    merged = 0
    previous: dict[tuple, dict] = {}
    for r in rows:
        anchor = r["country"] if r["claim_type"] == "travel" else counterpart_anchor(r["counterpart"] or "")
        key = (r["person_id"], r["claim_type"], anchor)
        prev = previous.get(key)
        if prev and r["occurred_at"] and prev["occurred_at"] and r["occurred_at"] - prev["occurred_at"] <= timedelta(days=3):
            merge_claims(conn, prev["id"], r["id"])
            merged += 1
            continue
        previous[key] = r
    conn.commit()
    return merged


@register
class Corroborator(Enricher):
    spec = DERIVED
    name = "enrich.corroborate"
    description = "Merge near-duplicate claims from different sources; cluster news stories"
    schedule = "9-59/10 * * * *"

    def run(self, ctx: RunContext) -> None:
        conn = ctx.conn
        ctx.stats["roles_merged"] = merge_roles(conn)
        ctx.stats["story_documents"] = cluster_stories(conn)
        ctx.stats["story_mentions_merged"] = merge_story_mentions(conn)
        ctx.stats["events_merged"] = merge_events(conn)
        return None