politics-tracking/backend/politrack/sources/political_ads.py

"""Political advertising on online platforms.

* Google Political Ads Transparency Report - public download bundle (zip with CSVs, ~300 MB). We keep
  ads served in Austria (Regions contains "AT"): advertiser, period, impressions bracket, spend range.
  Note: Google stopped serving political ads in the EU in October 2025 (EU TTPA regulation), so this
  is historical data (2019-2025).
* Meta Ad Library API (political and issue ads reaching Austria) - requires an identity-verified developer
  token (POLITRACK_META_AD_LIBRARY_TOKEN) and page ids in config/meta_pages.yaml; skipped otherwise.

Advertisers are resolved to parties/organisations by name where possible (e.g. "FPÖ", "SPÖ Wien").
"""

from __future__ import annotations

import csv
import io
import zipfile
from collections.abc import Iterator
from datetime import date
from typing import Any

import yaml

from politrack.config import get_settings
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
from politrack.core.records import Claim, Evidence, Money, OrgRef
from politrack.text import format_eur_de

GOOGLE_BUNDLE = "https://storage.googleapis.com/political-csv/google-political-ads-transparency-bundle.zip"
GOOGLE_SPEC = SourceSpec(
    slug="google-political-ads", name="Google Political Ads Transparency Report", kind="aggregator",
    homepage="https://adstransparency.google.com/political", license="Google transparency data (public download)",
    access="bulk", trust=85, keep_raw=False, min_interval=5.0,
    tos_notes="Public CSV bundle of political ads. ~300 MB; fetched monthly with conditional requests.",
)
META_SPEC = SourceSpec(
    slug="meta-adlibrary", name="Meta Ad Library", kind="social", homepage="https://www.facebook.com/ads/library/api",
    license="Meta Platform Terms", access="api", trust=80, min_interval=2.0,
    tos_notes="Official Ad Library API for political and issue ads (identity-verified developer token required).",
)
META_API = "https://graph.facebook.com/v21.0/ads_archive"


def col(row: dict[str, str], *names: str) -> str | None:
    lowered = {k.lower(): v for k, v in row.items()}
    for name in names:
        if lowered.get(name.lower()) not in (None, ""):
            return lowered[name.lower()]
    return None


@register
class GooglePoliticalAds(Adapter):
    spec = GOOGLE_SPEC
    name = "ads.google"
    description = "Political ads served in Austria on Google/YouTube (historical, until 10/2025)"
    schedule = "0 3 1 * *"
    queue = "bulk"

    def start(self, ctx: RunContext) -> Iterator[Request]:
        yield Request(url=GOOGLE_BUNDLE, kind="bundle")

    def parse(self, ctx: RunContext, response: Response) -> Iterator[Claim]:
        since = ctx.params.get("since") or get_settings().ad_archive_since
        with zipfile.ZipFile(io.BytesIO(response.body)) as bundle:
            name = next((n for n in bundle.namelist() if n.endswith("creative-stats.csv")), None)
            if name is None:
                ctx.stats["creative_stats_missing"] += 1
                return
            with bundle.open(name) as fh:
                for row in csv.DictReader(io.TextIOWrapper(fh, encoding="utf-8", errors="replace")):
                    regions = (col(row, "Regions") or "").replace(" ", "").split(",")
                    if "AT" not in regions:
                        continue
                    start = col(row, "Date_Range_Start") or ""
                    if start < since:
                        continue
                    advertiser = col(row, "Advertiser_Name") or "?"
                    low, high = col(row, "Spend_Range_Min_EUR"), col(row, "Spend_Range_Max_EUR")
                    ad_url = col(row, "Ad_URL") or GOOGLE_BUNDLE
                    payer = OrgRef(name=advertiser, kind="other", ids={"google_advertiser": col(row, "Advertiser_ID") or ""})
                    spend = f"{format_eur_de(low)}–{format_eur_de(high)} €" if low and high else None
                    yield Claim(
                        claim_type="money_flow", org=payer, occurred_at=start,
                        summary_de=f"Politische Google-Werbung von {advertiser} ({start} – {col(row, 'Date_Range_End')})"
                                   + (f", Ausgaben {spend}" if spend else ""),
                        summary_en=f"Political Google ad by {advertiser} ({start} – {col(row, 'Date_Range_End')})",
                        data={"impressions": col(row, "Impressions"), "ad_type": col(row, "Ad_Type"),
                              "age_targeting": col(row, "Age_Targeting"), "gender_targeting": col(row, "Gender_Targeting"),
                              "geo_included": col(row, "Geo_Targeting_Included")},
                        evidence=[Evidence(url=ad_url, doc_type="dataset_row", title=f"Google Ads Transparency: {advertiser}",
                                           published_at=start)],
                        facet=Money(kind="political_ad", payer=payer, payer_name=advertiser,
                                    payee=OrgRef(name="Google", kind="company"), payee_name="Google",
                                    paid_on=date.fromisoformat(start[:10]) if start else None,
                                    amount_category=spend, reference=col(row, "Ad_ID"), category=col(row, "Ad_Type")),
                    )


@register
class MetaPoliticalAds(Adapter):
    spec = META_SPEC
    name = "ads.meta"
    description = "Political/issue ads reaching Austria from Meta's Ad Library (token + page list required)"
    schedule = "0 4 * * 4"
    queue = "bulk"

    FIELDS = ("id,page_id,page_name,bylines,ad_delivery_start_time,ad_delivery_stop_time,spend,impressions,"
              "currency,ad_snapshot_url,ad_creative_bodies")

    def start(self, ctx: RunContext) -> Iterator[Request]:
        token = get_settings().meta_ad_library_token
        path = get_settings().config_dir / "meta_pages.yaml"
        if not token or not path.exists():
            ctx.stats["skipped_not_configured"] = 1
            return
        pages = [str(p["page_id"]) for p in (yaml.safe_load(path.read_text(encoding="utf-8")) or {}).get("pages", [])]
        for i in range(0, len(pages), 10):
            yield Request(url=META_API, kind="ads", conditional=False, params={
                "access_token": token, "ad_reached_countries": '["AT"]', "ad_type": "POLITICAL_AND_ISSUE_ADS",
                "search_page_ids": ",".join(pages[i:i + 10]), "fields": self.FIELDS, "limit": 250,
                "ad_delivery_date_min": ctx.params.get("since") or get_settings().ad_archive_since})

    def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
        data = response.json()
        next_url = (data.get("paging") or {}).get("next")
        if next_url:
            yield Request(url=next_url, kind="ads", conditional=False)
        for ad in data.get("data") or []:
            spend = ad.get("spend") or {}
            payer_name = ad.get("bylines") or ad.get("page_name") or "?"
            payer = OrgRef(name=ad.get("page_name") or payer_name, kind="other", ids={"facebook_page": ad.get("page_id")})
            rng = f"{spend.get('lower_bound')}–{spend.get('upper_bound')} {ad.get('currency') or 'EUR'}" if spend else None
            start = (ad.get("ad_delivery_start_time") or "")[:10]
            yield Claim(
                claim_type="money_flow", org=payer, occurred_at=start or None,
                summary_de=f"Politische Meta-Werbung von {payer_name} ab {start}" + (f", Ausgaben {rng}" if rng else ""),
                data={"impressions": ad.get("impressions"), "bodies": (ad.get("ad_creative_bodies") or [])[:2]},
                evidence=[Evidence(url=ad.get("ad_snapshot_url") or f"https://www.facebook.com/ads/library/?id={ad['id']}",
                                   doc_type="dataset_row", title=f"Meta Ad Library: {payer_name}", published_at=start or None)],
                facet=Money(kind="political_ad", payer=payer, payer_name=payer_name,
                            payee=OrgRef(name="Meta Platforms", kind="company"), payee_name="Meta",
                            paid_on=date.fromisoformat(start) if start else None, amount_category=rng,
                            reference=ad.get("id")),
            )