"""GDELT DOC 2.0 API: news coverage of politicians across many more outlets (metadata only). Free, open, max. one request every 5 s (we use 6 s and back off on the plain-text throttle notice). Coverage of German-language Austrian media is thin, so this is a supplement. Queries: active federal/EU politicians daily, former politicians in a weekly rotation ("what are they up to?"). """ from __future__ import annotations import json from collections.abc import Iterator from datetime import date from typing import Any from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register from politrack.core.records import Document, Evidence API = "https://api.gdeltproject.org/api/v2/doc/doc" SPEC = SourceSpec( slug="gdelt", name="Weitere Medien (via GDELT)", kind="aggregator", homepage="https://www.gdeltproject.org", license="GDELT open data (citation required)", access="api", trust=55, store_fulltext=False, keep_raw=False, min_interval=6.0, tos_notes="GDELT DOC 2.0 API: article metadata (URL, title, date). Max 1 request / 5 s.", ) DOMAIN_SOURCES = { "orf.at": "orf", "derstandard.at": "derstandard", "diepresse.com": "diepresse", "kurier.at": "kurier", "profil.at": "profil", "falter.at": "falter", "krone.at": "krone", "sn.at": "sn", "kleinezeitung.at": "kleinezeitung", "ots.at": "apa-ots", "vol.at": "vol", "vienna.at": "vol", } @register class Gdelt(Adapter): spec = SPEC name = "gdelt.coverage" description = "Supplementary news coverage via GDELT (active and former politicians)" schedule = "0 1 * * *" queue = "bulk" def start(self, ctx: RunContext) -> Iterator[Request]: if ctx.conn is None: return rows = ctx.conn.execute( """SELECT p.id, p.given_name, p.family_name, s.status FROM person p JOIN person_status s ON s.person_id = p.id WHERE p.given_name IS NOT NULL AND p.family_name IS NOT NULL AND ( (s.status = 'active' AND EXISTS (SELECT 1 FROM role r WHERE r.person_id = p.id AND r.level IN ('federal', 'eu') AND r.end_date IS NULL)) OR (s.status IN ('former', 'party_member') AND s.last_office_end > current_date - interval '25 years' AND p.id %% 7 = %s)) ORDER BY s.status, p.id LIMIT %s""", (date.today().weekday(), int(ctx.params.get("max_queries", 400))), ).fetchall() for row in rows: name = f"{row['given_name']} {row['family_name']}" yield Request(url=API, kind="artlist", meta={"person_id": row["id"]}, conditional=False, params={"query": f'"{name}" sourcecountry:AU', "mode": "artlist", "format": "json", "maxrecords": 50, "timespan": "7d" if row["status"] == "active" else "1m", "sort": "datedesc"}) def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: if not response.text.strip().startswith("{"): ctx.stats["throttled"] += 1 # plain-text rate-limit notice return for article in json.loads(response.text, strict=False).get("articles", []): domain = (article.get("domain") or "").removeprefix("www.") seen = article.get("seendate") or "" published = f"{seen[:4]}-{seen[4:6]}-{seen[6:8]}T{seen[9:11]}:{seen[11:13]}:00Z" if len(seen) >= 13 else None yield Document(Evidence(url=article["url"], source=DOMAIN_SOURCES.get(domain, "gdelt"), title=article.get("title"), doc_type="article", published_at=published, meta={"via": "gdelt", "domain": domain}))