simonandClaude Opus 5.5
583ccba31c
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
71 lines
3.7 KiB
Python
71 lines
3.7 KiB
Python
"""GDELT DOC 2.0 API: news coverage of politicians across many more outlets (metadata only).
|
|
|
|
Free, open, max. one request every 5 s (we use 6 s and back off on the plain-text throttle notice).
|
|
Coverage of German-language Austrian media is thin, so this is a supplement. Queries: active
|
|
federal/EU politicians daily, former politicians in a weekly rotation ("what are they up to?").
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from collections.abc import Iterator
|
|
from datetime import date
|
|
from typing import Any
|
|
|
|
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register
|
|
from politrack.core.records import Document, Evidence
|
|
|
|
API = "https://api.gdeltproject.org/api/v2/doc/doc"
|
|
SPEC = SourceSpec(
|
|
slug="gdelt", name="Weitere Medien (via GDELT)", kind="aggregator", homepage="https://www.gdeltproject.org",
|
|
license="GDELT open data (citation required)", access="api", trust=55, store_fulltext=False, keep_raw=False,
|
|
min_interval=6.0, tos_notes="GDELT DOC 2.0 API: article metadata (URL, title, date). Max 1 request / 5 s.",
|
|
)
|
|
DOMAIN_SOURCES = {
|
|
"orf.at": "orf", "derstandard.at": "derstandard", "diepresse.com": "diepresse", "kurier.at": "kurier",
|
|
"profil.at": "profil", "falter.at": "falter", "krone.at": "krone", "sn.at": "sn", "kleinezeitung.at": "kleinezeitung",
|
|
"ots.at": "apa-ots", "vol.at": "vol", "vienna.at": "vol",
|
|
}
|
|
|
|
|
|
@register
|
|
class Gdelt(Adapter):
|
|
spec = SPEC
|
|
name = "gdelt.coverage"
|
|
description = "Supplementary news coverage via GDELT (active and former politicians)"
|
|
schedule = "0 1 * * *"
|
|
queue = "bulk"
|
|
|
|
def start(self, ctx: RunContext) -> Iterator[Request]:
|
|
if ctx.conn is None:
|
|
return
|
|
rows = ctx.conn.execute(
|
|
"""SELECT p.id, p.given_name, p.family_name, s.status FROM person p
|
|
JOIN person_status s ON s.person_id = p.id
|
|
WHERE p.given_name IS NOT NULL AND p.family_name IS NOT NULL AND (
|
|
(s.status = 'active' AND EXISTS (SELECT 1 FROM role r WHERE r.person_id = p.id
|
|
AND r.level IN ('federal', 'eu') AND r.end_date IS NULL))
|
|
OR (s.status IN ('former', 'party_member') AND s.last_office_end > current_date - interval '25 years'
|
|
AND p.id %% 7 = %s))
|
|
ORDER BY s.status, p.id LIMIT %s""",
|
|
(date.today().weekday(), int(ctx.params.get("max_queries", 400))),
|
|
).fetchall()
|
|
for row in rows:
|
|
name = f"{row['given_name']} {row['family_name']}"
|
|
yield Request(url=API, kind="artlist", meta={"person_id": row["id"]}, conditional=False,
|
|
params={"query": f'"{name}" sourcecountry:AU', "mode": "artlist", "format": "json",
|
|
"maxrecords": 50, "timespan": "7d" if row["status"] == "active" else "1m",
|
|
"sort": "datedesc"})
|
|
|
|
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
|
if not response.text.strip().startswith("{"):
|
|
ctx.stats["throttled"] += 1 # plain-text rate-limit notice
|
|
return
|
|
for article in json.loads(response.text, strict=False).get("articles", []):
|
|
domain = (article.get("domain") or "").removeprefix("www.")
|
|
seen = article.get("seendate") or ""
|
|
published = f"{seen[:4]}-{seen[4:6]}-{seen[6:8]}T{seen[9:11]}:{seen[11:13]}:00Z" if len(seen) >= 13 else None
|
|
yield Document(Evidence(url=article["url"], source=DOMAIN_SOURCES.get(domain, "gdelt"),
|
|
title=article.get("title"), doc_type="article", published_at=published,
|
|
meta={"via": "gdelt", "domain": domain}))
|