Simon DiesenreiterandClaude Opus 5.5
9da5a1dc48
Initial import of Politrack, a self-hosted service that tracks Austrian politicians from federal to municipal level - roles, votes, money flows, statements, meetings, social media and legal proceedings - with every claim linked to its sources. - backend: FastAPI + Postgres/pgvector; adapter framework (fetch -> parse -> load -> enrich) with robots.txt/TDM-aware fetching, a raw-response archive for reparsing, and a claim/evidence model with corroboration - sources: Parliament, Wikidata, EU Parliament/Commission, Council of Ministers, Rechnungshof party reports and donations, procurement, RTR media transparency, political ad libraries, WKStA, news feeds, Wikipedia edits, Bluesky, Mastodon, public Telegram channels, YouTube and X (official API) - web: Next.js frontend in German and English with typed API client - mcp: MCP server for AI agents and RAG - docs: architecture, data model, sources, legal notes, Unraid deployment and contributor guides; Docker Compose setup for Unraid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
87 lines
4.6 KiB
Python
87 lines
4.6 KiB
Python
"""Headline monitoring via publishers' RSS feeds and Google-News sitemaps (listed in robots.txt).
|
|
|
|
Policy (docs/sources.md): only feeds/sitemaps publishers offer for automated discovery; store headline,
|
|
teaser/keywords and link - never article text (store_fulltext=False, keep_raw=False). If a site declares
|
|
a TDM reservation, the Loader drops even the teaser and linking uses the headline only.
|
|
|
|
The feed list lives in config/feeds.yaml (user-editable); every `source` there needs a SourceSpec below.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import xml.etree.ElementTree as ET
|
|
from collections.abc import Iterator
|
|
from datetime import datetime, timezone
|
|
from time import mktime
|
|
from typing import Any
|
|
|
|
import feedparser
|
|
import yaml
|
|
|
|
from politrack.config import get_settings
|
|
from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register, register_spec
|
|
from politrack.core.records import Document, Evidence
|
|
from politrack.text import html_to_text
|
|
|
|
|
|
def media(slug: str, name: str, homepage: str, trust: int, notes: str = "RSS/news sitemap only; headline + link.",
|
|
license: str = "All rights reserved") -> SourceSpec:
|
|
return register_spec(SourceSpec(slug=slug, name=name, kind="media", homepage=homepage, license=license,
|
|
access="rss", trust=trust, store_fulltext=False, keep_raw=False,
|
|
min_interval=2.0, tos_notes=notes))
|
|
|
|
|
|
ORF = media("orf", "ORF.at", "https://orf.at", 75, "Public RSS feeds (rss.orf.at). Headline, teaser, link.")
|
|
media("derstandard", "DER STANDARD", "https://www.derstandard.at", 75,
|
|
"RSS + news sitemap. robots.txt blocks AI crawlers by name; we only use feed/sitemap and link out.")
|
|
media("diepresse", "Die Presse", "https://www.diepresse.com", 75)
|
|
media("kurier", "KURIER", "https://kurier.at", 70)
|
|
media("profil", "profil", "https://www.profil.at", 75)
|
|
media("falter", "FALTER", "https://www.falter.at", 75)
|
|
media("krone", "Kronen Zeitung", "https://www.krone.at", 55)
|
|
media("sn", "Salzburger Nachrichten", "https://www.sn.at", 75)
|
|
media("kleinezeitung", "Kleine Zeitung", "https://www.kleinezeitung.at", 75)
|
|
media("vol", "Vorarlberg Online", "https://www.vol.at", 65)
|
|
media("apa-ots", "APA-OTS Presseaussendungen", "https://www.ots.at", 60,
|
|
"tdmrep.json declares tdm-reservation=1 for the whole site: headline + link only.")
|
|
|
|
NS = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9", "news": "http://www.google.com/schemas/sitemap-news/0.9"}
|
|
|
|
|
|
@register
|
|
class NewsFeeds(Adapter):
|
|
spec = ORF
|
|
name = "news.feeds"
|
|
description = "Headlines from outlets' RSS feeds and Google-News sitemaps (config/feeds.yaml)"
|
|
schedule = "*/20 * * * *"
|
|
|
|
def start(self, ctx: RunContext) -> Iterator[Request]:
|
|
config = yaml.safe_load((get_settings().config_dir / "feeds.yaml").read_text(encoding="utf-8"))
|
|
for feed in config.get("feeds") or []:
|
|
yield Request(url=feed["url"], kind=feed.get("type", "rss"), meta={"source": feed["source"]},
|
|
skip_unchanged=True)
|
|
|
|
def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]:
|
|
source = response.meta["source"]
|
|
doc_type = "press_release" if source == "apa-ots" else "article"
|
|
if response.request.kind == "sitemap":
|
|
root = ET.fromstring(response.body)
|
|
for node in root.findall("sm:url", NS):
|
|
link, title = node.findtext("sm:loc", namespaces=NS), node.findtext("news:news/news:title", namespaces=NS)
|
|
if not link or not title:
|
|
continue
|
|
keywords = node.findtext("news:news/news:keywords", namespaces=NS)
|
|
yield Document(Evidence(url=link.strip(), source=source, title=html_to_text(title), doc_type=doc_type,
|
|
published_at=node.findtext("news:news/news:publication_date", namespaces=NS),
|
|
excerpt=f"Schlagworte: {keywords}" if keywords else None,
|
|
meta={"sitemap": response.request.url}))
|
|
return
|
|
for entry in feedparser.parse(response.body).entries:
|
|
if not entry.get("link"):
|
|
continue
|
|
published = (datetime.fromtimestamp(mktime(entry.published_parsed), tz=timezone.utc)
|
|
if entry.get("published_parsed") else None)
|
|
yield Document(Evidence(url=entry["link"], source=source, title=html_to_text(entry.get("title")),
|
|
doc_type=doc_type, published_at=published, excerpt=html_to_text(entry.get("summary")),
|
|
meta={"feed": response.request.url}))
|