"""Headline monitoring via publishers' RSS feeds and Google-News sitemaps (listed in robots.txt). Policy (docs/sources.md): only feeds/sitemaps publishers offer for automated discovery; store headline, teaser/keywords and link - never article text (store_fulltext=False, keep_raw=False). If a site declares a TDM reservation, the Loader drops even the teaser and linking uses the headline only. The feed list lives in config/feeds.yaml (user-editable); every `source` there needs a SourceSpec below. """ from __future__ import annotations import xml.etree.ElementTree as ET from collections.abc import Iterator from datetime import datetime, timezone from time import mktime from typing import Any import feedparser import yaml from politrack.config import get_settings from politrack.core import Adapter, Request, Response, RunContext, SourceSpec, register, register_spec from politrack.core.records import Document, Evidence from politrack.text import html_to_text def media(slug: str, name: str, homepage: str, trust: int, notes: str = "RSS/news sitemap only; headline + link.", license: str = "All rights reserved") -> SourceSpec: return register_spec(SourceSpec(slug=slug, name=name, kind="media", homepage=homepage, license=license, access="rss", trust=trust, store_fulltext=False, keep_raw=False, min_interval=2.0, tos_notes=notes)) ORF = media("orf", "ORF.at", "https://orf.at", 75, "Public RSS feeds (rss.orf.at). Headline, teaser, link.") media("derstandard", "DER STANDARD", "https://www.derstandard.at", 75, "RSS + news sitemap. robots.txt blocks AI crawlers by name; we only use feed/sitemap and link out.") media("diepresse", "Die Presse", "https://www.diepresse.com", 75) media("kurier", "KURIER", "https://kurier.at", 70) media("profil", "profil", "https://www.profil.at", 75) media("falter", "FALTER", "https://www.falter.at", 75) media("krone", "Kronen Zeitung", "https://www.krone.at", 55) media("sn", "Salzburger Nachrichten", "https://www.sn.at", 75) media("kleinezeitung", "Kleine Zeitung", "https://www.kleinezeitung.at", 75) media("vol", "Vorarlberg Online", "https://www.vol.at", 65) media("apa-ots", "APA-OTS Presseaussendungen", "https://www.ots.at", 60, "tdmrep.json declares tdm-reservation=1 for the whole site: headline + link only.") NS = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9", "news": "http://www.google.com/schemas/sitemap-news/0.9"} @register class NewsFeeds(Adapter): spec = ORF name = "news.feeds" description = "Headlines from outlets' RSS feeds and Google-News sitemaps (config/feeds.yaml)" schedule = "*/20 * * * *" def start(self, ctx: RunContext) -> Iterator[Request]: config = yaml.safe_load((get_settings().config_dir / "feeds.yaml").read_text(encoding="utf-8")) for feed in config.get("feeds") or []: yield Request(url=feed["url"], kind=feed.get("type", "rss"), meta={"source": feed["source"]}, skip_unchanged=True) def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: source = response.meta["source"] doc_type = "press_release" if source == "apa-ots" else "article" if response.request.kind == "sitemap": root = ET.fromstring(response.body) for node in root.findall("sm:url", NS): link, title = node.findtext("sm:loc", namespaces=NS), node.findtext("news:news/news:title", namespaces=NS) if not link or not title: continue keywords = node.findtext("news:news/news:keywords", namespaces=NS) yield Document(Evidence(url=link.strip(), source=source, title=html_to_text(title), doc_type=doc_type, published_at=node.findtext("news:news/news:publication_date", namespaces=NS), excerpt=f"Schlagworte: {keywords}" if keywords else None, meta={"sitemap": response.request.url})) return for entry in feedparser.parse(response.body).entries: if not entry.get("link"): continue published = (datetime.fromtimestamp(mktime(entry.published_parsed), tz=timezone.utc) if entry.get("published_parsed") else None) yield Document(Evidence(url=entry["link"], source=source, title=html_to_text(entry.get("title")), doc_type=doc_type, published_at=published, excerpt=html_to_text(entry.get("summary")), meta={"feed": response.request.url}))