diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..c89a609 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Golden fixtures are compared byte for byte; never convert their line endings. +backend/tests/fixtures/** -text diff --git a/backend/politrack/sources/eu.py b/backend/politrack/sources/eu.py index 7ca12e9..6589c5b 100644 --- a/backend/politrack/sources/eu.py +++ b/backend/politrack/sources/eu.py @@ -25,6 +25,8 @@ from politrack.core.records import ( PersonRecord, PersonRef, Role, + VoteEvent, + VotePosition, ) from politrack.text import excerpt, fingerprint, normalize @@ -191,3 +193,123 @@ class EuropeanParliament(Adapter): evidence=[Evidence(url=url, doc_type="pdf", title=title, published_at=doc.get("document_date"))], topics=("Transparenz und Korruption",), ) + + +LD_JSON = "application/ld+json" +EP_DECISIONS_PAGE = 500 +EP_OUTCOME = {"ADOPTED": ("accepted", "angenommen", "adopted"), "REJECTED": ("rejected", "abgelehnt", "rejected")} +EP_VOTERS = (("had_voter_favor", "for", "Stimmte für", "Voted for"), + ("had_voter_against", "against", "Stimmte gegen", "Voted against"), + ("had_voter_abstention", "abstain", "Enthielt sich bei", "Abstained on")) +EP_INTENDED = (("had_voter_intended_favor", "for"), ("had_voter_intended_against", "against"), + ("had_voter_intended_abstention", "abstain")) + + +def ep_label(value: Any) -> str: + """German text of a multilingual EP field, falling back to English.""" + text = (value.get("de") or value.get("en") or "") if isinstance(value, dict) else (value or "") + return re.sub(r"\s+", " ", str(text)).strip() + + +def ep_person_ids(values: list[str] | None) -> set[str]: + return {v.rsplit("/", 1)[-1] for v in values or []} + + +@register +class ParliamentVotes(Adapter): + """Final roll-call votes (whole texts, no amendments) of the Austrian MEPs known to the database. + + Amendment votes are left out on purpose: a sitting has over a hundred of them, which would bury + the final votes on a member's page. The official result counts; corrections MEPs file afterwards are + kept in `data.intended`, because they don't change the result. + """ + + spec = EP_SPEC + name = "europarl.votes" + description = "Final roll-call votes of Austrian MEPs in the European Parliament plenary" + schedule = "45 5 * * *" + queue = "scrape" + + def start(self, ctx: RunContext) -> Iterator[Request]: + ep_ids = [] + if ctx.conn is not None: + ep_ids = sorted(row["value"] for row in ctx.conn.execute( + "SELECT value FROM identifier WHERE entity_type = 'person' AND scheme = 'ep'")) + if not ep_ids: + ctx.stats["no_meps_loaded"] = 1 + return + for year in range(int(ctx.params.get("since_year", 2024)), date.today().year + 1): + yield Request(url=f"{EP_API}/meetings", kind="meetings", params={"year": year, "format": LD_JSON, "limit": 200}, + meta={"ep_ids": ep_ids}) + + def parse(self, ctx: RunContext, response: Response) -> Iterator[Any]: + kind = response.request.kind + data = response.json().get("data") or [] + meta = response.meta + if kind == "meetings": + today = response.fetched_at.date().isoformat() + for meeting in data: # the list includes future sittings, which have no results yet + if str(meeting.get("activity_date", "9999"))[:10] <= today: + yield Request(url=f"{EP_API}/meetings/{meeting['activity_id']}/vote-results", kind="results", + params={"format": LD_JSON, "limit": 500}, + meta={"ep_ids": meta["ep_ids"], "meeting": meeting["activity_id"]}) + elif kind == "results": + titles = {item["id"]: ep_label(item.get("activity_label")) for item in data if item.get("activity_date")} + if titles: + yield self.decisions_request(meta["meeting"], 0, meta["ep_ids"], titles) + else: + yield from self.parse_decisions(ctx, response, data) + + @staticmethod + def decisions_request(meeting: str, offset: int, ep_ids: list[str], titles: dict[str, str]) -> Request: + return Request(url=f"{EP_API}/meetings/{meeting}/decisions", kind="decisions", + params={"format": LD_JSON, "limit": EP_DECISIONS_PAGE, "offset": offset}, + meta={"ep_ids": ep_ids, "meeting": meeting, "offset": offset, "titles": titles}) + + def parse_decisions(self, ctx: RunContext, response: Response, data: list[dict]) -> Iterator[Any]: + meta = response.meta + austrian = set(meta["ep_ids"]) + chamber = OrgRef(slug="europaeisches-parlament") + for decision in data: + if "ROLLCALL" not in str(decision.get("decision_method")) or "decisionAboutId" in decision: + ctx.stats["not_final_rollcall"] += 1 + continue + voters = {position: ep_person_ids(decision.get(field)) & austrian for field, position, _, _ in EP_VOTERS} + if not any(voters.values()): + ctx.stats["no_austrian_voters"] += 1 + continue + voted_on = date.fromisoformat(decision["activity_date"][:10]) + label = ep_label(decision.get("activity_label")) + items = decision.get("inverse_consists_of") or [None] + title = meta["titles"].get(items[0]) or label + stage = ep_label(decision.get("referenceText")) + status = str(decision.get("decision_outcome", "")).rsplit("/", 1)[-1] + result, outcome_de, outcome_en = EP_OUTCOME.get(status, ("unknown", "Ergebnis unbekannt", "outcome unknown")) + counts = {"yes": decision.get("number_of_votes_favor"), "no": decision.get("number_of_votes_against"), + "abstained": decision.get("number_of_votes_abstention")} + tally = f"{counts['yes']} Ja, {counts['no']} Nein, {counts['abstained']} Enthaltungen" + url = f"https://data.europarl.europa.eu/{decision['id']}" + record = next(iter(decision.get("recorded_in_a_realization_of") or []), "") + pv = re.search(r"PV-(\d+)-(\d{4}-\d{2}-\d{2})-RCV", record) + protocol = (f"https://www.europarl.europa.eu/doceo/document/PV-{pv.group(1)}-{pv.group(2)}-RCV_DE.html" + if pv else None) + event_key = fingerprint("ep-vote", decision["id"]) + yield VoteEvent(key=event_key, chamber=chamber, title=title, voted_on=voted_on, result=result, stage=stage, + item_ref=label.split(" – ", 1)[0], roll_call=True, url=url, protocol_url=protocol, + raw_text=f"{stage}: {tally}") + intended = {mep: position for field, position in EP_INTENDED for mep in ep_person_ids(decision.get(field))} + for _, position, verb_de, verb_en in EP_VOTERS: + for mep in sorted(voters[position]): + yield Claim( + claim_type="vote", person=PersonRef(ids={"ep": mep}), counterpart_org=chamber, occurred_at=voted_on, + summary_de=f"{verb_de}: {title} (Europäisches Parlament) – {outcome_de}", + summary_en=f"{verb_en}: {title} (European Parliament) – {outcome_en}", + data={"position": position, "result": result, "roll_call": True, "chamber": "EP", **counts, + **({"intended": intended[mep]} if mep in intended else {})}, + evidence=[Evidence(url=url, doc_type="api_record", title=title, published_at=voted_on, + quote=f"{stage}: {tally}", locator=f"vote:{decision['id']}")], + topics=("Europäische Union",), + facet=VotePosition(event_key=event_key, position=position), + ) + if len(data) == EP_DECISIONS_PAGE: + yield self.decisions_request(meta["meeting"], meta["offset"] + EP_DECISIONS_PAGE, meta["ep_ids"], meta["titles"]) diff --git a/backend/tests/fixtures/europarl.votes/decisions.expected.json b/backend/tests/fixtures/europarl.votes/decisions.expected.json new file mode 100644 index 0000000..6e7f6a2 --- /dev/null +++ b/backend/tests/fixtures/europarl.votes/decisions.expected.json @@ -0,0 +1,223 @@ +[ + { + "__type__": "VoteEvent", + "chamber": { + "__type__": "OrgRef", + "create": true, + "ids": {}, + "kind": "other", + "name": null, + "short_name": null, + "slug": "europaeisches-parlament" + }, + "item_ref": "A10-0259/2026", + "key": "3ab02bb907351dfa059987800a98ee3b1e2a77a9", + "protocol_url": "https://www.europarl.europa.eu/doceo/document/PV-10-2026-10-21-RCV_DE.html", + "raw_text": "Entschließungsantrag (gesamter Text): 504 Ja, 63 Nein, 99 Enthaltungen", + "result": "accepted", + "roll_call": true, + "stage": "Entschließungsantrag (gesamter Text)", + "title": "Förderung von Kompetenzen in der Union durch Möglichkeiten der beruflichen Aus- und Weiterbildung in Zeiten des Übergangs", + "url": "https://data.europarl.europa.eu/eli/dl/event/MTG-PL-2026-10-21-DEC-198302", + "voted_on": "2026-10-21" + }, + { + "__type__": "Claim", + "claim_type": "vote", + "counterpart_org": { + "__type__": "OrgRef", + "create": true, + "ids": {}, + "kind": "other", + "name": null, + "short_name": null, + "slug": "europaeisches-parlament" + }, + "counterpart_person": null, + "data": { + "abstained": 99, + "chamber": "EP", + "intended": "against", + "no": 63, + "position": "for", + "result": "accepted", + "roll_call": true, + "yes": 504 + }, + "evidence": [ + { + "__type__": "Evidence", + "confidence": 1.0, + "doc_type": "api_record", + "excerpt": null, + "full_text": null, + "locator": "vote:eli/dl/event/MTG-PL-2026-10-21-DEC-198302", + "meta": {}, + "published_at": "2026-10-21", + "quote": "Entschließungsantrag (gesamter Text): 504 Ja, 63 Nein, 99 Enthaltungen", + "source": null, + "stance": "supports", + "title": "Förderung von Kompetenzen in der Union durch Möglichkeiten der beruflichen Aus- und Weiterbildung in Zeiten des Übergangs", + "url": "https://data.europarl.europa.eu/eli/dl/event/MTG-PL-2026-10-21-DEC-198302" + } + ], + "facet": { + "__type__": "VotePosition", + "event_key": "3ab02bb907351dfa059987800a98ee3b1e2a77a9", + "party": null, + "position": "for" + }, + "key": null, + "occurred_at": "2026-10-21", + "org": null, + "person": { + "__type__": "PersonRef", + "hints": {}, + "ids": { + "ep": "257104" + }, + "name": null + }, + "review": false, + "sensitive": false, + "summary_de": "Stimmte für: Förderung von Kompetenzen in der Union durch Möglichkeiten der beruflichen Aus- und Weiterbildung in Zeiten des Übergangs (Europäisches Parlament) – angenommen", + "summary_en": "Voted for: Förderung von Kompetenzen in der Union durch Möglichkeiten der beruflichen Aus- und Weiterbildung in Zeiten des Übergangs (European Parliament) – adopted", + "topics": [ + "Europäische Union" + ], + "valid_from": null, + "valid_to": null + }, + { + "__type__": "Claim", + "claim_type": "vote", + "counterpart_org": { + "__type__": "OrgRef", + "create": true, + "ids": {}, + "kind": "other", + "name": null, + "short_name": null, + "slug": "europaeisches-parlament" + }, + "counterpart_person": null, + "data": { + "abstained": 99, + "chamber": "EP", + "no": 63, + "position": "against", + "result": "accepted", + "roll_call": true, + "yes": 504 + }, + "evidence": [ + { + "__type__": "Evidence", + "confidence": 1.0, + "doc_type": "api_record", + "excerpt": null, + "full_text": null, + "locator": "vote:eli/dl/event/MTG-PL-2026-10-21-DEC-198302", + "meta": {}, + "published_at": "2026-10-21", + "quote": "Entschließungsantrag (gesamter Text): 504 Ja, 63 Nein, 99 Enthaltungen", + "source": null, + "stance": "supports", + "title": "Förderung von Kompetenzen in der Union durch Möglichkeiten der beruflichen Aus- und Weiterbildung in Zeiten des Übergangs", + "url": "https://data.europarl.europa.eu/eli/dl/event/MTG-PL-2026-10-21-DEC-198302" + } + ], + "facet": { + "__type__": "VotePosition", + "event_key": "3ab02bb907351dfa059987800a98ee3b1e2a77a9", + "party": null, + "position": "against" + }, + "key": null, + "occurred_at": "2026-10-21", + "org": null, + "person": { + "__type__": "PersonRef", + "hints": {}, + "ids": { + "ep": "256996" + }, + "name": null + }, + "review": false, + "sensitive": false, + "summary_de": "Stimmte gegen: Förderung von Kompetenzen in der Union durch Möglichkeiten der beruflichen Aus- und Weiterbildung in Zeiten des Übergangs (Europäisches Parlament) – angenommen", + "summary_en": "Voted against: Förderung von Kompetenzen in der Union durch Möglichkeiten der beruflichen Aus- und Weiterbildung in Zeiten des Übergangs (European Parliament) – adopted", + "topics": [ + "Europäische Union" + ], + "valid_from": null, + "valid_to": null + }, + { + "__type__": "Claim", + "claim_type": "vote", + "counterpart_org": { + "__type__": "OrgRef", + "create": true, + "ids": {}, + "kind": "other", + "name": null, + "short_name": null, + "slug": "europaeisches-parlament" + }, + "counterpart_person": null, + "data": { + "abstained": 99, + "chamber": "EP", + "no": 63, + "position": "abstain", + "result": "accepted", + "roll_call": true, + "yes": 504 + }, + "evidence": [ + { + "__type__": "Evidence", + "confidence": 1.0, + "doc_type": "api_record", + "excerpt": null, + "full_text": null, + "locator": "vote:eli/dl/event/MTG-PL-2026-10-21-DEC-198302", + "meta": {}, + "published_at": "2026-10-21", + "quote": "Entschließungsantrag (gesamter Text): 504 Ja, 63 Nein, 99 Enthaltungen", + "source": null, + "stance": "supports", + "title": "Förderung von Kompetenzen in der Union durch Möglichkeiten der beruflichen Aus- und Weiterbildung in Zeiten des Übergangs", + "url": "https://data.europarl.europa.eu/eli/dl/event/MTG-PL-2026-10-21-DEC-198302" + } + ], + "facet": { + "__type__": "VotePosition", + "event_key": "3ab02bb907351dfa059987800a98ee3b1e2a77a9", + "party": null, + "position": "abstain" + }, + "key": null, + "occurred_at": "2026-10-21", + "org": null, + "person": { + "__type__": "PersonRef", + "hints": {}, + "ids": { + "ep": "213330" + }, + "name": null + }, + "review": false, + "sensitive": false, + "summary_de": "Enthielt sich bei: Förderung von Kompetenzen in der Union durch Möglichkeiten der beruflichen Aus- und Weiterbildung in Zeiten des Übergangs (Europäisches Parlament) – angenommen", + "summary_en": "Abstained on: Förderung von Kompetenzen in der Union durch Möglichkeiten der beruflichen Aus- und Weiterbildung in Zeiten des Übergangs (European Parliament) – adopted", + "topics": [ + "Europäische Union" + ], + "valid_from": null, + "valid_to": null + } +] diff --git a/backend/tests/fixtures/europarl.votes/decisions.json b/backend/tests/fixtures/europarl.votes/decisions.json new file mode 100644 index 0000000..dc50385 --- /dev/null +++ b/backend/tests/fixtures/europarl.votes/decisions.json @@ -0,0 +1,189 @@ +{ + "data": [ + { + "id": "eli/dl/event/MTG-PL-2026-10-21-DEC-198302", + "type": "Vote", + "activity_date": "2026-10-21", + "activity_id": "MTG-PL-2026-10-21-DEC-198302", + "activity_label": { + "de": "A10-0259/2026 – Brigitte van den Berg – Entschließungsantrag (gesamter Text)", + "en": "A10-0259/2026 – Brigitte van den Berg – Motion for a resolution (as a whole)" + }, + "activity_order": "7.0", + "activity_start_date": "2026-10-21T13:49:16+02:00", + "decision_method": "def/ep-decision-methods/VOTE_ELECTRONIC_ROLLCALL", + "decision_outcome": "def/ep-statuses/ADOPTED", + "had_activity_type": "def/ep-activities/PLENARY_OUTCOME", + "had_voter_abstention": [ + "person/213330" + ], + "had_voter_against": [ + "person/256996" + ], + "had_voter_favor": [ + "person/257104", + "person/125023" + ], + "number_of_attendees": 666, + "number_of_votes_abstention": 99, + "number_of_votes_against": 63, + "number_of_votes_favor": 504, + "recorded_in_a_realization_of": [ + "eli/dl/doc/PV-10-2026-10-21-RCV-ITM-151" + ], + "comment": { + "hu": "Nsz", + "lt": "VB", + "pt": "VN", + "es": "VN", + "cs": "JH", + "bg": "ПГ", + "de": "NA", + "lv": "ATKL", + "sl": "PG", + "mt": "VSI", + "fr": "AN", + "it": "AN", + "hr": "AN", + "da": "AN", + "ro": "AN", + "pl": "gi", + "ga": "VGR", + "nl": "HS", + "fi": "NHÄ", + "et": "NH", + "sk": "HPM", + "el": "OK", + "sv": "ONU", + "en": "RCV" + }, + "notation_votingId": "198302", + "referenceText": { + "de": "Entschließungsantrag (gesamter Text)", + "en": "Motion for a resolution (as a whole)" + }, + "inverse_consists_of": [ + "eli/dl/event/MTG-PL-2026-10-21-VOT-ITM-993266" + ], + "had_voter_intended_against": [ + "person/257104" + ] + }, + { + "id": "eli/dl/event/MTG-PL-2026-10-21-DEC-198977", + "type": "Vote", + "activity_date": "2026-10-21", + "activity_id": "MTG-PL-2026-10-21-DEC-198977", + "activity_label": { + "de": "A10-0272/2026 – Nils Ušakovs, Michalis Hadjipantela – nach § 3 – Änd. 70", + "en": "A10-0272/2026 – Nils Ušakovs, Michalis Hadjipantela – After § 3 – Am 70" + }, + "activity_order": "13.0", + "activity_start_date": "2026-10-21T13:20:25+02:00", + "decided_on_a_realization_of": [ + "eli/dl/doc/A-10-2026-0272-AM-70" + ], + "decision_method": "def/ep-decision-methods/VOTE_ELECTRONIC_ROLLCALL", + "decision_outcome": "def/ep-statuses/REJECTED", + "had_activity_type": "def/ep-activities/PLENARY_OUTCOME", + "number_of_attendees": 673, + "number_of_votes_abstention": 41, + "number_of_votes_against": 455, + "number_of_votes_favor": 177, + "recorded_in_a_realization_of": [ + "eli/dl/doc/PV-10-2026-10-21-RCV-ITM-023" + ], + "responsible_organization_label": { + "et": "PfE", + "hu": "PfE", + "pl": "PfE", + "sl": "PfE", + "el": "PfE", + "ro": "PfE", + "lv": "PfE", + "pt": "PfE", + "sk": "PfE", + "fi": "PfE", + "bg": "PfE", + "en": "PfE", + "fr": "PfE", + "hr": "PfE", + "nl": "PfE", + "mt": "PfE", + "da": "PfE", + "cs": "PfE", + "de": "PfE", + "ga": "PfE", + "it": "PfE", + "sv": "PfE", + "es": "PfE", + "lt": "PfE" + }, + "was_motivated_by": [ + "eli/dl/event/MTG-PL-2026-10-21-VOT-ITM-993294-RCV-3" + ], + "comment": { + "sl": "PG", + "sv": "ONU", + "cs": "JH", + "el": "OK", + "fr": "AN", + "it": "AN", + "hr": "AN", + "da": "AN", + "ro": "AN", + "lv": "ATKL", + "et": "NH", + "es": "VN", + "pt": "VN", + "ga": "VGR", + "bg": "ПГ", + "nl": "HS", + "mt": "VSI", + "pl": "gi", + "fi": "NHÄ", + "en": "RCV", + "de": "NA", + "sk": "HPM", + "hu": "Nsz", + "lt": "VB" + }, + "notation_votingId": "198977", + "decisionAboutId": "70", + "referenceText": { + "de": "Nach § 3", + "en": "After § 3" + }, + "inverse_consists_of": [ + "eli/dl/event/MTG-PL-2026-10-21-VOT-ITM-993294" + ], + "had_voter_favor": [ + "person/257104", + "person/125023" + ] + }, + { + "id": "eli/dl/event/MTG-PL-2026-10-21-DEC-199277", + "type": "Vote", + "activity_date": "2026-10-21", + "activity_id": "MTG-PL-2026-10-21-DEC-199277", + "activity_label": { + "de": "Block 16", + "en": "Block 16" + }, + "activity_order": "197.0", + "decision_method": "def/ep-decision-methods/VOTE_HAND", + "decision_outcome": "def/ep-statuses/ADOPTED", + "had_activity_type": "def/ep-activities/PLENARY_OUTCOME", + "notation_votingId": "199277", + "decisionAboutId": "707, 708, 709", + "referenceText": { + "de": "Block 16", + "en": "Block 16" + }, + "inverse_consists_of": [ + "eli/dl/event/MTG-PL-2026-10-21-VOT-ITM-993336" + ] + } + ] +} diff --git a/backend/tests/fixtures/europarl.votes/decisions.request.json b/backend/tests/fixtures/europarl.votes/decisions.request.json new file mode 100644 index 0000000..c8da1ee --- /dev/null +++ b/backend/tests/fixtures/europarl.votes/decisions.request.json @@ -0,0 +1,25 @@ +{ + "url": "https://data.europarl.europa.eu/api/v2/meetings/MTG-PL-2026-10-21/decisions", + "kind": "decisions", + "method": "GET", + "params": { + "format": "application/ld+json", + "limit": 500, + "offset": 0 + }, + "json": null, + "meta": { + "ep_ids": [ + "257104", + "256996", + "213330" + ], + "meeting": "MTG-PL-2026-10-21", + "offset": 0, + "titles": { + "eli/dl/event/MTG-PL-2026-10-21-VOT-ITM-993266": "Förderung von Kompetenzen in der Union durch Möglichkeiten der beruflichen Aus- und Weiterbildung in Zeiten des Übergangs" + } + }, + "content_type": "application/ld+json", + "body_file": "decisions.json" +} diff --git a/backend/tests/fixtures/europarl.votes/meetings.expected.json b/backend/tests/fixtures/europarl.votes/meetings.expected.json new file mode 100644 index 0000000..fe2a5ab --- /dev/null +++ b/backend/tests/fixtures/europarl.votes/meetings.expected.json @@ -0,0 +1,42 @@ +[ + { + "__type__": "Request", + "conditional": true, + "headers": null, + "json": null, + "kind": "results", + "meta": { + "ep_ids": [ + "197490" + ], + "meeting": "MTG-PL-2025-01-20" + }, + "method": "GET", + "params": { + "format": "application/ld+json", + "limit": 500 + }, + "skip_unchanged": true, + "url": "https://data.europarl.europa.eu/api/v2/meetings/MTG-PL-2025-01-20/vote-results" + }, + { + "__type__": "Request", + "conditional": true, + "headers": null, + "json": null, + "kind": "results", + "meta": { + "ep_ids": [ + "197490" + ], + "meeting": "MTG-PL-2025-01-21" + }, + "method": "GET", + "params": { + "format": "application/ld+json", + "limit": 500 + }, + "skip_unchanged": true, + "url": "https://data.europarl.europa.eu/api/v2/meetings/MTG-PL-2025-01-21/vote-results" + } +] diff --git a/backend/tests/fixtures/europarl.votes/meetings.json b/backend/tests/fixtures/europarl.votes/meetings.json new file mode 100644 index 0000000..e901ba9 --- /dev/null +++ b/backend/tests/fixtures/europarl.votes/meetings.json @@ -0,0 +1,34 @@ +{ + "data": [ + { + "id": "eli/dl/event/MTG-PL-2025-01-20", + "type": "Activity", + "activity_date": "2025-01-20", + "activity_id": "MTG-PL-2025-01-20", + "activity_label": { + "de": "Montag, 20. Januar 2025", + "en": "Monday, 20 January 2025" + } + }, + { + "id": "eli/dl/event/MTG-PL-2025-01-21", + "type": "Activity", + "activity_date": "2025-01-21", + "activity_id": "MTG-PL-2025-01-21", + "activity_label": { + "de": "Dienstag, 21. Januar 2025", + "en": "Tuesday, 21 January 2025" + } + }, + { + "id": "eli/dl/event/MTG-PL-2026-06-01", + "type": "Activity", + "activity_date": "2026-06-01", + "activity_id": "MTG-PL-2026-06-01", + "activity_label": { + "de": "Dienstag, 21. Januar 2025", + "en": "Tuesday, 21 January 2025" + } + } + ] +} diff --git a/backend/tests/fixtures/europarl.votes/meetings.request.json b/backend/tests/fixtures/europarl.votes/meetings.request.json new file mode 100644 index 0000000..ff3fbb6 --- /dev/null +++ b/backend/tests/fixtures/europarl.votes/meetings.request.json @@ -0,0 +1,18 @@ +{ + "url": "https://data.europarl.europa.eu/api/v2/meetings", + "kind": "meetings", + "method": "GET", + "params": { + "year": 2025, + "format": "application/ld+json", + "limit": 200 + }, + "json": null, + "meta": { + "ep_ids": [ + "197490" + ] + }, + "content_type": "application/ld+json", + "body_file": "meetings.json" +} diff --git a/backend/tests/fixtures/europarl.votes/results.expected.json b/backend/tests/fixtures/europarl.votes/results.expected.json new file mode 100644 index 0000000..26f0c97 --- /dev/null +++ b/backend/tests/fixtures/europarl.votes/results.expected.json @@ -0,0 +1,27 @@ +[ + { + "__type__": "Request", + "conditional": true, + "headers": null, + "json": null, + "kind": "decisions", + "meta": { + "ep_ids": [ + "X" + ], + "meeting": "MTG-PL-2026-10-21", + "offset": 0, + "titles": { + "eli/dl/event/MTG-PL-2026-10-21-VOT-ITM-993266": "Förderung von Kompetenzen in der Union durch Möglichkeiten der beruflichen Aus- und Weiterbildung in Zeiten des Übergangs" + } + }, + "method": "GET", + "params": { + "format": "application/ld+json", + "limit": 500, + "offset": 0 + }, + "skip_unchanged": true, + "url": "https://data.europarl.europa.eu/api/v2/meetings/MTG-PL-2026-10-21/decisions" + } +] diff --git a/backend/tests/fixtures/europarl.votes/results.json b/backend/tests/fixtures/europarl.votes/results.json new file mode 100644 index 0000000..b079b85 --- /dev/null +++ b/backend/tests/fixtures/europarl.votes/results.json @@ -0,0 +1,20 @@ +{ + "data": [ + { + "id": "eli/dl/event/MTG-PL-2026-10-21-VOT-ITM-993266", + "activity_date": "2026-10-21", + "activity_id": "MTG-PL-2026-10-21-VOT-ITM-993266", + "activity_label": { + "de": "Förderung von Kompetenzen in der Union durch Möglichkeiten der beruflichen Aus- und Weiterbildung in Zeiten des Übergangs", + "en": "Boosting skills in the Union – opportunities for vocational education and training in times of transition" + } + }, + { + "id": "eli/dl/event/no-date", + "activity_id": "no-date", + "activity_label": { + "de": "Eintrag ohne Datum" + } + } + ] +} diff --git a/backend/tests/fixtures/europarl.votes/results.request.json b/backend/tests/fixtures/europarl.votes/results.request.json new file mode 100644 index 0000000..6d6ff80 --- /dev/null +++ b/backend/tests/fixtures/europarl.votes/results.request.json @@ -0,0 +1,18 @@ +{ + "url": "https://data.europarl.europa.eu/api/v2/meetings/MTG-PL-2026-10-21/vote-results", + "kind": "results", + "method": "GET", + "params": { + "format": "application/ld+json", + "limit": 500 + }, + "json": null, + "meta": { + "ep_ids": [ + "X" + ], + "meeting": "MTG-PL-2026-10-21" + }, + "content_type": "application/ld+json", + "body_file": "results.json" +} diff --git a/docs/deployment-unraid.md b/docs/deployment-unraid.md index 9a09052..c152259 100644 --- a/docs/deployment-unraid.md +++ b/docs/deployment-unraid.md @@ -23,6 +23,10 @@ docker compose build docker compose up -d ``` + Docker creates `${DATA_DIR}/raw` as root, but the backend runs as a non-root user, and every fetch + then fails with "Permission denied". Hand it over once after the first `up`: + `chown -R "$(docker run --rm --entrypoint id politrack-backend -u):$(docker run --rm --entrypoint id politrack-backend -g)" "$DATA_DIR/raw"`. + `migrate` runs once (schema, seed data, source registry, job-queue tables), then `api`, `web`, `mcp` and three workers start: diff --git a/docs/roadmap.md b/docs/roadmap.md index 0bae59e..0d51cd6 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -32,6 +32,7 @@ Next.js frontend (de/en), Docker Compose for Unraid. - [x] Social: Bluesky, Mastodon, public Telegram channels, YouTube (key), X via official API (token); party/institution channels; curated `social_accounts.yaml` - [x] X deletion sync (`social.x_deletions`): posts gone from X keep a marked record without their text +- [x] EP final roll-call votes of Austrian MEPs (`europarl.votes`) - [x] Pages: organisations, money explorer, social feed, Wikipedia edits, person network ("follow the money"), say-vs-vote on roll calls @@ -45,7 +46,6 @@ Next.js frontend (de/en), Docker Compose for Unraid. **Data** - [ ] Full speech texts and interjections from the protocols; committee memberships (list 250) -- [ ] HowTheyVote.eu / EP roll-call votes of Austrian MEPs - [ ] EU Council participant lists, ministries' press releases, Federal President's schedule - [ ] Election results and candidate lists; AMA/TAM subsidy recipients (after checking bulk access and terms) - [ ] Cabinet staff and appointments to state-owned companies (from answers to parliamentary questions) diff --git a/docs/sources.md b/docs/sources.md index e915409..d55620a 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -45,6 +45,7 @@ Status: ✅ implemented (adapter name in `code`) · 🟡 planned · ⛔ delibera | Public procurement – data.gv.at "Vergabe-Kerndaten" | **contract awards**: buyer, winner, amount, CPV (eForms) | OGD hub catalogue → per-publisher XML | open data (CC BY 4.0) | ✅ `vergabe.awards` | | RTR / KommAustria Medientransparenz | **government advertising** and **media subsidies** per payer → medium | open-data API | free re-use, attribution | ✅ `rtr.medientransparenz` | | European Parliament | Austrian MEPs, roles, **declarations of interests** | open-data/website XML | CC BY 4.0 | ✅ `europarl.meps` | +| – plenary roll calls | **final votes** of Austrian MEPs (whole texts, no amendments), with totals and later corrections | Open Data API v2 (`/meetings/{id}/decisions`) | CC BY 4.0 | ✅ `europarl.votes` | | European Commission – meetings | meetings of Commissioners/cabinets with **interest representatives** (Austrians and Austrian organisations) | XLSX export | EU re-use policy | ✅ `eu.commission_meetings` | | Wikipedia (de/en) | anonymous edits from **parliament/government IP ranges** (`config/ip_ranges.yaml`) | MediaWiki API `usercontribs` | CC BY-SA (edit metadata) | ✅ `wikipedia.institution_edits` | | Google Political Ads Transparency | Austrian political ads: advertiser, spend bracket, impressions | public bundle (zip, CSV) | Google's transparency report terms | ✅ `ads.google` | @@ -66,7 +67,7 @@ Status: ✅ implemented (adapter name in `code`) · 🟡 planned · ⛔ delibera |---|---|---|---|---| | Wikidata | positions incl. Landtage and mayors, party membership, careers, birth dates, **social-media handles**, ids | SPARQL | CC0 | ✅ `wikidata.politicians` | | GDELT DOC 2.0 | article metadata for politicians' names, attributed to the original outlets | API, ≤ 1 request / 5 s | free | ✅ `gdelt.coverage` (supplement; thin for Austrian outlets) | -| HowTheyVote.eu | EP roll-call votes | CSV dump | check licence | 🟡 | +| HowTheyVote.eu | EP roll-call votes | CSV dump on GitHub (robots.txt blocks its download path), API | ODbL 1.0 (share-alike) | ⛔ not needed: `europarl.votes` takes the same votes from the EP under CC BY 4.0 | | OpenSanctions | PEP cross-references | bulk JSON | CC BY-NC 4.0 | 🟡 non-commercial only | | MeineAbgeordneten.at | curated dossiers | – | sui-generis database right (§76c UrhG) | ⛔ scraping → 🟡 ask for a partnership | | Parteispenden.at | party-finance analysis | – | ask | 🟡 partnership |