Add EP amendment and part votes, hidden from general lists by default

They come from the same responses as final votes. A sitting has over a
hundred, so they are flagged part_vote and only appear on request: the vote
lists have an option for them, feeds, counts and search chunks leave them out.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
simonandClaude Opus 5.5 committed 2026-11-09 13:17:10 +01:00
1 parent bf19e503f9
commit b06f17e587
20 files changed
+176 -47

No files matched your search

+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE vote_event ADD COLUMN part_vote BOOLEAN NOT NULL DEFAULT FALSE;
COMMENT ON COLUMN vote_event.part_vote IS 'TRUE for a vote on an amendment or a single part of a text (European Parliament), FALSE for a vote on the text as a whole. Claims of part votes carry data.part_vote = true and are left out of general lists by default, because a sitting has over a hundred of them.';
+7 -4
View File
@@ -133,8 +133,10 @@ def get_person(slug: str, conn: DB):
@app.get("/api/v1/persons/{slug}/votes", response_model=list[s.PersonVote])
def get_person_votes(slug: str, conn: DB, topics: str | None = None, limit: Limit = 100, offset: Offset = 0):
return queries.person_votes(conn, slug, topics=csv_list(topics), limit=limit, offset=offset)
def get_person_votes(slug: str, conn: DB, topics: str | None = None, include_parts: bool = False, limit: Limit = 100,
offset: Offset = 0):
return queries.person_votes(conn, slug, topics=csv_list(topics), include_parts=include_parts, limit=limit,
offset=offset)
@app.get("/api/v1/persons/{slug}/network", response_model=s.Network)
@@ -218,9 +220,10 @@ def get_wiki_edits(conn: DB, limit: Limit = 100, offset: Offset = 0):
@app.get("/api/v1/votes", response_model=list[s.VoteItem])
def get_votes(conn: DB, party: str | None = None, topics: str | None = None, roll_call: bool | None = None,
since: date | None = None, until: date | None = None, limit: Limit = 50, offset: Offset = 0):
since: date | None = None, until: date | None = None, include_parts: bool = False, limit: Limit = 50,
offset: Offset = 0):
return queries.list_votes(conn, party=party, topics=csv_list(topics), roll_call=roll_call, since=since,
until=until, limit=limit, offset=offset)
until=until, include_parts=include_parts, limit=limit, offset=offset)
@app.get("/api/v1/votes/{vote_id}", response_model=s.VoteDetail)
+4 -4
View File
@@ -250,14 +250,14 @@ class Loader:
chamber = self.resolver.org(rec.chamber, create=False) if rec.chamber else None
row = self.conn.execute(
"""INSERT INTO vote_event (event_key, chamber_id, item_ref, item_title, stage, voted_on, result, roll_call,
url, protocol_url, raw_text)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
url, protocol_url, raw_text, part_vote)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (event_key) DO UPDATE SET result = EXCLUDED.result, raw_text = EXCLUDED.raw_text,
roll_call = EXCLUDED.roll_call, item_title = EXCLUDED.item_title,
roll_call = EXCLUDED.roll_call, item_title = EXCLUDED.item_title, part_vote = EXCLUDED.part_vote,
protocol_url = COALESCE(EXCLUDED.protocol_url, vote_event.protocol_url)
RETURNING id""",
(rec.key, chamber, rec.item_ref, rec.title, rec.stage, rec.voted_on, rec.result, rec.roll_call, rec.url,
rec.protocol_url, rec.raw_text),
rec.protocol_url, rec.raw_text, rec.part_vote),
).fetchone()
self.ctx.stats["vote_events"] += 1
return row["id"]
+1
View File
@@ -149,6 +149,7 @@ class VoteEvent:
url: str | None = None
protocol_url: str | None = None
raw_text: str | None = None
part_vote: bool = False # a vote on an amendment or part of a text, not on the whole text
@dataclass
+1
View File
@@ -63,6 +63,7 @@ def build_chunks(conn: Connection, batch: int = 5000) -> int:
JOIN document d ON d.id = e.document_id
JOIN source s ON s.id = d.source_id
WHERE c.merged_into IS NULL AND c.review_status = 'published'
AND NOT COALESCE((c.data->>'part_vote')::boolean, FALSE)
AND NOT EXISTS (SELECT 1 FROM chunk ch WHERE ch.claim_id = c.id)
GROUP BY c.id, p.full_name, o.name
LIMIT %s
+4 -3
View File
@@ -170,11 +170,12 @@ def calendar(start: str, end: str, person: str | None = None, party: str | None
@mcp.tool()
def person_votes(slug: str, topics: list[str] | None = None, limit: int = 50) -> str:
def person_votes(slug: str, topics: list[str] | None = None, include_parts: bool = False, limit: int = 50) -> str:
"""Votes: individual roll-call votes (with dissent flag) and faction votes during the person's mandates.
`spoke` links a speech by the person on the same item (say vs. vote)."""
`spoke` links a speech by the person on the same item (say vs. vote). include_parts adds European
Parliament votes on amendments and parts of a text."""
with connection() as conn:
return _dump(queries.person_votes(conn, slug, topics=topics, limit=limit))
return _dump(queries.person_votes(conn, slug, topics=topics, include_parts=include_parts, limit=limit))
@mcp.tool()
+4 -3
View File
@@ -6,7 +6,7 @@ from datetime import date
from psycopg import Connection
from politrack.queries.common import VISIBLE, SourceFilter
from politrack.queries.common import NOT_PART_VOTE, VISIBLE, SourceFilter
def list_claims(
@@ -62,6 +62,7 @@ def list_claims(
WHERE e.claim_id = c.id AND {source_sql}
) ev
WHERE ev.evidence IS NOT NULL AND {VISIBLE}
AND (%(claim_id)s::bigint IS NOT NULL OR {NOT_PART_VOTE})
AND (%(person)s::text IS NULL OR p.slug = %(person)s OR cpp.slug = %(person)s)
AND (%(party)s::text IS NULL OR o.slug = %(party)s
OR c.counterpart_org_id = (SELECT id FROM organization WHERE slug = %(party)s))
@@ -97,7 +98,7 @@ def timeline_histogram(conn: Connection, *, sources: SourceFilter, person: str |
LEFT JOIN organization o ON o.id = c.organization_id
JOIN claim_evidence e ON e.claim_id = c.id JOIN document d ON d.id = e.document_id
JOIN source s ON s.id = d.source_id
WHERE {VISIBLE} AND {source_sql} AND COALESCE(c.occurred_at, c.valid_from::timestamptz) IS NOT NULL
WHERE {VISIBLE} AND {NOT_PART_VOTE} AND {source_sql} AND COALESCE(c.occurred_at, c.valid_from::timestamptz) IS NOT NULL
AND (%(person)s::text IS NULL OR p.slug = %(person)s OR cpp.slug = %(person)s)
AND (%(party)s::text IS NULL OR o.slug = %(party)s)
AND c.claim_type <> 'role'
@@ -149,7 +150,7 @@ def calendar(conn: Connection, *, sources: SourceFilter, start: date, end: date,
LEFT JOIN organization o ON o.id = c.organization_id
JOIN claim_evidence e ON e.claim_id = c.id JOIN document d ON d.id = e.document_id
JOIN source s ON s.id = d.source_id
WHERE {VISIBLE} AND {source_sql} AND c.occurred_at::date BETWEEN %(start)s AND %(end)s
WHERE {VISIBLE} AND {NOT_PART_VOTE} AND {source_sql} AND c.occurred_at::date BETWEEN %(start)s AND %(end)s
AND (%(person)s::text IS NULL OR p.slug = %(person)s OR cpp.slug = %(person)s)
AND (%(party)s::text IS NULL OR o.slug = %(party)s)
GROUP BY 1, 2 ORDER BY 1
+2
View File
@@ -13,6 +13,8 @@ from typing import Any
VISIBLE = ("c.merged_into IS NULL AND (c.review_status = 'published' OR "
"(%(include_pending)s AND c.review_status = 'pending'))")
# Votes on amendments and parts of a text only appear where someone asks for them: they would bury everything else.
NOT_PART_VOTE = "NOT COALESCE((c.data->>'part_vote')::boolean, FALSE)"
@dataclass
+1 -1
View File
@@ -72,7 +72,7 @@ def get_person(conn: Connection, slug: str) -> dict | None:
r["claim_type"]: r["count"]
for r in conn.execute(
"SELECT claim_type, count(*) AS count FROM claim WHERE person_id = %s AND merged_into IS NULL "
"AND review_status = 'published' GROUP BY claim_type",
"AND review_status = 'published' AND NOT COALESCE((data->>'part_vote')::boolean, FALSE) GROUP BY claim_type",
(person["id"],),
)
}
+10 -8
View File
@@ -17,9 +17,9 @@ SPOKE = """
"""
def person_votes(conn: Connection, slug: str, *, topics: list[str] | None = None, limit: int = 100,
offset: int = 0) -> list[dict]:
"""Votes concerning a person.
def person_votes(conn: Connection, slug: str, *, topics: list[str] | None = None, include_parts: bool = False,
limit: int = 100, offset: int = 0) -> list[dict]:
"""Votes concerning a person (include_parts: also votes on amendments and parts of a text).
* individual=true: roll-call votes with the person's own vote; dissent=true when it differs from the
majority of their faction (vote_position.party_id = their faction that day).
@@ -41,7 +41,8 @@ def person_votes(conn: Connection, slug: str, *, topics: list[str] | None = None
JOIN organization ch ON ch.id = ve.chamber_id
LEFT JOIN organization fo ON fo.id = vp.party_id
LEFT JOIN vote_position fp ON fp.vote_event_id = ve.id AND fp.organization_id = vp.party_id
WHERE (%(topics)s::text[] IS NULL OR EXISTS (
WHERE (%(include_parts)s OR NOT ve.part_vote)
AND (%(topics)s::text[] IS NULL OR EXISTS (
SELECT 1 FROM topic_assignment ta JOIN topic t ON t.id = ta.topic_id
WHERE ta.claim_id = vp.claim_id AND t.slug = ANY(%(topics)s)))
UNION ALL
@@ -63,13 +64,13 @@ def person_votes(conn: Connection, slug: str, *, topics: list[str] | None = None
ORDER BY voted_on DESC NULLS LAST
LIMIT %(limit)s OFFSET %(offset)s
""",
{"slug": slug, "topics": topics, "limit": limit, "offset": offset},
{"slug": slug, "topics": topics, "include_parts": include_parts, "limit": limit, "offset": offset},
).fetchall()
def list_votes(conn: Connection, *, party: str | None = None, topics: list[str] | None = None,
roll_call: bool | None = None, since: date | None = None, until: date | None = None,
limit: int = 50, offset: int = 0) -> list[dict]:
include_parts: bool = False, limit: int = 50, offset: int = 0) -> list[dict]:
return conn.execute(
"""
SELECT ve.id, ve.item_title, ve.item_ref, ve.stage, ve.voted_on, ve.result, ve.url, ve.protocol_url,
@@ -83,7 +84,8 @@ def list_votes(conn: Connection, *, party: str | None = None, topics: list[str]
JOIN organization ch ON ch.id = ve.chamber_id
LEFT JOIN vote_position vp ON vp.vote_event_id = ve.id AND vp.organization_id IS NOT NULL
LEFT JOIN organization o ON o.id = vp.organization_id
WHERE (%(party)s::text IS NULL OR EXISTS (
WHERE (%(include_parts)s OR NOT ve.part_vote)
AND (%(party)s::text IS NULL OR EXISTS (
SELECT 1 FROM vote_position v2 JOIN organization o2 ON o2.id = v2.organization_id
WHERE v2.vote_event_id = ve.id AND o2.slug = %(party)s))
AND (%(topics)s::text[] IS NULL OR EXISTS (
@@ -97,7 +99,7 @@ def list_votes(conn: Connection, *, party: str | None = None, topics: list[str]
LIMIT %(limit)s OFFSET %(offset)s
""",
{"party": party, "topics": topics, "roll_call": roll_call, "since": since, "until": until,
"limit": limit, "offset": offset},
"include_parts": include_parts, "limit": limit, "offset": offset},
).fetchall()
+20 -12
View File
@@ -200,6 +200,7 @@ class EuropeanParliament(Adapter):
LD_JSON = "application/ld+json"
EP_DECISIONS_PAGE = 500
EP_REFERENCE = re.compile(r"^[AB]\d+-\d+/\d{4}$")
EP_OUTCOME = {"ADOPTED": ("accepted", "angenommen", "adopted"), "REJECTED": ("rejected", "abgelehnt", "rejected")}
EP_VOTERS = (("had_voter_favor", "for", "Stimmte für", "Voted for"),
("had_voter_against", "against", "Stimmte gegen", "Voted against"),
@@ -220,18 +221,19 @@ def ep_person_ids(values: list[str] | None) -> set[str]:
@register
class ParliamentVotes(Adapter):
"""Final roll-call votes (whole texts, no amendments) of the Austrian MEPs known to the database.
"""Roll-call votes of the Austrian MEPs known to the database: on whole texts and on amendments or parts.
Amendment votes are left out on purpose: a sitting has over a hundred of them, which would bury
the final votes on a member's page. The official result counts; corrections MEPs file afterwards are
kept in `data.intended`, because they don't change the result.
Part votes are flagged (`VoteEvent.part_vote`, `data.part_vote`), because a sitting has over a hundred of
them and they would otherwise bury the final votes. The official result counts; corrections MEPs file
afterwards are kept in `data.intended`, because they don't change the result.
"""
spec = EP_SPEC
name = "europarl.votes"
description = "Final roll-call votes of Austrian MEPs in the European Parliament plenary"
description = "Roll-call votes of Austrian MEPs in the European Parliament plenary, including amendments"
schedule = "45 5 * * *"
queue = "scrape"
parser_version = 2 # amendment and part votes
def start(self, ctx: RunContext) -> Iterator[Request]:
ep_ids = []
@@ -277,9 +279,10 @@ class ParliamentVotes(Adapter):
austrian = set(meta["ep_ids"])
chamber = OrgRef(slug="europaeisches-parlament")
for decision in data:
if "ROLLCALL" not in str(decision.get("decision_method")) or "decisionAboutId" in decision:
ctx.stats["not_final_rollcall"] += 1
if "ROLLCALL" not in str(decision.get("decision_method")):
ctx.stats["not_rollcall"] += 1
continue
part_vote = "decisionAboutId" in decision
voters = {position: ep_person_ids(decision.get(field)) & austrian for field, position, _, _ in EP_VOTERS}
if not any(voters.values()):
ctx.stats["no_austrian_voters"] += 1
@@ -289,6 +292,9 @@ class ParliamentVotes(Adapter):
items = decision.get("inverse_consists_of") or [None]
title = meta["titles"].get(items[0]) or label
stage = ep_label(decision.get("referenceText"))
label_parts = label.split(" – ", 2) # "A10-0272/2026 – rapporteur – § 13, nach Buchstabe a – Änd. 9"
if part_vote and len(label_parts) == 3:
stage = label_parts[2]
status = str(decision.get("decision_outcome", "")).rsplit("/", 1)[-1]
result, outcome_de, outcome_en = EP_OUTCOME.get(status, ("unknown", "Ergebnis unbekannt", "outcome unknown"))
counts = {"yes": decision.get("number_of_votes_favor"), "no": decision.get("number_of_votes_against"),
@@ -301,17 +307,19 @@ class ParliamentVotes(Adapter):
if pv else None)
event_key = fingerprint("ep-vote", decision["id"])
yield VoteEvent(key=event_key, chamber=chamber, title=title, voted_on=voted_on, result=result, stage=stage,
item_ref=label.split(" – ", 1)[0], roll_call=True, url=url, protocol_url=protocol,
raw_text=f"{stage}: {tally}")
item_ref=label_parts[0] if EP_REFERENCE.match(label_parts[0]) else None, roll_call=True,
url=url, protocol_url=protocol, raw_text=f"{stage}: {tally}", part_vote=part_vote)
subject = f"{title}, {stage}" if part_vote and title != label else title # a label already names the part
intended = {mep: position for field, position in EP_INTENDED for mep in ep_person_ids(decision.get(field))}
for _, position, verb_de, verb_en in EP_VOTERS:
for mep in sorted(voters[position]):
yield Claim(
claim_type="vote", person=PersonRef(ids={"ep": mep}), counterpart_org=chamber, occurred_at=voted_on,
summary_de=f"{verb_de}: {title} (Europäisches Parlament) – {outcome_de}",
summary_en=f"{verb_en}: {title} (European Parliament) – {outcome_en}",
summary_de=f"{verb_de}: {subject} (Europäisches Parlament) – {outcome_de}",
summary_en=f"{verb_en}: {subject} (European Parliament) – {outcome_en}",
data={"position": position, "result": result, "roll_call": True, "chamber": "EP", **counts,
**({"intended": intended[mep]} if mep in intended else {})},
**({"intended": intended[mep]} if mep in intended else {}),
**({"part_vote": True} if part_vote else {})},
evidence=[Evidence(url=url, doc_type="api_record", title=title, published_at=voted_on,
quote=f"{stage}: {tally}", locator=f"vote:{decision['id']}")],
topics=("Europäische Union",),
@@ -12,6 +12,7 @@
},
"item_ref": "A10-0259/2026",
"key": "3ab02bb907351dfa059987800a98ee3b1e2a77a9",
"part_vote": false,
"protocol_url": "https://www.europarl.europa.eu/doceo/document/PV-10-2026-10-21-RCV_DE.html",
"raw_text": "Entschließungsantrag (gesamter Text): 504 Ja, 63 Nein, 99 Enthaltungen",
"result": "accepted",
@@ -219,5 +220,95 @@
],
"valid_from": null,
"valid_to": null
},
{
"__type__": "VoteEvent",
"chamber": {
"__type__": "OrgRef",
"create": true,
"ids": {},
"kind": "other",
"name": null,
"short_name": null,
"slug": "europaeisches-parlament"
},
"item_ref": "A10-0272/2026",
"key": "6f5c1cd4005ecf97b7b450c671850a66d9fd5943",
"part_vote": true,
"protocol_url": "https://www.europarl.europa.eu/doceo/document/PV-10-2026-10-21-RCV_DE.html",
"raw_text": "nach § 3 – Änd. 70: 177 Ja, 455 Nein, 41 Enthaltungen",
"result": "rejected",
"roll_call": true,
"stage": "nach § 3 – Änd. 70",
"title": "A10-0272/2026 – Nils Ušakovs, Michalis Hadjipantela – nach § 3 – Änd. 70",
"url": "https://data.europarl.europa.eu/eli/dl/event/MTG-PL-2026-10-21-DEC-198977",
"voted_on": "2026-10-21"
},
{
"__type__": "Claim",
"claim_type": "vote",
"counterpart_org": {
"__type__": "OrgRef",
"create": true,
"ids": {},
"kind": "other",
"name": null,
"short_name": null,
"slug": "europaeisches-parlament"
},
"counterpart_person": null,
"data": {
"abstained": 41,
"chamber": "EP",
"no": 455,
"part_vote": true,
"position": "for",
"result": "rejected",
"roll_call": true,
"yes": 177
},
"evidence": [
{
"__type__": "Evidence",
"confidence": 1.0,
"doc_type": "api_record",
"excerpt": null,
"full_text": null,
"locator": "vote:eli/dl/event/MTG-PL-2026-10-21-DEC-198977",
"meta": {},
"published_at": "2026-10-21",
"quote": "nach § 3 – Änd. 70: 177 Ja, 455 Nein, 41 Enthaltungen",
"source": null,
"stance": "supports",
"title": "A10-0272/2026 – Nils Ušakovs, Michalis Hadjipantela – nach § 3 – Änd. 70",
"url": "https://data.europarl.europa.eu/eli/dl/event/MTG-PL-2026-10-21-DEC-198977"
}
],
"facet": {
"__type__": "VotePosition",
"event_key": "6f5c1cd4005ecf97b7b450c671850a66d9fd5943",
"party": null,
"position": "for"
},
"key": null,
"occurred_at": "2026-10-21",
"org": null,
"person": {
"__type__": "PersonRef",
"hints": {},
"ids": {
"ep": "257104"
},
"name": null
},
"review": false,
"sensitive": false,
"summary_de": "Stimmte für: A10-0272/2026 – Nils Ušakovs, Michalis Hadjipantela – nach § 3 – Änd. 70 (Europäisches Parlament) – abgelehnt",
"summary_en": "Voted for: A10-0272/2026 – Nils Ušakovs, Michalis Hadjipantela – nach § 3 – Änd. 70 (European Parliament) – rejected",
"topics": [
"Europäische Union"
],
"valid_from": null,
"valid_to": null
}
]
@@ -12,6 +12,7 @@
},
"item_ref": "416/A",
"key": "931917bd2feea2b95c21f3e15a86267702bce32f",
"part_vote": false,
"protocol_url": "https://www.parlament.gv.at/dokument/XXVIII/NRSITZ/44/fnameorig_1764795.html#160",
"raw_text": "44. Sitzung des Nationalrates: Gesetzesvorschlag in zweiter Lesung angenommen Dafür: ÖVP, SPÖ, NEOS, GRÜNE, dagegen: FPÖ",
"result": "accepted",
@@ -459,6 +460,7 @@
},
"item_ref": "416/A",
"key": "106b0163e733dbe54ccf3a7c148a2041ee607b6b",
"part_vote": false,
"protocol_url": "https://www.parlament.gv.at/dokument/XXVIII/NRSITZ/44/fnameorig_1764795.html#160",
"raw_text": "44. Sitzung des Nationalrates: Gesetzesvorschlag in dritter Lesung angenommen Dafür: ÖVP, SPÖ, NEOS, GRÜNE, dagegen: FPÖ",
"result": "accepted",
@@ -906,6 +908,7 @@
},
"item_ref": "416/A",
"key": "bc19d7af2b29acce57bfa6ad139b44c43aacc98a",
"part_vote": false,
"protocol_url": "https://www.parlament.gv.at/dokument/BR/BRSITZ/982/fnameorig_1768518.html#45",
"raw_text": "982. Sitzung: Antrag, keinen Einspruch zu erheben, angenommen Dafür: ÖVP, SPÖ, GRÜNE, NEOS, dagegen: FPÖ",
"result": "accepted",