Landtag lists: match existing persons by birth date, Landtag membership and name variants

- person_by_name rules out homonyms born on another day and prefers the one who sat in
  that Landtag; failing an exact name, accepts a unique variant among that Landtag's
  members (Rudi/Rudolf, Chiara-Sophia/Chiara, new double surnames)
- normalize() maps ß to ss like SQL unaccent(); migration 0008 aligns stored aliases
- clean_name drops image captions, parenthesised suffixes, LTP, other abbreviations
  and zero-width characters

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
simonandClaude Opus 5.5 committed 2026-11-17 22:50:18 +01:00
1 parent faf9572b9f
commit 68be25a953
10 files changed
+114 -13

No files matched your search

@@ -0,0 +1,7 @@
-- normalize() now maps ß to ss, as SQL unaccent() does, so 'Stroißnig' and 'Stroissnig' are one alias.
-- Bring stored aliases in line (dropping those that would collide with an existing ss spelling).
DELETE FROM alias a
WHERE a.alias_norm LIKE '%%ß%%'
AND EXISTS (SELECT 1 FROM alias b WHERE b.entity_type = a.entity_type AND b.entity_id = a.entity_id
AND b.alias_norm = replace(a.alias_norm, 'ß', 'ss'));
UPDATE alias SET alias_norm = replace(alias_norm, 'ß', 'ss') WHERE alias_norm LIKE '%%ß%%';
+1 -1
View File
@@ -162,7 +162,7 @@ class Loader:
def person(self, rec: PersonRecord) -> int | None:
person_id = self.resolver.person_by_ids(rec.ref.ids)
if person_id is None and rec.match_by_name:
person_id = self.resolver.person_by_name(rec.full_name)
person_id = self.resolver.person_by_name(rec.full_name, rec.birth_date, rec.match_landtag)
if person_id is None:
if not self.ctx.creates_persons:
self.ctx.stats["persons_not_created"] += 1
+3
View File
@@ -98,6 +98,9 @@ class PersonRecord:
#: if no identifier matches, merge with an existing person of exactly this (unambiguous) name before
#: creating a new one - for authoritative sources that share no id with others (e.g. EP)
match_by_name: bool = False
#: with match_by_name: state key ('noe', ...) of a Landtag list, to tell homonyms apart and accept name
#: variants among that Landtag's members (Resolver.person_by_name)
match_landtag: str | None = None
@dataclass
+60 -6
View File
@@ -28,6 +28,24 @@ OFFICE_FILLER = re.compile(r"\b(der republik osterreich|der bundesrepublik oster
r"\(osterreich\)")
def name_parts(name: str) -> tuple[str, str]:
"""'Mag. Chiara-Sophia Glawogger' -> ('chiara', 'glawogger'): first given name and surname, normalised."""
tokens = normalize(strip_titles(name)).replace(",", " ").split()
if len(tokens) < 2:
return "", ""
return tokens[0].split("-")[0], tokens[-1]
def variant_of(a: str, b: str) -> bool:
"""Same person written differently: surnames equal or one is part of the other's double surname
('Kirchmayr' / 'Kirchmayr-Felbermaier'), first names share their first three letters ('Rudi' / 'Rudolf')."""
(ga, sa), (gb, sb) = name_parts(a), name_parts(b)
if not (ga and sa and gb and sb):
return False
surname = sa == sb or sa in sb.split("-") or sb in sa.split("-")
return surname and len(ga) >= 3 and len(gb) >= 3 and ga[:3] == gb[:3]
def office_key(title: str) -> str:
"""'Bundesministerin für Justiz' / 'Bundeskanzler der Republik Österreich' -> 'bundesminister fur justiz' /
'bundeskanzler' (gender-neutral, without filler, punctuation-free) for comparing office titles."""
@@ -109,18 +127,54 @@ class Resolver:
self._person_cache[cache_key] = found
return found
def person_by_name(self, name: str) -> int | None:
def person_by_name(self, name: str, birth_date: date | None = None, landtag: str | None = None) -> int | None:
"""Exact alias match. A known birth date rules out persons born on another day; `landtag` (state key such
as 'noe') prefers, among homonyms, the one who has sat in that Landtag, and falls back to name variants
among that Landtag's members (landtag_member_by_variant)."""
norm = normalize(strip_titles(name))
rows = self.conn.execute(
"""SELECT DISTINCT a.entity_id AS id, ps.status
FROM alias a LEFT JOIN person_status ps ON ps.person_id = a.entity_id
"""SELECT DISTINCT a.entity_id AS id, ps.status, p.birth_date,
%s::text IS NOT NULL AND EXISTS (
SELECT 1 FROM role r JOIN claim c ON c.id = r.claim_id
WHERE r.person_id = a.entity_id AND c.merged_into IS NULL AND r.level = 'state'
AND (landtag_key(r.role) = %s OR c.created_by = 'landtag.' || %s)) AS in_landtag
FROM alias a JOIN person p ON p.id = a.entity_id
LEFT JOIN person_status ps ON ps.person_id = a.entity_id
WHERE a.entity_type = 'person' AND a.alias_norm = %s""",
(norm,),
(landtag, landtag, landtag, norm),
).fetchall()
if birth_date:
same = [r for r in rows if r["birth_date"] == birth_date]
if len(same) == 1:
return same[0]["id"]
rows = [r for r in rows if r["birth_date"] is None]
if len(rows) == 1:
return rows[0]["id"]
active = [r for r in rows if r["status"] == "active"]
return active[0]["id"] if len(active) == 1 else None
for preferred in ([r for r in rows if r["in_landtag"]], [r for r in rows if r["status"] == "active"]):
if len(preferred) == 1:
return preferred[0]["id"]
if not rows and landtag:
return self.landtag_member_by_variant(name, landtag, birth_date)
return None
def landtag_member_by_variant(self, name: str, landtag: str, birth_date: date | None = None) -> int | None:
"""A Landtag list writes 'Rudi Hemetsberger', 'Chiara-Sophia Glawogger' or 'Helena Kirchmayr-Felbermaier'
where Wikidata has 'Rudolf Hemetsberger', 'Chiara Glawogger' and 'Helena Kirchmayr'. Among persons who
have sat in that Landtag, accept the single one whose surname matches (or is one part of a double surname)
and whose first name shares its first three letters, unless the birth dates differ."""
given, surname = name_parts(name)
if not given or not surname:
return None
rows = self.conn.execute(
"""SELECT DISTINCT p.id, p.full_name, p.birth_date FROM person p
JOIN role r ON r.person_id = p.id JOIN claim c ON c.id = r.claim_id
WHERE c.merged_into IS NULL AND r.level = 'state' AND landtag_key(r.role) = %s""",
(landtag,),
).fetchall()
hits = {row["id"] for row in rows
if not (birth_date and row["birth_date"] and row["birth_date"] != birth_date)
and variant_of(name, row["full_name"])}
return hits.pop() if len(hits) == 1 else None
def _person_by_hints(self, ref: PersonRef) -> int | None:
hints = ref.hints
+13 -4
View File
@@ -53,22 +53,30 @@ PARTY_ALIASES = {"KEIN KLUB": None, "VP TIROL": "ÖVP", "VP": "ÖVP", "VOLKSPART
# Functions written in front of names: Klubobmann, Landtagspräsidentin, KO-Stellvertreter, ...
ROLE_PREFIX = re.compile(
r"^(?:(?:Landtagspräsident(?:in)?|Präsident(?:in)?|(?:Zweite|Dritte)[rn]? Präsident(?:in)?|Klubobmann|Klubobfrau|"
r"Klubobleute|KO-Stellvertreter(?:in)?|KO|VP(?:in)?|LTPräs(?:in)?|LAbg\.|Abg\.|Bgm\.?(?:in)?|KommR|HR|Prof\.)\s+)+")
r"Klubobleute|KO-Stellvertreter(?:in)?|KO|VP(?:in)?|LTPräs(?:in)?|LTP(?:in)?|LAbg\.|Abg\.|Bgm\.?(?:in)?|KommR|HR|Prof\.)\s+)+")
TITLE_TOKENS = {"Mag", "Maga", "MMag", "Dr", "DI", "DIin", "Dipl", "Ing", "Prof", "HR", "BA", "MA", "BSc", "MSc", "MBA",
"BEd", "B.Ed", "B.A", "M.A", "LL.M", "LLM", "MPA", "PhD", "KommR", "Bgm", "Päd", "Univ", "Mag.a",
"Dr.in", "Mag.(FH)", "(FH)", "MAS", "MSc.", "M.A.I.S", "Dipl.-Ing", "Dipl.Soz.-Wiss.in", "Ing.in"}
SUFFIX_TOKENS = {"a", "in"} # "Mag. a", "Dr. in"
# Image alt texts ("Portraitfoto von Karin Berger") and invisible characters some lists contain (zero-width space)
CAPTION_PREFIX = re.compile(r"^(?:Portrait|Porträt|Foto|Bild)(?:foto)?(?: von| of)?\s+", re.IGNORECASE)
INVISIBLE = re.compile("[\u00ad\u200b-\u200f\u2060\ufeff]")
def _is_title(token: str) -> bool:
t = token.strip(",.")
return t in TITLE_TOKENS or re.fullmatch(r"(?:Dipl|Univ|Mag|Dr)\.[\w.\-]*", token.strip(",")) is not None
if t in TITLE_TOKENS or re.fullmatch(r"(?:Dipl|Univ|Mag|Dr)\.[\w.\-]*", token.strip(",")):
return True
# any other abbreviation of two or more letters ("MMst.", "Bakk."); a name never ends with a dot, an
# initial ("J.") is kept
return re.fullmatch(r"[^\W\d_]{2,}\.", token.strip(",")) is not None
def clean_name(name: str) -> str:
"""'KO Mag. Markus Abwerzger' -> 'Markus Abwerzger'; drops functions, academic titles and stray punctuation.
Titles are matched as whole tokens, so a first name like "Ingrid" is never taken for "Ing."."""
name = ROLE_PREFIX.sub("", re.sub(r"\s+", " ", name).strip())
name = re.sub(r"\([^()]*\)", " ", INVISIBLE.sub("", name)) # "(WU)" after "MSc", "(FH)"
name = ROLE_PREFIX.sub("", CAPTION_PREFIX.sub("", re.sub(r"\s+", " ", name).strip()))
name = strip_titles(name)
kept = []
for token in name.replace(",", " , ").split():
@@ -253,7 +261,8 @@ class LandtagMembers(Adapter):
region, body_name, _, list_url = LANDTAGE[self.key]
scheme = f"landtag-{self.key}"
ref = PersonRef(ids={scheme: m["id"]}, name=m["name"])
yield PersonRecord(ref=ref, full_name=m["name"], birth_date=m["birth_date"], match_by_name=True)
yield PersonRecord(ref=ref, full_name=m["name"], birth_date=m["birth_date"], match_by_name=True,
match_landtag=self.key)
body = OrgRef(slug=f"landtag-{self.key}", name=body_name, kind="parliament")
party = OrgRef(short_name=m["party"], kind="party", create=False) if m["party"] else None
label = f" ({m['party']})" if m["party"] else ""
+2 -2
View File
@@ -14,10 +14,10 @@ ACADEMIC_TITLES = re.compile(
def normalize(text: str) -> str:
"""Lowercase, strip accents and collapse whitespace (matches SQL lower(unaccent()))."""
"""Lowercase, strip accents, ß -> ss and collapse whitespace (matches SQL lower(unaccent()))."""
text = unicodedata.normalize("NFKD", text)
text = "".join(c for c in text if not unicodedata.combining(c))
return re.sub(r"\s+", " ", text).strip().lower()
return re.sub(r"\s+", " ", text).strip().lower().replace("ß", "ss")
def strip_titles(name: str) -> str:
@@ -11,6 +11,7 @@
"honorific_prefix": null,
"honorific_suffix": null,
"match_by_name": false,
"match_landtag": null,
"photo_url": "https://www.europarl.europa.eu/mepphoto/220897.jpg",
"ref": {
"__type__": "PersonRef",
@@ -13,6 +13,7 @@
"honorific_prefix": null,
"honorific_suffix": null,
"match_by_name": true,
"match_landtag": null,
"photo_url": null,
"ref": {
"__type__": "PersonRef",
@@ -74,6 +75,7 @@
"honorific_prefix": null,
"honorific_suffix": null,
"match_by_name": true,
"match_landtag": null,
"photo_url": null,
"ref": {
"__type__": "PersonRef",
@@ -135,6 +137,7 @@
"honorific_prefix": null,
"honorific_suffix": null,
"match_by_name": true,
"match_landtag": null,
"photo_url": null,
"ref": {
"__type__": "PersonRef",
@@ -196,6 +199,7 @@
"honorific_prefix": null,
"honorific_suffix": null,
"match_by_name": true,
"match_landtag": null,
"photo_url": null,
"ref": {
"__type__": "PersonRef",
@@ -257,6 +261,7 @@
"honorific_prefix": null,
"honorific_suffix": null,
"match_by_name": true,
"match_landtag": null,
"photo_url": null,
"ref": {
"__type__": "PersonRef",
@@ -318,6 +323,7 @@
"honorific_prefix": null,
"honorific_suffix": null,
"match_by_name": true,
"match_landtag": null,
"photo_url": null,
"ref": {
"__type__": "PersonRef",
@@ -379,6 +385,7 @@
"honorific_prefix": null,
"honorific_suffix": null,
"match_by_name": true,
"match_landtag": null,
"photo_url": null,
"ref": {
"__type__": "PersonRef",
@@ -440,6 +447,7 @@
"honorific_prefix": null,
"honorific_suffix": null,
"match_by_name": true,
"match_landtag": null,
"photo_url": null,
"ref": {
"__type__": "PersonRef",
@@ -11,6 +11,7 @@
"honorific_prefix": null,
"honorific_suffix": null,
"match_by_name": false,
"match_landtag": null,
"photo_url": "https://www.parlament.gv.at/dokument/bild/201426/20142639_500.jpg",
"ref": {
"__type__": "PersonRef",
+18
View File
@@ -10,6 +10,12 @@ def test_clean_name_drops_functions_and_titles_but_not_names():
assert L.clean_name("Dipl. Päd., DI Carina Laschober-Luif") == "Carina Laschober-Luif"
assert L.clean_name("Mag. a Astrid Eisenkopf") == "Astrid Eisenkopf"
assert L.clean_name("Ingrid Korosec") == "Ingrid Korosec" # "Ing" is a title, "Ingrid" is not
assert L.clean_name("Portraitfoto von Simon Heilig-Hofbauer") == "Simon Heilig-Hofbauer"
assert L.clean_name("LTP Jakob Wolf") == "Jakob Wolf"
assert L.clean_name("MMst. Doris Schmidl") == "Doris Schmidl"
assert L.clean_name("Claudia Gamon, MSc (WU)") == "Claudia Gamon"
assert L.clean_name("Johannes" + chr(0x200B) + " Schwarz") == "Johannes Schwarz"
assert L.clean_name("Michael J. Fox") == "Michael J. Fox" # initials stay
assert L.surname_first("Höglinger Tobias, Mag.") == "Tobias Höglinger"
assert L.surname_first("EISEL-EISELSBERG Detlev") == "Detlev Eisel-Eiselsberg"
@@ -66,3 +72,15 @@ def test_robots_excluded_states_have_no_adapter():
from politrack.core.registry import ADAPTERS
assert "landtag.wien" not in ADAPTERS and "landtag.ktn" not in ADAPTERS
assert "landtag.bgld" in ADAPTERS
def test_name_variants_and_sharp_s():
from politrack.core.resolve import variant_of
from politrack.text import normalize
assert normalize("Klaudia Stroißnig") == normalize("Klaudia Stroissnig")
for listed, known in [("Rudi Hemetsberger", "Rudolf Hemetsberger"), ("Max Hiegelsberger", "Maximilian Hiegelsberger"),
("Chiara-Sophia Glawogger", "Chiara Glawogger"), ("Helena Kirchmayr-Felbermaier", "Helena Kirchmayr"),
("Michael Stefan Wagner", "Michael Wagner"), ("Gebi Mair", "Gebhard Mair")]:
assert variant_of(listed, known), (listed, known)
for listed, known in [("Michael Weber", "Michael Huber"), ("Doris Kampus", "Doris Hahn"), ("Jo Mair", "Josef Mair")]:
assert not variant_of(listed, known), (listed, known)