Landtag lists: match existing persons by birth date, Landtag membership and name variants

- person_by_name rules out homonyms born on another day and prefers the one who sat in
  that Landtag; failing an exact name, accepts a unique variant among that Landtag's
  members (Rudi/Rudolf, Chiara-Sophia/Chiara, new double surnames)
- normalize() maps ß to ss like SQL unaccent(); migration 0008 aligns stored aliases
- clean_name drops image captions, parenthesised suffixes, LTP, other abbreviations
  and zero-width characters

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
simonandClaude Opus 5.5 committed 2026-11-17 22:50:18 +01:00
1 parent faf9572b9f
commit 68be25a953
10 files changed
+114 -13

No files matched your search

@@ -0,0 +1,7 @@
-- normalize() now maps ß to ss, as SQL unaccent() does, so 'Stroißnig' and 'Stroissnig' are one alias.
-- Bring stored aliases in line (dropping those that would collide with an existing ss spelling).
DELETE FROM alias a
WHERE a.alias_norm LIKE '%%ß%%'
AND EXISTS (SELECT 1 FROM alias b WHERE b.entity_type = a.entity_type AND b.entity_id = a.entity_id
AND b.alias_norm = replace(a.alias_norm, 'ß', 'ss'));
UPDATE alias SET alias_norm = replace(alias_norm, 'ß', 'ss') WHERE alias_norm LIKE '%%ß%%';
+1 -1
View File
@@ -162,7 +162,7 @@ class Loader:
def person(self, rec: PersonRecord) -> int | None: def person(self, rec: PersonRecord) -> int | None:
person_id = self.resolver.person_by_ids(rec.ref.ids) person_id = self.resolver.person_by_ids(rec.ref.ids)
if person_id is None and rec.match_by_name: if person_id is None and rec.match_by_name:
person_id = self.resolver.person_by_name(rec.full_name) person_id = self.resolver.person_by_name(rec.full_name, rec.birth_date, rec.match_landtag)
if person_id is None: if person_id is None:
if not self.ctx.creates_persons: if not self.ctx.creates_persons:
self.ctx.stats["persons_not_created"] += 1 self.ctx.stats["persons_not_created"] += 1
+3
View File
@@ -98,6 +98,9 @@ class PersonRecord:
#: if no identifier matches, merge with an existing person of exactly this (unambiguous) name before #: if no identifier matches, merge with an existing person of exactly this (unambiguous) name before
#: creating a new one - for authoritative sources that share no id with others (e.g. EP) #: creating a new one - for authoritative sources that share no id with others (e.g. EP)
match_by_name: bool = False match_by_name: bool = False
#: with match_by_name: state key ('noe', ...) of a Landtag list, to tell homonyms apart and accept name
#: variants among that Landtag's members (Resolver.person_by_name)
match_landtag: str | None = None
@dataclass @dataclass
+60 -6
View File
@@ -28,6 +28,24 @@ OFFICE_FILLER = re.compile(r"\b(der republik osterreich|der bundesrepublik oster
r"\(osterreich\)") r"\(osterreich\)")
def name_parts(name: str) -> tuple[str, str]:
"""'Mag. Chiara-Sophia Glawogger' -> ('chiara', 'glawogger'): first given name and surname, normalised."""
tokens = normalize(strip_titles(name)).replace(",", " ").split()
if len(tokens) < 2:
return "", ""
return tokens[0].split("-")[0], tokens[-1]
def variant_of(a: str, b: str) -> bool:
"""Same person written differently: surnames equal or one is part of the other's double surname
('Kirchmayr' / 'Kirchmayr-Felbermaier'), first names share their first three letters ('Rudi' / 'Rudolf')."""
(ga, sa), (gb, sb) = name_parts(a), name_parts(b)
if not (ga and sa and gb and sb):
return False
surname = sa == sb or sa in sb.split("-") or sb in sa.split("-")
return surname and len(ga) >= 3 and len(gb) >= 3 and ga[:3] == gb[:3]
def office_key(title: str) -> str: def office_key(title: str) -> str:
"""'Bundesministerin für Justiz' / 'Bundeskanzler der Republik Österreich' -> 'bundesminister fur justiz' / """'Bundesministerin für Justiz' / 'Bundeskanzler der Republik Österreich' -> 'bundesminister fur justiz' /
'bundeskanzler' (gender-neutral, without filler, punctuation-free) for comparing office titles.""" 'bundeskanzler' (gender-neutral, without filler, punctuation-free) for comparing office titles."""
@@ -109,18 +127,54 @@ class Resolver:
self._person_cache[cache_key] = found self._person_cache[cache_key] = found
return found return found
def person_by_name(self, name: str) -> int | None: def person_by_name(self, name: str, birth_date: date | None = None, landtag: str | None = None) -> int | None:
"""Exact alias match. A known birth date rules out persons born on another day; `landtag` (state key such
as 'noe') prefers, among homonyms, the one who has sat in that Landtag, and falls back to name variants
among that Landtag's members (landtag_member_by_variant)."""
norm = normalize(strip_titles(name)) norm = normalize(strip_titles(name))
rows = self.conn.execute( rows = self.conn.execute(
"""SELECT DISTINCT a.entity_id AS id, ps.status """SELECT DISTINCT a.entity_id AS id, ps.status, p.birth_date,
FROM alias a LEFT JOIN person_status ps ON ps.person_id = a.entity_id %s::text IS NOT NULL AND EXISTS (
SELECT 1 FROM role r JOIN claim c ON c.id = r.claim_id
WHERE r.person_id = a.entity_id AND c.merged_into IS NULL AND r.level = 'state'
AND (landtag_key(r.role) = %s OR c.created_by = 'landtag.' || %s)) AS in_landtag
FROM alias a JOIN person p ON p.id = a.entity_id
LEFT JOIN person_status ps ON ps.person_id = a.entity_id
WHERE a.entity_type = 'person' AND a.alias_norm = %s""", WHERE a.entity_type = 'person' AND a.alias_norm = %s""",
(norm,), (landtag, landtag, landtag, norm),
).fetchall() ).fetchall()
if birth_date:
same = [r for r in rows if r["birth_date"] == birth_date]
if len(same) == 1:
return same[0]["id"]
rows = [r for r in rows if r["birth_date"] is None]
if len(rows) == 1: if len(rows) == 1:
return rows[0]["id"] return rows[0]["id"]
active = [r for r in rows if r["status"] == "active"] for preferred in ([r for r in rows if r["in_landtag"]], [r for r in rows if r["status"] == "active"]):
return active[0]["id"] if len(active) == 1 else None if len(preferred) == 1:
return preferred[0]["id"]
if not rows and landtag:
return self.landtag_member_by_variant(name, landtag, birth_date)
return None
def landtag_member_by_variant(self, name: str, landtag: str, birth_date: date | None = None) -> int | None:
"""A Landtag list writes 'Rudi Hemetsberger', 'Chiara-Sophia Glawogger' or 'Helena Kirchmayr-Felbermaier'
where Wikidata has 'Rudolf Hemetsberger', 'Chiara Glawogger' and 'Helena Kirchmayr'. Among persons who
have sat in that Landtag, accept the single one whose surname matches (or is one part of a double surname)
and whose first name shares its first three letters, unless the birth dates differ."""
given, surname = name_parts(name)
if not given or not surname:
return None
rows = self.conn.execute(
"""SELECT DISTINCT p.id, p.full_name, p.birth_date FROM person p
JOIN role r ON r.person_id = p.id JOIN claim c ON c.id = r.claim_id
WHERE c.merged_into IS NULL AND r.level = 'state' AND landtag_key(r.role) = %s""",
(landtag,),
).fetchall()
hits = {row["id"] for row in rows
if not (birth_date and row["birth_date"] and row["birth_date"] != birth_date)
and variant_of(name, row["full_name"])}
return hits.pop() if len(hits) == 1 else None
def _person_by_hints(self, ref: PersonRef) -> int | None: def _person_by_hints(self, ref: PersonRef) -> int | None:
hints = ref.hints hints = ref.hints
+13 -4
View File
@@ -53,22 +53,30 @@ PARTY_ALIASES = {"KEIN KLUB": None, "VP TIROL": "ÖVP", "VP": "ÖVP", "VOLKSPART
# Functions written in front of names: Klubobmann, Landtagspräsidentin, KO-Stellvertreter, ... # Functions written in front of names: Klubobmann, Landtagspräsidentin, KO-Stellvertreter, ...
ROLE_PREFIX = re.compile( ROLE_PREFIX = re.compile(
r"^(?:(?:Landtagspräsident(?:in)?|Präsident(?:in)?|(?:Zweite|Dritte)[rn]? Präsident(?:in)?|Klubobmann|Klubobfrau|" r"^(?:(?:Landtagspräsident(?:in)?|Präsident(?:in)?|(?:Zweite|Dritte)[rn]? Präsident(?:in)?|Klubobmann|Klubobfrau|"
r"Klubobleute|KO-Stellvertreter(?:in)?|KO|VP(?:in)?|LTPräs(?:in)?|LAbg\.|Abg\.|Bgm\.?(?:in)?|KommR|HR|Prof\.)\s+)+") r"Klubobleute|KO-Stellvertreter(?:in)?|KO|VP(?:in)?|LTPräs(?:in)?|LTP(?:in)?|LAbg\.|Abg\.|Bgm\.?(?:in)?|KommR|HR|Prof\.)\s+)+")
TITLE_TOKENS = {"Mag", "Maga", "MMag", "Dr", "DI", "DIin", "Dipl", "Ing", "Prof", "HR", "BA", "MA", "BSc", "MSc", "MBA", TITLE_TOKENS = {"Mag", "Maga", "MMag", "Dr", "DI", "DIin", "Dipl", "Ing", "Prof", "HR", "BA", "MA", "BSc", "MSc", "MBA",
"BEd", "B.Ed", "B.A", "M.A", "LL.M", "LLM", "MPA", "PhD", "KommR", "Bgm", "Päd", "Univ", "Mag.a", "BEd", "B.Ed", "B.A", "M.A", "LL.M", "LLM", "MPA", "PhD", "KommR", "Bgm", "Päd", "Univ", "Mag.a",
"Dr.in", "Mag.(FH)", "(FH)", "MAS", "MSc.", "M.A.I.S", "Dipl.-Ing", "Dipl.Soz.-Wiss.in", "Ing.in"} "Dr.in", "Mag.(FH)", "(FH)", "MAS", "MSc.", "M.A.I.S", "Dipl.-Ing", "Dipl.Soz.-Wiss.in", "Ing.in"}
SUFFIX_TOKENS = {"a", "in"} # "Mag. a", "Dr. in" SUFFIX_TOKENS = {"a", "in"} # "Mag. a", "Dr. in"
# Image alt texts ("Portraitfoto von Karin Berger") and invisible characters some lists contain (zero-width space)
CAPTION_PREFIX = re.compile(r"^(?:Portrait|Porträt|Foto|Bild)(?:foto)?(?: von| of)?\s+", re.IGNORECASE)
INVISIBLE = re.compile("[\u00ad\u200b-\u200f\u2060\ufeff]")
def _is_title(token: str) -> bool: def _is_title(token: str) -> bool:
t = token.strip(",.") t = token.strip(",.")
return t in TITLE_TOKENS or re.fullmatch(r"(?:Dipl|Univ|Mag|Dr)\.[\w.\-]*", token.strip(",")) is not None if t in TITLE_TOKENS or re.fullmatch(r"(?:Dipl|Univ|Mag|Dr)\.[\w.\-]*", token.strip(",")):
return True
# any other abbreviation of two or more letters ("MMst.", "Bakk."); a name never ends with a dot, an
# initial ("J.") is kept
return re.fullmatch(r"[^\W\d_]{2,}\.", token.strip(",")) is not None
def clean_name(name: str) -> str: def clean_name(name: str) -> str:
"""'KO Mag. Markus Abwerzger' -> 'Markus Abwerzger'; drops functions, academic titles and stray punctuation. """'KO Mag. Markus Abwerzger' -> 'Markus Abwerzger'; drops functions, academic titles and stray punctuation.
Titles are matched as whole tokens, so a first name like "Ingrid" is never taken for "Ing.".""" Titles are matched as whole tokens, so a first name like "Ingrid" is never taken for "Ing."."""
name = ROLE_PREFIX.sub("", re.sub(r"\s+", " ", name).strip()) name = re.sub(r"\([^()]*\)", " ", INVISIBLE.sub("", name)) # "(WU)" after "MSc", "(FH)"
name = ROLE_PREFIX.sub("", CAPTION_PREFIX.sub("", re.sub(r"\s+", " ", name).strip()))
name = strip_titles(name) name = strip_titles(name)
kept = [] kept = []
for token in name.replace(",", " , ").split(): for token in name.replace(",", " , ").split():
@@ -253,7 +261,8 @@ class LandtagMembers(Adapter):
region, body_name, _, list_url = LANDTAGE[self.key] region, body_name, _, list_url = LANDTAGE[self.key]
scheme = f"landtag-{self.key}" scheme = f"landtag-{self.key}"
ref = PersonRef(ids={scheme: m["id"]}, name=m["name"]) ref = PersonRef(ids={scheme: m["id"]}, name=m["name"])
yield PersonRecord(ref=ref, full_name=m["name"], birth_date=m["birth_date"], match_by_name=True) yield PersonRecord(ref=ref, full_name=m["name"], birth_date=m["birth_date"], match_by_name=True,
match_landtag=self.key)
body = OrgRef(slug=f"landtag-{self.key}", name=body_name, kind="parliament") body = OrgRef(slug=f"landtag-{self.key}", name=body_name, kind="parliament")
party = OrgRef(short_name=m["party"], kind="party", create=False) if m["party"] else None party = OrgRef(short_name=m["party"], kind="party", create=False) if m["party"] else None
label = f" ({m['party']})" if m["party"] else "" label = f" ({m['party']})" if m["party"] else ""
+2 -2
View File
@@ -14,10 +14,10 @@ ACADEMIC_TITLES = re.compile(
def normalize(text: str) -> str: def normalize(text: str) -> str:
"""Lowercase, strip accents and collapse whitespace (matches SQL lower(unaccent())).""" """Lowercase, strip accents, ß -> ss and collapse whitespace (matches SQL lower(unaccent()))."""
text = unicodedata.normalize("NFKD", text) text = unicodedata.normalize("NFKD", text)
text = "".join(c for c in text if not unicodedata.combining(c)) text = "".join(c for c in text if not unicodedata.combining(c))
return re.sub(r"\s+", " ", text).strip().lower() return re.sub(r"\s+", " ", text).strip().lower().replace("ß", "ss")
def strip_titles(name: str) -> str: def strip_titles(name: str) -> str:
@@ -11,6 +11,7 @@
"honorific_prefix": null, "honorific_prefix": null,
"honorific_suffix": null, "honorific_suffix": null,
"match_by_name": false, "match_by_name": false,
"match_landtag": null,
"photo_url": "https://www.europarl.europa.eu/mepphoto/220897.jpg", "photo_url": "https://www.europarl.europa.eu/mepphoto/220897.jpg",
"ref": { "ref": {
"__type__": "PersonRef", "__type__": "PersonRef",
@@ -13,6 +13,7 @@
"honorific_prefix": null, "honorific_prefix": null,
"honorific_suffix": null, "honorific_suffix": null,
"match_by_name": true, "match_by_name": true,
"match_landtag": null,
"photo_url": null, "photo_url": null,
"ref": { "ref": {
"__type__": "PersonRef", "__type__": "PersonRef",
@@ -74,6 +75,7 @@
"honorific_prefix": null, "honorific_prefix": null,
"honorific_suffix": null, "honorific_suffix": null,
"match_by_name": true, "match_by_name": true,
"match_landtag": null,
"photo_url": null, "photo_url": null,
"ref": { "ref": {
"__type__": "PersonRef", "__type__": "PersonRef",
@@ -135,6 +137,7 @@
"honorific_prefix": null, "honorific_prefix": null,
"honorific_suffix": null, "honorific_suffix": null,
"match_by_name": true, "match_by_name": true,
"match_landtag": null,
"photo_url": null, "photo_url": null,
"ref": { "ref": {
"__type__": "PersonRef", "__type__": "PersonRef",
@@ -196,6 +199,7 @@
"honorific_prefix": null, "honorific_prefix": null,
"honorific_suffix": null, "honorific_suffix": null,
"match_by_name": true, "match_by_name": true,
"match_landtag": null,
"photo_url": null, "photo_url": null,
"ref": { "ref": {
"__type__": "PersonRef", "__type__": "PersonRef",
@@ -257,6 +261,7 @@
"honorific_prefix": null, "honorific_prefix": null,
"honorific_suffix": null, "honorific_suffix": null,
"match_by_name": true, "match_by_name": true,
"match_landtag": null,
"photo_url": null, "photo_url": null,
"ref": { "ref": {
"__type__": "PersonRef", "__type__": "PersonRef",
@@ -318,6 +323,7 @@
"honorific_prefix": null, "honorific_prefix": null,
"honorific_suffix": null, "honorific_suffix": null,
"match_by_name": true, "match_by_name": true,
"match_landtag": null,
"photo_url": null, "photo_url": null,
"ref": { "ref": {
"__type__": "PersonRef", "__type__": "PersonRef",
@@ -379,6 +385,7 @@
"honorific_prefix": null, "honorific_prefix": null,
"honorific_suffix": null, "honorific_suffix": null,
"match_by_name": true, "match_by_name": true,
"match_landtag": null,
"photo_url": null, "photo_url": null,
"ref": { "ref": {
"__type__": "PersonRef", "__type__": "PersonRef",
@@ -440,6 +447,7 @@
"honorific_prefix": null, "honorific_prefix": null,
"honorific_suffix": null, "honorific_suffix": null,
"match_by_name": true, "match_by_name": true,
"match_landtag": null,
"photo_url": null, "photo_url": null,
"ref": { "ref": {
"__type__": "PersonRef", "__type__": "PersonRef",
@@ -11,6 +11,7 @@
"honorific_prefix": null, "honorific_prefix": null,
"honorific_suffix": null, "honorific_suffix": null,
"match_by_name": false, "match_by_name": false,
"match_landtag": null,
"photo_url": "https://www.parlament.gv.at/dokument/bild/201426/20142639_500.jpg", "photo_url": "https://www.parlament.gv.at/dokument/bild/201426/20142639_500.jpg",
"ref": { "ref": {
"__type__": "PersonRef", "__type__": "PersonRef",
+18
View File
@@ -10,6 +10,12 @@ def test_clean_name_drops_functions_and_titles_but_not_names():
assert L.clean_name("Dipl. Päd., DI Carina Laschober-Luif") == "Carina Laschober-Luif" assert L.clean_name("Dipl. Päd., DI Carina Laschober-Luif") == "Carina Laschober-Luif"
assert L.clean_name("Mag. a Astrid Eisenkopf") == "Astrid Eisenkopf" assert L.clean_name("Mag. a Astrid Eisenkopf") == "Astrid Eisenkopf"
assert L.clean_name("Ingrid Korosec") == "Ingrid Korosec" # "Ing" is a title, "Ingrid" is not assert L.clean_name("Ingrid Korosec") == "Ingrid Korosec" # "Ing" is a title, "Ingrid" is not
assert L.clean_name("Portraitfoto von Simon Heilig-Hofbauer") == "Simon Heilig-Hofbauer"
assert L.clean_name("LTP Jakob Wolf") == "Jakob Wolf"
assert L.clean_name("MMst. Doris Schmidl") == "Doris Schmidl"
assert L.clean_name("Claudia Gamon, MSc (WU)") == "Claudia Gamon"
assert L.clean_name("Johannes" + chr(0x200B) + " Schwarz") == "Johannes Schwarz"
assert L.clean_name("Michael J. Fox") == "Michael J. Fox" # initials stay
assert L.surname_first("Höglinger Tobias, Mag.") == "Tobias Höglinger" assert L.surname_first("Höglinger Tobias, Mag.") == "Tobias Höglinger"
assert L.surname_first("EISEL-EISELSBERG Detlev") == "Detlev Eisel-Eiselsberg" assert L.surname_first("EISEL-EISELSBERG Detlev") == "Detlev Eisel-Eiselsberg"
@@ -66,3 +72,15 @@ def test_robots_excluded_states_have_no_adapter():
from politrack.core.registry import ADAPTERS from politrack.core.registry import ADAPTERS
assert "landtag.wien" not in ADAPTERS and "landtag.ktn" not in ADAPTERS assert "landtag.wien" not in ADAPTERS and "landtag.ktn" not in ADAPTERS
assert "landtag.bgld" in ADAPTERS assert "landtag.bgld" in ADAPTERS
def test_name_variants_and_sharp_s():
from politrack.core.resolve import variant_of
from politrack.text import normalize
assert normalize("Klaudia Stroißnig") == normalize("Klaudia Stroissnig")
for listed, known in [("Rudi Hemetsberger", "Rudolf Hemetsberger"), ("Max Hiegelsberger", "Maximilian Hiegelsberger"),
("Chiara-Sophia Glawogger", "Chiara Glawogger"), ("Helena Kirchmayr-Felbermaier", "Helena Kirchmayr"),
("Michael Stefan Wagner", "Michael Wagner"), ("Gebi Mair", "Gebhard Mair")]:
assert variant_of(listed, known), (listed, known)
for listed, known in [("Michael Weber", "Michael Huber"), ("Doris Kampus", "Doris Hahn"), ("Jo Mair", "Josef Mair")]:
assert not variant_of(listed, known), (listed, known)