diff --git a/backend/migrations/0008_alias_sharp_s.sql b/backend/migrations/0008_alias_sharp_s.sql new file mode 100644 index 0000000..f9c9559 --- /dev/null +++ b/backend/migrations/0008_alias_sharp_s.sql @@ -0,0 +1,7 @@ +-- normalize() now maps ß to ss, as SQL unaccent() does, so 'Stroißnig' and 'Stroissnig' are one alias. +-- Bring stored aliases in line (dropping those that would collide with an existing ss spelling). +DELETE FROM alias a +WHERE a.alias_norm LIKE '%%ß%%' + AND EXISTS (SELECT 1 FROM alias b WHERE b.entity_type = a.entity_type AND b.entity_id = a.entity_id + AND b.alias_norm = replace(a.alias_norm, 'ß', 'ss')); +UPDATE alias SET alias_norm = replace(alias_norm, 'ß', 'ss') WHERE alias_norm LIKE '%%ß%%'; diff --git a/backend/politrack/core/loader.py b/backend/politrack/core/loader.py index bc8fde1..13e29c1 100644 --- a/backend/politrack/core/loader.py +++ b/backend/politrack/core/loader.py @@ -162,7 +162,7 @@ class Loader: def person(self, rec: PersonRecord) -> int | None: person_id = self.resolver.person_by_ids(rec.ref.ids) if person_id is None and rec.match_by_name: - person_id = self.resolver.person_by_name(rec.full_name) + person_id = self.resolver.person_by_name(rec.full_name, rec.birth_date, rec.match_landtag) if person_id is None: if not self.ctx.creates_persons: self.ctx.stats["persons_not_created"] += 1 diff --git a/backend/politrack/core/records.py b/backend/politrack/core/records.py index 67d1fb4..82da84e 100644 --- a/backend/politrack/core/records.py +++ b/backend/politrack/core/records.py @@ -98,6 +98,9 @@ class PersonRecord: #: if no identifier matches, merge with an existing person of exactly this (unambiguous) name before #: creating a new one - for authoritative sources that share no id with others (e.g. EP) match_by_name: bool = False + #: with match_by_name: state key ('noe', ...) of a Landtag list, to tell homonyms apart and accept name + #: variants among that Landtag's members (Resolver.person_by_name) + match_landtag: str | None = None @dataclass diff --git a/backend/politrack/core/resolve.py b/backend/politrack/core/resolve.py index 3712f13..d342a96 100644 --- a/backend/politrack/core/resolve.py +++ b/backend/politrack/core/resolve.py @@ -28,6 +28,24 @@ OFFICE_FILLER = re.compile(r"\b(der republik osterreich|der bundesrepublik oster r"\(osterreich\)") +def name_parts(name: str) -> tuple[str, str]: + """'Mag. Chiara-Sophia Glawogger' -> ('chiara', 'glawogger'): first given name and surname, normalised.""" + tokens = normalize(strip_titles(name)).replace(",", " ").split() + if len(tokens) < 2: + return "", "" + return tokens[0].split("-")[0], tokens[-1] + + +def variant_of(a: str, b: str) -> bool: + """Same person written differently: surnames equal or one is part of the other's double surname + ('Kirchmayr' / 'Kirchmayr-Felbermaier'), first names share their first three letters ('Rudi' / 'Rudolf').""" + (ga, sa), (gb, sb) = name_parts(a), name_parts(b) + if not (ga and sa and gb and sb): + return False + surname = sa == sb or sa in sb.split("-") or sb in sa.split("-") + return surname and len(ga) >= 3 and len(gb) >= 3 and ga[:3] == gb[:3] + + def office_key(title: str) -> str: """'Bundesministerin für Justiz' / 'Bundeskanzler der Republik Österreich' -> 'bundesminister fur justiz' / 'bundeskanzler' (gender-neutral, without filler, punctuation-free) for comparing office titles.""" @@ -109,18 +127,54 @@ class Resolver: self._person_cache[cache_key] = found return found - def person_by_name(self, name: str) -> int | None: + def person_by_name(self, name: str, birth_date: date | None = None, landtag: str | None = None) -> int | None: + """Exact alias match. A known birth date rules out persons born on another day; `landtag` (state key such + as 'noe') prefers, among homonyms, the one who has sat in that Landtag, and falls back to name variants + among that Landtag's members (landtag_member_by_variant).""" norm = normalize(strip_titles(name)) rows = self.conn.execute( - """SELECT DISTINCT a.entity_id AS id, ps.status - FROM alias a LEFT JOIN person_status ps ON ps.person_id = a.entity_id + """SELECT DISTINCT a.entity_id AS id, ps.status, p.birth_date, + %s::text IS NOT NULL AND EXISTS ( + SELECT 1 FROM role r JOIN claim c ON c.id = r.claim_id + WHERE r.person_id = a.entity_id AND c.merged_into IS NULL AND r.level = 'state' + AND (landtag_key(r.role) = %s OR c.created_by = 'landtag.' || %s)) AS in_landtag + FROM alias a JOIN person p ON p.id = a.entity_id + LEFT JOIN person_status ps ON ps.person_id = a.entity_id WHERE a.entity_type = 'person' AND a.alias_norm = %s""", - (norm,), + (landtag, landtag, landtag, norm), ).fetchall() + if birth_date: + same = [r for r in rows if r["birth_date"] == birth_date] + if len(same) == 1: + return same[0]["id"] + rows = [r for r in rows if r["birth_date"] is None] if len(rows) == 1: return rows[0]["id"] - active = [r for r in rows if r["status"] == "active"] - return active[0]["id"] if len(active) == 1 else None + for preferred in ([r for r in rows if r["in_landtag"]], [r for r in rows if r["status"] == "active"]): + if len(preferred) == 1: + return preferred[0]["id"] + if not rows and landtag: + return self.landtag_member_by_variant(name, landtag, birth_date) + return None + + def landtag_member_by_variant(self, name: str, landtag: str, birth_date: date | None = None) -> int | None: + """A Landtag list writes 'Rudi Hemetsberger', 'Chiara-Sophia Glawogger' or 'Helena Kirchmayr-Felbermaier' + where Wikidata has 'Rudolf Hemetsberger', 'Chiara Glawogger' and 'Helena Kirchmayr'. Among persons who + have sat in that Landtag, accept the single one whose surname matches (or is one part of a double surname) + and whose first name shares its first three letters, unless the birth dates differ.""" + given, surname = name_parts(name) + if not given or not surname: + return None + rows = self.conn.execute( + """SELECT DISTINCT p.id, p.full_name, p.birth_date FROM person p + JOIN role r ON r.person_id = p.id JOIN claim c ON c.id = r.claim_id + WHERE c.merged_into IS NULL AND r.level = 'state' AND landtag_key(r.role) = %s""", + (landtag,), + ).fetchall() + hits = {row["id"] for row in rows + if not (birth_date and row["birth_date"] and row["birth_date"] != birth_date) + and variant_of(name, row["full_name"])} + return hits.pop() if len(hits) == 1 else None def _person_by_hints(self, ref: PersonRef) -> int | None: hints = ref.hints diff --git a/backend/politrack/sources/landtage.py b/backend/politrack/sources/landtage.py index a96fd87..359f401 100644 --- a/backend/politrack/sources/landtage.py +++ b/backend/politrack/sources/landtage.py @@ -53,22 +53,30 @@ PARTY_ALIASES = {"KEIN KLUB": None, "VP TIROL": "ÖVP", "VP": "ÖVP", "VOLKSPART # Functions written in front of names: Klubobmann, Landtagspräsidentin, KO-Stellvertreter, ... ROLE_PREFIX = re.compile( r"^(?:(?:Landtagspräsident(?:in)?|Präsident(?:in)?|(?:Zweite|Dritte)[rn]? Präsident(?:in)?|Klubobmann|Klubobfrau|" - r"Klubobleute|KO-Stellvertreter(?:in)?|KO|VP(?:in)?|LTPräs(?:in)?|LAbg\.|Abg\.|Bgm\.?(?:in)?|KommR|HR|Prof\.)\s+)+") + r"Klubobleute|KO-Stellvertreter(?:in)?|KO|VP(?:in)?|LTPräs(?:in)?|LTP(?:in)?|LAbg\.|Abg\.|Bgm\.?(?:in)?|KommR|HR|Prof\.)\s+)+") TITLE_TOKENS = {"Mag", "Maga", "MMag", "Dr", "DI", "DIin", "Dipl", "Ing", "Prof", "HR", "BA", "MA", "BSc", "MSc", "MBA", "BEd", "B.Ed", "B.A", "M.A", "LL.M", "LLM", "MPA", "PhD", "KommR", "Bgm", "Päd", "Univ", "Mag.a", "Dr.in", "Mag.(FH)", "(FH)", "MAS", "MSc.", "M.A.I.S", "Dipl.-Ing", "Dipl.Soz.-Wiss.in", "Ing.in"} SUFFIX_TOKENS = {"a", "in"} # "Mag. a", "Dr. in" +# Image alt texts ("Portraitfoto von Karin Berger") and invisible characters some lists contain (zero-width space) +CAPTION_PREFIX = re.compile(r"^(?:Portrait|Porträt|Foto|Bild)(?:foto)?(?: von| of)?\s+", re.IGNORECASE) +INVISIBLE = re.compile("[\u00ad\u200b-\u200f\u2060\ufeff]") def _is_title(token: str) -> bool: t = token.strip(",.") - return t in TITLE_TOKENS or re.fullmatch(r"(?:Dipl|Univ|Mag|Dr)\.[\w.\-]*", token.strip(",")) is not None + if t in TITLE_TOKENS or re.fullmatch(r"(?:Dipl|Univ|Mag|Dr)\.[\w.\-]*", token.strip(",")): + return True + # any other abbreviation of two or more letters ("MMst.", "Bakk."); a name never ends with a dot, an + # initial ("J.") is kept + return re.fullmatch(r"[^\W\d_]{2,}\.", token.strip(",")) is not None def clean_name(name: str) -> str: """'KO Mag. Markus Abwerzger' -> 'Markus Abwerzger'; drops functions, academic titles and stray punctuation. Titles are matched as whole tokens, so a first name like "Ingrid" is never taken for "Ing.".""" - name = ROLE_PREFIX.sub("", re.sub(r"\s+", " ", name).strip()) + name = re.sub(r"\([^()]*\)", " ", INVISIBLE.sub("", name)) # "(WU)" after "MSc", "(FH)" + name = ROLE_PREFIX.sub("", CAPTION_PREFIX.sub("", re.sub(r"\s+", " ", name).strip())) name = strip_titles(name) kept = [] for token in name.replace(",", " , ").split(): @@ -253,7 +261,8 @@ class LandtagMembers(Adapter): region, body_name, _, list_url = LANDTAGE[self.key] scheme = f"landtag-{self.key}" ref = PersonRef(ids={scheme: m["id"]}, name=m["name"]) - yield PersonRecord(ref=ref, full_name=m["name"], birth_date=m["birth_date"], match_by_name=True) + yield PersonRecord(ref=ref, full_name=m["name"], birth_date=m["birth_date"], match_by_name=True, + match_landtag=self.key) body = OrgRef(slug=f"landtag-{self.key}", name=body_name, kind="parliament") party = OrgRef(short_name=m["party"], kind="party", create=False) if m["party"] else None label = f" ({m['party']})" if m["party"] else "" diff --git a/backend/politrack/text.py b/backend/politrack/text.py index 985027b..6e53324 100644 --- a/backend/politrack/text.py +++ b/backend/politrack/text.py @@ -14,10 +14,10 @@ ACADEMIC_TITLES = re.compile( def normalize(text: str) -> str: - """Lowercase, strip accents and collapse whitespace (matches SQL lower(unaccent())).""" + """Lowercase, strip accents, ß -> ss and collapse whitespace (matches SQL lower(unaccent())).""" text = unicodedata.normalize("NFKD", text) text = "".join(c for c in text if not unicodedata.combining(c)) - return re.sub(r"\s+", " ", text).strip().lower() + return re.sub(r"\s+", " ", text).strip().lower().replace("ß", "ss") def strip_titles(name: str) -> str: diff --git a/backend/tests/fixtures/europarl.meps/mep.expected.json b/backend/tests/fixtures/europarl.meps/mep.expected.json index 92a14de..598989a 100644 --- a/backend/tests/fixtures/europarl.meps/mep.expected.json +++ b/backend/tests/fixtures/europarl.meps/mep.expected.json @@ -11,6 +11,7 @@ "honorific_prefix": null, "honorific_suffix": null, "match_by_name": false, + "match_landtag": null, "photo_url": "https://www.europarl.europa.eu/mepphoto/220897.jpg", "ref": { "__type__": "PersonRef", diff --git a/backend/tests/fixtures/europarl.meps/meps.expected.json b/backend/tests/fixtures/europarl.meps/meps.expected.json index df584a9..1ba8842 100644 --- a/backend/tests/fixtures/europarl.meps/meps.expected.json +++ b/backend/tests/fixtures/europarl.meps/meps.expected.json @@ -13,6 +13,7 @@ "honorific_prefix": null, "honorific_suffix": null, "match_by_name": true, + "match_landtag": null, "photo_url": null, "ref": { "__type__": "PersonRef", @@ -74,6 +75,7 @@ "honorific_prefix": null, "honorific_suffix": null, "match_by_name": true, + "match_landtag": null, "photo_url": null, "ref": { "__type__": "PersonRef", @@ -135,6 +137,7 @@ "honorific_prefix": null, "honorific_suffix": null, "match_by_name": true, + "match_landtag": null, "photo_url": null, "ref": { "__type__": "PersonRef", @@ -196,6 +199,7 @@ "honorific_prefix": null, "honorific_suffix": null, "match_by_name": true, + "match_landtag": null, "photo_url": null, "ref": { "__type__": "PersonRef", @@ -257,6 +261,7 @@ "honorific_prefix": null, "honorific_suffix": null, "match_by_name": true, + "match_landtag": null, "photo_url": null, "ref": { "__type__": "PersonRef", @@ -318,6 +323,7 @@ "honorific_prefix": null, "honorific_suffix": null, "match_by_name": true, + "match_landtag": null, "photo_url": null, "ref": { "__type__": "PersonRef", @@ -379,6 +385,7 @@ "honorific_prefix": null, "honorific_suffix": null, "match_by_name": true, + "match_landtag": null, "photo_url": null, "ref": { "__type__": "PersonRef", @@ -440,6 +447,7 @@ "honorific_prefix": null, "honorific_suffix": null, "match_by_name": true, + "match_landtag": null, "photo_url": null, "ref": { "__type__": "PersonRef", diff --git a/backend/tests/fixtures/parlament.persons/person_detail.expected.json b/backend/tests/fixtures/parlament.persons/person_detail.expected.json index 14203ad..04e93f4 100644 --- a/backend/tests/fixtures/parlament.persons/person_detail.expected.json +++ b/backend/tests/fixtures/parlament.persons/person_detail.expected.json @@ -11,6 +11,7 @@ "honorific_prefix": null, "honorific_suffix": null, "match_by_name": false, + "match_landtag": null, "photo_url": "https://www.parlament.gv.at/dokument/bild/201426/20142639_500.jpg", "ref": { "__type__": "PersonRef", diff --git a/backend/tests/test_landtage.py b/backend/tests/test_landtage.py index d1c3b65..2667df1 100644 --- a/backend/tests/test_landtage.py +++ b/backend/tests/test_landtage.py @@ -10,6 +10,12 @@ def test_clean_name_drops_functions_and_titles_but_not_names(): assert L.clean_name("Dipl. Päd., DI Carina Laschober-Luif") == "Carina Laschober-Luif" assert L.clean_name("Mag. a Astrid Eisenkopf") == "Astrid Eisenkopf" assert L.clean_name("Ingrid Korosec") == "Ingrid Korosec" # "Ing" is a title, "Ingrid" is not + assert L.clean_name("Portraitfoto von Simon Heilig-Hofbauer") == "Simon Heilig-Hofbauer" + assert L.clean_name("LTP Jakob Wolf") == "Jakob Wolf" + assert L.clean_name("MMst. Doris Schmidl") == "Doris Schmidl" + assert L.clean_name("Claudia Gamon, MSc (WU)") == "Claudia Gamon" + assert L.clean_name("Johannes" + chr(0x200B) + " Schwarz") == "Johannes Schwarz" + assert L.clean_name("Michael J. Fox") == "Michael J. Fox" # initials stay assert L.surname_first("Höglinger Tobias, Mag.") == "Tobias Höglinger" assert L.surname_first("EISEL-EISELSBERG Detlev") == "Detlev Eisel-Eiselsberg" @@ -66,3 +72,15 @@ def test_robots_excluded_states_have_no_adapter(): from politrack.core.registry import ADAPTERS assert "landtag.wien" not in ADAPTERS and "landtag.ktn" not in ADAPTERS assert "landtag.bgld" in ADAPTERS + + +def test_name_variants_and_sharp_s(): + from politrack.core.resolve import variant_of + from politrack.text import normalize + assert normalize("Klaudia Stroißnig") == normalize("Klaudia Stroissnig") + for listed, known in [("Rudi Hemetsberger", "Rudolf Hemetsberger"), ("Max Hiegelsberger", "Maximilian Hiegelsberger"), + ("Chiara-Sophia Glawogger", "Chiara Glawogger"), ("Helena Kirchmayr-Felbermaier", "Helena Kirchmayr"), + ("Michael Stefan Wagner", "Michael Wagner"), ("Gebi Mair", "Gebhard Mair")]: + assert variant_of(listed, known), (listed, known) + for listed, known in [("Michael Weber", "Michael Huber"), ("Doris Kampus", "Doris Hahn"), ("Jo Mair", "Josef Mair")]: + assert not variant_of(listed, known), (listed, known)