Landtag lists: match existing persons by birth date, Landtag membership and name variants
- person_by_name rules out homonyms born on another day and prefers the one who sat in that Landtag; failing an exact name, accepts a unique variant among that Landtag's members (Rudi/Rudolf, Chiara-Sophia/Chiara, new double surnames) - normalize() maps ß to ss like SQL unaccent(); migration 0008 aligns stored aliases - clean_name drops image captions, parenthesised suffixes, LTP, other abbreviations and zero-width characters Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
faf9572b9f
commit
68be25a953
10 files changed
+114
-13
No files matched your search
@@ -0,0 +1,7 @@
|
||||
-- normalize() now maps ß to ss, as SQL unaccent() does, so 'Stroißnig' and 'Stroissnig' are one alias.
|
||||
-- Bring stored aliases in line (dropping those that would collide with an existing ss spelling).
|
||||
DELETE FROM alias a
|
||||
WHERE a.alias_norm LIKE '%%ß%%'
|
||||
AND EXISTS (SELECT 1 FROM alias b WHERE b.entity_type = a.entity_type AND b.entity_id = a.entity_id
|
||||
AND b.alias_norm = replace(a.alias_norm, 'ß', 'ss'));
|
||||
UPDATE alias SET alias_norm = replace(alias_norm, 'ß', 'ss') WHERE alias_norm LIKE '%%ß%%';
|
||||
@@ -162,7 +162,7 @@ class Loader:
|
||||
def person(self, rec: PersonRecord) -> int | None:
|
||||
person_id = self.resolver.person_by_ids(rec.ref.ids)
|
||||
if person_id is None and rec.match_by_name:
|
||||
person_id = self.resolver.person_by_name(rec.full_name)
|
||||
person_id = self.resolver.person_by_name(rec.full_name, rec.birth_date, rec.match_landtag)
|
||||
if person_id is None:
|
||||
if not self.ctx.creates_persons:
|
||||
self.ctx.stats["persons_not_created"] += 1
|
||||
|
||||
@@ -98,6 +98,9 @@ class PersonRecord:
|
||||
#: if no identifier matches, merge with an existing person of exactly this (unambiguous) name before
|
||||
#: creating a new one - for authoritative sources that share no id with others (e.g. EP)
|
||||
match_by_name: bool = False
|
||||
#: with match_by_name: state key ('noe', ...) of a Landtag list, to tell homonyms apart and accept name
|
||||
#: variants among that Landtag's members (Resolver.person_by_name)
|
||||
match_landtag: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -28,6 +28,24 @@ OFFICE_FILLER = re.compile(r"\b(der republik osterreich|der bundesrepublik oster
|
||||
r"\(osterreich\)")
|
||||
|
||||
|
||||
def name_parts(name: str) -> tuple[str, str]:
|
||||
"""'Mag. Chiara-Sophia Glawogger' -> ('chiara', 'glawogger'): first given name and surname, normalised."""
|
||||
tokens = normalize(strip_titles(name)).replace(",", " ").split()
|
||||
if len(tokens) < 2:
|
||||
return "", ""
|
||||
return tokens[0].split("-")[0], tokens[-1]
|
||||
|
||||
|
||||
def variant_of(a: str, b: str) -> bool:
|
||||
"""Same person written differently: surnames equal or one is part of the other's double surname
|
||||
('Kirchmayr' / 'Kirchmayr-Felbermaier'), first names share their first three letters ('Rudi' / 'Rudolf')."""
|
||||
(ga, sa), (gb, sb) = name_parts(a), name_parts(b)
|
||||
if not (ga and sa and gb and sb):
|
||||
return False
|
||||
surname = sa == sb or sa in sb.split("-") or sb in sa.split("-")
|
||||
return surname and len(ga) >= 3 and len(gb) >= 3 and ga[:3] == gb[:3]
|
||||
|
||||
|
||||
def office_key(title: str) -> str:
|
||||
"""'Bundesministerin für Justiz' / 'Bundeskanzler der Republik Österreich' -> 'bundesminister fur justiz' /
|
||||
'bundeskanzler' (gender-neutral, without filler, punctuation-free) for comparing office titles."""
|
||||
@@ -109,18 +127,54 @@ class Resolver:
|
||||
self._person_cache[cache_key] = found
|
||||
return found
|
||||
|
||||
def person_by_name(self, name: str) -> int | None:
|
||||
def person_by_name(self, name: str, birth_date: date | None = None, landtag: str | None = None) -> int | None:
|
||||
"""Exact alias match. A known birth date rules out persons born on another day; `landtag` (state key such
|
||||
as 'noe') prefers, among homonyms, the one who has sat in that Landtag, and falls back to name variants
|
||||
among that Landtag's members (landtag_member_by_variant)."""
|
||||
norm = normalize(strip_titles(name))
|
||||
rows = self.conn.execute(
|
||||
"""SELECT DISTINCT a.entity_id AS id, ps.status
|
||||
FROM alias a LEFT JOIN person_status ps ON ps.person_id = a.entity_id
|
||||
"""SELECT DISTINCT a.entity_id AS id, ps.status, p.birth_date,
|
||||
%s::text IS NOT NULL AND EXISTS (
|
||||
SELECT 1 FROM role r JOIN claim c ON c.id = r.claim_id
|
||||
WHERE r.person_id = a.entity_id AND c.merged_into IS NULL AND r.level = 'state'
|
||||
AND (landtag_key(r.role) = %s OR c.created_by = 'landtag.' || %s)) AS in_landtag
|
||||
FROM alias a JOIN person p ON p.id = a.entity_id
|
||||
LEFT JOIN person_status ps ON ps.person_id = a.entity_id
|
||||
WHERE a.entity_type = 'person' AND a.alias_norm = %s""",
|
||||
(norm,),
|
||||
(landtag, landtag, landtag, norm),
|
||||
).fetchall()
|
||||
if birth_date:
|
||||
same = [r for r in rows if r["birth_date"] == birth_date]
|
||||
if len(same) == 1:
|
||||
return same[0]["id"]
|
||||
rows = [r for r in rows if r["birth_date"] is None]
|
||||
if len(rows) == 1:
|
||||
return rows[0]["id"]
|
||||
active = [r for r in rows if r["status"] == "active"]
|
||||
return active[0]["id"] if len(active) == 1 else None
|
||||
for preferred in ([r for r in rows if r["in_landtag"]], [r for r in rows if r["status"] == "active"]):
|
||||
if len(preferred) == 1:
|
||||
return preferred[0]["id"]
|
||||
if not rows and landtag:
|
||||
return self.landtag_member_by_variant(name, landtag, birth_date)
|
||||
return None
|
||||
|
||||
def landtag_member_by_variant(self, name: str, landtag: str, birth_date: date | None = None) -> int | None:
|
||||
"""A Landtag list writes 'Rudi Hemetsberger', 'Chiara-Sophia Glawogger' or 'Helena Kirchmayr-Felbermaier'
|
||||
where Wikidata has 'Rudolf Hemetsberger', 'Chiara Glawogger' and 'Helena Kirchmayr'. Among persons who
|
||||
have sat in that Landtag, accept the single one whose surname matches (or is one part of a double surname)
|
||||
and whose first name shares its first three letters, unless the birth dates differ."""
|
||||
given, surname = name_parts(name)
|
||||
if not given or not surname:
|
||||
return None
|
||||
rows = self.conn.execute(
|
||||
"""SELECT DISTINCT p.id, p.full_name, p.birth_date FROM person p
|
||||
JOIN role r ON r.person_id = p.id JOIN claim c ON c.id = r.claim_id
|
||||
WHERE c.merged_into IS NULL AND r.level = 'state' AND landtag_key(r.role) = %s""",
|
||||
(landtag,),
|
||||
).fetchall()
|
||||
hits = {row["id"] for row in rows
|
||||
if not (birth_date and row["birth_date"] and row["birth_date"] != birth_date)
|
||||
and variant_of(name, row["full_name"])}
|
||||
return hits.pop() if len(hits) == 1 else None
|
||||
|
||||
def _person_by_hints(self, ref: PersonRef) -> int | None:
|
||||
hints = ref.hints
|
||||
|
||||
@@ -53,22 +53,30 @@ PARTY_ALIASES = {"KEIN KLUB": None, "VP TIROL": "ÖVP", "VP": "ÖVP", "VOLKSPART
|
||||
# Functions written in front of names: Klubobmann, Landtagspräsidentin, KO-Stellvertreter, ...
|
||||
ROLE_PREFIX = re.compile(
|
||||
r"^(?:(?:Landtagspräsident(?:in)?|Präsident(?:in)?|(?:Zweite|Dritte)[rn]? Präsident(?:in)?|Klubobmann|Klubobfrau|"
|
||||
r"Klubobleute|KO-Stellvertreter(?:in)?|KO|VP(?:in)?|LTPräs(?:in)?|LAbg\.|Abg\.|Bgm\.?(?:in)?|KommR|HR|Prof\.)\s+)+")
|
||||
r"Klubobleute|KO-Stellvertreter(?:in)?|KO|VP(?:in)?|LTPräs(?:in)?|LTP(?:in)?|LAbg\.|Abg\.|Bgm\.?(?:in)?|KommR|HR|Prof\.)\s+)+")
|
||||
TITLE_TOKENS = {"Mag", "Maga", "MMag", "Dr", "DI", "DIin", "Dipl", "Ing", "Prof", "HR", "BA", "MA", "BSc", "MSc", "MBA",
|
||||
"BEd", "B.Ed", "B.A", "M.A", "LL.M", "LLM", "MPA", "PhD", "KommR", "Bgm", "Päd", "Univ", "Mag.a",
|
||||
"Dr.in", "Mag.(FH)", "(FH)", "MAS", "MSc.", "M.A.I.S", "Dipl.-Ing", "Dipl.Soz.-Wiss.in", "Ing.in"}
|
||||
SUFFIX_TOKENS = {"a", "in"} # "Mag. a", "Dr. in"
|
||||
# Image alt texts ("Portraitfoto von Karin Berger") and invisible characters some lists contain (zero-width space)
|
||||
CAPTION_PREFIX = re.compile(r"^(?:Portrait|Porträt|Foto|Bild)(?:foto)?(?: von| of)?\s+", re.IGNORECASE)
|
||||
INVISIBLE = re.compile("[\u00ad\u200b-\u200f\u2060\ufeff]")
|
||||
|
||||
|
||||
def _is_title(token: str) -> bool:
|
||||
t = token.strip(",.")
|
||||
return t in TITLE_TOKENS or re.fullmatch(r"(?:Dipl|Univ|Mag|Dr)\.[\w.\-]*", token.strip(",")) is not None
|
||||
if t in TITLE_TOKENS or re.fullmatch(r"(?:Dipl|Univ|Mag|Dr)\.[\w.\-]*", token.strip(",")):
|
||||
return True
|
||||
# any other abbreviation of two or more letters ("MMst.", "Bakk."); a name never ends with a dot, an
|
||||
# initial ("J.") is kept
|
||||
return re.fullmatch(r"[^\W\d_]{2,}\.", token.strip(",")) is not None
|
||||
|
||||
|
||||
def clean_name(name: str) -> str:
|
||||
"""'KO Mag. Markus Abwerzger' -> 'Markus Abwerzger'; drops functions, academic titles and stray punctuation.
|
||||
Titles are matched as whole tokens, so a first name like "Ingrid" is never taken for "Ing."."""
|
||||
name = ROLE_PREFIX.sub("", re.sub(r"\s+", " ", name).strip())
|
||||
name = re.sub(r"\([^()]*\)", " ", INVISIBLE.sub("", name)) # "(WU)" after "MSc", "(FH)"
|
||||
name = ROLE_PREFIX.sub("", CAPTION_PREFIX.sub("", re.sub(r"\s+", " ", name).strip()))
|
||||
name = strip_titles(name)
|
||||
kept = []
|
||||
for token in name.replace(",", " , ").split():
|
||||
@@ -253,7 +261,8 @@ class LandtagMembers(Adapter):
|
||||
region, body_name, _, list_url = LANDTAGE[self.key]
|
||||
scheme = f"landtag-{self.key}"
|
||||
ref = PersonRef(ids={scheme: m["id"]}, name=m["name"])
|
||||
yield PersonRecord(ref=ref, full_name=m["name"], birth_date=m["birth_date"], match_by_name=True)
|
||||
yield PersonRecord(ref=ref, full_name=m["name"], birth_date=m["birth_date"], match_by_name=True,
|
||||
match_landtag=self.key)
|
||||
body = OrgRef(slug=f"landtag-{self.key}", name=body_name, kind="parliament")
|
||||
party = OrgRef(short_name=m["party"], kind="party", create=False) if m["party"] else None
|
||||
label = f" ({m['party']})" if m["party"] else ""
|
||||
|
||||
@@ -14,10 +14,10 @@ ACADEMIC_TITLES = re.compile(
|
||||
|
||||
|
||||
def normalize(text: str) -> str:
|
||||
"""Lowercase, strip accents and collapse whitespace (matches SQL lower(unaccent()))."""
|
||||
"""Lowercase, strip accents, ß -> ss and collapse whitespace (matches SQL lower(unaccent()))."""
|
||||
text = unicodedata.normalize("NFKD", text)
|
||||
text = "".join(c for c in text if not unicodedata.combining(c))
|
||||
return re.sub(r"\s+", " ", text).strip().lower()
|
||||
return re.sub(r"\s+", " ", text).strip().lower().replace("ß", "ss")
|
||||
|
||||
|
||||
def strip_titles(name: str) -> str:
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
"honorific_prefix": null,
|
||||
"honorific_suffix": null,
|
||||
"match_by_name": false,
|
||||
"match_landtag": null,
|
||||
"photo_url": "https://www.europarl.europa.eu/mepphoto/220897.jpg",
|
||||
"ref": {
|
||||
"__type__": "PersonRef",
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
"honorific_prefix": null,
|
||||
"honorific_suffix": null,
|
||||
"match_by_name": true,
|
||||
"match_landtag": null,
|
||||
"photo_url": null,
|
||||
"ref": {
|
||||
"__type__": "PersonRef",
|
||||
@@ -74,6 +75,7 @@
|
||||
"honorific_prefix": null,
|
||||
"honorific_suffix": null,
|
||||
"match_by_name": true,
|
||||
"match_landtag": null,
|
||||
"photo_url": null,
|
||||
"ref": {
|
||||
"__type__": "PersonRef",
|
||||
@@ -135,6 +137,7 @@
|
||||
"honorific_prefix": null,
|
||||
"honorific_suffix": null,
|
||||
"match_by_name": true,
|
||||
"match_landtag": null,
|
||||
"photo_url": null,
|
||||
"ref": {
|
||||
"__type__": "PersonRef",
|
||||
@@ -196,6 +199,7 @@
|
||||
"honorific_prefix": null,
|
||||
"honorific_suffix": null,
|
||||
"match_by_name": true,
|
||||
"match_landtag": null,
|
||||
"photo_url": null,
|
||||
"ref": {
|
||||
"__type__": "PersonRef",
|
||||
@@ -257,6 +261,7 @@
|
||||
"honorific_prefix": null,
|
||||
"honorific_suffix": null,
|
||||
"match_by_name": true,
|
||||
"match_landtag": null,
|
||||
"photo_url": null,
|
||||
"ref": {
|
||||
"__type__": "PersonRef",
|
||||
@@ -318,6 +323,7 @@
|
||||
"honorific_prefix": null,
|
||||
"honorific_suffix": null,
|
||||
"match_by_name": true,
|
||||
"match_landtag": null,
|
||||
"photo_url": null,
|
||||
"ref": {
|
||||
"__type__": "PersonRef",
|
||||
@@ -379,6 +385,7 @@
|
||||
"honorific_prefix": null,
|
||||
"honorific_suffix": null,
|
||||
"match_by_name": true,
|
||||
"match_landtag": null,
|
||||
"photo_url": null,
|
||||
"ref": {
|
||||
"__type__": "PersonRef",
|
||||
@@ -440,6 +447,7 @@
|
||||
"honorific_prefix": null,
|
||||
"honorific_suffix": null,
|
||||
"match_by_name": true,
|
||||
"match_landtag": null,
|
||||
"photo_url": null,
|
||||
"ref": {
|
||||
"__type__": "PersonRef",
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
"honorific_prefix": null,
|
||||
"honorific_suffix": null,
|
||||
"match_by_name": false,
|
||||
"match_landtag": null,
|
||||
"photo_url": "https://www.parlament.gv.at/dokument/bild/201426/20142639_500.jpg",
|
||||
"ref": {
|
||||
"__type__": "PersonRef",
|
||||
|
||||
@@ -10,6 +10,12 @@ def test_clean_name_drops_functions_and_titles_but_not_names():
|
||||
assert L.clean_name("Dipl. Päd., DI Carina Laschober-Luif") == "Carina Laschober-Luif"
|
||||
assert L.clean_name("Mag. a Astrid Eisenkopf") == "Astrid Eisenkopf"
|
||||
assert L.clean_name("Ingrid Korosec") == "Ingrid Korosec" # "Ing" is a title, "Ingrid" is not
|
||||
assert L.clean_name("Portraitfoto von Simon Heilig-Hofbauer") == "Simon Heilig-Hofbauer"
|
||||
assert L.clean_name("LTP Jakob Wolf") == "Jakob Wolf"
|
||||
assert L.clean_name("MMst. Doris Schmidl") == "Doris Schmidl"
|
||||
assert L.clean_name("Claudia Gamon, MSc (WU)") == "Claudia Gamon"
|
||||
assert L.clean_name("Johannes" + chr(0x200B) + " Schwarz") == "Johannes Schwarz"
|
||||
assert L.clean_name("Michael J. Fox") == "Michael J. Fox" # initials stay
|
||||
assert L.surname_first("Höglinger Tobias, Mag.") == "Tobias Höglinger"
|
||||
assert L.surname_first("EISEL-EISELSBERG Detlev") == "Detlev Eisel-Eiselsberg"
|
||||
|
||||
@@ -66,3 +72,15 @@ def test_robots_excluded_states_have_no_adapter():
|
||||
from politrack.core.registry import ADAPTERS
|
||||
assert "landtag.wien" not in ADAPTERS and "landtag.ktn" not in ADAPTERS
|
||||
assert "landtag.bgld" in ADAPTERS
|
||||
|
||||
|
||||
def test_name_variants_and_sharp_s():
|
||||
from politrack.core.resolve import variant_of
|
||||
from politrack.text import normalize
|
||||
assert normalize("Klaudia Stroißnig") == normalize("Klaudia Stroissnig")
|
||||
for listed, known in [("Rudi Hemetsberger", "Rudolf Hemetsberger"), ("Max Hiegelsberger", "Maximilian Hiegelsberger"),
|
||||
("Chiara-Sophia Glawogger", "Chiara Glawogger"), ("Helena Kirchmayr-Felbermaier", "Helena Kirchmayr"),
|
||||
("Michael Stefan Wagner", "Michael Wagner"), ("Gebi Mair", "Gebhard Mair")]:
|
||||
assert variant_of(listed, known), (listed, known)
|
||||
for listed, known in [("Michael Weber", "Michael Huber"), ("Doris Kampus", "Doris Hahn"), ("Jo Mair", "Josef Mair")]:
|
||||
assert not variant_of(listed, known), (listed, known)
|
||||
Reference in new issue
Block a user