Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/nameparser/_lexicon.py: 55%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Immutable vocabulary configuration for the 2.0 API.
3Layering: may import nameparser.config DATA modules (the imports in
4_default_lexicon() are the authoritative list) as the single source of
5vocabulary during 2.x -- never nameparser.config itself, never
6nameparser.parser. Enforced by tests/v2/test_layering.py.
7"""
8from __future__ import annotations
10import dataclasses
11import functools
12import sys
13import unicodedata
14import warnings
15from collections.abc import Iterable, Mapping, Set
16from dataclasses import dataclass, field
17from types import FrameType, MappingProxyType
18from typing import cast
20#: Vocabulary set fields, in declaration order. add()/remove() operate
21#: on exactly these and reject capitalization_exceptions (its entries
22#: are pairs -- use dataclasses.replace); __or__ unions these AND merges
23#: capitalization_exceptions right-biased.
24_VOCAB_FIELDS = (
25 "titles", "given_name_titles", "suffix_acronyms", "suffix_words",
26 "suffix_acronyms_ambiguous", "particles", "particles_ambiguous",
27 "conjunctions", "conjunctions_ambiguous", "bound_given_names",
28 "maiden_markers", "surnames", "honorific_tails",
29)
31#: (marker, base, why) triples. Each marker QUALIFIES how entries of
32#: its base vocabulary are read and carries no vocabulary of its own,
33#: so an entry outside the base is a configuration mistake -- but the
34#: mistake differs per pair, and the reason is recorded here rather
35#: than generalized, because an orphan is NOT simply inert. Nor is the
36#: qualification one-directional: the first two NARROW their base (an
37#: entry is read as vocabulary in fewer places), while honorific_tails
38#: WIDENS it, granting a suffix word the glued position on top of the
39#: whole-token match every suffix word already gets.
40#:
41#: * particles_ambiguous: _assign keys on the tag alone, so an orphan
42#: makes the parse emit a spurious particle-or-given ambiguity.
43#: * suffix_acronyms_ambiguous: _vocab returns True on the ambiguous
44#: set before testing suffix_acronyms, so an orphan silently turns a
45#: word into a period-gated suffix.
46#: * honorific_tails: script_segment peels the tail into its own token
47#: before classify ever runs, so an orphan splits the name and leaves
48#: the fragment stranded inside it -- worse than not peeling at all.
49#: Its base is deliberately NARROWER than what actually claims the
50#: peeled piece: suffix_as_written ORs suffix_words with the
51#: non-ambiguous suffix_acronyms, so a tail listed only as an acronym
52#: would classify fine yet is rejected here. Accepted, and a decision
53#: rather than an oversight -- the three-term predicate is easy to
54#: get wrong in the dangerous direction (an ambiguous acronym admitted
55#: as a tail would peel a period-gated word off a real name), and
56#: nothing needs the acronym half: the shipped tails are CJK
57#: honorifics, which are words.
58#: The same relation is asserted a second time in config/suffixes.py,
59#: over the raw GLUED_HONORIFICS/SUFFIX_WORDS constants at
60#: import. The two are not redundant in the way they look: that one
61#: is an `assert`, stripped under `python -O`, while the check here
62#: raises unconditionally -- so under -O this is what still holds the
63#: SHIPPED vocabulary to the invariant, as it is the only thing that
64#: ever held a caller's own.
65#:
66#: Two fields are deliberately NOT here and have no check of their own.
67#: given_name_titles -- see the note in __post_init__ for why every
68#: attempt at one rejected working configurations. conjunctions_ambiguous
69#: -- an orphan is never consulted because both the classify fork and
70#: its ambiguity emitter require the base entry as well, so an orphan
71#: decides nothing, and AGENTS.md's invariants rule guards harm, not
72#: no-ops.
73_SUBSET_FIELDS = (
74 ("particles_ambiguous", "particles",
75 "an orphan emits a spurious particle-or-given ambiguity"),
76 ("suffix_acronyms_ambiguous", "suffix_acronyms",
77 "an orphan silently becomes a period-gated suffix"),
78 ("honorific_tails", "suffix_words",
79 "an orphan splits the name and leaves the tail inside it"),
80)
83#: The fields whose entries may be PHRASES: stored space-joined and
84#: folded per word (_title_key), and exempt from the dead-entry warning
85#: below. Every other field is matched one word at a time, where a
86#: multi-word entry can never match.
87#:
88#: The two are not the same mechanism, and the difference is why the
89#: exemption is a list rather than a rule. A given_name_titles run is
90#: identified per word FIRST -- 'lt' and 'col' are each title
91#: vocabulary -- and only then looked up, as the whole run's key or as
92#: its LAST word's (_run_addresses_by_given, #489). The whole-run arm
93#: is the one a phrase entry is matched by; the last-word arm is a
94#: single word and cannot reach a phrase, so storage stays what it was
95#: (the two are asked together, not in precedence). A maiden marker
96#: phrase has no such per-word foothold: 'z' and 'domu' are not markers
97#: individually, and adding them separately (which this warning used to
98#: advise) reads 'Maria Kowalska z domu Nowak' as maiden 'domu Nowak'
99#: and strips 'Anna z Nowak' of its family name. So markers are matched
100#: by a genuine multi-token lookahead instead --
101#: _pipeline._vocab.maiden_marker_run, longest first -- and only the
102#: STORAGE rule is shared with titles.
103_PHRASE_FIELDS = ("given_name_titles", "maiden_markers")
106#: Every character the parser reads as a full stop at a WORD'S EDGE
107#: (#322/#323): the ASCII period, the fullwidth FULL STOP U+FF0E, the
108#: IDEOGRAPHIC FULL STOP U+3002 and its halfwidth form U+FF61. One
109#: string, four readers -- the lookup fold below, _vocab's script
110#: classification fold, _script_segment's surname site and honorific
111#: tail match -- so that "which characters are a period" is answered
112#: once; the bundle's fifth site, _pieces' opening-abbreviation shape,
113#: reads no stop set at all but the no-initials repertoire
114#: (_policy._NO_INITIALS) that licenses every reader here. NFKC is
115#: NOT a substitute for listing them: it folds U+FF0E to '.' and
116#: U+FF61 to U+3002, and leaves U+3002 as it is.
117#: Not every period test reads this set, and the ones that do not are
118#: named so the claim is checkable: the interior-period shapes -- the
119#: dotted acronym 'M.A.' and the split 'Ph. D.' (_vocab) -- and FOUR
120#: ASCII-only edge tests on Latin shapes, across THREE modules -- the
121#: initial 'J.' (_vocab), the bracketed credential '(Mgr.)' and the
122#: word-internal apostrophe rule (_extract), and the trailing
123#: middle-initial carve-out 'V.' (_assign) -- stay ASCII. That is a
124#: fact about those TESTS and not about this set's reach: the fold
125#: below is script-agnostic, so a LATIN word wearing a wide stop
126#: reaches vocabulary the ASCII veto would have taken it out of.
127#: 'V。' folds to 'v' and does not end in the ASCII period the
128#: trailing-suffix carve-out tests (_assign, not is_initial), so
129#: it is roman five -- suffix -- where 'V.' stays a middle
130#: initial through that same carve-out; 'Jr。' simply reaches its
131#: entry, and 'Dr。' its title.
132#: decisions.md#cjk-full-stops carries the measurements.
133FULL_STOPS = ".．。｡"
136def _normalize(word: str) -> str:
137 """Lowercase, NFC-compose, strip whitespace and EDGE full stops
138 (FULL_STOPS, not the ASCII period alone -- #322). Interior periods
139 survive on purpose: 'J.R.' must not collapse to 'jr' and hit the
140 periodless vocabulary (v1 parity, pinned live 2026-07-17).
141 Suffix-ACRONYM membership alone uses the period-free form (see
142 _vocab.suffix_as_written), mirroring v1's is_suffix, which removed
143 periods only for the acronym test.
145 lower(), NOT casefold(): casefold's caseless-matching folds mutate
146 the stored vocabulary itself -- 'κος' becomes the misspelling 'κοσ'
147 (final sigma flattened) and 'großfürst' becomes 'grossfürst' --
148 while lower() applies Unicode SpecialCasing contextually and keeps
149 both as authored. This function is the single fold for storage AND
150 match-time lookups, so matching stays symmetric either way; lower()
151 is what v1's lc() used, preserving which cross-spellings match.
153 NFC, since #322, and only for a non-ASCII word (ASCII is already
154 NFC, and this runs per token on the parse path --
155 decisions.md#parse-cost). NFD hangul decomposes to jamo and missed
156 every vocabulary entry; composing at lookup AND at storage keeps
157 the two symmetric, and no shipped entry changes under it (measured
158 2026-09-09: fold every string member of every Lexicon.default()
159 field through NFC and count the changes; zero of 1735). This is
160 the same NFC composition script classification already applies
161 (_vocab._normalized_for_script); token text is never rewritten.
163 Strips to a FIXED POINT. A single strip().strip(FULL_STOPS) leaves
164 stops-around-whitespace half done ('. a .' -> ' a '), so a value
165 that is re-normalized later -- on unpickle, or by a second add() --
166 would change under its owner. v1 never re-normalized, so this only
167 matters now that storage and match-time share one fold."""
168 word = word.lower()
169 if not word.isascii():
170 word = unicodedata.normalize("NFC", word)
171 while True:
172 stripped = word.strip().strip(FULL_STOPS)
173 if stripped == word:
174 return word
175 word = stripped
178def _fold_words(words: Iterable[str]) -> list[str]:
179 """The words of a title run, folded for storage and lookup.
181 A multi-word title is matched as one key ('lt col'), so the fold has
182 to run per word -- _normalize on the whole phrase would leave
183 interior periods.
185 Words that fold away are DROPPED, not kept as empty. Keeping the
186 gap makes the key non-idempotent -- 'lt .' would store 'lt ', which
187 match time can never build: a run CAN carry a lone '.' (the
188 conjunction merge puts one there), but the fold drops the empty
189 word, so the key is 'lt' and the stored entry is inert. Storage
190 re-runs this fold on unpickle and on every dataclasses.replace, so a
191 value that changes under a second pass is one Lexicon later rejects as
192 "not written by this version". _normalize converges for the same
193 reason; so must anything built on top of it.
195 A LIST, so that _run_addresses_by_given's last-word arm can be the
196 last word of this fold rather than a re-split of the joined key --
197 "the last word of the FOLDED key" is then structural, and the two
198 arms read one fold between them.
200 map/filter rather than a comprehension: both are C calls where a
201 comprehension is a Python frame on 3.11, and this runs on the parse
202 path for every name carrying a title run (decisions.md#parse-cost).
203 """
204 return list(filter(None, map(_normalize, words)))
207def _title_key(words: Iterable[str]) -> str:
208 """The given_name_titles lookup key for a run of title words: the
209 folded words, space-joined.
211 Defined once because it is built at match time
212 (_run_addresses_by_given, which is how post_rules' H1 and group's
213 P5 licence both reach it) and at translation time (_config_shim's
214 first_name_titles), and a divergence between them fails silently:
215 the entry simply stops matching."""
216 return " ".join(_fold_words(words))
219def _run_addresses_by_given(words: Iterable[str],
220 vocabulary: Set[str]) -> bool:
221 """Whether a run of title words addresses by the GIVEN name.
223 Two sites ask it and they must agree -- post_rules for H1, group
224 for the P5 licence -- because a run read two ways is a rule
225 contradicting itself (decisions.md#P5, the 2026-08-22 #369 entry).
226 One predicate, so they cannot drift.
228 The WHOLE run's key, or that key's LAST word (#489): a run is
229 written as several titles and addresses the way its final one does,
230 so 'Her Majesty Queen' addresses by given name because 'queen'
231 does. The two arms are asked of one key and neither is the other's
232 fallback -- the `or` short-circuits but decides nothing, since for
233 a one-word run the key IS its last word. Both arms read one
234 _fold_words list, so the last word is the last word of the FOLDED
235 key by construction rather than by agreement, and a run token that
236 folds away cannot empty that arm: the conjunction merge can put a
237 lone '.' in the run ('Sir and . John'), and the fold drops it.
238 What the drop leaves as the last word can then be the CONJUNCTION
239 -- 'Sir and . John' keys 'sir and' and reads family 'John', where
240 'Sir and Dame John' keys 'sir and dame' and reads given. Harmless
241 on the shipped vocabulary, which holds no entry ending in a
242 connective, and a caller who stored one would be asking for it
243 (measured 2026-09-09).
245 The whole-run arm is what keeps a caller's multi-word phrase entry
246 working: 'lt col' is stored as one key and matched as one run. Over
247 the SHIPPED vocabulary it is dead -- every shipped entry is a single
248 word (asserted in test_lexicon.py), so it can only ever match a
249 one-word run, which is a run the last-word arm reads the same way.
251 H2's unlisted abbreviations ride in the run. One can never match as
252 the last-word key, being in no vocabulary by definition -- written
253 as the inputs that produce those runs, 'Xyz. Sir John' keys 'xyz
254 sir', matches on 'sir' and reads given 'John', while 'Sir Xyz.
255 John' keys 'sir xyz', matches on neither arm and reads family
256 'John'. It CAN sit inside a whole-run key that matches, because
257 given_name_titles is deliberately not validated against titles: a
258 caller who stores 'sir xyz' makes 'Sir Xyz. John' read given.
260 The vocabulary is passed in rather than read off a default: a
261 caller's own Lexicon is the one that has to be consulted, and this
262 module is where Lexicon is defined."""
263 folded = _fold_words(words)
264 if not folded:
265 # every word folded away, so there is no key and no last word:
266 # the joined spelling built "" here, which matched nothing
267 return False
268 return " ".join(folded) in vocabulary or folded[-1] in vocabulary
271def _reject_buffer(value: object, label: str, plural: str) -> None:
272 """Binary sequences iterate to INTS, so every downstream entry check
273 reports a byte value -- "must be strings, got 100" for b'dr', where
274 100 is 'd'. That names neither the cause nor the fix. v1 shipped a
275 decode hint for this (#238), and parse() and the facade both carry
276 one, so no config entry point should be the odd one out.
277 """
278 if isinstance(value, (bytes, bytearray, memoryview)):
279 raise TypeError(
280 f"{label} must be an iterable of {plural}, not "
281 f"{type(value).__name__} -- decode first, e.g. "
282 f"raw.decode('utf-8')"
283 )
286def _warn_dead_entry(message: str) -> None:
287 # A fixed stacklevel always lands on library internals: the call
288 # depth differs per entry point (Lexicon(), add(), unpickle,
289 # dataclasses.replace, and the v1 shim's lazy snapshot -- built on
290 # the first parse after a Constants mutation, several facade frames
291 # below the user's own add()). Walk out of this module (and
292 # dataclasses' replace frames, and the facade layer that builds
293 # lexicons on the caller's behalf) so the warning points at the
294 # caller's own line.
295 level = 2
296 frame: FrameType | None = sys._getframe(1)
297 while frame is not None and frame.f_globals.get("__name__") in (
298 __name__, "dataclasses",
299 "nameparser._config_shim", "nameparser._facade"):
300 frame, level = frame.f_back, level + 1
301 warnings.warn(message, UserWarning, stacklevel=level)
304def _normset(
305 entries: Iterable[str], field_name: str, warn: bool = True,
306) -> frozenset[str]:
307 # Reject a bare str before iterating: iterating "dr" would silently
308 # yield the single characters {'d', 'r'} -- the set(str) footgun on
309 # the primary customization surface.
310 if isinstance(entries, str):
311 raise TypeError(
312 f"Lexicon.{field_name} must be an iterable of strings, "
313 f"not a bare string"
314 )
315 _reject_buffer(entries, f"Lexicon.{field_name}", "strings")
316 # A Mapping would silently contribute only its keys; a dict here
317 # almost always means the caller confused this field with
318 # capitalization_exceptions.
319 if isinstance(entries, Mapping):
320 raise TypeError(
321 f"Lexicon.{field_name} must be an iterable of strings, not a "
322 f"mapping (only capitalization_exceptions holds key->value pairs)"
323 )
324 try:
325 items = tuple(entries) # materialize once; entries may be a generator
326 except TypeError:
327 raise TypeError(
328 f"Lexicon.{field_name} must be an iterable of strings, "
329 f"got {entries!r}"
330 ) from None
331 normalized = set()
332 for w in items:
333 if not isinstance(w, str):
334 raise TypeError(
335 f"Lexicon.{field_name} entries must be strings, got {w!r}"
336 )
337 # A phrase field's entries are matched as a multi-word run, so
338 # they fold per word: stored as the same key the match site
339 # builds, or 'lt. col' (and 'z. domu') would be kept verbatim
340 # and never match anything -- a silent no-op on the config
341 # surface. Every other field holds single words, where the two
342 # folds agree. See _PHRASE_FIELDS for how the two differ once
343 # stored.
344 n = _title_key(w.split()) if field_name in _PHRASE_FIELDS \
345 else _normalize(w)
346 # "." or "" is a data bug (stray split artifact, empty CSV
347 # cell); dropping it silently would also let a data-module typo
348 # vanish instead of failing CI.
349 if not n:
350 raise ValueError(
351 f"Lexicon.{field_name} entry {w!r} normalizes to empty "
352 f"(lowercase + strip full stops/whitespace leaves nothing)"
353 )
354 # Every field outside _PHRASE_FIELDS is matched one word at a
355 # time, so a multi-word entry can never match -- the library
356 # itself shipped eight such dead entries for years (repaired
357 # 2026-07-26). Warn, never raise: an inert entry produces
358 # nothing, and the given_name_titles precedent says a raise
359 # here costs working configurations (see __post_init__).
360 # warn=False is _edit()'s pass (both ops): add() warns via the
361 # new instance's __post_init__; remove() stores nothing, so
362 # warning there would name entries the caller is trying to get
363 # RID of, with "split it" advice that makes no sense for a
364 # no-op. Both invariants are indifferent to WHICH fields are
365 # exempt: the exemption only decides whether a warning exists to
366 # be emitted once or suppressed.
367 if (warn and field_name not in _PHRASE_FIELDS
368 # interior whitespace test; split() covers all Unicode
369 # whitespace
370 and n != "".join(n.split())):
371 _warn_dead_entry(
372 f"Lexicon.{field_name} entries are matched one word at "
373 f"a time; multi-word entry {w!r} can never match. "
374 f"Split it into separate entries")
375 normalized.add(n)
376 return frozenset(normalized)
379def _normpairs(
380 raw: Mapping[str, str] | Iterable[tuple[str, str]],
381) -> tuple[tuple[str, str], ...]:
382 """Canonicalize capitalization_exceptions input: _normset's sibling
383 for the one pair-valued field. Dedupes on the NORMALIZED key so the
384 tuple and the derived map always agree ("Ph.D." and "phd" collide
385 after normalization); last occurrence wins, matching dict semantics
386 and the right-bias rule used elsewhere."""
387 if isinstance(raw, str):
388 raise TypeError(
389 "capitalization_exceptions must be a mapping or an "
390 "iterable of (key, value) pairs, not a bare string"
391 )
392 _reject_buffer(raw, "capitalization_exceptions", "(key, value) pairs")
393 pairs = raw.items() if isinstance(raw, Mapping) else raw
394 try:
395 pairs = iter(pairs)
396 except TypeError:
397 raise TypeError(
398 "capitalization_exceptions must be a mapping or an "
399 f"iterable of (key, value) pairs, got {raw!r}"
400 ) from None
401 deduped: dict[str, str] = {}
402 for entry in pairs:
403 # A 2-char str entry would unpack "ab" into ("a", "b")
404 # silently, so reject str outright; other mis-shapes would
405 # otherwise surface as bare unpack errors.
406 if isinstance(entry, str):
407 raise TypeError(
408 f"capitalization_exceptions entries must be "
409 f"(key, value) pairs, got {entry!r}"
410 )
411 try:
412 k, v = entry
413 except (TypeError, ValueError):
414 raise TypeError(
415 f"capitalization_exceptions entries must be "
416 f"(key, value) pairs, got {entry!r}"
417 ) from None
418 if not isinstance(k, str) or not isinstance(v, str):
419 raise TypeError(
420 f"capitalization_exceptions entries must be "
421 f"str -> str, got {k!r}: {v!r}"
422 )
423 normalized_key = _normalize(k)
424 if not normalized_key:
425 raise ValueError(
426 f"capitalization_exceptions key {k!r} normalizes to "
427 f"empty (lowercase + strip full stops/whitespace leaves "
428 f"nothing)"
429 )
430 # capitalized() looks words up one at a time (the _WORD regex
431 # never yields spaces), so a multi-word key is unreachable.
432 # interior whitespace test; split() covers all Unicode whitespace
433 if normalized_key != "".join(normalized_key.split()):
434 _warn_dead_entry(
435 f"capitalization_exceptions keys are matched one word "
436 f"at a time; multi-word key {k!r} can never match. "
437 f"Split it into per-word entries")
438 deduped[normalized_key] = v
439 return tuple(sorted(deduped.items()))
442@dataclass(frozen=True, slots=True)
443class Lexicon:
444 """The vocabulary a parser matches against: which words are
445 titles, particles, suffixes, and so on. Immutable and hashable.
446 Start from :meth:`default` (the shipped vocabulary) or
447 :meth:`empty`, derive variants with :meth:`add` / :meth:`remove` /
448 ``|`` (union), and pass the result to ``Parser(lexicon=...)``.
449 Entries are normalized at construction -- lowercased, NFC-composed,
450 edge full stops stripped -- so matching is case-insensitive.
451 Vocabulary entries are single words -- a multi-word entry warns at
452 construction and can never match. Two fields are exempt, and they
453 differ in HOW they match: ``given_name_titles`` is looked up
454 against the run of words the parse has ALREADY read as titles --
455 the whole run space-joined, or that run's last word -- while
456 ``maiden_markers`` is matched by lookahead, longest first, over
457 words that need not be markers on their own (``"z domu"``).
458 Field docs below show examples, not full
459 contents; inspect any field's shipped vocabulary directly, e.g.
460 ``Lexicon.default().conjunctions``."""
462 #: Pre-nominal titles ("dr", "sir", "capt", ...). Full default
463 #: list: :data:`~nameparser.config.titles.TITLES`.
464 titles: frozenset[str] = frozenset()
465 #: Titles whose single following name reads as a GIVEN name
466 #: ("sheikh", "sister", ...) rather than a family name. Full
467 #: default list: :data:`~nameparser.config.titles.GIVEN_NAME_TITLES`.
468 given_name_titles: frozenset[str] = frozenset()
469 #: Post-nominal acronym suffixes, matched with or without periods
470 #: ("phd" matches "PhD" and "Ph.D."). Full default list:
471 #: :data:`~nameparser.config.suffixes.SUFFIX_ACRONYMS`.
472 suffix_acronyms: frozenset[str] = frozenset()
473 #: Post-nominal word suffixes ("jr", "esquire", "iii", ...). Full
474 #: default list:
475 #: :data:`~nameparser.config.suffixes.SUFFIX_WORDS`.
476 suffix_words: frozenset[str] = frozenset()
477 #: Subset of suffix_acronyms counted as suffixes only when written
478 #: WITH periods -- their bare forms are common surnames ("ma",
479 #: "do": "Jack Ma" keeps his family name). Full default list:
480 #: :data:`~nameparser.config.suffixes.SUFFIX_ACRONYMS_AMBIGUOUS`.
481 suffix_acronyms_ambiguous: frozenset[str] = frozenset()
482 #: Family-name particles that chain onto the following piece
483 #: ("van", "de", "bin", ...). Full default list:
484 #: :data:`~nameparser.config.particles.PARTICLES`.
485 particles: frozenset[str] = frozenset()
486 #: Subset of particles that can also BE a given name ("Van
487 #: Johnson", but also "Van Buren"). Membership decides nothing
488 #: about chaining: the prefix chain skips the name's first piece
489 #: and never consults this set, so it leaves a leading particle a
490 #: piece of its own whether listed or not -- "de Mesnil" groups
491 #: into two pieces exactly as "van Gogh" does, and since #367 the
492 #: NAME in "Dr. de Mesnil" and "Dr. Van Johnson" groups into those
493 #: same two pieces behind the title piece, a title not being
494 #: part of the name it precedes. What membership
495 #: decides is what becomes of that piece afterwards. Under ANY
496 #: ``name_order`` a member records a particle-or-given ambiguity
497 #: and a non-member records none, and a non-member is additionally
498 #: folded back into the family name once roles exist WHERE IT OPENS
499 #: THE NAME, so the whole name is the surname ("de Mesnil" -- a bare "de", with nothing to
500 #: fold into, is left alone). That fold is order-independent too
501 #: (#359) -- not because the word could not be a given name, which
502 #: it can where position forces it (``parse("de")`` reports given
503 #: "de"), but because a particle OPENING the name has the rest of
504 #: the name to join forward to, and that evidence is positional
505 #: rather than vocabulary, so no declared order contradicts it. Which field a MEMBER's piece
506 #: lands in is ``name_order``'s question, not this set's.
507 #: No constant of its own -- the default derives
508 #: as particles minus
509 #: :data:`~nameparser.config.particles.NON_GIVEN_NAME_PARTICLES`
510 #: (which marks the opposite, never-given subset).
511 particles_ambiguous: frozenset[str] = frozenset()
512 #: Words or characters that join surrounding pieces into one
513 #: ("and", "&", "y", "и", ...). Full default list:
514 #: :data:`~nameparser.config.conjunctions.CONJUNCTIONS`.
515 conjunctions: frozenset[str] = frozenset()
516 #: Subset of conjunctions that read as an INITIAL rather than a
517 #: connective in a name written wholly in one case ("e": "jose e
518 #: maria santos" reads middle "e maria", where "juan garcia y
519 #: lopez" joins because "y" is not a member). Mixed-case input is
520 #: decided by the writing instead and never consults this set, and
521 #: neither does a caseless letter (Arabic و), which has no case to
522 #: read. A member additionally reports
523 #: :attr:`~nameparser.AmbiguityKind.CONJUNCTION_OR_INITIAL`; a
524 #: non-member reports nothing, its reading not being in doubt.
525 #: Full default list:
526 #: :data:`~nameparser.config.conjunctions.CONJUNCTIONS_AMBIGUOUS`.
527 #: Entries need not also be in ``conjunctions``; one that is not is
528 #: simply never consulted, so it is inert rather than an error.
529 conjunctions_ambiguous: frozenset[str] = frozenset()
530 #: Given-name prefixes that bind to the following word to form one
531 #: given name ("abdul" -> "Abdul Salam"); never standalone names.
532 #: Full default list:
533 #: :data:`~nameparser.config.bound_given_names.BOUND_GIVEN_NAMES`.
534 bound_given_names: frozenset[str] = frozenset()
535 #: Marker words introducing a birth surname, routed to the maiden
536 #: field ("née", "geb.", "rozená", ...). An entry may be a PHRASE
537 #: ("z domu"): entries are matched by lookahead, longest first, so
538 #: a phrase wins where it matches and a word entry that starts one
539 #: still matches on its own everywhere else. A phrase matches only
540 #: where its words stand together -- a bracketed clause or a comma
541 #: between them ends the run. Full default list:
542 #: :data:`~nameparser.config.maiden_markers.MAIDEN_MARKERS`.
543 maiden_markers: frozenset[str] = frozenset()
544 #: Family names for the unspaced-name segmentation stage (#271),
545 #: matched longest-first against the start of the FIRST token
546 #: written wholly in a script :attr:`Policy.segment_scripts
547 #: <nameparser.Policy.segment_scripts>` activates. The default
548 #: carries the Korean census list
549 #: (:data:`~nameparser.config.surnames.KOREAN_SURNAMES`); Chinese
550 #: surnames ship in locales.ZH because Han segmentation is opt-in.
551 surnames: frozenset[str] = frozenset()
552 #: Honorifics that may be peeled off the END of a name token
553 #: (#308), matched longest-first: 田中さん splits into 田中 and さん
554 #: before the tokens are classified. Every entry must also be a
555 #: :attr:`suffix_words` entry -- the peeled tail is claimed by
556 #: suffix classification like any other post-nominal. Deliberately
557 #: NOT gated on :attr:`Policy.segment_scripts
558 #: <nameparser.Policy.segment_scripts>` (unlike :attr:`surnames`
559 #: above): 田中さん peels under the default policy, where HAN is in
560 #: no activation set, because a tail entry carries its own license
561 #: to fire. Entries are matched against the RAW token text, and
562 #: only within a name containing a non-ASCII character, so an ASCII
563 #: or mixed-case entry is at best conditionally active -- a ``"Jr"``
564 #: entry is stored ``"jr"`` and matches only lowercase text. The
565 #: field is effectively CJK-scoped in 2.1, which is what the shipped
566 #: vocabulary is. Full default list:
567 #: :data:`~nameparser.config.suffixes.GLUED_HONORIFICS`.
568 honorific_tails: frozenset[str] = frozenset()
569 #: Lowercase word -> exact-cased replacement used by capitalized()
570 #: ("phd" -> "Ph.D."). Pair-valued: change it with
571 #: dataclasses.replace(), not add()/remove(); read it as a mapping
572 #: via capitalization_exceptions_map. Full default mapping:
573 #: :data:`~nameparser.config.capitalization.CAPITALIZATION_EXCEPTIONS`.
574 # Canonical storage: sorted tuple of (key, value) pairs. The
575 # constructor tolerates any Mapping (or pair iterable) at runtime and
576 # canonicalizes here; this closes the caller-aliasing hole and keeps
577 # Lexicon hashable. Read via capitalization_exceptions_map.
578 capitalization_exceptions: tuple[tuple[str, str], ...] = ()
579 _cap_map: Mapping[str, str] = field(
580 init=False, repr=False, compare=False, hash=False,
581 default_factory=lambda: MappingProxyType({}))
583 def __post_init__(self) -> None:
584 for name in _VOCAB_FIELDS:
585 object.__setattr__(self, name, _normset(getattr(self, name), name))
586 canonical = _normpairs(self.capitalization_exceptions)
587 object.__setattr__(self, "capitalization_exceptions", canonical)
588 object.__setattr__(self, "_cap_map", MappingProxyType(dict(canonical)))
589 for marker, base, why in _SUBSET_FIELDS:
590 orphans = getattr(self, marker) - getattr(self, base)
591 if orphans:
592 raise ValueError(
593 f"{marker} marks a subset of {base}; "
594 f"not in {base}: {', '.join(sorted(orphans))} "
595 f"({why}). Add them to {base}, or drop them from "
596 f"{marker}"
597 )
598 # NOT validated: given_name_titles against titles. The lookup
599 # key is the space-joined run of Role.TITLE tokens, built by the
600 # parse rather than drawn from this vocabulary, and a
601 # conjunction inside a run is itself tagged Role.TITLE -- so
602 # "sir and dame" is a matchable key whose middle word is in
603 # conjunctions, not titles. A whole-entry check rejected
604 # multi-word entries; a per-word check rejected that one. An
605 # unreachable entry here is inert: nothing consults it, nothing
606 # misparses. That is the cheap failure, and guarding it proved
607 # the expensive one -- three working configurations broken
608 # across two attempts. Do not add a third.
609 #
610 # The v2 form of particles.py's NON_GIVEN_NAME_PARTICLES-disjoint-
611 # from-BOUND_GIVEN_NAMES assertion. That module guards its own
612 # data at import; this guards vocabulary a caller supplies.
613 contradictory = (
614 self.bound_given_names & self.particles) - self.particles_ambiguous
615 if contradictory:
616 raise ValueError(
617 f"bound_given_names entries that are also particles must be "
618 f"in particles_ambiguous; not in particles_ambiguous: "
619 f"{', '.join(sorted(contradictory))}. A particle that never "
620 f"starts a given name cannot also bind one — add them to "
621 f"particles_ambiguous, or drop them from bound_given_names"
622 )
623 # suffix_as_written ORs the acronym and word branches, so a word
624 # membership bypasses the period gate the ambiguous set exists
625 # to impose: listing 'ma' in suffix_words would make "Jack Ma"
626 # read as a suffix and lose the family name.
627 gate_bypassed = self.suffix_acronyms_ambiguous & self.suffix_words
628 if gate_bypassed:
629 raise ValueError(
630 f"an ambiguous suffix acronym must not also be a suffix "
631 f"word; in both: {', '.join(sorted(gate_bypassed))}. The "
632 f"word branch matches without periods, which bypasses the "
633 f"period gate suffix_acronyms_ambiguous exists to impose"
634 )
636 # -- constructors ----------------------------------------------------
638 @classmethod
639 def empty(cls) -> Lexicon:
640 return cls()
642 @classmethod
643 def default(cls) -> Lexicon:
644 return _default_lexicon()
646 # -- dunders ----------------------------------------------------------
648 def _deltas_from(self, baseline: Lexicon) -> list[tuple[str, int, int]]:
649 deltas = []
650 for name in _VOCAB_FIELDS + ("capitalization_exceptions",):
651 mine = set(getattr(self, name))
652 theirs = set(getattr(baseline, name))
653 added, removed = len(mine - theirs), len(theirs - mine)
654 if added or removed:
655 deltas.append((name, added, removed))
656 return deltas
658 def __repr__(self) -> str:
659 # Bounded: renders only which fields deviate from the nearer of
660 # the two named constructors and by how many entries -- never the
661 # entries themselves (design rule, see nameparser._types module
662 # docstring). Diffing empty()-built lexicons against default()
663 # would tell the wrong story ("default minus the entire
664 # default vocabulary").
665 if self == Lexicon.default():
666 return "Lexicon(default)"
667 if self == Lexicon.empty():
668 return "Lexicon(empty)"
669 candidates = [(label, self._deltas_from(baseline))
670 for label, baseline in (("default", Lexicon.default()),
671 ("empty", Lexicon.empty()))]
672 label, deltas = min(
673 candidates, key=lambda c: sum(a + r for _, a, r in c[1]))
674 rendered = ", ".join(
675 name + ": " + "".join(
676 part for part, n in ((f"+{a}", a), (f"-{r}", r)) if n)
677 for name, a, r in deltas)
678 return f"Lexicon({label} + {rendered})"
680 def __getstate__(self) -> dict[str, object]:
681 # _cap_map is a MappingProxyType, which pickle rejects; ship every
682 # other slot and rebuild the proxy from the canonical tuple on load.
683 return {f.name: getattr(self, f.name)
684 for f in dataclasses.fields(self) if f.name != "_cap_map"}
686 def __setstate__(self, state: dict[str, object]) -> None:
687 # Fail at the unpickle site if the state comes from a different
688 # Lexicon field layout (version skew) -- silently loading it
689 # would defer the failure to some distant attribute read.
690 # Message kept in sync with _types._guarded_setstate by design
691 # (layering keeps this module import-free of _types).
692 expected = {f.name for f in dataclasses.fields(Lexicon)} - {"_cap_map"}
693 if set(state) != expected:
694 missing = ", ".join(sorted(expected - set(state))) or "none"
695 unexpected = ", ".join(sorted(set(state) - expected)) or "none"
696 raise ValueError(
697 f"incompatible Lexicon pickle: missing fields: {missing}; "
698 f"unexpected fields: {unexpected}"
699 )
700 for name, value in state.items():
701 object.__setattr__(self, name, value)
702 # Re-run construction validation rather than trusting the blob.
703 # The layout check above catches SHAPE skew; this catches
704 # CONTENT skew, which is likelier -- particles_ambiguous flipped
705 # meaning between v1's never-given set and v2's may-be-given set
706 # without changing its name, so an old pickle would otherwise
707 # load a semantically inverted lexicon in silence. Rebuilds
708 # _cap_map for free, which this used to do by hand.
709 self.__post_init__()
710 # __post_init__ also normalizes, and quietly accepting a
711 # rewritten value would make this a place where caller data
712 # changes without a word. A pickle this library wrote is always
713 # normalized already (_normalize converges), so a difference
714 # here means the state came from somewhere else -- say so while
715 # the offending entries can still be named.
716 # One pass per field, in _VOCAB_FIELDS declaration order. cast is
717 # safe: __post_init__ just ran _normset over each of these and
718 # raised for anything that was not an iterable of str.
719 drifted = []
720 for name in _VOCAB_FIELDS:
721 lost = frozenset(
722 cast("Iterable[str]", state[name])) - getattr(self, name)
723 if lost:
724 drifted.append(f"{name}: {', '.join(sorted(lost))}")
725 # the pair field too, or ten fields raise and the eleventh is
726 # quietly re-canonicalized (keys normalized, sorted, deduped)
727 given_pairs = cast("Iterable[tuple[str, str]]",
728 state["capitalization_exceptions"])
729 if tuple(given_pairs) != self.capitalization_exceptions:
730 drifted.append("capitalization_exceptions")
731 if drifted:
732 raise ValueError(
733 "incompatible Lexicon pickle: entries are not normalized "
734 f"({'; '.join(drifted)}); this state was not written by "
735 "this version of nameparser"
736 )
738 def __or__(self, other: Lexicon) -> Lexicon:
739 if not isinstance(other, Lexicon):
740 return NotImplemented
741 updates: dict[str, object] = {
742 name: getattr(self, name) | getattr(other, name)
743 for name in _VOCAB_FIELDS
744 }
745 # right-biased on key conflicts, mirroring later-wins for scalars
746 merged = dict(self._cap_map) | dict(other._cap_map)
747 updates["capitalization_exceptions"] = tuple(sorted(merged.items()))
748 return dataclasses.replace(self, **updates) # type: ignore[arg-type]
750 # -- properties -------------------------------------------------------
752 @property
753 def capitalization_exceptions_map(self) -> Mapping[str, str]:
754 return self._cap_map
756 # -- editing ----------------------------------------------------------
758 def _edit(self, op: str, entries: Mapping[str, Iterable[str]]) -> Lexicon:
759 updates: dict[str, frozenset[str]] = {}
760 for name, words in entries.items():
761 if name == "capitalization_exceptions":
762 raise TypeError(
763 "capitalization_exceptions holds key->value pairs; "
764 "use dataclasses.replace(lexicon, "
765 "capitalization_exceptions={...}) instead of "
766 f"{op}()"
767 )
768 if name not in _VOCAB_FIELDS:
769 raise TypeError(
770 f"unknown Lexicon field {name!r}; valid fields: "
771 f"{', '.join(_VOCAB_FIELDS)}"
772 )
773 current: frozenset[str] = getattr(self, name)
774 # warn=False: this pass only computes the new set membership,
775 # never stores it directly. add() still warns exactly once,
776 # from the replaced instance's own __post_init__ -> _normset;
777 # remove() never reaches __post_init__ with the dead entry
778 # (it is subtracted out here), so it stays silent -- correct,
779 # since a removal stores nothing a warning could be about.
780 # That silence covers the entry BEING REMOVED only: a
781 # different multi-word entry still stored re-warns from the
782 # derived instance's __post_init__, since the warning is
783 # per-construction by design.
784 normalized = _normset(words, name, warn=False)
785 updates[name] = (current | normalized if op == "add"
786 else current - normalized)
787 # mypy's dataclasses.replace() typing checks a **dict's single
788 # value type against every field's type (it can't see which keys
789 # are actually present behind the unpack), so a homogeneous
790 # frozenset[str] dict is flagged against the tuple/Mapping-typed
791 # capitalization_exceptions/_cap_map fields even though this dict
792 # never contains those keys (guarded above).
793 return dataclasses.replace(self, **updates) # type: ignore[arg-type]
795 def add(self, **entries: Iterable[str]) -> Lexicon:
796 return self._edit("add", entries)
798 def remove(self, **entries: Iterable[str]) -> Lexicon:
799 return self._edit("remove", entries)
802@functools.cache
803def _default_lexicon() -> Lexicon:
804 # v1 data modules are the single source of vocabulary through 2.x.
805 from nameparser.config.bound_given_names import BOUND_GIVEN_NAMES
806 from nameparser.config.capitalization import CAPITALIZATION_EXCEPTIONS
807 from nameparser.config.conjunctions import (
808 CONJUNCTIONS, CONJUNCTIONS_AMBIGUOUS)
809 from nameparser.config.maiden_markers import MAIDEN_MARKERS
810 from nameparser.config.particles import NON_GIVEN_NAME_PARTICLES, PARTICLES
811 from nameparser.config.suffixes import (
812 GLUED_HONORIFICS, SUFFIX_ACRONYMS, SUFFIX_ACRONYMS_AMBIGUOUS,
813 SUFFIX_WORDS,
814 )
815 from nameparser.config.surnames import KOREAN_SURNAMES
816 from nameparser.config.titles import GIVEN_NAME_TITLES, TITLES
818 # every vocabulary constant is a frozenset since #293, so each one
819 # feeds its strictly-typed frozenset[str] field as it stands -- and
820 # this cache reading them ONCE is the reason they are frozen. A
821 # mutated module set always reached a freshly built Constants, and
822 # reached this Lexicon only when the edit landed before the first
823 # call; after it, the cache was already built and the same edit was
824 # invisible here. Which of the two a program got was not something
825 # the code doing the mutating could see.
826 # keep in sync with _config_shim.Constants._snapshot() (pinned by the
827 # default-Constants equality test in tests/v2/test_config_shim.py)
828 return Lexicon(
829 titles=TITLES,
830 given_name_titles=GIVEN_NAME_TITLES,
831 suffix_acronyms=SUFFIX_ACRONYMS,
832 suffix_words=SUFFIX_WORDS,
833 suffix_acronyms_ambiguous=SUFFIX_ACRONYMS_AMBIGUOUS,
834 particles=PARTICLES,
835 # FLIPPED from v1: v1 marks the never-given subset; v2 marks the
836 # may-be-given subset (migration: complement translation).
837 particles_ambiguous=PARTICLES - NON_GIVEN_NAME_PARTICLES,
838 conjunctions=CONJUNCTIONS,
839 conjunctions_ambiguous=CONJUNCTIONS_AMBIGUOUS,
840 bound_given_names=BOUND_GIVEN_NAMES,
841 maiden_markers=MAIDEN_MARKERS,
842 surnames=KOREAN_SURNAMES,
843 honorific_tails=GLUED_HONORIFICS,
844 # pass canonical pair-tuples so this strictly-typed call site never
845 # feeds a Mapping to the tuple-annotated field; __post_init__
846 # still tolerates a Mapping at runtime for interactive use
847 capitalization_exceptions=tuple(sorted(CAPITALIZATION_EXCEPTIONS.items())),
848 )