Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/nameparser/_lexicon.py: 23%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Immutable vocabulary configuration: a Lexicon holds readings, what
2each word may be read as, and compiles the sets a parse reads from them
3(3.0's 4i-b; decisions/vocabulary-model, 2026-11-02 #754).
5Layering: may import nameparser.config DATA modules as the single
6source of vocabulary -- never the nameparser.config package itself --
7and nameparser._reading_index, the shared compiler and the index's
8reader, at call time only: tools/vocab/build.py loads this module by
9path. Enforced by tests/v2/test_layering.py.
10"""
11from __future__ import annotations
13import dataclasses
14import functools
15import sys
16import unicodedata
17import warnings
18from collections.abc import Iterable, Mapping, Set
19from dataclasses import dataclass, field
20from types import FrameType, MappingProxyType, ModuleType
21from typing import TYPE_CHECKING, Any, NamedTuple, cast
23#: Vocabulary set fields, in declaration order. add()/remove() operate
24#: on exactly these and reject capitalization_exceptions (its entries
25#: are pairs -- use dataclasses.replace); __or__ unions these AND merges
26#: capitalization_exceptions right-biased.
27_VOCAB_FIELDS = (
28 "titles", "given_name_titles", "suffix_acronyms", "suffix_words",
29 "suffix_acronyms_ambiguous", "particles", "particles_ambiguous",
30 "conjunctions", "conjunctions_ambiguous", "bound_given_names",
31 "given_name_abbreviations",
32 "maiden_markers", "surnames", "honorific_tails", "after_given_titles",
33 "suffix_words_ambiguous", "agnomens", "titles_ambiguous",
34 "title_modifiers", "title_styles", "title_phrases", "given_names",
35)
37#: The metadata key a Lexicon field added after `Lexicon` first shipped
38#: carries, naming the release that added it, with a default: a pickle
39#: written before that release lacks the field and loads with the
40#: default. The same string as `_types.ADDED_IN`, the key the other
41#: frozen types' shared guard reads; layering keeps this module from
42#: importing `_types`, so tests/v2/test_lexicon.py holds the two equal.
43_ADDED_IN = "nameparser.added_in"
45#: Every character the parser reads as a full stop at a WORD'S EDGE
46#: (#322/#323): the ASCII period, the fullwidth FULL STOP U+FF0E, the
47#: IDEOGRAPHIC FULL STOP U+3002 and its halfwidth form U+FF61. One
48#: string, four readers -- the lookup fold below, _vocab's script
49#: classification fold, _script_segment's surname site and honorific
50#: tail match -- so that "which characters are a period" is answered
51#: once; the bundle's fifth site, _pieces' opening-abbreviation shape,
52#: reads no stop set at all but the no-initials repertoire
53#: (_policy._NO_INITIALS) that licenses every reader here. NFKC is
54#: NOT a substitute for listing them: it folds U+FF0E to '.' and
55#: U+FF61 to U+3002, and leaves U+3002 as it is.
56#: Not every period test reads this set, and the ones that do not are
57#: named so the claim is checkable: the interior-period shapes -- the
58#: dotted acronym 'M.A.' and the split 'Ph. D.' (_vocab) -- and FOUR
59#: ASCII-only edge tests on Latin shapes, across THREE modules -- the
60#: initial 'J.' (_vocab), the bracketed credential '(Mgr.)' and the
61#: word-internal apostrophe rule (_extract), and the trailing
62#: middle-initial carve-out 'V.' (_assign) -- stay ASCII. That is a
63#: fact about those TESTS and not about this set's reach: the fold
64#: below is script-agnostic, so a LATIN word wearing a wide stop
65#: reaches vocabulary the ASCII veto would have taken it out of.
66#: 'V。' folds to 'v' and does not end in the ASCII period the
67#: trailing-suffix carve-out tests (_assign, not is_initial), so
68#: it is roman five -- suffix -- where 'V.' stays a middle
69#: initial through that same carve-out; 'Jr。' simply reaches its
70#: entry, and 'Dr。' its title.
71#: decisions/cjk-full-stops carries the measurements.
72FULL_STOPS = ".．。｡"
75# The Arabic tatweel (kashida) stretches a word for looks and carries no
76# letter, so deleting it changes no word's identity (#1151: `الدكتــور`
77# is the title `الدكتور`). Token text is never rewritten.
78_TATWEEL = "\u0640"
81def _normalize(word: str) -> str:
82 """Lowercase, NFC-compose, strip whitespace and EDGE full stops
83 (FULL_STOPS, not the ASCII period alone -- #322). Interior periods
84 survive on purpose: 'J.R.' must not collapse to 'jr' and hit the
85 periodless vocabulary (v1 parity, pinned live 2026-07-17).
86 Suffix-ACRONYM membership alone uses the period-free form (see
87 _vocab.suffix_as_written), mirroring v1's is_suffix, which removed
88 periods only for the acronym test.
90 lower(), NOT casefold(): casefold's caseless-matching folds mutate
91 the stored vocabulary itself -- 'κος' becomes the misspelling 'κοσ'
92 (final sigma flattened) and 'großfürst' becomes 'grossfürst' --
93 while lower() applies Unicode SpecialCasing contextually and keeps
94 both as authored. This function is the single fold for storage AND
95 match-time lookups, so matching stays symmetric either way; lower()
96 is what v1's lc() used, preserving which cross-spellings match.
98 Tatweel (U+0640) is deleted, in a non-ASCII word only (#1151).
100 NFC, since #322, and only for a non-ASCII word (ASCII is already
101 NFC, and this runs per token on the parse path --
102 decisions/parse-cost). NFD hangul decomposes to jamo and missed
103 every vocabulary entry; composing at lookup AND at storage keeps
104 the two symmetric, and no shipped entry changes under it (measured
105 2026-09-09: fold every string member of every Lexicon.default()
106 field through NFC and count the changes; zero of 1735). This is
107 the same NFC composition script classification already applies
108 (_vocab._normalized_for_script); token text is never rewritten.
110 Strips to a FIXED POINT. A single strip().strip(FULL_STOPS) leaves
111 stops-around-whitespace half done ('. a .' -> ' a '), so a value
112 that is re-normalized later -- on unpickle, or by a second add() --
113 would change under its owner. v1 never re-normalized, so this only
114 matters now that storage and match-time share one fold."""
115 word = word.lower()
116 if not word.isascii():
117 word = unicodedata.normalize("NFC", word)
118 if _TATWEEL in word:
119 word = word.replace(_TATWEEL, "")
120 while True:
121 stripped = word.strip().strip(FULL_STOPS)
122 if stripped == word:
123 return word
124 word = stripped
127# rules/P7: "A one-letter word written with its period is an
128# initial and not a particle"
129def _spells_an_initial(n: str, text: str) -> bool:
130 """Whether `text` is ONE letter written with its period ('Ó.',
131 'j.'), which makes it an initial whatever particle the letter also
132 spells (#604). `n` is `_normalize(text)`, folded once by the caller.
134 Every particle test asks this AFTER its membership hit, so a word
135 no particle set holds never pays the frame. It lives here rather
136 than in _pipeline._vocab because _render's case repair, which
137 re-reads the particle vocabulary over a parsed name, must ask it
138 too and may not import the pipeline (the 2.x facade's initials and
139 last-name split were a second such reader).
141 The ASCII period only, as `_vocab.is_initial` takes it: a CJK full
142 stop marks no initial ('Ó。' is the particle). The other half of
143 is_initial, the scripts that write no initials (#320), is NOT asked
144 -- its script table is _policy's, which this module may not import
145 -- so a caller's one-character particle in such a script, written
146 with a period, is vetoed where is_initial would call it no initial
147 (decisions/P7). No shipped particle is one character of those
148 scripts."""
149 return len(n) == 1 and text[-1] == "."
152def _fold_words(words: Iterable[str]) -> list[str]:
153 """The words of a title run, folded for storage and lookup.
155 A multi-word title is matched as one key ('lt col'), so the fold has
156 to run per word -- _normalize on the whole phrase would leave
157 interior periods.
159 Words that fold away are DROPPED, not kept as empty. Keeping the
160 gap makes the key non-idempotent -- 'lt .' would store 'lt ', which
161 match time can never build: a run CAN carry a lone '.' (the
162 conjunction merge puts one there), but the fold drops the empty
163 word, so the key is 'lt' and the stored entry is inert. Storage
164 re-runs this fold on unpickle and on every dataclasses.replace, so a
165 value that changes under a second pass is one Lexicon later rejects as
166 "not written by this version". _normalize converges for the same
167 reason; so must anything built on top of it.
169 A LIST, so that _run_addresses_by_given's last-word arm can be the
170 last word of this fold rather than a re-split of the joined key --
171 "the last word of the FOLDED key" is then structural, and the two
172 arms read one fold between them.
174 map/filter rather than a comprehension: both are C calls where a
175 comprehension is a Python frame on 3.11, and this runs on the parse
176 path for every name carrying a title run (decisions/parse-cost).
177 """
178 return list(filter(None, map(_normalize, words)))
181def _title_key(words: Iterable[str]) -> str:
182 """The given_name_titles lookup key for a run of title words: the
183 folded words, space-joined.
185 Defined once because it is built at match time
186 (_run_addresses_by_given, which is how post_rules' H1 and group's
187 P5 licence both reach it) and wherever a caller's word is stored or
188 looked up (_edit_words, which folds every word add() and remove()
189 take, a phrase included, and readings_of), and a divergence between
190 them fails silently: the entry simply stops matching."""
191 return " ".join(_fold_words(words))
194def _run_addresses_by_given(words: Iterable[str],
195 vocabulary: Set[str]) -> bool:
196 """Whether a run of title words addresses by the GIVEN name.
198 Two sites ask it and they must agree -- post_rules for H1, group
199 for the P5 licence -- because a run read two ways is a rule
200 contradicting itself (decisions/P5, the 2026-08-22 #369 entry).
201 One predicate, so they cannot drift.
203 The WHOLE run's key, or that key's LAST word (#489): a run is
204 written as several titles and addresses the way its final one does,
205 so 'Her Majesty Queen' addresses by given name because 'queen'
206 does. The two arms are asked of one key and neither is the other's
207 fallback -- the `or` short-circuits but decides nothing, since for
208 a one-word run the key IS its last word. Both arms read one
209 _fold_words list, so the last word is the last word of the FOLDED
210 key by construction rather than by agreement, and a run token that
211 folds away cannot empty that arm: the conjunction merge can put a
212 lone '.' in the run ('Sir and . John'), and the fold drops it.
213 What the drop leaves as the last word can then be the CONJUNCTION
214 -- 'Sir and . John' keys 'sir and' and reads family 'John', where
215 'Sir and Dame John' keys 'sir and dame' and reads given. Harmless
216 on the shipped vocabulary, which holds no entry ending in a
217 connective, and a caller who stored one would be asking for it
218 (measured 2026-09-09).
220 The whole-run arm is what keeps a caller's multi-word phrase entry
221 working: 'lt col' is stored as one key and matched as one run. Over
222 the SHIPPED vocabulary it is dead -- every shipped entry is a single
223 word (asserted in test_lexicon.py), so it can only ever match a
224 one-word run, which is a run the last-word arm reads the same way.
226 H2's unlisted abbreviations ride in the run. One can never match as
227 the last-word key, being in no vocabulary by definition -- written
228 as the inputs that produce those runs, 'Xyz. Sir John' keys 'xyz
229 sir', matches on 'sir' and reads given 'John', while 'Sir Xyz.
230 John' keys 'sir xyz', matches on neither arm and reads family
231 'John'. It CAN sit inside a whole-run key that matches, because
232 given_name_titles is deliberately not validated against titles: a
233 caller who stores 'sir xyz' makes 'Sir Xyz. John' read given.
235 The vocabulary is passed in rather than read off a default: a
236 caller's own Lexicon is the one that has to be consulted, and this
237 module is where Lexicon is defined."""
238 folded = _fold_words(words)
239 if not folded:
240 # every word folded away, so there is no key and no last word:
241 # the joined spelling built "" here, which matched nothing
242 return False
243 return " ".join(folded) in vocabulary or folded[-1] in vocabulary
246def _reject_buffer(value: object, label: str, plural: str) -> None:
247 """Binary sequences iterate to INTS, so every downstream entry check
248 reports a byte value -- "must be strings, got 100" for b'dr', where
249 100 is 'd'. That names neither the cause nor the fix. v1 shipped a
250 decode hint for this (#238), and parse() carries one, so no config
251 entry point should be the odd one out.
252 """
253 if isinstance(value, (bytes, bytearray, memoryview)):
254 raise TypeError(
255 f"{label} must be an iterable of {plural}, not "
256 f"{type(value).__name__} -- decode first, e.g. "
257 f"raw.decode('utf-8')"
258 )
261def _warn_at_caller(message: str) -> None:
262 # A fixed stacklevel always lands on library internals: the call
263 # depth differs per entry point (Lexicon(), add(), unpickle,
264 # dataclasses.replace). Walk out of this module (and dataclasses'
265 # replace frames) so the warning points at the caller's own line.
266 level = 2
267 frame: FrameType | None = sys._getframe(1)
268 while frame is not None and frame.f_globals.get("__name__") in (
269 __name__, "dataclasses"):
270 frame, level = frame.f_back, level + 1
271 warnings.warn(message, UserWarning, stacklevel=level)
274def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]:
275 # Reject a bare str before iterating: iterating "dr" would silently
276 # yield the single characters {'d', 'r'} -- the set(str) footgun on
277 # the primary customization surface.
278 if isinstance(entries, str):
279 raise TypeError(
280 f"Lexicon.{field_name} must be an iterable of strings, "
281 f"not a bare string"
282 )
283 _reject_buffer(entries, f"Lexicon.{field_name}", "strings")
284 # A Mapping would silently contribute only its keys; a dict here
285 # almost always means the caller confused this field with
286 # capitalization_exceptions.
287 if isinstance(entries, Mapping):
288 raise TypeError(
289 f"Lexicon.{field_name} must be an iterable of strings, not a "
290 f"mapping (only capitalization_exceptions holds key->value pairs)"
291 )
292 try:
293 items = tuple(entries) # materialize once; entries may be a generator
294 except TypeError:
295 raise TypeError(
296 f"Lexicon.{field_name} must be an iterable of strings, "
297 f"got {entries!r}"
298 ) from None
299 normalized = set()
300 for w in items:
301 if not isinstance(w, str):
302 raise TypeError(
303 f"Lexicon.{field_name} entries must be strings, got {w!r}"
304 )
305 n = _normalize(w)
306 # "." or "" is a data bug (stray split artifact, empty CSV
307 # cell); dropping it silently would also let a data-module typo
308 # vanish instead of failing CI.
309 if not n:
310 raise ValueError(
311 f"Lexicon.{field_name} entry {w!r} normalizes to empty "
312 f"(lowercase + strip full stops/whitespace leaves nothing)"
313 )
314 # A list is matched one word at a time, so a multi-word entry
315 # can never match -- the library itself shipped eight such dead
316 # entries for years (repaired 2026-07-26). Warn, never raise: an
317 # inert entry produces nothing, while a raise costs a working
318 # configuration (AGENTS.md's gotcha on given-name title runs,
319 # the precedent). split() covers all Unicode whitespace.
320 if n != "".join(n.split()):
321 _warn_at_caller(
322 f"Lexicon.{field_name} entries are matched one word at "
323 f"a time; multi-word entry {w!r} can never match. "
324 f"Split it into separate entries")
325 normalized.add(n)
326 return frozenset(normalized)
329def _alnum(text: str) -> str:
330 return "".join(c for c in text if c.isalnum())
333def _offered_mask(normalized_key: str) -> str:
334 """The working value a mismatched-mask error offers: the
335 key upper-cased, or the key unchanged -- an identity mask, always
336 legal -- where .upper() changes the letter count ('straße' ->
337 'STRASSE') and would fail the check if pasted back in."""
338 upper = normalized_key.upper()
339 return upper if _alnum(_normalize(upper)) == _alnum(
340 normalized_key) else normalized_key
343class _MaskValueError(ValueError):
344 """_normpairs' error for a value that does not spell its key.
345 Built in the UnicodeDecodeError idiom -- structure in, message
346 out: `key`, `normalized_key` and `value` are the offending inputs;
347 `offered` (the fix quoted in the message below) is computed from
348 them, not carried separately.
350 __reduce__ is explicit because the inherited one re-calls
351 __init__ with self.args, which holds the message alone, not the
352 three fields __init__ needs -- and a ProcessPoolExecutor pickles a
353 worker's exception to deliver it, surfacing a construction failure
354 as BrokenProcessPool. Passing self.__dict__ as reduce's third
355 element (restored via a plain __dict__.update, there being no
356 __setstate__ here) carries along anything added after construction
357 -- e.g. add_note() -- that reconstructing from the three fields
358 alone would not. (An exception, so _types.py's frozen-dataclass
359 pickle guards do not apply.)"""
361 def __init__(self, key: str, normalized_key: str, value: str) -> None:
362 self.key = key
363 self.normalized_key = normalized_key
364 self.value = value
365 self.offered = _offered_mask(normalized_key)
366 super().__init__(
367 f"capitalization_exceptions value {value!r} for key {key!r} "
368 f"does not spell the key's letters and digits: a value "
369 f"is a case mask, the key's own letters and digits "
370 f"recased -- e.g. "
371 f"capitalization_exceptions=(({normalized_key!r}, "
372 f"{self.offered!r}),)")
374 def __reduce__(
375 self,
376 ) -> tuple[type[_MaskValueError], tuple[str, str, str], dict[str, object]]:
377 return (type(self), (self.key, self.normalized_key, self.value),
378 self.__dict__)
381def _normpairs(
382 raw: Mapping[str, str] | Iterable[tuple[str, str]],
383) -> tuple[tuple[str, str], ...]:
384 """Canonicalize capitalization_exceptions input: _normset's sibling
385 for the one pair-valued field. Dedupes on the NORMALIZED key so the
386 tuple and the derived map always agree ("PHD" and "phd" collide
387 after normalization; "Ph.D." does not, the fold keeping interior
388 periods, so it is a key of its own, "ph.d"); last occurrence wins,
389 matching dict semantics and the right-bias rule used elsewhere."""
390 if isinstance(raw, str):
391 raise TypeError(
392 "capitalization_exceptions must be a mapping or an "
393 "iterable of (key, value) pairs, not a bare string"
394 )
395 _reject_buffer(raw, "capitalization_exceptions", "(key, value) pairs")
396 pairs = raw.items() if isinstance(raw, Mapping) else raw
397 try:
398 pairs = iter(pairs)
399 except TypeError:
400 raise TypeError(
401 "capitalization_exceptions must be a mapping or an "
402 f"iterable of (key, value) pairs, got {raw!r}"
403 ) from None
404 deduped: dict[str, str] = {}
405 for entry in pairs:
406 # A 2-char str entry would unpack "ab" into ("a", "b")
407 # silently, so reject str outright; other mis-shapes would
408 # otherwise surface as bare unpack errors.
409 if isinstance(entry, str):
410 raise TypeError(
411 f"capitalization_exceptions entries must be "
412 f"(key, value) pairs, got {entry!r}"
413 )
414 try:
415 k, v = entry
416 except (TypeError, ValueError):
417 raise TypeError(
418 f"capitalization_exceptions entries must be "
419 f"(key, value) pairs, got {entry!r}"
420 ) from None
421 if not isinstance(k, str) or not isinstance(v, str):
422 raise TypeError(
423 f"capitalization_exceptions entries must be "
424 f"str -> str, got {k!r}: {v!r}"
425 )
426 normalized_key = _normalize(k)
427 if not normalized_key:
428 raise ValueError(
429 f"capitalization_exceptions key {k!r} normalizes to "
430 f"empty (lowercase + strip full stops/whitespace leaves "
431 f"nothing)"
432 )
433 # Stored NFC-composed, case kept (unicodedata, not _normalize):
434 # one stored spelling per value, whichever form the caller
435 # wrote. _apply_mask reads both forms alike since #542
436 # (_render._beside looks past combining marks); before it, a
437 # decomposed letter read as a base letter split off beside a
438 # non-alpha combining mark. The mask check below
439 # passes either spelling, _normalize composing both of its
440 # sides. `written` keeps the caller's own spelling -- composed
441 # or not -- for the mismatch error below: a decomposed value
442 # (e.g. 'e' + a combining acute) and its NFC form print
443 # identically to a reader's eye but are different str objects,
444 # so echoing the composed spelling back would show the caller
445 # a value they did not write.
446 written = v
447 v = unicodedata.normalize("NFC", v)
448 # capitalized() looks words up one at a time (the _WORD regex
449 # never yields spaces), so a multi-word key is unreachable --
450 # checked FIRST, ahead of the mask check below, because an
451 # invariant guards harm and a value that can never be read
452 # does no harm no matter what it says (AGENTS.md: "a check
453 # guards harm, not no-ops"). Raising the mask error for such a
454 # key -- as a naive mask-first order does -- would refuse a
455 # config that was always going to be a no-op.
456 # interior whitespace test; split() covers all Unicode whitespace
457 if normalized_key != "".join(normalized_key.split()):
458 _warn_at_caller(
459 f"capitalization_exceptions keys are matched one word "
460 f"at a time; multi-word key {k!r} can never match. "
461 f"Split it into per-word entries")
462 deduped[normalized_key] = v
463 continue
464 # A value is a case MASK (#459): the key's own letters and
465 # digits recased, compared through the key's own fold. Its
466 # punctuation marks which letters are joined into one run
467 # (_render._apply_mask reads that) and is never written into
468 # the word. A raise, not a warning: a mismatched value used to
469 # be SUBSTITUTED for the word, and rules/R4: "Repair
470 # changes case and nothing else", so there is no reading of
471 # one that repair can honor.
472 if _alnum(_normalize(v)) != _alnum(normalized_key):
473 # key is the RAW key, not normalized_key, and `written`,
474 # not `v`: the message names the entry as the caller wrote
475 # it, not as the fold or NFC composed it.
476 raise _MaskValueError(k, normalized_key, written)
477 deduped[normalized_key] = v
478 return tuple(sorted(deduped.items()))
481#: The two lists outside the per-word model (decisions/vocabulary-model's
482#: Standing line on name readings): `Lexicon` fields, changed through
483#: ``|`` with a fragment or ``dataclasses.replace``, and copied into
484#: `_LexiconSets` beside the compiled sets.
485_LIST_FIELDS = ("surnames", "given_names")
487#: The 20 sets a compile of readings gives, under the names the 2.x
488#: `Lexicon` fields had: internal since 3.0 (#754, 4i-b), read through
489#: `Lexicon._sets`, and the names an old spelling is refused by.
490_SET_FIELDS = tuple(f for f in _VOCAB_FIELDS if f not in _LIST_FIELDS)
492#: What those sets do not record: the kinds the views read (rules/R8,
493#: rules/R9), and the titles written short that rules/H5's trailing
494#: chain and rules/H9's article partner read where one may also be a
495#: given name (#999), which is no
496#: kind; the name is the tuple's first use.
497#: `_LexiconSets` attributes after the `_VOCAB_FIELDS` ones, each a
498#: compiled constant of its own.
499_KIND_FIELDS = ("generation_words", "credential_words", "honorific_words",
500 "patronymic_particles", "title_abbreviations",
501 "title_case_name_words", "shared_names",
502 "particle_given_names", "saint_particles",
503 "capitalized_particles", "capital_opening_particles",
504 "dotted_titles", "glued_titles", "bound_phrases")
506@dataclass(frozen=True, slots=True, eq=False, repr=False)
507class _LexiconSets:
508 """The vocabulary sets a parse reads: the pipeline, `_render`,
509 `_parser` and the tools read `lexicon._sets.titles` (#754, 4i-a;
510 tests/v2/test_layering.py holds it), the 2.x public fields having
511 become internal in 4i-b. One attribute per `_VOCAB_FIELDS` entry, in
512 that order, then one per `_KIND_FIELDS` entry, each a frozenset, so
513 a membership test stays a C-level attribute read and costs no frame.
515 Built once per `Lexicon`, at construction: the compiled sets from
516 its readings by `compile_readings`, or for `Lexicon.default()` from
517 nameparser/config/_compiled.py, so the reading index stays unloaded
518 on `parse("John Smith")`; the two lists from its fields. Never
519 compared, hashed or pickled: a `Lexicon`'s equality and pickle are
520 its fields', and a load or a `dataclasses.replace` builds this
521 again.
523 Not a mapping and no repr of its own: it holds whole word lists,
524 and a repr of them would be unbounded."""
526 titles: frozenset[str]
527 given_name_titles: frozenset[str]
528 suffix_acronyms: frozenset[str]
529 suffix_words: frozenset[str]
530 suffix_acronyms_ambiguous: frozenset[str]
531 particles: frozenset[str]
532 particles_ambiguous: frozenset[str]
533 conjunctions: frozenset[str]
534 conjunctions_ambiguous: frozenset[str]
535 bound_given_names: frozenset[str]
536 given_name_abbreviations: frozenset[str]
537 maiden_markers: frozenset[str]
538 surnames: frozenset[str]
539 honorific_tails: frozenset[str]
540 after_given_titles: frozenset[str]
541 suffix_words_ambiguous: frozenset[str]
542 agnomens: frozenset[str]
543 titles_ambiguous: frozenset[str]
544 title_modifiers: frozenset[str]
545 title_styles: frozenset[str]
546 title_phrases: frozenset[str]
547 given_names: frozenset[str]
548 #: the suffix words whose suffix reading is a generation (`jr`,
549 #: `iii`), a credential written as a word (`kc`), or an honorific
550 #: (`さん`, `先生`), and the particles read as patronymics (`bin`):
551 #: the kinds rules/R8's and rules/R9's views read, which the sets
552 #: above do not record
553 generation_words: frozenset[str]
554 credential_words: frozenset[str]
555 honorific_words: frozenset[str]
556 patronymic_particles: frozenset[str]
557 #: the words whose title reading is written `abbreviation=True`
558 #: (`maj`); rules/H5's trailing chain reads one only beside
559 #: `titles_ambiguous`, through classify's TRAILING_TITLE_TAG (#999)
560 title_abbreviations: frozenset[str]
561 #: the suffix words written `title_case_is_name=True` (`fils`,
562 #: `kc`), whose writing leans where a name reading gates them:
563 #: lower case is the suffix, Title case the name (rules/S6, #1039)
564 title_case_name_words: frozenset[str]
565 #: the names written `shared=True` (Sikh `kaur`, `singh`), which a
566 #: community gives every member: rules/O6 and O7 neither take nor
567 #: complete one, and P10 completes none (#704)
568 shared_names: frozenset[str]
569 #: the given names written `after_particle=True` (`carmen`), the
570 #: second of a compound given name written behind a particle
571 #: (`María del Carmen`): rules/P10 reads one as a middle name behind
572 #: a one-word given name, and O6 leaves it there (#704)
573 particle_given_names: frozenset[str]
574 #: the particles of the `saint` kind (`san`, `santa`, `st`, `ste`): no given
575 #: name phrase opens with one, `Santa María` being a surname (#704)
576 saint_particles: frozenset[str]
577 #: the particles of the kinds written with a capital, `saint` and
578 #: `rank` (`st`, `san`, `freiherr`, `graf`): rules/R4's case repair
579 #: writes one as a name word, not in lowercase (#1069)
580 capitalized_particles: frozenset[str]
581 #: the particles written `capital_opening=True` (`ter`, `der`):
582 #: rules/R12's case repair writes one with a capital where no
583 #: particle stands before it in the family name, nor, where it opens
584 #: the family name, written right before it in any role (`Levon Ter
585 #: Petrosyan`, but `Van der Walt`)
586 capital_opening_particles: frozenset[str]
587 #: the titles written `dotted=True` (`us`, `uk`, `pd`): rules/H10
588 #: reads a run of single letters joined by full stops that spells
589 #: one, at the head of a name (the given part's title run, or H6's
590 #: before a family comma), as that title where a title the run keeps
591 #: stands behind it (`U.S. District Judge`, #490)
592 dotted_titles: frozenset[str]
593 #: the titles written `attached="before"` (Thai `นาย`, `นางสาว`,
594 #: `อาจารย์`): rules/W6 splits one off the start of the name's first
595 #: word it is glued to, the mirror of `honorific_tails` (#317)
596 glued_titles: frozenset[str]
597 #: the two-word given names a bound word's `completed_by` lists
598 #: ('عبد الله'), which rules/P12 joins past the first name word (#694)
599 bound_phrases: frozenset[str]
602#: Lexicon field -> the compiled constant it is read from, for every
603#: set the vocabulary compiles (nameparser/_reading_index.py's
604#: Compiled); `particles_ambiguous` is derived (_vocab_fields below).
605#: tests/v2/test_parser.py holds that these, read from the compile of
606#: no choice, are Lexicon.default()'s.
607_COMPILED_FIELDS = {
608 "titles": "TITLES",
609 "given_name_titles": "GIVEN_NAME_TITLES",
610 "suffix_acronyms": "SUFFIX_ACRONYMS",
611 "suffix_words": "SUFFIX_WORDS",
612 "suffix_acronyms_ambiguous": "SUFFIX_ACRONYMS_AMBIGUOUS",
613 "particles": "PARTICLES",
614 "conjunctions": "CONJUNCTIONS",
615 "conjunctions_ambiguous": "CONJUNCTIONS_AMBIGUOUS",
616 "bound_given_names": "BOUND_GIVEN_NAMES",
617 "given_name_abbreviations": "GIVEN_NAME_ABBREVIATIONS",
618 "maiden_markers": "MAIDEN_MARKERS",
619 "honorific_tails": "GLUED_HONORIFICS",
620 "after_given_titles": "AFTER_GIVEN_TITLES",
621 "suffix_words_ambiguous": "SUFFIX_WORDS_AMBIGUOUS",
622 "agnomens": "AGNOMENS",
623 "titles_ambiguous": "TITLES_AMBIGUOUS",
624 "title_modifiers": "TITLE_MODIFIERS",
625 "title_styles": "TITLE_STYLES",
626 "title_phrases": "TITLE_PHRASES",
627 "generation_words": "GENERATION_WORDS",
628 "credential_words": "CREDENTIAL_WORDS",
629 "honorific_words": "HONORIFIC_WORDS",
630 "patronymic_particles": "PATRONYMIC_PARTICLES",
631 "title_abbreviations": "TITLE_ABBREVIATIONS",
632 "title_case_name_words": "TITLE_CASE_NAME_WORDS",
633 "shared_names": "SHARED_NAMES",
634 "particle_given_names": "PARTICLE_GIVEN_NAMES",
635 "saint_particles": "SAINT_PARTICLES",
636 "capitalized_particles": "CAPITALIZED_PARTICLES",
637 "capital_opening_particles": "CAPITAL_OPENING_PARTICLES",
638 "dotted_titles": "DOTTED_TITLES",
639 "glued_titles": "GLUED_TITLES",
640 "bound_phrases": "BOUND_PHRASES",
641}
644#: The groups `SAME_AS` is keyed by, one per kind of role a name match
645#: compares (`_reading_index.SAME_AS_GROUPS` maps a reading's kind to
646#: one; tests/v2/test_lexicon.py holds the two to one list).
647_SAME_AS_GROUPS = frozenset({"given", "family", "title", "suffix"})
649#: A group -> word -> the words it is a listed spelling of.
650SameAs = Mapping[str, Mapping[str, frozenset[str]]]
653def _same_as_of(compiled: Mapping[str, Any]) -> SameAs:
654 """The compile's `SAME_AS` as the lexicon keeps it: per group, each
655 word's full forms normalized, as a caller's `same_as` is written
656 free-form."""
657 return MappingProxyType({
658 group: MappingProxyType({
659 w: frozenset(_normalize(h) for h in heads)
660 for w, heads in words.items()})
661 for group, words in compiled["SAME_AS"].items()})
664def _vocab_fields(compiled: Mapping[str, Any]) -> dict[str, frozenset[str]]:
665 """The sets a compile of the vocabulary gives, by `_LexiconSets`
666 name (the constants of nameparser/config/_compiled.py, or a compile
667 of readings), the two lists aside."""
668 out = {name: compiled[key] for name, key in _COMPILED_FIELDS.items()}
669 # FLIPPED from v1: v1 marked the never-given subset; the sets mark
670 # the may-be-given one
671 out["particles_ambiguous"] = (compiled["PARTICLES"]
672 - compiled["NON_GIVEN_NAME_PARTICLES"])
673 return out
676#: A group -> the words of it with a listed spelling that are a name of
677#: their own too (`SAME_AS_NAMED`).
678SameAsNamed = Mapping[str, frozenset[str]]
681# -- the name match's fold, which the familiar-form index shares --------
683#: Full stops and apostrophes: the typed one, the typographic U+2019,
684#: the modifier letter U+02BC, and Uzbek Latin's other spellings of
685#: the `ʻ` in `oʻ` and `gʻ` (U+02BB, U+2018, the backtick), so
686#: `O’Brien` folds to `obrien` and `Gʻofurov` to `gofurov`
687#: (decisions/name-matching, 2026-11-09). And the Arabic tatweel U+0640,
688#: which `_normalize` deletes too (#1151), so a name's two spellings that
689#: differ by one fold alike where the vocabulary and the match meet.
690_DROPPED = str.maketrans("", "", ".'`\u2019\u02bc\u02bb\u2018\u0640")
691#: The scripts whose combining marks `_fold` drops as accents.
692_ACCENTED_SCRIPTS = ("LATIN ", "GREEK ", "CYRILLIC ")
694def _fold(text: str) -> str:
695 """The text with case, accents, full stops and apostrophes folded
696 away: NFC, `.` and the apostrophes and their stand-ins (`_DROPPED`) removed,
697 NFD, casefolded, then NFC again, so the folded text, and every key
698 cut from it, is precomposed (a Hangul syllable, never a conjoining
699 jamo). Hyphens and spaces are kept. A combining mark (Mn) is dropped
700 only where the letter it attaches to is Latin, Greek or Cyrillic,
701 where it is an accent; in another script (a Devanagari virama,
702 anusvara or vowel sign) it is spelling and is kept. A letter counts
703 as in those scripts where it or its casefolded form is, so a letter
704 that casefolds into one (the micro sign U+00B5 into Greek mu, U+2183
705 into Latin) loses its mark on the first fold, which keeps the fold
706 idempotent; so does removing the full stops and apostrophes before
707 the marks are read, a mark after one then attaching to the letter
708 before it, and again after casefolding, which can write one (U+0149
709 `ʼn` casefolds to `ʼn`). The iota subscript U+0345 is a mark like any
710 other, so `ᾳ` folds to `α`."""
711 if text.isascii():
712 # no accent, no composition, and no apostrophe but the typed
713 # one and the backtick
714 return text.casefold().translate(_DROPPED)
715 decomposed = unicodedata.normalize(
716 "NFD", unicodedata.normalize("NFC", text).translate(_DROPPED))
717 kept = []
718 accented = False
719 for c in decomposed:
720 if unicodedata.category(c) == "Mn":
721 if accented:
722 continue
723 else:
724 accented = (unicodedata.name(c, "").startswith(_ACCENTED_SCRIPTS)
725 or unicodedata.name(c.casefold()[:1] or c, "")
726 .startswith(_ACCENTED_SCRIPTS))
727 kept.append(c)
728 # again after casefolding, which can write an apostrophe (U+0149)
729 return unicodedata.normalize(
730 "NFC", "".join(kept).casefold().translate(_DROPPED))
733#: A given name -> every name it is paired with as a familiar form, in
734#: either direction (`FAMILIAR`, read both ways).
735Familiar = Mapping[str, frozenset[str]]
738def _familiar_of(compiled: Mapping[str, Any]) -> Familiar:
739 """The compile's `FAMILIAR`, a familiar form -> the longer names its
740 `short_for` lists, read both ways: each word with every name it is
741 paired with directly, as the shorter name or the longer. The
742 sources do not always write the shorter name first (nam_dict's
743 `Maria` / `Masha`, the nicknames list's `barbara` / `babs`), so the
744 match reads a pair whichever way it was written, and never chains
745 two pairs (decisions/name-matching, 2026-11-08
746 FAMILIAR FORMS). Each name is indexed under its fold too (`_fold`,
747 the match's own), so a pair written accented (`józsi`, `józsef`)
748 is found from either name written without its accents."""
749 paired: dict[str, set[str]] = {}
750 for short, longs in compiled["FAMILIAR"].items():
751 for long in longs:
752 for key in {short, _fold(short)}:
753 paired.setdefault(key, set()).add(long)
754 for key in {long, _fold(long)}:
755 paired.setdefault(key, set()).add(short)
756 return MappingProxyType({w: frozenset(ns) for w, ns in paired.items()})
759#: What a lexicon's readings compile to: the vocabulary sets by
760#: `_LexiconSets` name, the listed-spelling map, the words in it that
761#: are a name of their own as well, and the familiar forms.
762_Compiled = tuple[Mapping[str, frozenset[str]], SameAs, SameAsNamed,
763 Familiar]
766def _compiled_of(compiled: Mapping[str, Any]) -> _Compiled:
767 return (MappingProxyType(_vocab_fields(compiled)), _same_as_of(compiled),
768 MappingProxyType(dict(compiled["SAME_AS_NAMED"])),
769 _familiar_of(compiled))
772@functools.cache
773def _default_compiled() -> _Compiled:
774 # the compiled data module is the default's one source, so the
775 # default never loads the reading index (decisions/vocabulary-
776 # model, 2026-10-29 3a; its 2026-11-02 #754 entry, item 4)
777 import nameparser.config._compiled as compiled
778 return _compiled_of(vars(compiled))
781_NO_SETS: Mapping[str, frozenset[str]] = MappingProxyType(
782 {name: frozenset() for name in _COMPILED_FIELDS} | {
783 "particles_ambiguous": frozenset()})
784#: What the empty base compiles to: no sets and no listed spellings.
785_NOTHING_COMPILED: _Compiled = (_NO_SETS, MappingProxyType(
786 {g: MappingProxyType({}) for g in sorted(_SAME_AS_GROUPS)}),
787 MappingProxyType({g: frozenset() for g in sorted(_SAME_AS_GROUPS)}),
788 MappingProxyType({}))
791# -- readings ------------------------------------------------------------
793#: A reading's properties as the log and the repr compare them: sorted
794#: (key, value) pairs, a list value a sorted tuple.
795_Props = tuple[tuple[str, Any], ...]
798class _Edit(NamedTuple):
799 """One `add` or `remove`, stored frozen as written (decisions/
800 vocabulary-model, 2026-11-02 #754, item 3): its words normalized,
801 its kind (None on a `remove` of every reading), its properties
802 sorted pairs, a property given its unwritten value held as None,
803 which removes it. Compared and hashed as a tuple."""
805 op: str
806 words: tuple[str, ...]
807 kind: str | None
808 props: _Props
811def _index_module() -> ModuleType:
812 # the reading index's reader, imported at call time: tools/vocab/
813 # build.py loads this module by path, before the index it rewrites
814 # can be trusted to import (root AGENTS.md's layering bullet)
815 import nameparser._reading_index as ri
816 return ri
819def _clean(props: Mapping[str, Any]) -> _Props:
820 """A reading's properties as a caller states them: `default` (a
821 caller's reading is on) and `cased` (a case mask, configured through
822 `capitalization_exceptions`) left out."""
823 return tuple(sorted((k, v) for k, v in props.items()
824 if k not in ("default", "cased")))
827def _unwritten(kind: str, prop: str) -> object:
828 """The value `prop` takes on a `kind` reading that does not write
829 it, read through _reading_index.ReadingProperties: `acronym` is
830 true on a credential, `addresses` the family name."""
831 bare = _index_module().IndexReading("", "", kind, MappingProxyType({}))
832 if prop == "acronym":
833 return bare.acronym
834 if prop == "addresses":
835 return bare.addresses
836 return bare.flag(prop)
839#: The properties whose value is a list of words, each with what its
840#: entries are and an example, for the messages refusing a value.
841_LIST_PROPS: Mapping[str, str] = MappingProxyType({
842 "attaches": "title kinds or title words, e.g. attaches=['military']",
843 "completed_by": "single words, e.g. completed_by=['الله']",
844 "short_for": "given names, e.g. short_for=['maxwell']",
845})
848def _prop_value(kind: str, prop: str, value: object) -> object:
849 """`value` canonical for the log, None where it is the unwritten
850 one (which removes the property)."""
851 if value is None or (isinstance(value, (bool, str))
852 and value == _unwritten(kind, prop)):
853 return None
854 if prop in ("abbreviation", "acronym", "after_given", "may_be_initial",
855 "title_case_is_name", "shared", "after_particle",
856 "capital_opening", "dotted"):
857 if not isinstance(value, bool):
858 raise TypeError(f"{prop} must be True or False, got {value!r}")
859 return value
860 if prop == "addresses":
861 if value != "given":
862 raise ValueError(f"addresses must be 'given' or 'family', "
863 f"got {value!r}")
864 return value
865 if prop == "attached":
866 # a suffix glued after the name (rules/W2) or a title glued in
867 # front of it (rules/W6), the side its category reads
868 side = ("before" if _index_module().CATEGORY.get(kind) == "title"
869 else "after")
870 if value != side:
871 raise ValueError(f"attached on a {kind!r} reading must be "
872 f"{side!r} (a {'title peeled off the start' if side == 'before' else 'tail peeled off the end'} "
873 f"of a name token) or None, got {value!r}")
874 return value
875 if prop in _LIST_PROPS:
876 what = _LIST_PROPS[prop]
877 if isinstance(value, (str, bytes)) or isinstance(value, Mapping):
878 raise TypeError(f"{prop} must be an iterable of {what}, "
879 f"got {value!r}")
880 try:
881 items: tuple[object, ...] = tuple(cast("Iterable[object]", value))
882 except TypeError:
883 raise TypeError(f"{prop} must be an iterable of {what}, "
884 f"got {value!r}") from None
885 words = [x for x in items if isinstance(x, str)]
886 if len(words) != len(items):
887 raise TypeError(f"{prop} entries must be strings, got "
888 f"{value!r}")
889 if prop == "short_for" and (
890 not words or any(not _normalize(x) for x in words)):
891 # the build's check 3: each a given name, none empty
892 raise ValueError(
893 f"short_for must be a non-empty list of the given names "
894 f"the word stands for, none normalizing to nothing, got "
895 f"{value!r}; write e.g. short_for=['maxwell']")
896 folded = tuple(sorted({_normalize(x) for x in words} - {""}))
897 if prop == "completed_by" and any(" " in w for w in folded):
898 # rules/P12 joins a bound word to the ONE word after it
899 raise ValueError(f"completed_by entries must be single words, "
900 f"got {value!r}; list each completing word "
901 f"on its own")
902 if prop == "short_for" and any(" " in w for w in folded):
903 # the match compares a given name word by word
904 raise ValueError(f"short_for entries must be single given "
905 f"names, got {value!r}; list each on its own")
906 return folded or None
907 # same_as: the word this one is a listed spelling of, a given name
908 # written short's full name (#789) or a title's or suffix's full form
909 if not isinstance(value, str) or not value.strip():
910 raise TypeError(f"same_as must be the word this one is a spelling "
911 f"of, a non-empty string, got {value!r}")
912 return value
915def _check_kind(kind: object, op: str) -> str | None:
916 """`kind` checked for `op`: a kind, or on `remove` and `words` a
917 category too; None (every reading) only on `remove`."""
918 ri = _index_module()
919 if kind is None and op == "remove":
920 return None
921 if not isinstance(kind, str):
922 raise TypeError(f"{op}() takes kind as a string, a vocabulary "
923 f"kind such as 'title' or 'credential', got "
924 f"{kind!r}")
925 if kind in ri.CATEGORY:
926 return kind
927 categories = set(ri.CATEGORY.values())
928 if kind in categories:
929 if op in ("remove", "words"):
930 return kind
931 raise ValueError(
932 f"{kind!r} is a category with no kind of its name; add() "
933 f"takes one of its kinds: " + ", ".join(
934 k for k, c in ri.CATEGORY.items() if c == kind))
935 raise ValueError(
936 f"unknown reading kind {kind!r}; the kinds are "
937 + ", ".join(ri.CATEGORY)
938 + (f" (or a category, on {op}: "
939 + ", ".join(sorted(categories)) + ")"
940 if op in ("remove", "words") else ""))
943def _edit_words(words: tuple[object, ...], op: str,
944 kind: str | None) -> tuple[str, ...]:
945 if not words:
946 if op == "remove" and kind is not None:
947 # every word's readings of `kind`: a remove of a whole kind,
948 # which `remove` writes only with every=True, so that an
949 # empty `remove(*words, kind=...)` never reaches it
950 return ()
951 raise TypeError(
952 f"{op}() takes at least one word, e.g. {op}('dean', kind="
953 f"'title')" + (", or every=True with a kind to take out every "
954 "word's readings of it, e.g. remove(kind="
955 "'title', every=True)" if op == "remove" else ""))
956 out = []
957 for w in words:
958 _reject_buffer(w, f"{op}() words", "strings")
959 if not isinstance(w, str):
960 raise TypeError(
961 f"{op}() takes each word as its own argument, a string, "
962 f"got {w!r}; unpack a collection: {op}(*words, kind=...)")
963 n = _title_key(w.split())
964 if not n:
965 raise ValueError(f"{op}() word {w!r} normalizes to empty "
966 f"(lowercase + strip full stops/whitespace "
967 f"leaves nothing)")
968 out.append(n)
969 return tuple(out)
972def _edit_props(kind: str, props: Mapping[str, object]) -> _Props:
973 ri = _index_module()
974 allowed = ri.indexed(kind) - {"default", "cased"}
975 if kind == "family":
976 # a name category's property that only a given name takes: the
977 # raise below names it, so the generic message leaves it out
978 allowed -= {"after_particle"}
979 out: dict[str, object] = {}
980 for prop, value in props.items():
981 if prop == "default":
982 raise ValueError(
983 "add() takes no default: a caller's reading is on, which "
984 "is the caller's choice; take one out with remove(word, "
985 "kind=...)")
986 if prop == "cased":
987 raise ValueError(
988 "add() takes no cased: a case mask is configured through "
989 "capitalization_exceptions, e.g. dataclasses.replace("
990 "lexicon, capitalization_exceptions=(*lexicon."
991 "capitalization_exceptions, ('phd', 'PhD')))")
992 if prop == "after_particle" and kind == "family":
993 raise ValueError(
994 "after_particle names a given name written behind a "
995 "particle (rules/O7), so only a given reading takes it: "
996 "write add(word, kind='given', after_particle=True)")
997 if prop not in allowed:
998 raise ValueError(
999 f"a {kind} reading takes "
1000 + (", ".join(sorted(allowed)) or "no properties")
1001 + f"; got {prop!r}")
1002 out[prop] = _prop_value(kind, prop, value)
1003 return tuple(sorted(out.items()))
1006def _make_edit(op: object, words: object, kind: object,
1007 props: object) -> _Edit:
1008 """An edit checked and canonical: what `add` and `remove` store and
1009 what a constructor's `_edits` must already be."""
1010 if op not in ("add", "remove"):
1011 raise ValueError(f"an edit is 'add' or 'remove', got {op!r}")
1012 if not isinstance(words, tuple):
1013 raise TypeError(f"an edit's words are a tuple, got {words!r}")
1014 k = _check_kind(kind, op)
1015 ws = _edit_words(words, op, k)
1016 if op == "remove":
1017 if props:
1018 raise ValueError(f"remove() takes no properties, got {props!r}")
1019 return _Edit(op, ws, k, ())
1020 assert k is not None
1021 if not isinstance(props, (tuple, Mapping)):
1022 raise TypeError(f"an edit's properties are pairs, got {props!r}")
1023 p = _edit_props(k, dict(props))
1024 ri = _index_module()
1025 category = ri.CATEGORY[k]
1026 if category == "familiar":
1027 # the build's check 3: a familiar reading lists the longer names
1028 longs = dict(p).get("short_for")
1029 if not longs:
1030 raise ValueError(
1031 "a familiar reading needs short_for, a non-empty list of "
1032 "the given names the word stands for, e.g. add('max', "
1033 "kind='familiar', short_for=['maxwell'])")
1034 for w in ws:
1035 if w in longs:
1036 raise ValueError(
1037 f"short_for names the word itself, {w!r}; it lists "
1038 f"the other given names {w!r} stands for")
1039 for w in ws:
1040 # the build's check 8: a phrase is matched whole only as a birth
1041 # marker or a run of titles addressing the given name
1042 if " " in w and not (k == "birth_marker" or (
1043 category == "title" and ("addresses", "given") in p)):
1044 raise ValueError(
1045 f"{w!r} is a phrase, which can never match as a {k}: only "
1046 f"a birth_marker and a title with addresses='given' are "
1047 f"matched as a whole run; add its words one by one")
1048 # the build refuses attached on a title phrase: the glued-title
1049 # peel splits one word (rules/W6)
1050 if " " in w and category == "title" and ("attached", "before") in p:
1051 raise ValueError(
1052 f"{w!r} is a phrase; attached='before' marks a one-word "
1053 f"title that the glued-title peel splits off a word")
1054 # the build refuses may_be_initial on anything but a cased single
1055 # letter: no other word reads as an initial (rules/P8)
1056 if ("may_be_initial", True) in p and not (
1057 len(w) == 1 and w.upper() != w.lower()):
1058 raise ValueError(
1059 f"may_be_initial marks a single cased letter that may read "
1060 f"as an initial; {w!r} is not one")
1061 return _Edit(op, ws, k, p)
1064def _apply_edit(pool: dict[str, tuple[Any, ...]],
1065 edit: _Edit) -> tuple[str, ...]:
1066 """Apply `edit` to `pool` (word -> its readings) in place, returning
1067 the words it touched. An add of a kind the word holds updates that
1068 reading's properties in every file holding it, and turns a reading
1069 written `default = false` on; otherwise it adds the caller's own
1070 reading, which names no file. A remove with no words is one of a
1071 whole kind, reaching every word in the pool that holds a reading of
1072 it, whatever turned the reading on."""
1073 ri = _index_module()
1074 words = edit.words or tuple(
1075 w for w, rs in pool.items()
1076 if any(r.kind == edit.kind or r.category == edit.kind for r in rs))
1077 for w in words:
1078 rs = pool.get(w, ())
1079 if edit.op == "add" and edit.kind == "familiar":
1080 # EXTENDS: the caller's longer names go in a reading of the
1081 # caller's own, attributed to the edit and to no file, and
1082 # the compile unions it with each file's, which stay as
1083 # they are (decisions/name-matching, 2026-11-08 FAMILIAR
1084 # FORMS); an explicit replace is remove, then add
1085 added = set(dict(edit.props)["short_for"])
1086 mine = [r for r in rs if r.kind == "familiar" and not r.tag]
1087 if mine:
1088 added |= set(mine[0].props.get("short_for", ()))
1089 reading = ri.IndexReading(w, "", "familiar", MappingProxyType(
1090 {"short_for": tuple(sorted(added))}))
1091 pool[w] = tuple(r for r in rs if r not in mine) + (reading,)
1092 continue
1093 if edit.op == "remove":
1094 kept = tuple(r for r in rs if edit.kind is not None
1095 and r.kind != edit.kind and r.category != edit.kind)
1096 elif any(r.kind == edit.kind for r in rs):
1097 kept = tuple(
1098 ri.IndexReading(w, r.tag, r.kind, MappingProxyType(
1099 _merged(r.props, edit.props)))
1100 if r.kind == edit.kind else r for r in rs)
1101 else:
1102 kept = rs + (ri.IndexReading(w, "", edit.kind, MappingProxyType(
1103 {k: v for k, v in edit.props if v is not None})),)
1104 if kept:
1105 pool[w] = kept
1106 else:
1107 pool.pop(w, None)
1108 return words
1111def _merged(props: Mapping[str, Any], update: _Props) -> dict[str, Any]:
1112 out = {k: v for k, v in props.items() if k != "default"}
1113 for k, v in update:
1114 if v is None:
1115 out.pop(k, None)
1116 else:
1117 out[k] = v
1118 return out
1121Pool = Mapping[str, tuple[Any, ...]]
1124@functools.lru_cache(maxsize=8)
1125def _base_pool(base: object) -> Pool:
1126 """A base's readings by word, on by default or not: the shipped
1127 index for `"default"`, none for `"empty"`, a snapshot's own rows."""
1128 if base == "empty":
1129 return MappingProxyType({})
1130 ri = _index_module()
1131 rows = ri._by_text().items() if base == "default" else base
1132 return MappingProxyType({
1133 text: tuple(ri.from_row(text, row) for row in rs)
1134 for text, rs in rows}) # type: ignore[union-attr]
1137def _check_pool(pool: Pool, words: Iterable[str]) -> None:
1138 """The build's checks a caller's edits can reach, over the words
1139 they touch (checks 3 and 8 run per edit, in _make_edit): one suffix
1140 kind per word (check 4), an agnomen has a name reading (check 11's
1141 agnomen clause), and a bound given name that is also a particle has
1142 a given reading (check 9)."""
1143 for w in dict.fromkeys(words):
1144 on = [r for r in pool.get(w, ()) if r.on_by_default]
1145 suffix = {r.kind: r for r in on if r.category == "suffix"}
1146 if len(suffix) > 1:
1147 held = sorted(k for k, r in suffix.items() if r.tag)
1148 drop = held[0] if held else sorted(suffix)[0]
1149 raise ValueError(
1150 f"{w!r} would hold two suffix kinds, "
1151 f"{' and '.join(sorted(suffix))}, and a word holds one; "
1152 f"remove the one it holds first: .remove({w!r}, "
1153 f"kind={drop!r})")
1154 kinds = {r.category for r in on} | {r.kind for r in on}
1155 # check 11's agnomen clause: an agnomen is written in the case of
1156 # the name around it, so only the count gates it, and the gate is
1157 # a name reading beside it (rules/S11)
1158 if "agnomen" in kinds and "name" not in kinds:
1159 raise ValueError(
1160 f"{w!r} is an agnomen with no name reading, and an agnomen "
1161 f"is read as the suffix only where a name reading gates "
1162 f"it by the count: write .add({w!r}, kind='family') "
1163 f"first, then the agnomen")
1164 if {"bound_given", "particle"} <= kinds and "given" not in kinds:
1165 raise ValueError(
1166 f"{w!r} is a bound given name and a particle with no given "
1167 f"reading, and a particle that never starts a given name "
1168 f"cannot bind one: .add({w!r}, kind='given'), or "
1169 f".remove({w!r}, kind='bound_given')")
1172@functools.lru_cache(maxsize=32)
1173def _chosen_pool(base: object, chosen: frozenset[str], strict: bool) -> Pool:
1174 """A base's readings under the choice of the file tags `chosen`,
1175 before the caller's edits (decisions/vocabulary-model, 2026-11-02
1176 #754, item 4): `choose` over the base's readings, under `strict`
1177 only the chosen files', mul.toml's and native-script ones; a
1178 reading an
1179 unchosen file leaves off stays in the pool off, so a caller's add
1180 of its kind still turns it on, as it does with no choice."""
1181 pool = _base_pool(base)
1182 if not chosen:
1183 return pool
1184 ri = _index_module()
1185 readings = [r for rs in pool.values() for r in rs]
1186 if strict:
1187 readings = ri.strict_kept(readings, chosen)
1188 out: dict[str, list[Any]] = {}
1189 for r in ri.choose(readings, chosen):
1190 out.setdefault(r.text, []).append(r)
1191 for r in readings:
1192 if not r.on_by_default and r.tag not in chosen:
1193 out.setdefault(r.text, []).append(r)
1194 return MappingProxyType({t: tuple(rs) for t, rs in out.items()})
1197@functools.lru_cache(maxsize=128)
1198def _built(base: object, edits: tuple[_Edit, ...], chosen: frozenset[str],
1199 strict: bool) -> tuple[Pool, Mapping[str, frozenset[str]],
1200 SameAs, SameAsNamed, Familiar]:
1201 """A lexicon's readings by word and the sets they compile to, kept
1202 per base, log and choice: a compile costs milliseconds, paid once
1203 per configuration (decisions/vocabulary-model, 2026-11-01 #754).
1204 The choice filters the base's readings and the log applies after
1205 it, so a caller's reading survives any choice; one that clashes
1206 with a reading the choice turns on is refused, naming the edit
1207 (2026-11-02 #754, item 3)."""
1208 pool = dict(_chosen_pool(base, chosen, strict))
1209 touched = [(edit, _apply_edit(pool, edit)) for edit in edits]
1210 for w in dict.fromkeys(w for _, ws in touched for w in ws):
1211 try:
1212 _check_pool(pool, (w,))
1213 except ValueError as exc:
1214 if not chosen or _clashes_unchosen(base, edits):
1215 raise
1216 mine = " and ".join(_rendered(e for e, ws in touched if w in ws))
1217 raise ValueError(
1218 f"the lexicon's {mine} clashes with a reading the choice "
1219 f"turns on: {exc}") from None
1220 ri = _index_module()
1221 try:
1222 compiled = ri.compile_readings(
1223 r for rs in pool.values() for r in rs)
1224 except ri.CompileError as exc:
1225 raise ValueError(str(exc)) from None
1226 return (MappingProxyType(pool), *_compiled_of(compiled))
1229def _clashes_unchosen(base: object, edits: tuple[_Edit, ...]) -> bool:
1230 """Whether `edits` clash on `base` with no choice, so that a clash
1231 under one is the log's own rather than the choice's."""
1232 try:
1233 _built(base, edits, frozenset(), False)
1234 except ValueError:
1235 return True
1236 return False
1239def _compiled_for(base: object, edits: tuple[_Edit, ...],
1240 chosen: frozenset[str] = frozenset(),
1241 strict: bool = False) -> _Compiled:
1242 if not edits and base == "default":
1243 # under a choice, the shipped index compiled once per choice by
1244 # the reader every pack shares
1245 return (_choice_compiled(chosen, strict) if chosen
1246 else _default_compiled())
1247 if not edits and not chosen and base == "empty":
1248 return _NOTHING_COMPILED
1249 _, sets, same_as, named, familiar = _built(base, edits, chosen, strict)
1250 return sets, same_as, named, familiar
1253@functools.lru_cache(maxsize=32)
1254def _choice_compiled(chosen: frozenset[str], strict: bool) -> _Compiled:
1255 ri = _index_module()
1256 return _compiled_of(ri.compile_choice(chosen, strict))
1259class _ReadingFiles:
1260 """The files whose readings are on in one lexicon, by word: what
1261 ``Reading.languages`` names (decisions/vocabulary-model, 2026-11-02
1262 #754, sub-phase 4i-d). One per base, log and choice
1263 (`_reading_files`), held by each `Lexicon` it describes and never
1264 pickled; it hashes by identity, in C, so assemble's memo keyed by
1265 it costs no frame on a hit.
1267 The default answers from the reading index's rows
1268 (`_reading_index.languages_of`), and a choice with no log from the
1269 word's own rows under `choose`, which is exact because a choice's
1270 preference is per word; neither builds the lexicon's whole pool,
1271 which a log or a snapshot base has built already, at construction."""
1273 __slots__ = ("_base", "_edits", "_chosen", "_strict")
1275 def __init__(self, base: object, edits: tuple[_Edit, ...],
1276 chosen: frozenset[str], strict: bool) -> None:
1277 self._base, self._edits = base, edits
1278 self._chosen, self._strict = chosen, strict
1280 def pool(self) -> Pool:
1281 """The lexicon's readings by word (`Lexicon._pool`)."""
1282 if not self._edits:
1283 return _chosen_pool(self._base, self._chosen, self._strict)
1284 return _built(self._base, self._edits, self._chosen,
1285 self._strict)[0]
1287 def languages(self, text: str, kinds: frozenset[str]) -> tuple[str, ...]:
1288 """The tags of the files holding a reading of `text` (as the
1289 vocabulary stores it) of one of `kinds`, on in this lexicon,
1290 sorted; a caller's own reading names none."""
1291 ri = _index_module()
1292 readings: Iterable[Any]
1293 if self._edits or self._base not in ("default", "empty"):
1294 readings = self.pool().get(text, ())
1295 elif self._base == "empty":
1296 return ()
1297 elif not self._chosen:
1298 return cast("tuple[str, ...]", ri.languages_of(text, kinds))
1299 else:
1300 readings = ri.chosen_readings(ri.readings_of(text), self._chosen,
1301 self._strict)
1302 return tuple(sorted({r.tag for r in readings
1303 if r.tag and r.kind in kinds
1304 and r.on_by_default}))
1307@functools.lru_cache(maxsize=128)
1308def _reading_files(base: object, edits: tuple[_Edit, ...],
1309 chosen: frozenset[str], strict: bool) -> _ReadingFiles:
1310 # one per configuration, so equal lexicons share assemble's memo
1311 return _ReadingFiles(base, edits, chosen, strict)
1314def _check_choice(languages: object, strict: object) -> tuple[
1315 frozenset[str], frozenset[str]]:
1316 """A recorded choice validated as `parser_for` validates one: its
1317 languages in canonical case and the file tags they select, the
1318 selection raising for a tag no file serves (decisions/
1319 vocabulary-model, 2026-11-02 #754, item 3)."""
1320 if not isinstance(strict, bool):
1321 raise TypeError(f"_strict must be True or False, got {strict!r}")
1322 if (isinstance(languages, (str, bytes, bytearray, Mapping))
1323 or not isinstance(languages, Iterable)):
1324 raise TypeError(f"_languages must be a set of BCP 47 tags, e.g. "
1325 f"frozenset({{'pt'}}), got {languages!r}")
1326 tags = frozenset(cast("Iterable[object]", languages))
1327 for tag in tags:
1328 if not isinstance(tag, str):
1329 raise TypeError(f"_languages entries must be str BCP 47 tags, "
1330 f"got {tag!r}")
1331 if not tags:
1332 if strict:
1333 raise ValueError(
1334 "a strict choice keeps only the chosen languages' "
1335 "vocabulary and the shared words (mul.toml), and "
1336 "_languages chooses none; choose through "
1337 "parser_for(..., strict=True) with a pack that chooses one")
1338 return frozenset(), frozenset()
1339 canon = frozenset(_canonical(cast(str, t)) for t in tags)
1340 return canon, _index_module().selected(canon, strict=strict)
1343def _canonical(tag: str) -> str:
1344 """`tag` in BCP 47's canonical case: the language subtag lowercase,
1345 a four-letter script subtag title case, a two-letter region
1346 uppercase, every other subtag lowercase (RFC 5646, 2.1.1). Equal
1347 choices then compare and hash equal, whatever case they were
1348 written in."""
1349 first, *rest = tag.split("-")
1350 out = [first.lower()]
1351 for sub in rest:
1352 if len(sub) == 4 and sub.isalpha():
1353 out.append(sub.title())
1354 elif len(sub) == 2 and sub.isalpha():
1355 out.append(sub.upper())
1356 else:
1357 out.append(sub.lower())
1358 return "-".join(out)
1361#: The most tags a repr lists (a `Lexicon`'s choice, a `Locale`'s); a
1362#: longer choice is shown by its size, so the repr stays bounded however
1363#: many languages are chosen.
1364_REPR_TAGS = 3
1367def _shown_choice(languages: frozenset[str]) -> str:
1368 if len(languages) > _REPR_TAGS:
1369 return f"languages: {len(languages)}"
1370 return "languages " + "+".join(sorted(languages))
1373def _check_base(base: object) -> object:
1374 """`"default"`, `"empty"`, or a snapshot of index rows from a pickle
1375 (decisions/vocabulary-model, 2026-11-02 #754, items 3 and 5), which
1376 becomes `"default"` where it is the running release's index."""
1377 if base in ("default", "empty"):
1378 return base
1379 if not isinstance(base, tuple):
1380 raise TypeError(f"a Lexicon's base is 'default', 'empty' or a "
1381 f"snapshot of index rows, got {base!r}")
1382 ri = _index_module()
1383 from nameparser.config._index import READINGS
1384 if base == READINGS:
1385 return "default"
1386 shape = ("a Lexicon snapshot is (word, rows) pairs, each row (tag, "
1387 "kind, properties), the properties (name, value) pairs")
1388 for entry in base:
1389 if not (isinstance(entry, tuple) and len(entry) == 2
1390 and isinstance(entry[0], str)
1391 and isinstance(entry[1], tuple)):
1392 raise TypeError(f"{shape}; got {entry!r}")
1393 word, rows = entry
1394 for row in rows:
1395 if not (isinstance(row, tuple) and len(row) == 3):
1396 raise TypeError(f"{shape}; {word!r} has the row {row!r}")
1397 tag, kind, pairs = row
1398 if not isinstance(tag, str) or not isinstance(kind, str):
1399 raise TypeError(f"{shape}, the tag and the kind strings; "
1400 f"{word!r} has the row {row!r}")
1401 if kind not in ri.CATEGORY:
1402 raise ValueError(
1403 f"a Lexicon snapshot holds the reading kind {kind!r}, "
1404 f"which this release does not know")
1405 if not isinstance(pairs, tuple) or not all(
1406 isinstance(p, tuple) and len(p) == 2
1407 and isinstance(p[0], str) for p in pairs):
1408 raise TypeError(f"{shape}; {word!r} has the row {row!r}")
1409 for name, value in pairs:
1410 if name not in ri.indexed(kind):
1411 raise ValueError(
1412 f"a Lexicon snapshot gives {word!r}'s {kind} reading "
1413 f"the property {name!r}, which a {kind} reading "
1414 f"does not take in this release")
1415 try:
1416 hash(value)
1417 except TypeError:
1418 raise TypeError(
1419 f"a Lexicon snapshot gives {word!r}'s {kind} reading "
1420 f"{name}={value!r}; a value is a string, a bool or "
1421 f"a tuple") from None
1422 return base
1425@functools.cache
1426def _choice_masks(chosen: frozenset[str], strict: bool = False) -> tuple[
1427 frozenset[str], tuple[tuple[str, str], ...]]:
1428 """The case masks the choice of the file tags `chosen` moves against
1429 the default's compile: the ones that leave and the ones set. A mask
1430 is no reading (a `Lexicon` keeps its masks in a field of their own),
1431 so `parser_for` moves them in that field, as 3b's delta did."""
1432 ri = _index_module()
1433 old = ri.compile_choice(frozenset())["CAPITALIZATION_EXCEPTIONS"]
1434 new = ri.compile_choice(chosen, strict)["CAPITALIZATION_EXCEPTIONS"]
1435 return (frozenset(old.keys() - new.keys()),
1436 tuple(sorted((k, v) for k, v in new.items() if old.get(k) != v)))
1439def _rechosen_masks(lexicon: Lexicon, chosen: frozenset[str],
1440 strict: bool) -> tuple[tuple[str, str], ...]:
1441 """`lexicon`'s masks moved from its own choice's to `chosen`'s: the
1442 base's choice's moves undone, then the new choice's applied, so a
1443 chained choice moves them as the union does."""
1444 cap = dict(lexicon.capitalization_exceptions)
1445 if lexicon.languages:
1446 default = _index_module().compile_choice(
1447 frozenset())["CAPITALIZATION_EXCEPTIONS"]
1448 gone, joined = _choice_masks(lexicon._chosen, lexicon.strict)
1449 for k in gone:
1450 cap.setdefault(k, default[k])
1451 for k, v in joined:
1452 if cap.get(k) == v:
1453 if k in default:
1454 cap[k] = default[k]
1455 else:
1456 del cap[k]
1457 gone, joined = _choice_masks(chosen, strict)
1458 for k in gone:
1459 cap.pop(k, None)
1460 cap.update(joined)
1461 return tuple(sorted(cap.items()))
1464def _fields_of(sets: _LexiconSets | Mapping[str, frozenset[str]],
1465 ) -> dict[str, frozenset[str]]:
1466 get = (sets.__getitem__ if isinstance(sets, Mapping)
1467 else functools.partial(getattr, sets))
1468 return {f: get(f) for f in _SET_FIELDS + _KIND_FIELDS}
1471def _rendered(edits: Iterable[_Edit]) -> list[str]:
1472 """Edits as the calls that make them, consecutive ones of one kind
1473 and properties joined into one call."""
1474 calls: list[tuple[str, list[str], str | None, _Props]] = []
1475 for e in edits:
1476 # a remove of a whole kind (no words) joins no call of words
1477 if (calls and e.words and calls[-1][1]
1478 and calls[-1][0] == e.op
1479 and calls[-1][2:] == (e.kind, e.props)):
1480 calls[-1][1].extend(e.words)
1481 else:
1482 calls.append((e.op, list(e.words), e.kind, e.props))
1483 out = []
1484 for op, words, kind, props in calls:
1485 args = [repr(w) for w in words]
1486 if kind is not None:
1487 args.append(f"kind={kind!r}")
1488 args += [f"{k}={_shown_value(kind, k, v)}" for k, v in props]
1489 if op == "remove" and not words:
1490 args.append("every=True")
1491 out.append(f".{op}({', '.join(args)})")
1492 return out
1495def _shown_value(kind: str | None, prop: str, value: object) -> str:
1496 """A logged property value as a caller writes it: a removed one as
1497 its unwritten value where it has one (`may_be_initial=False`)."""
1498 if value is None and kind is not None and prop in (
1499 "abbreviation", "acronym", "addresses", "after_given",
1500 "may_be_initial", "title_case_is_name", "shared",
1501 "after_particle", "capital_opening", "dotted"):
1502 value = _unwritten(kind, prop)
1503 if isinstance(value, tuple) and len(value) > _SHOWN_WORDS:
1504 # bounded: a reading's list may name every title word it
1505 # reaches
1506 return (repr(list(value[:_SHOWN_WORDS]))[:-1]
1507 + f", ... {len(value) - _SHOWN_WORDS} more]")
1508 return repr(list(value) if isinstance(value, tuple) else value)
1511#: The most words a clash message shows of one property's list.
1512_SHOWN_WORDS = 6
1515def _refusal(where: str, removed: Mapping[str, object]) -> TypeError:
1516 """The TypeError for a 2.x spelling: a list or the masks with the
1517 call that changes them, a set field with docs/migrate.rst's table,
1518 which names the kind of each (decisions/vocabulary-model,
1519 2026-11-08 #1003)."""
1520 unknown = [k for k in removed
1521 if k not in _SET_FIELDS + _LIST_FIELDS
1522 and k != "capitalization_exceptions"]
1523 if unknown:
1524 return TypeError(f"{where} got an unexpected keyword argument "
1525 f"{unknown[0]!r}")
1526 if "capitalization_exceptions" in removed:
1527 return TypeError(
1528 "capitalization_exceptions holds key->value pairs; use "
1529 "dataclasses.replace(lexicon, capitalization_exceptions=(*lexicon."
1530 "capitalization_exceptions, ('key', 'KEY'))) instead of "
1531 f"{where}")
1532 lists = [k for k in removed if k in _LIST_FIELDS]
1533 if lists:
1534 name = lists[0]
1535 given = removed[name]
1536 words = (sorted(given) if isinstance(given, Iterable)
1537 and not isinstance(given, (str, bytes, Mapping))
1538 and all(isinstance(w, str) for w in given) and given
1539 else ["word"])
1540 shown = "{" + ", ".join(map(repr, words)) + "}"
1541 code = (f"dataclasses.replace(lexicon, {name}=lexicon.{name} - "
1542 f"{shown})" if where.startswith("remove")
1543 else f"lexicon | Lexicon({name}={shown})")
1544 return TypeError(
1545 f"{name} is a list, changed with a fragment since 3.0, not "
1546 f"{where}: write {code}")
1547 return TypeError(
1548 f"{where} takes no {', '.join(removed)} since 3.0: a Lexicon is "
1549 f"configured by reading, add(*words, kind=..., **properties) and "
1550 f"remove(*words, kind=...); docs/migrate.rst's Lexicon table "
1551 f"gives the kind for each 2.x field")
1554#: The keys of the one 2.x `Lexicon` layout 3.0 loaded, 2.7's and
1555#: 2.8's, read off a 2.8.0 `Lexicon.default()` state; 3.0 admitted
1556#: `given_names` beside them. A record of what shipped, so it never
1557#: follows `_VOCAB_FIELDS`, which a new vocabulary field joins.
1558_LAYOUT_3_0_LOADED = frozenset({
1559 "after_given_titles", "agnomens", "bound_given_names",
1560 "capitalization_exceptions", "conjunctions", "conjunctions_ambiguous",
1561 "given_name_abbreviations", "given_name_titles", "honorific_tails",
1562 "maiden_markers", "particles", "particles_ambiguous", "suffix_acronyms",
1563 "suffix_acronyms_ambiguous", "suffix_words", "suffix_words_ambiguous",
1564 "surnames", "title_modifiers", "title_phrases", "title_styles",
1565 "titles", "titles_ambiguous"})
1568def _refused_2x(state: Mapping[str, object]) -> ValueError:
1569 """The ValueError for a 2.x `Lexicon` state, its remedy told by the
1570 state's layout, since a caller holding an old pickle seldom knows
1571 which release wrote it (decisions/vocabulary-model, 2026-11-08
1572 #1003). It is raised wherever the `Lexicon` is, so a `Parser` or
1573 locale pack holding one is named too. Any other layout is named by
1574 the keys it lacks and holds beyond that one, as 3.0 named them."""
1575 held = "the Lexicon, or the Parser or locale pack holding it"
1576 missing = sorted(_LAYOUT_3_0_LOADED - set(state))
1577 unexpected = sorted(set(state) - _LAYOUT_3_0_LOADED - {"given_names"})
1578 if not missing and not unexpected:
1579 return ValueError(
1580 f"this Lexicon was pickled by nameparser 2.7 or 2.8, which "
1581 f"3.1 no longer loads: load the pickle under 3.0 and pickle "
1582 f"the loaded object ({held}) again; if 3.0 refuses the "
1583 f"pickle too, build the object again")
1584 return ValueError(
1585 f"this Lexicon was pickled by nameparser 2.x in a layout no 3.x "
1586 f"release loads (missing: {', '.join(missing) or 'none'}; "
1587 f"unexpected: {', '.join(unexpected) or 'none'}), as 2.6 and "
1588 f"earlier wrote: build the object ({held}) again")
1591# -- the public types ----------------------------------------------------
1593@dataclass(frozen=True, slots=True)
1594class LexiconReading:
1595 """One reading a :class:`Lexicon` holds for a word, as
1596 :meth:`Lexicon.readings_of` returns it: what the word may be read
1597 as, in the vocabulary files' terms (decisions/vocabulary-model,
1598 2026-11-02 #754, item 2).
1600 ``kind`` is a vocabulary kind as the language files write it
1601 (``"office"``, ``"credential"``, ``"preposition"``, ``"given"``),
1602 finer than :class:`ReadingKind`, which is the parse's; ``category``
1603 its category (``"title"``, ``"suffix"``, ``"particle"``,
1604 ``"conjunction"``, ``"bound_given"``, ``"relation"``, ``"name"``);
1605 ``properties`` its behavioral properties as sorted ``(name, value)``
1606 pairs, spelled as the files spell them (``("addresses", "given")``),
1607 a list value a sorted tuple; ``languages`` the BCP 47 tags of the
1608 shipped files holding it, sorted, empty for a reading a caller
1609 added. Whether the parse READ a word this way is the parse's
1610 question, :attr:`Token.reading`: a title reading beside a ``given``
1611 one is a title only inside a title phrase (rules/H9). Added in
1612 3.0."""
1614 text: str
1615 kind: str
1616 category: str
1617 properties: tuple[tuple[str, Any], ...] = ()
1618 languages: tuple[str, ...] = ()
1620 def __post_init__(self) -> None:
1621 ri = _index_module()
1622 for name in ("text", "kind", "category"):
1623 if not isinstance(getattr(self, name), str):
1624 raise TypeError(f"LexiconReading.{name} must be a string, "
1625 f"got {getattr(self, name)!r}")
1626 if self.kind not in ri.CATEGORY:
1627 raise ValueError(f"unknown reading kind {self.kind!r}; the "
1628 f"kinds are {', '.join(ri.CATEGORY)}")
1629 if self.category != ri.CATEGORY[self.kind]:
1630 raise ValueError(
1631 f"a {self.kind} reading's category is "
1632 f"{ri.CATEGORY[self.kind]!r}, got {self.category!r}")
1633 for name in ("properties", "languages"):
1634 if not isinstance(getattr(self, name), tuple):
1635 raise TypeError(f"LexiconReading.{name} must be a tuple, "
1636 f"got {getattr(self, name)!r}")
1638 def __getstate__(self) -> dict[str, object]:
1639 return {f.name: getattr(self, f.name)
1640 for f in dataclasses.fields(self)}
1642 def __setstate__(self, state: dict[str, object]) -> None:
1643 # the layout check the other frozen types share
1644 # (_types._guarded_setstate), held here by layering
1645 expected = {f.name for f in dataclasses.fields(LexiconReading)}
1646 if set(state) != expected:
1647 missing = ", ".join(sorted(expected - set(state))) or "none"
1648 unexpected = ", ".join(sorted(set(state) - expected)) or "none"
1649 raise ValueError(
1650 f"incompatible LexiconReading pickle: missing fields: "
1651 f"{missing}; unexpected fields: {unexpected}")
1652 for name, value in state.items():
1653 object.__setattr__(self, name, value)
1656@dataclass(frozen=True, slots=True, init=False, repr=False)
1657class Lexicon:
1658 """The vocabulary a parser matches against: which words may be read
1659 as titles, particles, suffixes, and so on. Immutable and hashable.
1660 Start from :meth:`default` (the shipped vocabulary) or
1661 :meth:`empty`, configure it by READING -- :meth:`add` and
1662 :meth:`remove` take words and a vocabulary kind as the language
1663 files write it, ``add("leed", "ap", kind="credential")`` -- join
1664 fragments with ``|``, and pass the result to ``Parser(lexicon=...)``.
1665 Ask what a word may be read as with :meth:`readings_of`.
1667 Words are normalized as they are stored -- lowercased, NFC-composed,
1668 edge full stops stripped -- so matching is case-insensitive.
1670 Since 3.0 the sets a parse reads are compiled from the readings and
1671 are internal; the 2.x set fields (``titles``, ``suffix_acronyms``,
1672 ...) are refused wherever they were written, each refusal naming
1673 the field and pointing at docs/migrate.rst's table of the kind each
1674 field's words take. Three fields stay, being outside the per-word model:
1675 ``surnames``, ``given_names`` and ``capitalization_exceptions``.
1677 State is a base (the shipped vocabulary, none, or a pickled
1678 release's), the choice of languages :func:`~nameparser.parser_for`
1679 records (:attr:`languages`, :attr:`strict`), the ordered log of
1680 :meth:`add` and :meth:`remove` calls applied after the choice, and
1681 the three fields; two lexicons are equal when all of them are, the log compared call by call as written, so two
1682 logs that read alike can still differ (decisions/vocabulary-model,
1683 2026-11-02 #754, item 3)."""
1685 #: Family names for the unspaced-name segmentation stage (#271),
1686 #: matched longest-first against the start of the FIRST token
1687 #: written wholly in a script :attr:`Policy.segment_scripts
1688 #: <nameparser.Policy.segment_scripts>` activates. The default
1689 #: carries the Korean census list
1690 #: (``nameparser/config/_ko_surnames.py``); Chinese
1691 #: surnames ship in locales.ZH because Han segmentation is opt-in.
1692 #: A list, outside the per-word model: add to it with a fragment,
1693 #: ``lexicon | Lexicon(surnames={...})``.
1694 surnames: frozenset[str] = frozenset()
1695 #: Given names read by two-family-name handling alone
1696 #: (:attr:`Policy.two_family_names
1697 #: <nameparser.Policy.two_family_names>`, rules/O6): the word that
1698 #: rule would take into the family stays where it is when listed, so
1699 #: "José Luis Pérez" keeps family "Pérez". A word matches as
1700 #: written or with its acute and grave accents and diaeresis
1701 #: dropped (the tilde on n and the cedilla on c kept), so an entry
1702 #: written "jose" matches "José" too. No other rule reads it. Empty
1703 #: by default, so the switch reads as without it; a caller passes
1704 #: a list of its own, and ``locales.ES`` carries a Spanish list, with
1705 #: the switch on. Added in 3.0.
1706 given_names: frozenset[str] = frozenset()
1707 #: Lowercase word -> case mask used by capitalized(): the word's
1708 #: own letters and digits, each in the case it takes ("phd" ->
1709 #: "PhD"), laid over the word as written, so the one entry
1710 #: repairs "ph.d." to "Ph.D.". The mask's own punctuation marks
1711 #: where its letters are joined into one run versus split apart,
1712 #: and is never written into the word -- repair keeps the writer's
1713 #: own punctuation. A value that does not spell the key's letters
1714 #: and digits raises ValueError. Pair-valued: change it with
1715 #: dataclasses.replace(); read it as a mapping via
1716 #: capitalization_exceptions_map; the default's are
1717 #: ``Lexicon.default().capitalization_exceptions``.
1718 # Canonical storage: sorted tuple of (key, value) pairs. The
1719 # constructor tolerates any Mapping (or pair iterable) at runtime and
1720 # canonicalizes here; this closes the caller-aliasing hole and keeps
1721 # Lexicon hashable. Read via capitalization_exceptions_map.
1722 capitalization_exceptions: tuple[tuple[str, str], ...] = ()
1723 # The base, the log and the choice: constructor parameters under
1724 # underscore names, so that dataclasses.replace and copy.replace
1725 # carry them, and no public API (decisions/vocabulary-model,
1726 # 2026-11-02 #754, item 3); a choice is set publicly by parser_for
1727 # alone, and read through `languages` and `strict`.
1728 _base: Any = "empty"
1729 _edits: tuple[_Edit, ...] = ()
1730 # marked added: a 4i-b pickle, from no release, holds no choice and
1731 # loads choosing nothing, its log already holding what a choice of
1732 # 3b's delta had written into it
1733 _languages: frozenset[str] = field(
1734 default=frozenset(), metadata={_ADDED_IN: "3.0.0"})
1735 _strict: bool = field(default=False, metadata={_ADDED_IN: "3.0.0"})
1736 # the file tags the choice selects, derived from it
1737 _chosen: frozenset[str] = field(
1738 init=False, repr=False, compare=False, hash=False)
1739 _cap_map: Mapping[str, str] = field(
1740 init=False, repr=False, compare=False, hash=False)
1741 _sets: _LexiconSets = field(
1742 init=False, repr=False, compare=False, hash=False)
1743 # group -> word -> the words it is a listed spelling of (`given`:
1744 # `wm` -> {`william`}), from every reading's `same_as` in the group
1745 # of its kind; built at construction, never pickled, like `_cap_map`
1746 _same_as: SameAs = field(
1747 init=False, repr=False, compare=False, hash=False)
1748 # group -> the words with a link there that are a name of their own
1749 # too (`jo` among given names), whose links need the period
1750 # (`_headwords`); built like `_same_as`
1751 _same_as_named: SameAsNamed = field(
1752 init=False, repr=False, compare=False, hash=False)
1753 # a given name -> every name it is paired with as a familiar form,
1754 # either way (`max` -> {`maksim`, `maxim`, `maximilian`},
1755 # `maximilian` -> {`max`}), from every `familiar` reading's
1756 # `short_for`, read by the name match alone (`_familiar_forms`);
1757 # built like `_same_as`
1758 _familiar: Familiar = field(
1759 init=False, repr=False, compare=False, hash=False)
1760 # what assemble keys Reading.languages by: one per base, log and
1761 # choice (_ReadingFiles)
1762 _files: _ReadingFiles = field(
1763 init=False, repr=False, compare=False, hash=False)
1765 def __init__(
1766 self, *,
1767 surnames: Iterable[str] = frozenset(),
1768 given_names: Iterable[str] = frozenset(),
1769 capitalization_exceptions: (Mapping[str, str]
1770 | Iterable[tuple[str, str]]) = (),
1771 _base: object = "empty",
1772 _edits: Iterable[_Edit] = (),
1773 _languages: Iterable[str] = frozenset(),
1774 _strict: bool = False,
1775 **removed: object,
1776 ) -> None:
1777 put = functools.partial(object.__setattr__, self)
1778 put("surnames", _normset(surnames, "surnames"))
1779 put("given_names", _normset(given_names, "given_names"))
1780 canonical = _normpairs(capitalization_exceptions)
1781 put("capitalization_exceptions", canonical)
1782 put("_cap_map", MappingProxyType(dict(canonical)))
1783 put("_base", _check_base(_base))
1784 if isinstance(_edits, (str, bytes)) or not isinstance(
1785 _edits, Iterable):
1786 raise TypeError(f"_edits must be a tuple of edits, got "
1787 f"{_edits!r}")
1788 edits = tuple(_edits)
1789 for e in edits:
1790 if not isinstance(e, tuple) or len(e) != 4:
1791 raise TypeError(f"an edit is (op, words, kind, props), "
1792 f"got {e!r}")
1793 canon = tuple(_make_edit(*e) for e in edits)
1794 if canon != edits:
1795 raise ValueError(
1796 "_edits holds an edit not in canonical form; build a "
1797 "Lexicon's log with add() and remove()")
1798 put("_edits", canon)
1799 languages, chosen = _check_choice(_languages, _strict)
1800 put("_languages", languages)
1801 put("_strict", _strict)
1802 put("_chosen", chosen)
1803 compiled, same_as, named, familiar = _compiled_for(
1804 self._base, canon, chosen, _strict)
1805 put("_files", _reading_files(self._base, canon, chosen, _strict))
1806 put("_same_as", same_as)
1807 put("_same_as_named", named)
1808 put("_familiar", familiar)
1809 put("_sets", _LexiconSets(
1810 **compiled, surnames=self.surnames, given_names=self.given_names))
1811 if removed:
1812 raise _refusal("Lexicon() (and dataclasses.replace)", removed)
1814 # -- constructors ----------------------------------------------------
1816 @classmethod
1817 def empty(cls) -> Lexicon:
1818 """A lexicon holding no readings, no surnames, no given names
1819 and no case masks, equal to ``Lexicon()``: the start for a
1820 fragment to join onto another with ``|``, or for a vocabulary
1821 built from nothing. A parser over it still reads by shape and
1822 position (``Dr. John Smith`` keeps title ``Dr.``, a period-marked
1823 word opening a name), and, the default :class:`Policy
1824 <nameparser.Policy>` dividing unspaced Hangul, warns at
1825 construction that no surnames cover it."""
1826 return cls()
1828 @classmethod
1829 def default(cls) -> Lexicon:
1830 """The shipped vocabulary, the lexicon ``Parser()`` uses when
1831 given none: every reading the language files turn on by
1832 default, the Korean census surnames and the case masks, with no
1833 :attr:`given_names` and no choice of languages. Cached, so
1834 every call returns the same object, which being immutable is
1835 safe to share; configure a copy with :meth:`add`,
1836 :meth:`remove` or ``|``."""
1837 return _default_lexicon()
1839 # -- dunders ----------------------------------------------------------
1841 # hidden from a type checker, which would otherwise read any
1842 # attribute as defined and stop flagging `lexicon.titles` itself
1843 if not TYPE_CHECKING:
1844 def __getattr__(self, name: str) -> object:
1845 if name in _SET_FIELDS:
1846 raise AttributeError(
1847 f"Lexicon.{name} is internal since 3.0, compiled from "
1848 f"the lexicon's readings: ask lexicon.readings_of(word) "
1849 f"what a word may be read as, and a parse's "
1850 f"Token.reading what it was read as")
1851 raise AttributeError(
1852 f"{type(self).__name__!r} object has no attribute {name!r}")
1854 def __repr__(self) -> str:
1855 # Bounded: counts readings by kind against the nearer of the two
1856 # named lexicons, never naming a word (the design rule in
1857 # nameparser._types' module docstring)
1858 # A choice is shown by its tags, and the readings counted against
1859 # the named lexicon under the same choice
1860 choice = ""
1861 named = [("default", Lexicon.default()), ("empty", Lexicon.empty())]
1862 if self._languages:
1863 choice = ", " + _shown_choice(self._languages) + (
1864 ", strict" if self._strict else "")
1865 named = [(label, dataclasses.replace(
1866 other, _languages=self._languages, _strict=self._strict,
1867 capitalization_exceptions=_rechosen_masks(
1868 other, self._chosen, self._strict)))
1869 for label, other in named]
1870 for label, other in named:
1871 if self == other:
1872 return f"Lexicon({label}{choice})"
1873 candidates = [(label, self._deltas_from(other))
1874 for label, other in named]
1875 label, (kinds, lists) = min(
1876 candidates,
1877 key=lambda c: sum(a + r for _, a, r in c[1][0] + c[1][1]))
1879 def counts(deltas: list[tuple[str, int, int]]) -> str:
1880 return ", ".join(
1881 name + ": " + "".join(
1882 part for part, n in ((f"+{a}", a), (f"-{r}", r)) if n)
1883 for name, a, r in deltas)
1885 rendered = "; ".join(filter(None, (counts(kinds), counts(lists))))
1886 return (f"Lexicon({label} + {rendered or f'edits: {len(self._edits)}'}"
1887 f"{choice})")
1889 def __getstate__(self) -> dict[str, object]:
1890 # the base as a snapshot of its rows, so a pickle parses as it
1891 # did where it was written (decisions/vocabulary-model,
1892 # 2026-11-02 #754, item 5)
1893 base = self._base
1894 if base == "default":
1895 from nameparser.config._index import READINGS
1896 base = READINGS
1897 return {"surnames": self.surnames, "given_names": self.given_names,
1898 "capitalization_exceptions": self.capitalization_exceptions,
1899 "_base": base, "_edits": tuple(tuple(e) for e in self._edits),
1900 "_languages": self._languages, "_strict": self._strict}
1902 def __setstate__(self, state: dict[str, object]) -> None:
1903 # A 2.x state holds the compiled sets, refused since 3.1
1904 # (decisions/vocabulary-model, 2026-11-08 #1003); a 3.0 one is
1905 # checked for its layout as _types._guarded_setstate checks the
1906 # other frozen types', the one skew admitted a field marked
1907 # `_ADDED_IN` with a default (`_languages`, `_strict`), and then
1908 # built through the constructor, which validates every value as
1909 # for a caller's.
1910 if "titles" in state:
1911 raise _refused_2x(state)
1912 fields = [f for f in dataclasses.fields(Lexicon) if f.init]
1913 expected = {f.name for f in fields}
1914 added = {f.name: f.default for f in fields
1915 if _ADDED_IN in f.metadata
1916 and f.default is not dataclasses.MISSING}
1917 missing_set = expected - set(state) - set(added)
1918 if missing_set or not set(state) <= expected:
1919 missing = ", ".join(sorted(missing_set)) or "none"
1920 unexpected = ", ".join(sorted(set(state) - expected)) or "none"
1921 raise ValueError(
1922 f"incompatible Lexicon pickle: missing fields: {missing}; "
1923 f"unexpected fields: {unexpected}")
1924 defaulted = {name for name in added if name not in state}
1925 state = {**{name: added[name] for name in defaulted}, **state}
1926 Lexicon.__init__(self, **state) # type: ignore[misc, arg-type]
1927 # The constructor normalizes, and quietly accepting a rewritten
1928 # value would make this a place where caller data changes
1929 # without a word. A pickle this library wrote is always
1930 # normalized already (_normalize converges), so a difference
1931 # here means the state came from somewhere else -- say so while
1932 # the offending entries can still be named.
1933 drifted = []
1934 for name in _LIST_FIELDS:
1935 lost = frozenset(
1936 cast("Iterable[str]", state[name])) - getattr(self, name)
1937 if lost:
1938 drifted.append(f"{name}: {', '.join(sorted(lost))}")
1939 given_pairs = state["capitalization_exceptions"]
1940 if (isinstance(given_pairs, tuple)
1941 and given_pairs != self.capitalization_exceptions):
1942 drifted.append("capitalization_exceptions")
1943 faulty = [d for d in drifted if d.split(":")[0] in defaulted]
1944 if faulty:
1945 raise ValueError(
1946 "Lexicon field default is not normalized "
1947 f"({'; '.join(faulty)}): a field marked _ADDED_IN loads "
1948 "its default into every pickle written before it, so "
1949 "the default must be normalized entries; this is a "
1950 "defect in nameparser, not in the pickle")
1951 if drifted:
1952 raise ValueError(
1953 "incompatible Lexicon pickle: entries are not normalized "
1954 f"({'; '.join(drifted)}); this state was not written by "
1955 "this version of nameparser")
1957 def __or__(self, other: Lexicon) -> Lexicon:
1958 """The union of two lexicons: the lists unioned, the masks taken
1959 right-biased, and the logs concatenated, this one's first, so a
1960 later lexicon's edits apply after an earlier one's (decisions/
1961 vocabulary-model, 2026-11-02 #754, item 3); the base is the one
1962 that is not empty and the choice the one that is made, and two
1963 different bases, or two different choices, raise."""
1964 if not isinstance(other, Lexicon):
1965 return NotImplemented
1966 if (self._languages and other._languages
1967 and (self._languages, self._strict)
1968 != (other._languages, other._strict)):
1969 raise ValueError(
1970 "cannot join two lexicons compiled under different choices "
1971 "of languages or of strictness; combine choices with "
1972 "parser_for, which unions them: parser_for(*packs, "
1973 "base=parser)")
1974 mine = self if self._languages else other
1975 if "empty" in (self._base, other._base) or self._base == other._base:
1976 base = other._base if self._base == "empty" else self._base
1977 else:
1978 raise ValueError(
1979 "cannot join two lexicons built on different bases (the "
1980 "shipped vocabulary of two releases); rebuild one from "
1981 "Lexicon.default() or Lexicon.empty()")
1982 merged = dict(self._cap_map) | dict(other._cap_map)
1983 return Lexicon(
1984 surnames=self.surnames | other.surnames,
1985 given_names=self.given_names | other.given_names,
1986 capitalization_exceptions=tuple(sorted(merged.items())),
1987 _base=base, _edits=self._edits + other._edits,
1988 _languages=mine._languages, _strict=mine._strict)
1990 # -- properties -------------------------------------------------------
1992 @property
1993 def capitalization_exceptions_map(self) -> Mapping[str, str]:
1994 return self._cap_map
1996 @property
1997 def languages(self) -> frozenset[str]:
1998 """The languages this lexicon's readings are chosen under, as
1999 BCP 47 tags; empty where none was chosen. Set by
2000 :func:`~nameparser.parser_for`, which unions a base's choice
2001 with its packs' (decisions/vocabulary-model, 2026-11-02 #754,
2002 item 4): the shipped readings are filtered by the choice, and
2003 the lexicon's own :meth:`add` and :meth:`remove` calls apply
2004 after it. Added in 3.0."""
2005 return self._languages
2007 @property
2008 def strict(self) -> bool:
2009 """Whether the choice is strict: only the chosen languages'
2010 vocabulary files, mul.toml and native-script words' readings (as a
2011 plain choice of the same languages keeps them) are read. Added in 3.0."""
2012 return self._strict
2014 # -- access -----------------------------------------------------------
2016 def readings_of(self, word: str) -> tuple[LexiconReading, ...]:
2017 """The readings this lexicon holds for `word`: one per kind and
2018 properties, each with the shipped files that hold it, a reading
2019 a caller added naming none; empty for a word it holds no reading
2020 of. `word` is normalized as the lexicon stores it, a phrase
2021 (``"z domu"``) whole. "Is X a title here?" is ``any(r.category
2022 == "title" for r in lexicon.readings_of(x))``; whether a parse
2023 read it as one is :attr:`Token.reading`'s. Added in 3.0."""
2024 _reject_buffer(word, "readings_of()", "characters")
2025 if not isinstance(word, str):
2026 raise TypeError(f"readings_of() takes a word, a string, got "
2027 f"{word!r}")
2028 text = _title_key(word.split())
2029 ri = _index_module()
2030 grouped: dict[tuple[str, _Props], list[str]] = {}
2031 for r in self._pool().get(text, ()):
2032 if r.on_by_default:
2033 grouped.setdefault((r.kind, _clean(r.props)), []).append(
2034 r.tag)
2035 order = list(ri.CATEGORY)
2036 return tuple(
2037 LexiconReading(text, kind, ri.CATEGORY[kind], props,
2038 tuple(sorted(t for t in set(tags) if t)))
2039 for (kind, props), tags in sorted(
2040 grouped.items(),
2041 key=lambda item: (order.index(item[0][0]), repr(item[0][1]))))
2043 def words(self, *, kind: str, **properties: object) -> tuple[str, ...]:
2044 """The words holding a reading of `kind` on in this lexicon,
2045 sorted: a kind or a category, as :meth:`remove` takes it
2046 (``words(kind="title")`` lists every title word, ``dean``
2047 among them, whichever set its readings compile to), and
2048 `properties`, which a kind takes (``title``, ``suffix`` and
2049 ``particle`` are kinds as well as categories, and take their
2050 own kind's properties), narrowing to the readings that carry
2051 them as :meth:`add` spells them, each compared in its own
2052 reading's kind (so through a category an unwritten value means
2053 each kind's own: ``acronym=None`` under ``suffix`` lists the
2054 credentials, unwritten-true, and the suffix words, unwritten-
2055 false, alike)
2056 (``words(kind="title", addresses="given")``, the titles that
2057 address a person by the given name). A phrase the lexicon holds
2058 whole (``"z domu"``) is one entry. Added in 3.0."""
2059 k = _check_kind(kind, "words")
2060 assert k is not None
2061 ri = _index_module()
2062 if properties:
2063 if k not in ri.CATEGORY:
2064 raise ValueError(
2065 f"words() takes properties with a kind, not the "
2066 f"category {k!r}; name one of its kinds: " + ", ".join(
2067 x for x, c in ri.CATEGORY.items() if c == k))
2068 # checked against the kind named; compared below against
2069 # each reading's own kind, since a kind that is also a
2070 # category reaches kinds whose unwritten value differs
2071 # (`acronym` is unwritten-true on a credential, false on a
2072 # suffix word)
2073 _edit_props(k, properties)
2075 return tuple(sorted(
2076 w for w, rs in self._pool().items()
2077 if any(r.on_by_default and (r.kind == k or r.category == k)
2078 # a reading holds its properties canonical, as the
2079 # build and add write them, the unwritten value never
2080 and all(r.props.get(p) == _prop_value(r.kind, p, v)
2081 for p, v in properties.items())
2082 for r in rs)))
2084 # -- editing ----------------------------------------------------------
2086 # `kind` is required, and a type checker reads it so: the runtime
2087 # default exists only to catch a 2.x call, which names no kind, and
2088 # refuse it pointing at docs/migrate.rst's table of kinds rather than
2089 # with Python's own missing-argument error
2090 if TYPE_CHECKING:
2091 def add(self, *words: str, kind: str,
2092 **properties: object) -> Lexicon: ...
2093 else:
2094 def add(self, *words: str, kind: str | None = None,
2095 **properties: object) -> Lexicon:
2096 """This lexicon with a reading of `kind` added to each word.
2097 `kind` is required; the signature's default exists only so
2098 that a 2.x call, which names no kind, is refused pointing at
2099 docs/migrate.rst's table of kinds. It is a vocabulary kind as
2100 the language files write it, a key of
2101 ``nameparser._reading_index.CATEGORY`` (``"title"``,
2102 ``"office"``, ``"credential"``, ``"generation"``,
2103 ``"preposition"``, ``"conjunction"``, ``"bound_given"``,
2104 ``"birth_marker"``, ``"given"``, ``"family"``, ...), and
2105 `properties` that kind's behavioral properties as the files
2106 spell them (``addresses="given"``, ``after_given=True``,
2107 ``abbreviation=True``, ``attaches=[...]``, ``acronym=...``, ``attached="after"`` on a suffix word reading
2108 and ``attached="before"`` on a one-word title reading (rules/W2
2109 and W6),
2110 ``may_be_initial=True``, ``completed_by=[...]`` on a
2111 ``bound_given`` reading, ``after_particle=True`` on a
2112 ``given`` reading, ``shared=True`` on a name reading,
2113 ``capital_opening=True`` on a particle reading,
2114 ``dotted=True`` on a title reading, and
2115 ``same_as=...`` on a reading of any kind: the word this one
2116 is a spelling of, which name matching reads as a
2117 ``listed-spelling`` and keys alike in the roles that read
2118 the kind's words (a ``given`` reading's in the given name,
2119 middle name and nickname, a ``family`` reading's in the
2120 family and maiden names, a title's and a suffix's in theirs),
2121 and which on a ``given`` reading also marks a given name
2122 written short; written on a word, it applies to the word's
2123 given readings, so the word keeps no plain given reading
2124 and its link counts written bare, where a word the
2125 vocabulary also holds as a name of its own counts only
2126 written with its period), and ``short_for=[...]`` on a
2127 ``familiar`` reading, the longer given names a familiar
2128 form stands for, which name matching reads as a
2129 ``familiar-form`` where that outcome is accepted and no
2130 parse reads (``add("max", kind="familiar",
2131 short_for=["maxwell"])``), which EXTENDS the longer names
2132 the word's shipped familiar readings list, in a reading of
2133 the caller's own that names no file, rather than replacing
2134 them (unlike an add of any other kind the word holds); to
2135 replace them, ``remove(word, kind="familiar")`` first. A
2136 pair lives on the word whose reading lists the other (its
2137 shorter name, as the files write it): to cut
2138 ``Max`` / ``Maximilian``, remove ``max``'s familiar
2139 reading and add it back without ``maximilian``; a property
2140 given its unwritten value (``None``, ``False``) is removed. Where the word holds a reading of `kind`
2141 already, the add updates its properties in every file holding
2142 it; otherwise it adds the caller's own reading, which names no
2143 file. Warns where it makes a title reading that a ``given``
2144 reading keeps a title only inside a title phrase (rules/H9),
2145 naming the ``remove`` that makes it a title everywhere."""
2146 old = {k: properties.pop(k) for k in list(properties)
2147 if k in _SET_FIELDS + _LIST_FIELDS
2148 or k == "capitalization_exceptions"}
2149 if old:
2150 raise _refusal("add()", old)
2151 if kind is None:
2152 raise TypeError("add() missing required keyword argument "
2153 "'kind', e.g. add('leed', kind='credential')")
2154 edit = _make_edit("add", words, kind, properties)
2155 added = dataclasses.replace(self, _edits=self._edits + (edit,))
2156 ri = _index_module()
2157 if ri.CATEGORY[edit.kind] == "title": # type: ignore[index]
2158 before = self._pool()
2159 for w in edit.words:
2160 if (w in added._sets.titles_ambiguous
2161 and not _updates(before.get(w, ()), edit)):
2162 _warn_at_caller(
2163 f"{w!r} holds a given reading, so it reads as a "
2164 f"title only inside a title phrase (rules/H9) and "
2165 f"this add reads no name differently elsewhere; to "
2166 f"read it as a title everywhere, write "
2167 f".remove({w!r}, kind='given')")
2168 return added
2170 def remove(self, *words: str, kind: str | None = None,
2171 every: bool = False, **removed: object) -> Lexicon:
2172 """This lexicon without each word's readings of `kind`: a kind,
2173 or a category, matching every kind of it (``kind="title"`` takes
2174 out ``dean``'s ``office`` reading, ``kind="name"`` its ``given``
2175 and ``family`` ones), or None, taking out every reading, so the
2176 word leaves the vocabulary. With ``every=True``, no words and a
2177 `kind`, every word's readings of it go (``remove(kind="title",
2178 every=True)``), those a later choice of languages turns on
2179 included; a call with no words and no `every` raises, so an
2180 empty ``remove(*words, kind=...)`` never takes out a whole kind.
2181 Removing a reading the word does not hold changes nothing now,
2182 and is kept: a later choice of languages may turn that reading
2183 on."""
2184 if removed:
2185 raise _refusal("remove()", removed)
2186 if not isinstance(every, bool):
2187 raise TypeError(f"remove() takes every as True or False, got "
2188 f"{every!r}")
2189 if every and words:
2190 raise TypeError(
2191 f"remove() takes words or every=True, not both: every=True "
2192 f"takes out every word's readings of the kind; got "
2193 f"{len(words)} word(s)")
2194 if every and kind is None:
2195 raise TypeError(
2196 "remove(every=True) takes a kind, e.g. remove(kind='title', "
2197 "every=True); to remove everything, start from "
2198 "Lexicon.empty()")
2199 if not every and not words:
2200 raise TypeError(
2201 "remove() takes at least one word, e.g. remove('dean', "
2202 "kind='title'), or every=True with a kind to take out every "
2203 "word's readings of it, e.g. remove(kind='title', "
2204 "every=True)")
2205 edit = _make_edit("remove", words, kind, ())
2206 return dataclasses.replace(self, _edits=self._edits + (edit,))
2208 # -- private helpers --------------------------------------------------
2210 def _pool(self) -> Pool:
2211 return self._files.pool()
2213 def _headwords(self, word: str, group: str, *,
2214 period: bool = True) -> frozenset[str]:
2215 """Every word reachable from `word`, normalized, through the
2216 `same_as` links of `group` (`given`, `family`, `title` or
2217 `suffix`, the group of the readings' kinds), the word itself
2218 included: `Wm.` among given names is {`wm`, `william`}, and `sr`
2219 among titles {`sr`, `señor`, `senhor`}. A word's readings of
2220 another kind are another group's (`ma` is María written short
2221 among given names and only itself among surnames), and a word
2222 with no listed spelling is itself alone. A loop, which a
2223 caller's edits can write where the build refuses it in the
2224 files, yields every word in it. The name match reads two words
2225 as listed spellings of each other where their sets meet. `period`
2226 says whether the word was written with its trailing full stop,
2227 which normalizing drops: a word that is a name of its own in
2228 `group` as well as a spelling (`jo`, José written short and the
2229 given name Jo) reaches its links only where it was, and is itself
2230 alone where it was not."""
2231 links = self._same_as.get(group)
2232 if links is None:
2233 raise ValueError(f"unknown same_as group {group!r}; the groups "
2234 f"are {', '.join(sorted(_SAME_AS_GROUPS))}")
2235 start = _normalize(word)
2236 if not period and start in self._same_as_named[group]:
2237 return frozenset({start})
2238 seen = {start}
2239 todo = [start]
2240 while todo:
2241 for head in links.get(todo.pop(), ()):
2242 if head not in seen:
2243 seen.add(head)
2244 todo.append(head)
2245 return frozenset(seen)
2247 def _familiar_forms(self, word: str) -> frozenset[str]:
2248 """The given names `word`, normalized, is paired with directly
2249 as a familiar form, in either direction (`max` -> {`maksim`,
2250 `maxim`, `maximilian`}, `maximilian` -> {`max`}), empty for a
2251 word in no pair. Never chained: two longer names paired with
2252 one shorter one (`maximilian`, `maxim`) do not reach each
2253 other. Read by the name match alone, never by a parse."""
2254 return self._familiar.get(_normalize(word), frozenset())
2256 def _deltas_from(self, other: Lexicon) -> tuple[
2257 list[tuple[str, int, int]], list[tuple[str, int, int]]]:
2258 ri = _index_module()
2259 mine, theirs = _keyed(self._pool()), _keyed(other._pool())
2260 kinds = []
2261 for kind in ri.CATEGORY:
2262 a = sum(1 for key in mine - theirs if key[1] == kind)
2263 r = sum(1 for key in theirs - mine if key[1] == kind)
2264 if a or r:
2265 kinds.append((kind, a, r))
2266 lists = []
2267 for name in _LIST_FIELDS + ("capitalization_exceptions",):
2268 m, t = set(getattr(self, name)), set(getattr(other, name))
2269 if m != t:
2270 lists.append((name, len(m - t), len(t - m)))
2271 return kinds, lists
2274def _keyed(pool: Pool) -> set[tuple[str, str, _Props]]:
2275 """The readings on in `pool`, each once, as the repr counts them:
2276 by word, kind and properties, whatever file holds them."""
2277 return {(r.text, r.kind, _clean(r.props))
2278 for rs in pool.values() for r in rs if r.on_by_default}
2281def _updates(held: Iterable[Any], edit: _Edit) -> bool:
2282 """Whether `edit` only updates the properties of a reading of its
2283 kind the word holds: the title warning's exemption (decisions/
2284 vocabulary-model, 2026-11-01 #754, the title warning)."""
2285 return any(r.kind == edit.kind
2286 and _clean(_merged(r.props, edit.props)) != _clean(r.props)
2287 for r in held)
2290@functools.cache
2291def _default_lexicon() -> Lexicon:
2292 # nameparser/config/_compiled.py, compiled from nameparser/vocab/, is
2293 # the single source of the default vocabulary: its sets are read from
2294 # it, never compiled from the index here (_compiled_for), so
2295 # Lexicon.default() loads no index.
2296 from nameparser.config._compiled import CAPITALIZATION_EXCEPTIONS
2297 from nameparser.config._ko_surnames import KOREAN_SURNAMES
2299 # every vocabulary constant is a frozenset since #293, and this
2300 # cache reading them ONCE is the reason they are frozen.
2301 return Lexicon(
2302 surnames=KOREAN_SURNAMES,
2303 # pass canonical pair-tuples so this strictly-typed call site never
2304 # feeds a Mapping to the tuple-annotated field; __init__ still
2305 # tolerates a Mapping at runtime for interactive use
2306 capitalization_exceptions=tuple(sorted(CAPITALIZATION_EXCEPTIONS.items())),
2307 _base="default",
2308 )