Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Not a stage: converts the final ParseState into a public ParsedName.
3Consumes: tokens (all roles set, and their tags, which name each
4token's reading), dropped, order (for ParsedName.order, and for the
5field a report's alternative names under rules/P1's family-first fold)
6and structure (for ParsedName.order, and for rules/T7's family comma,
7which names a family where C15 reads none),
8ambiguities (by index;
9one carrying ``field_tail`` is worded from its word's final role,
10#626), segments, for the part a withdrawn connective report's
11letter ends (rules/P8, #721), and comma_offsets, for the comma part
12rules/R9's Turkic arm keeps a marker and its father's name in.
13Produces: a validated ParsedName, a given name written as a run of
14initials parted into them (rules/T7), each token carrying the reading the
15parse took (`Token.reading`), a word of the family's second unit
16marked `Reading.second_surname` where O6's or W5's tag says so
17(rules/R10) and `Reading.first_surname` where a first-surname tag
18does (rules/R11) --
19the constructor re-checks every
20invariant (span order/bounds, ambiguity subset), so a pipeline bug
21that would produce an invalid result dies HERE, not in a renderer
22three layers away.
23Reads: Policy.middle_as_family, folding a report's other field as the
24policy folds a middle, and with Policy.two_given_names keeping a given
25name written as a run of initials whole (rules/T7); Policy.name_order, for ParsedName.order where
26no positional read happened; Policy.patronymic_rules, for the words
27an ending rule reads as a patronymic (rules/R9); and Lexicon.surnames, suffix_acronyms,
28suffix_acronyms_ambiguous and given_name_abbreviations (the last
29through _vocab.is_title_shaped), for the source of a token's reading
30where no tag gives it; and the Lexicon's readings by word (Lexicon._files),
31for Reading.languages, the files whose reading is on in it, with
32every vocabulary field (`_lexicon._VOCAB_FIELDS`, `_listed_whole`)
33for whether the lexicon lists a word whole rather than by its parts;
34and every vocabulary field and Lexicon.capitalization_exceptions
35(`_listed_run`) for a run of initials listed whole (rules/T7).
37Structural tokens (dropped maiden markers) are omitted, like delimiter
38characters. A main-stream token that somehow reaches here with no role
39takes Role.GIVEN -- parse must never raise on the *content* of a name,
40so the fallback is deliberately boring rather than an exception. This
41should never fire in practice (assign/group must set a role on every
42main-stream token); it exists as a last-resort safety net against a
43pipeline bug, not as sanctioned behavior for real input -- see
44test_assemble_falls_back_to_given_for_unassigned_role in
45tests/v2/pipeline/test_assemble.py.
46"""
47from __future__ import annotations
49import functools
50import unicodedata
51from collections.abc import Sequence
52from types import MappingProxyType
54from nameparser._lexicon import (
55 _VOCAB_FIELDS, Lexicon, _normalize, _ReadingFiles,
56)
57from nameparser._pipeline._state import (
58 AGNOMEN_TAG, AMBIGUOUS_ACRONYM_TAG, AMBIGUOUS_WORD_TAG, ATTACHED_TAG,
59 CREDENTIAL_WORD_TAG, FIELD_READING, GENERATION_WORD_TAG, HONORIFIC_WORD_TAG,
60 NAME_ROLES, PATRONYMIC_TAG, SECOND_SURNAME_TAG, SHAPE_ACRONYM_TAG,
61 SHARED_NAME_TAG, PARTICLE_GIVEN_TAG, BOUND_PHRASE_TAG,
62 FIRST_SURNAME_TAG, SURNAME_PAIR_TAG,
63 SPLIT_CREDENTIAL_TAG, SPLIT_MEMBER_TAG, PAST_REACH_TAG,
64 CHAIN_TITLE_TAG, DOTTED_TITLE_TAG, JOINED_TITLE_TAG, TITLE_AMBIGUOUS_TAG, TRAILING_TITLE_TAG, Beside, Deferred, ParseState, PendingAmbiguity,
65 Structure, WorkToken, comma_bucket, reported_suffix_fork,
66)
67from nameparser._pipeline._vocab import (
68 _LETTER_RUN, _RUN_LETTER,
69 has_patronymic_ending, is_patronymic_marker, is_title_shaped,
70 name_source, suffix_kind,
71)
72from nameparser._policy import PatronymicRule
73from nameparser._reading_index import CATEGORY
74from nameparser._types import (
75 FOLDED_TAG, GIVEN_FIRST, UNJOINED_CONJUNCTION_TAG, UNJOINED_TAG,
76 Ambiguity, AmbiguityKind, ParsedName, Reading, ReadingKind,
77 ReadingSource, Role, Span, Token,
78)
80#: Every Reading a token can take, built once: a `Reading` is three
81#: enum members, so the parse hands out shared instances rather than
82#: constructing one per token, which would cost two frames a word
83#: (decisions/parse-cost).
84_READINGS = MappingProxyType({
85 (kind, role, source): Reading(kind, role, source)
86 for kind in ReadingKind for role in Role for source in ReadingSource})
88def _kinds_of(*categories: str) -> frozenset[str]:
89 return frozenset(k for k, c in CATEGORY.items() if c in categories)
92#: The vocabulary kinds (_reading_index.CATEGORY's keys) whose rows
93#: hold a reading of each kind. A suffix reading takes the word's
94#: suffix rows whatever their finer kind, since a word has one suffix
95#: kind (the build's check 4), which `_vocab.suffix_kind` has already
96#: read the reading's kind by (4b).
97#: A name reading's rows depend on its field (`_NAME_KINDS`). A
98#: patronymic's are the patronymic particle's (rules/R9); the words its
99#: run takes, and a patronymic an ending rule reads, are read by
100#: position or shape, as an initial is, never from a file.
101_SUFFIX_KINDS = _kinds_of("suffix")
102_KINDS: MappingProxyType[ReadingKind, frozenset[str]] = MappingProxyType({
103 ReadingKind.TITLE: _kinds_of("title"),
104 ReadingKind.PARTICLE: _kinds_of("particle"),
105 ReadingKind.CONJUNCTION: _kinds_of("conjunction"),
106 ReadingKind.PATRONYMIC: frozenset({"patronymic"}),
107 ReadingKind.CREDENTIAL: _SUFFIX_KINDS,
108 ReadingKind.GENERATION: _SUFFIX_KINDS,
109 ReadingKind.AGNOMEN: _SUFFIX_KINDS,
110 ReadingKind.POST_NOMINAL: _SUFFIX_KINDS,
111 ReadingKind.HONORIFIC: _SUFFIX_KINDS,
112})
114#: A name reading's rows by the field it lands in: a family word the
115#: files listing it as a family name, a given or middle word those
116#: listing it as a given name ('Ma' as a family name names `en`, not
117#: `fil`, which lists it only as given), and a bound given name
118#: (rules/P5) in any field, its rows being the reason it was read so;
119#: a name in another field (a nickname) any name rows.
120_GIVEN_KINDS = frozenset({"given", "bound_given"})
121_FAMILY_KINDS = frozenset({"family", "bound_given"})
122_NAME_KINDS: MappingProxyType[Role, frozenset[str]] = MappingProxyType({
123 Role.GIVEN: _GIVEN_KINDS, Role.MIDDLE: _GIVEN_KINDS,
124 Role.FAMILY: _FAMILY_KINDS, Role.MAIDEN: _FAMILY_KINDS,
125})
126_ANY_NAME_KINDS = _kinds_of("name", "bound_given")
128#: The sources a file can hold: a listed word, and a name the
129#: vocabulary declares beside another reading.
130_LISTED = frozenset({ReadingSource.VOCABULARY, ReadingSource.DECLARED})
133def _kinds(kind: ReadingKind, role: Role) -> frozenset[str]:
134 # an initial or a patronymic matches no row
135 return (_NAME_KINDS.get(role, _ANY_NAME_KINDS)
136 if kind is ReadingKind.NAME else _KINDS.get(kind, frozenset()))
139# The lookup is a memo, C-level on a hit, so a word read before costs
140# no frame (decisions/vocabulary-model, 2026-10-29; 2026-11-05 #1085,
141# which made the lookup one memo): a pure function of
142# the word and the lexicon's readings, which `files` stands for, one
143# object per base, log and choice hashed by identity
144# (decisions/vocabulary-model, 2026-11-02 #754, sub-phase 4i-d), and
145# bounded (maxsize) because the caller's text keys it; so is `_reading`,
146# of the readings that name files.
147@functools.lru_cache(maxsize=4096)
148def _languages_of(text: str, kind: ReadingKind, role: Role,
149 files: _ReadingFiles) -> tuple[tuple[str, ...], bool]:
150 """Reading.languages for the `kind` reading of `text` in `role`:
151 the files holding the word whole (`_languages`), else those holding
152 its parts (`_part_languages`); and whether they are its parts',
153 which the caller drops where the parse's own lexicon lists the word
154 whole (`_listed_whole`): that reads the lexicon's `surnames` and
155 `given_names` lists too, which `files` does not stand for, so the
156 memo cannot answer it."""
157 found = _languages(text, kind, role, files)
158 if found:
159 return found, False
160 found = _part_languages(text, kind, role, files)
161 return found, bool(found)
164#: `_languages_of`'s answer for a reading by position or shape
165#: (`source not in _LISTED`), which names no file.
166_UNLISTED: tuple[tuple[str, ...], bool] = ((), False)
169def _languages(text: str, kind: ReadingKind, role: Role,
170 files: _ReadingFiles) -> tuple[str, ...]:
171 """The files holding the `kind` reading of `text` as written, in
172 `role`, on in the parse's lexicon (`files`, its `_files`),
173 Reading.languages: by the vocabulary's own form of the
174 word, and then with every period removed, the form a credential is
175 matched by ('M.A' in 'John Smith (M.A)') and a word listed whole
176 without them ('Lt.Gen.' as 'ltgen')."""
177 kinds = _kinds(kind, role)
178 n = _normalize(text)
179 found = files.languages(n, kinds)
180 if not found and "." in n:
181 found = files.languages(n.replace(".", ""), kinds)
182 return found
185def _part_languages(text: str, kind: ReadingKind, role: Role,
186 files: _ReadingFiles) -> tuple[str, ...]:
187 """The files of the listed words `text` joins with punctuation
188 ('Lt.Gov.', 'MD/PhD', 'MD;'), sorted: a suffix's parts divided by
189 any mark but the period first, each an acronym ('M.D./Ph.D.'),
190 then any word's by every mark. Asked only where `_languages` finds
191 nothing (`_languages_of`); its answer is dropped where the parse's
192 own lexicon lists the word whole (`_listed_whole`)."""
193 kinds = _kinds(kind, role)
194 n = _normalize(text)
195 found = (_by_parts(n, kinds, ".", files) if kinds is _SUFFIX_KINDS
196 else ())
197 return found or _by_parts(n, kinds, "", files)
200def _by_parts(n: str, kinds: frozenset[str], keep: str,
201 files: _ReadingFiles) -> tuple[str, ...]:
202 """The files of the parts of `n`, sorted: divided by punctuation,
203 symbols and spaces, but for the marks in `keep`, which are removed
204 from each part instead; never by a combining mark, which a `[^\\w]`
205 would count ('डॉ' in 'प्रोफे.डॉ.')."""
206 parts = "".join(" " if c not in keep
207 and unicodedata.category(c)[0] in "PSZ" else c
208 for c in n).split()
209 return tuple(sorted({
210 tag for part in parts
211 for tag in files.languages(
212 part.replace(keep, "") if keep else part, kinds)}))
215@functools.lru_cache(maxsize=4096)
216def _forms(text: str) -> tuple[str, str]:
217 """`text` as the vocabulary stores it, and without its periods."""
218 n = _normalize(text)
219 return n, n.replace(".", "")
222def _listed_whole(text: str, lexicon: Lexicon) -> bool:
223 """Whether the parse's own lexicon lists `text` whole, in either
224 form, so that the parse read it whole and not by its parts: a
225 caller's 'zzq-dr' title names none of the files of 'dr'. Asked
226 only of a word whose whole form no file holds and whose parts some
227 file does (`_part_languages`), so other words pay no frame."""
228 n, bare = _forms(text)
229 sets = lexicon._sets
230 for name in _VOCAB_FIELDS:
231 field = getattr(sets, name)
232 if n in field or bare in field:
233 return True
234 return False
237def _next_word(tokens: Sequence[WorkToken], dropped: set[int], i: int,
238 step: int) -> int:
239 """The index of the nearest token past `i` in direction `step` (1
240 or -1) that is neither dropped nor a nickname: -1 or len(tokens)
241 where none is. Asked only on the rare shapes its callers name."""
242 j = i + step
243 while 0 <= j < len(tokens) and (j in dropped
244 or tokens[j].role is Role.NICKNAME):
245 j += step
246 return j
249def _split_partner(tokens: tuple[WorkToken, ...], dropped: set[int],
250 i: int) -> str:
251 """The text of the second word of the credential split across a
252 space that token `i` opens: the next word past a nickname written
253 between them ('Ph. "Bob" D.'), or nothing where that word is not
254 the joined suffix word that completes it. Asked only of a split
255 credential's first word, so other names pay nothing."""
256 # classify tags a split's words only as a pair, so its second word
257 # always follows; the bound test keeps parse total without a
258 # branch no input reaches
259 j = _next_word(tokens, dropped, i, 1)
260 return (tokens[j].text if j < len(tokens) and tokens[j].role is Role.SUFFIX
261 and "joined" in tokens[j].tags else "")
264_reading = functools.lru_cache(maxsize=1024)(Reading)
267#: Tags that serve the stages and reach no published token: classify's
268#: kind tags for a listed suffix word (rules/R8) and a patronymic
269#: particle, and P6's attachment mark (rules/R9), whose kind
270#: `Token.reading` carries out; and the written-short mark rules/H5's
271#: chain reads (#999), for which the published token keeps
272#: TITLE_AMBIGUOUS_TAG; and the shared-name and particle-given marks
273#: rules/O6 and O7 read (#704), the bound-phrase mark rules/P12
274#: reads (#694), the past-reach mark on a particle chain past rules/P1's
275#: fold (#828), and the joined-title mark (#1104) and the
276#: chain-title mark (#1171) rules/H5's chain reads.
277_KIND_TAGS = frozenset({GENERATION_WORD_TAG, CREDENTIAL_WORD_TAG, HONORIFIC_WORD_TAG,
278 PATRONYMIC_TAG, ATTACHED_TAG, TRAILING_TITLE_TAG,
279 SHARED_NAME_TAG, PARTICLE_GIVEN_TAG, PAST_REACH_TAG,
280 BOUND_PHRASE_TAG, JOINED_TITLE_TAG, CHAIN_TITLE_TAG,
281 SURNAME_PAIR_TAG})
283#: The second-surname mark O6 or W5 writes (rules/R10) and the
284#: first-unit mark O6, W5 or O3 writes (rules/R11), which no published
285#: token carries either:
286#: `Reading.second_surname` and `Reading.first_surname` name them.
287_SURNAME_UNITS = frozenset({SECOND_SURNAME_TAG, FIRST_SURNAME_TAG})
289#: The tags that say a list or a shape made a word a suffix: without
290#: one, a suffix word stands there by position, unless the vocabulary
291#: lists it (a bracketed member, which classify leaves untagged).
292_SUFFIX_EVIDENCE = frozenset({
293 "vocab:suffix", "vocab:suffix-word", AMBIGUOUS_ACRONYM_TAG,
294 AMBIGUOUS_WORD_TAG, AGNOMEN_TAG, SHAPE_ACRONYM_TAG,
295 SPLIT_CREDENTIAL_TAG, SPLIT_MEMBER_TAG})
297#: The tags that say the vocabulary lists a word as a title, in any of
298#: the title sets, a dotted run's spelling without its stops included
299#: (rules/H10).
300_TITLE_TAGS = frozenset({
301 "vocab:title", TITLE_AMBIGUOUS_TAG, "vocab:given-title",
302 "vocab:after-given-title", DOTTED_TITLE_TAG})
305def _alternative(pending: PendingAmbiguity, materialized: tuple[Token, ...],
306 final: dict[int, Token],
307 state: ParseState) -> Reading | None:
308 """The public Reading of a report's PendingReading, its field named
309 from the final roles where the emitter deferred it (O5's other
310 primary field) and folded as `Policy.middle_as_family` folds every middle
311 (rules/O6's neighbour, the policy's own fold), and its languages
312 the files holding it for any word the report names."""
313 alt = pending.alternative
314 if alt is None or not materialized:
315 return None
316 role = alt.role
317 if role is Deferred.OTHER_PRIMARY:
318 role = (Role.FAMILY if materialized[0].role is Role.GIVEN
319 else Role.GIVEN)
320 elif isinstance(role, Beside) and role.index in final:
321 beside = role
322 front = final[beside.index]
323 role = (beside.unfolded
324 if FOLDED_TAG in front.tags and beside.unfolded is not None
325 else front.role)
326 # a chain whose last word stayed the family when its head left
327 # it (rules/P10's middle name, which O7 may then take) lands the
328 # word there; any other split names the head's field, and so
329 # does a last word P6 attached to the family (FOLDED_TAG) (#704)
330 if (role is not Role.FAMILY and beside.last is not None
331 and beside.last != beside.index and beside.last in final
332 and final[beside.last].role is Role.FAMILY
333 and FOLDED_TAG not in final[beside.last].tags):
334 role = Role.FAMILY
335 # a chain opening a family-first name holding a name word of its
336 # own: rules/P1's fold takes its particles and that ONE unit
337 # (#828), so the word, joining it last, is laid out by the order
338 # behind the family -- the other primary where it would be the
339 # one word left, else the order's last name position -- unless
340 # another word past the fold's reach was read against the whole
341 # chain, which the fold then leaves whole (the one predicate the
342 # fold asks)
343 elif (role is Role.FAMILY and state.order is not None
344 and state.order[0] is Role.FAMILY and beside.last is not None
345 and beside.last in final
346 and "particle" in front.tags
347 and "particle" not in final[beside.last].tags
348 and not reported_suffix_fork(
349 state.ambiguities,
350 [i for i in range(beside.index, pending.indices[0])
351 if PAST_REACH_TAG in state.tokens[i].tags],
352 skip=pending)):
353 alone = (final[beside.last].role is Role.FAMILY
354 and not any(beside.last < i < pending.indices[0]
355 for i in final))
356 role = Role.GIVEN if alone else state.order[2]
357 elif isinstance(role, (Deferred, Beside)):
358 # assign found no name position to name it from
359 return None
360 # the files holding the reading for any of its words, as a token's
361 # own reading names them, by the field before the policy's fold of
362 # the middle (a middle word names the files listing it as given)
363 found: set[str] = set()
364 if alt.source in _LISTED:
365 # a loop, not a generator, which would resume a frame per word
366 for t in materialized:
367 langs, by_parts = _languages_of(t.text, alt.kind, role,
368 state.lexicon._files)
369 if not (by_parts and _listed_whole(t.text, state.lexicon)):
370 found.update(langs)
371 if state.policy.middle_as_family and role is Role.MIDDLE:
372 role = Role.FAMILY
373 return Reading(alt.kind, role, alt.source, tuple(sorted(found)))
376def _opens_its_segment(state: ParseState, i: int) -> bool:
377 """Whether token `i` is the first word of its segment that is not a
378 title: where rules/P5 joins a bound given name. Asked only of a
379 word that is both a bound given name and a particle, read into the
380 given part, so other names pay nothing."""
381 dropped = state.dropped
382 # every given-part token stands in a segment; the empty default
383 # keeps parse total all the same
384 seg = next((s for s in state.segments if i in s), ())
385 return next((j for j in seg if j not in dropped
386 and state.tokens[j].role is not Role.TITLE), None) == i
389#: The fields a Turkic marker and its father's name may stand in: the
390#: name's own and the maiden name, a name of the same shape.
391_PATRONYMIC_FIELDS = frozenset({*NAME_ROLES, Role.MAIDEN})
394# rules/R9: "a standalone patronymic marker, O2's, with the name word
395# before it in its comma part" -- the Turkic arm (decisions/R9, #754)
396def _turkic_patronymics(state: ParseState,
397 dropped: set[int]) -> dict[int, int]:
398 """The markers rules/R9's Turkic arm may read as a patronymic, each
399 with the father's name before it: a marker standing in a name field
400 or the maiden name, with the word before it, a nickname aside,
401 standing in one too, the maiden name's own where the marker stands
402 there, and in the marker's own comma part. The father is read once
403 its marker reads as a name word. Asked only where the policy turns
404 the rule on."""
405 markers: dict[int, int] = {}
406 tokens = state.tokens
407 commas = state.comma_offsets
408 for i, t in enumerate(tokens):
409 if (i in dropped or t.role not in _PATRONYMIC_FIELDS
410 or not is_patronymic_marker(t.text)):
411 continue
412 j = _next_word(tokens, dropped, i, -1)
413 if (j >= 0 and tokens[j].role in _PATRONYMIC_FIELDS
414 and (tokens[j].role is Role.MAIDEN) is (t.role is Role.MAIDEN)
415 and comma_bucket(tokens[j].span.start, commas)
416 == comma_bucket(t.span.start, commas)):
417 markers[i] = j
418 return markers
421def _patronymic_opening(state: ParseState, dropped: set[int], i: int,
422 field: Role, folds_middle: bool) -> tuple[bool, bool]:
423 """Whether the patronymic particle at `i`, read as a particle, names
424 a parent (rules/R9), and whether it names it as P6's attachment: a
425 particle P6 attached to the family written before it ('Mohamad,
426 Mahathir bin', 'Bakar, Ali bin Abu') names that family, the words
427 attached after it with it, where the family holds a word not
428 folded; any other names the words after it in
429 its field, a nickname aside, and one ending its field names nobody,
430 a family word spelled as one ('Bar' in 'Fritz von Bar'). Asked only
431 of a patronymic particle, so other names pay nothing."""
432 tokens = state.tokens
433 if ATTACHED_TAG in tokens[i].tags:
434 # the family it attaches to must hold a word of its own: behind
435 # an empty head P6 attaches it as the whole family (', Siti
436 # Aminah binti'), which O3's fold can give a base, and a
437 # particle with no family to name names nobody
438 # (a loop, not a generator, which would resume a frame per word)
439 for k, other in enumerate(tokens):
440 if (other.role is Role.FAMILY and FOLDED_TAG not in other.tags
441 and k not in dropped):
442 return True, True
443 return False, False
444 j = _next_word(tokens, dropped, i, 1)
445 return (j < len(tokens)
446 and _field_of(tokens[j], folds_middle) is field), False
449def _field_of(token: WorkToken, folds_middle: bool) -> Role:
450 """The field a token stands as: its role, save a middle word the
451 policy folded into the family, which stands as the middle; a word
452 P6 attached carries the folded mark too and stands as the family."""
453 role = token.role if token.role is not None else Role.GIVEN
454 return (Role.MIDDLE if role is Role.FAMILY and folds_middle
455 and FOLDED_TAG in token.tags and ATTACHED_TAG not in token.tags
456 else role)
459def _listed_suffix(text: str, lexicon: Lexicon) -> bool:
460 """Whether the suffix vocabulary lists, its periods removed, an
461 acronym no tag marks: a bracketed member written with an interior
462 period ('M.A' in 'John Smith (M.A)'), which rules/S13 settles by its
463 writing and classify leaves untagged. A listed suffix WORD is
464 always tagged, bracketed or not. Asked only of a suffix word with
465 no suffix tag, so other names pay nothing."""
466 n = _normalize(text).replace(".", "")
467 return n in lexicon._sets.suffix_acronyms or n in lexicon._sets.suffix_acronyms_ambiguous
470# rules/A2: "a name with no name content parses to the empty
471# name — every field empty, false as a boolean — while ambiguities
472# born from its punctuation survive on the empty result"
473# (history: decisions/A2)
474def assemble(state: ParseState) -> ParsedName:
475 dropped = set(state.dropped)
476 final: dict[int, Token] = {}
477 surnames = state.lexicon._sets.surnames
478 files = state.lexicon._files
479 folds_middle = state.policy.middle_as_family
480 tokens = state.tokens
481 last = len(tokens)
482 # the suffix reading of the token before, and whether it opens a
483 # credential split across a space, whose second word takes it
484 before: Reading | None = None
485 prev_split = False
486 # the field of the patronymic a patronymic particle opened, whose
487 # run takes the words after it there; and the ending rules' words,
488 # where a policy turns one on
489 run_field: Role | None = None
490 # a patronymic particle rules/P6 attached to the family written
491 # before it, whose run is that family's own words
492 attached = False
493 rules = state.policy.patronymic_rules
494 slavic = PatronymicRule.EAST_SLAVIC in rules
495 markers = (_turkic_patronymics(state, dropped)
496 if PatronymicRule.TURKIC in rules else {})
497 # each marker's father, held unbuilt until its marker is read: the
498 # word before it reads `patronymic` only where the marker reads as
499 # a name word, and building its Token twice costs frames
500 held: dict[int, tuple[str, Span | None, Role, frozenset[str], Reading]] = {}
501 fathers = frozenset(markers.values())
502 for i, t in enumerate(state.tokens):
503 if i in dropped:
504 continue
505 role = t.role if t.role is not None else Role.GIVEN
506 tags = t.tags
507 # rules/R10: "the family's second name unit, the one written
508 # second, makes the second-surname view" -- O6's mark, on family
509 # words only; the word is read below as it reads without it,
510 # and its reading then carries the mark (decisions/R10, #754)
511 second = SECOND_SURNAME_TAG in tags
512 # rules/R11: "the family's first name unit, the one written
513 # first, makes the first-surname view" -- the first-unit mark O6
514 # or O3 writes, read and carried out the same way (decisions/R11,
515 # #1022)
516 first = FIRST_SURNAME_TAG in tags
517 if second or first:
518 tags = tags - _SURNAME_UNITS
519 # The reading the parse took (Token.reading): the final role
520 # says which decision placed the word, and the tags the
521 # deciding stages left (classify's, the `joined` and unjoined
522 # marks) say which reading of it that decision weighed. Read
523 # here, where roles are final, so a rule that moves a word
524 # later (P1's fold, P6's attachment, the policy folds) needs
525 # no reading of its own to keep in step. Per token, measured
526 # 2026-10-29: no frame for an untagged non-suffix word, the
527 # commonest; one for a listed or shaped suffix (`suffix_kind`,
528 # reading the kind classify tagged) and for a tagged name word that reaches `name_source` or a
529 # family word glued to the next (`_normalize`); three for a
530 # suffix word nothing tags or lists (`suffix_kind`, then
531 # `_listed_suffix` and its `_normalize`). `_vocab`'s two predicates
532 # also name the sources of the reading NOT taken
533 # (Ambiguity.alternative).
534 # Reading.languages, the files holding the reading taken, on in
535 # the parse's lexicon, is read below for a listed or declared
536 # reading, through `_languages_of`, whose hit costs no frame,
537 # save `_listed_whole` for a word read by its parts
538 # (decisions/vocabulary-model, 2026-10-29; 2026-11-02, 4i-d;
539 # 2026-11-05 #1085).
540 reading: Reading | None = None
541 if role is Role.SUFFIX:
542 # a suffix ends a particle's run: the family can resume
543 # behind it ('Hasan' in 'bin Ahmad Jr. Hasan, Ali')
544 run_field = None
545 kind, source = suffix_kind(t.text, tags)
546 if (source is ReadingSource.VOCABULARY
547 and tags.isdisjoint(_SUFFIX_EVIDENCE)):
548 if "conjunction" in tags:
549 # a connective joining two entries ('and' in 'John
550 # Smith, PhD and MD'), as in a title or name part
551 kind = ReadingKind.CONJUNCTION
552 elif not _listed_suffix(t.text, state.lexicon):
553 # nothing lists or shapes it: a word read into the
554 # suffix by where it stands
555 source = ReadingSource.POSITION
556 if ("joined" in tags and before is not None
557 and prev_split):
558 # the second word of a credential split across
559 # a space, which is one credential ('Ph.' 'D.');
560 # any other word a run took stays a credential
561 # by position ('Jones' in 'John Smith III Jones'),
562 # files and all
563 reading = before
564 prev_split = (SPLIT_CREDENTIAL_TAG in tags
565 or SPLIT_MEMBER_TAG in tags)
566 if reading is None:
567 # a credential split across a space is listed whole
568 # ('Ph.' 'D.' as 'phd'): its first word reads the files
569 # of the two words, which the second then takes
570 written = (t.text + _split_partner(tokens, dropped, i)
571 if prev_split else t.text)
572 langs, by_parts = (_languages_of(written, kind, role, files)
573 if source in _LISTED else _UNLISTED)
574 if by_parts and _listed_whole(written, state.lexicon):
575 langs = ()
576 reading = (_reading(kind, role, source, langs) if langs
577 else _READINGS[kind, role, source])
578 before = reading
579 else:
580 if role is not Role.NICKNAME:
581 # a nickname written inside a split credential ('Ph.
582 # "Bob" D.') leaves the run to its second word
583 before = None
584 if role is Role.TITLE:
585 if "conjunction" in tags:
586 kind, source = (ReadingKind.CONJUNCTION,
587 ReadingSource.VOCABULARY)
588 elif not tags.isdisjoint(_TITLE_TAGS):
589 kind, source = ReadingKind.TITLE, ReadingSource.VOCABULARY
590 elif is_title_shaped(t.text, state.lexicon):
591 # rules/H2: "an abbreviation the vocabulary reads as
592 # a given name written short", "word in one of the
593 # ten Brahmic scripts from Devanagari to Sinhala" and
594 # "has no period abbreviations either" -- H2's shape,
595 # asked through its own predicate; a word it refuses
596 # is a title only by where it stands (decisions/A1,
597 # 2026-11-09)
598 kind, source = ReadingKind.TITLE, ReadingSource.SHAPE
599 else:
600 # a title phrase's connective object ('of
601 # Parliament'), a title by where it stands
602 kind, source = ReadingKind.TITLE, ReadingSource.POSITION
603 elif not tags:
604 kind, source = ReadingKind.NAME, ReadingSource.POSITION
605 # a surname the segmentation split off an unspaced
606 # name by the surname list ('김' in '김민준'): a family
607 # word touching the next with nothing written between
608 # (C-level). Spaced, the order placed the family and the
609 # list decided nothing ('김' in '김 민준')
610 if (role is Role.FAMILY and i + 1 < last
611 and tokens[i + 1].span.start == t.span.end
612 and _normalize(t.text) in surnames):
613 source = ReadingSource.VOCABULARY
614 elif "initial" in tags:
615 kind, source = ReadingKind.INITIAL, ReadingSource.SHAPE
616 elif ("conjunction" in tags
617 and UNJOINED_CONJUNCTION_TAG not in tags):
618 kind, source = (ReadingKind.CONJUNCTION,
619 ReadingSource.VOCABULARY)
620 elif ("particle" in tags and UNJOINED_TAG not in tags
621 # a nickname has no family for a particle to join,
622 # and the unjoined marks are written in the name
623 # parts only ('"Van" Smith' reads a name)
624 and role is not Role.NICKNAME
625 and not ("vocab:bound-given" in tags
626 and role is Role.GIVEN
627 and _opens_its_segment(state, i))):
628 kind, source = ReadingKind.PARTICLE, ReadingSource.VOCABULARY
629 elif "vocab:bound-given" in tags:
630 # rules/P5's bound given name, joined ('Abdul Aziz', and
631 # 'Abu Bakar' in 'Abu Bakar Smith') or not ('Abdul' as a
632 # middle name in 'John Abdul Smith'), a word the
633 # vocabulary lists as a given name. One that is also a
634 # particle ('abu') is P5's only in the given part and
635 # only where P5 joins, the first word of its segment
636 # past the titles; elsewhere the particle chain took it
637 # (the family in 'John Abu Smith', the given part in
638 # 'Salim Ali abu Bakar Hassan' under
639 # FAMILY_FIRST_GIVEN_LAST) and it read the arm above
640 kind, source = ReadingKind.NAME, ReadingSource.VOCABULARY
641 else:
642 # a particle or connective with nothing to join reads
643 # as the name word it stands as ('van' in 'van Gogh')
644 kind, source = ReadingKind.NAME, name_source(tags)
645 # a name's files by its field, a middle word the policy
646 # folded into the family by the middle it stands as
647 # (inline, not `_field_of`: a frame per token, measured
648 # 2026-10-29 at 7 on the reference row)
649 field = (Role.MIDDLE if role is Role.FAMILY and folds_middle
650 and FOLDED_TAG in tags and ATTACHED_TAG not in tags
651 else role)
652 # rules/R9: "a particle the lexicon holds as a patronymic",
653 # read as a particle, opens a patronymic that
654 # takes the words after it in its field; an ending rule a
655 # policy turns on reads its own words (decisions/R9, #754)
656 opens = attaches = False
657 if kind is ReadingKind.PARTICLE and PATRONYMIC_TAG in tags:
658 opens, attaches = _patronymic_opening(state, dropped, i,
659 field, folds_middle)
660 if opens:
661 kind = ReadingKind.PATRONYMIC
662 run_field = field
663 attached = attached or attaches
664 elif field is run_field:
665 kind, source = ReadingKind.PATRONYMIC, ReadingSource.POSITION
666 else:
667 if role is not Role.NICKNAME:
668 run_field = None
669 # a marker or an ending read as a name word: one a
670 # caller's Lexicon lists as a connective keeps that
671 # reading, and its father with it
672 if kind is ReadingKind.NAME and (
673 i in markers or slavic and field is Role.MIDDLE
674 and has_patronymic_ending(t.text)):
675 kind, source = ReadingKind.PATRONYMIC, ReadingSource.SHAPE
676 if i in markers:
677 text, span, frole, ftags, freading = held.pop(markers[i])
678 if (kind is ReadingKind.PATRONYMIC
679 and freading.kind is not ReadingKind.PATRONYMIC):
680 # a father in the second surname keeps the mark
681 freading = _reading(ReadingKind.PATRONYMIC, frole,
682 ReadingSource.POSITION, (),
683 freading.second_surname,
684 freading.first_surname)
685 final[markers[i]] = Token(text, span, frole, ftags, freading)
686 langs, by_parts = (_languages_of(t.text, kind, field, files)
687 if source in _LISTED else _UNLISTED)
688 if by_parts and _listed_whole(t.text, state.lexicon):
689 langs = ()
690 reading = (_reading(kind, role, source, langs, second, first)
691 if langs or second or first
692 else _READINGS[kind, role, source])
693 if not tags.isdisjoint(_KIND_TAGS):
694 # the kind tags serve the stages and this one, whatever the
695 # word's field (a lone 'Jr' is a given name), the reading
696 # carrying the kind out; the written-short mark serves the
697 # stages alone; and a published tag would make a name
698 # unequal to its own 2.x pickle (Token compares tags)
699 tags = tags - _KIND_TAGS
700 if i in fathers:
701 held[i] = (t.text, t.span, role, tags, reading)
702 else:
703 final[i] = Token(t.text, t.span, role, tags, reading)
704 if markers:
705 # a father built at its marker went in after the words between
706 # them (a nickname); `final` is read in token order
707 final = dict(sorted(final.items()))
708 if attached:
709 # rules/R9: "A trailing patronymic particle P6 attaches to the
710 # family written before it takes that family's words as the
711 # parent's name" (decisions/R9, #754)
712 for j, tok in final.items():
713 if (tok.role is Role.FAMILY and FOLDED_TAG not in tok.tags
714 and tok.reading is not None
715 and tok.reading.kind is not ReadingKind.PATRONYMIC):
716 # a word of the second surname keeps the mark
717 final[j] = Token(tok.text, tok.span, tok.role, tok.tags,
718 _reading(ReadingKind.PATRONYMIC,
719 Role.FAMILY,
720 ReadingSource.POSITION, (),
721 tok.reading.second_surname,
722 tok.reading.first_surname))
723 # No alphanumeric character among the SURVIVING tokens means no
724 # name: a bare '.' or '- -' is not a person. v1 kept such input
725 # (parse('.') -> first '.'); 2.0 empties it so bool() stays an
726 # honest "did I get a name?" check. isalnum() is Unicode-aware, so
727 # every real name in any script has content and only pure
728 # punctuation/symbols empty out. (Embedded junk in a name with
729 # content -- 'John . Smith' -- is left alone: that parse is truthy,
730 # so bool() is not misled.)
731 #
732 # This read "anywhere" until #329, and the two said the same thing
733 # for as long as nothing could take a content-bearing token away.
734 # Dropping a maiden marker inside a delimited clause can, and
735 # "(née —)" under Policy(maiden_delimiters=...) is the input where
736 # the readings part: the marker is the clause's only alnum token,
737 # so once it goes structural the em dash is all that survives and
738 # the whole parse empties -- where 1.4.0 and pre-#329 both gave
739 # maiden 'née —'. Deliberate, not fallout: a dropped marker is
740 # structural like a delimiter character, and brackets plus a marker
741 # plus a dash name no one, exactly as "(-)" already named no one.
742 # A guard keyed on what ELSE is in the clause would be a different
743 # rule, and would leave maiden holding marker-plus-punctuation.
744 # Pinned by cases.py's maiden_marker_delimited_content_free and,
745 # for the other side of it -- the same clause inside a name, where
746 # the em dash survives as the maiden value because Jane Smith
747 # carries the content -- ..._content_free_in_a_name beside it.
748 #
749 # The TOKENS go, not the diagnostics: this drops the name, and
750 # "was the input malformed?" is the one question still worth
751 # answering about it -- most of all here, where there is no parse
752 # left to infer it from. parse('(') keeps its unbalanced-delimiter
753 # report, pointing at no token because no token survived.
754 contentless = not any(
755 c.isalnum() for t in final.values() for c in t.text)
756 if contentless:
757 final = {}
758 ambiguities = []
759 segment_of: dict[int, tuple[int, ...]] | None = None
760 for pending in state.ambiguities:
761 materialized = tuple(final[i] for i in pending.indices
762 if i in final)
763 if pending.indices and not materialized and not contentless:
764 # every referent was dropped: the ambiguity describes
765 # nothing that survives assembly. Born-empty ambiguities
766 # (unbalanced delimiters) are token-independent and kept.
767 # Not so when the whole name was emptied just above -- the
768 # referents did not lose a contest, they were discarded
769 # wholesale, and the report still describes the input.
770 continue
771 # rules/A1: "a report names the reading the parse took, so
772 # a report whose fork the rest of the parse then resolved to
773 # NEITHER branch is withdrawn rather than carried beside a
774 # reading it contradicts". classify's connective-or-initial
775 # fork offers exactly two readings and its detail says which
776 # it took ("it is read as an initial"); a generation and an
777 # honorific are neither. So where the parse goes on to role
778 # that letter SUFFIX or TITLE the report is false on its
779 # face, and 'JOHN QUINCY SMITH I' carried it beside a
780 # suffix-or-name saying the same token reads as a
781 # generational suffix (#397 second review). 'i' is the first
782 # word that is both a marked connective and suffix
783 # vocabulary, so no parse before this cycle could reach the
784 # shape.
785 #
786 # WITHDRAWN HERE and not emitted later, which is the narrow
787 # reading of mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE
788 # rather than an exception to it: "Emit at the site that
789 # takes the branch, not where an ambiguous tag sits" governs
790 # the EMISSION, classify still owns the fork, and what this
791 # drops is a report whose subject the parse went on to read
792 # as something else. Here is the one place that knows --
793 # roles are final and nothing downstream moves them -- and
794 # the loop already withdraws a report whose referents did not
795 # survive, just above.
796 if (pending.kind is AmbiguityKind.CONJUNCTION_OR_INITIAL
797 and any(t.role in (Role.SUFFIX, Role.TITLE)
798 for t in materialized)):
799 continue
800 # rules/P8: "unless no name word of its part stands to its
801 # right" -- nothing after the letter in its segment, or only
802 # suffixes, titles, a maiden clause or a nickname: it connects
803 # nothing, so the report goes (#721). Asked here for the reason
804 # above: the roles are final. Where the letter is also a
805 # numeral the fork read, its own report (rules/S10) is the one
806 # left, so one token never carries both. Built only for a
807 # connective report, so other names pay nothing.
808 if pending.kind is AmbiguityKind.CONJUNCTION_OR_INITIAL:
809 if segment_of is None:
810 segment_of = {j: seg for seg in state.segments
811 for j in seg}
812 i = pending.indices[0]
813 if not any(j > i and j in final
814 and final[j].role in NAME_ROLES
815 for j in segment_of.get(i, ())):
816 continue
817 detail = pending.detail
818 if pending.field_tail is not None:
819 # A report naming a field, in a name emptied above (no
820 # emitter of one reaches that today: every referent is a
821 # name word): no field is left to name, so it goes rather
822 # than ship the sentence's first half.
823 if not materialized:
824 continue
825 detail = (f"{detail}{FIELD_READING[materialized[0].role]}"
826 f"{pending.field_tail}")
827 ambiguities.append(Ambiguity(
828 pending.kind, detail, materialized,
829 _alternative(pending, materialized, final, state)))
830 # rules/R6: "the name is laid out in the order it was read in" --
831 # ParsedName.order records that order for the rendering to read:
832 # the order assign read the name in, given-first after a family
833 # comma (which fixes the family without consulting an order), and
834 # the declared name_order where no positional read happened; inline,
835 # since every parse would pay a frame for a helper (decisions/R6,
836 # #754)
837 order = (state.order if state.order is not None
838 else GIVEN_FIRST if state.structure is Structure.FAMILY_COMMA
839 else state.policy.name_order)
840 # rules/T7: "the given name is a run of single letters joined by
841 # full stops" -- found inline (a helper would cost every parse a
842 # frame, decisions/parse-cost): the one given word, a stop in it,
843 # and a family beside it, a family comma marking one where it
844 # names none (rules/C15) -- decisions/T7 (#1296)
845 run_at = None
846 has_family = state.structure is Structure.FAMILY_COMMA
847 capitals = False
848 for j, tok in final.items():
849 if tok.role is Role.GIVEN:
850 run_at = (-1 if run_at is not None or "." not in tok.text
851 else j)
852 elif tok.role is Role.FAMILY:
853 has_family = True
854 if tok.role in NAME_ROLES and not tok.text.islower():
855 capitals = True
856 out = tuple(final.values())
857 if (run_at is not None and run_at >= 0 and has_family
858 and not state.policy.middle_as_family
859 and not state.policy.two_given_names
860 and final[run_at].span is not None
861 and _LETTER_RUN.fullmatch(final[run_at].text)
862 # written in capitals, or among given, middle and family
863 # names written wholly in lower case, which read as their
864 # capitals do (a title is not asked): a run in lower case
865 # among capitals may be an abbreviation of its own ('Dr.
866 # h.c. Hans Meier', honoris causa)
867 and (final[run_at].text.isupper() or not capitals)
868 # a run the vocabulary lists or masks whole is that word
869 and not _listed_run(final[run_at].text, state.lexicon)):
870 out, ambiguities = _part_letter_run(final[run_at], out,
871 ambiguities, order)
872 return ParsedName(original=state.original,
873 tokens=out,
874 ambiguities=tuple(ambiguities),
875 order=order)
878# rules/T7: "it reads as its letters written with a space between
879# them would: the letter in the given name's place in the order is the
880# given name, and the others are middle names" -- decisions/T7 (#1296)
881def _part_letter_run(whole: Token, tokens: tuple[Token, ...],
882 ambiguities: list[Ambiguity],
883 order: tuple[Role, Role, Role],
884 ) -> tuple[tuple[Token, ...], list[Ambiguity]]:
885 """`tokens` with `whole`, a given name written as a run of single
886 letters, parted into one initial per letter, and every report
887 naming it naming its letters instead ('Smith, J.R.', a credential's
888 shape after a comma), worded for them: the field it named is now
889 the name's initials. The given name is the first letter, or the
890 last where the order lays the given name out after the middle
891 names."""
892 assert whole.span is not None
893 letters = _RUN_LETTER.findall(whole.text)
894 given = (len(letters) - 1
895 if order.index(Role.GIVEN) > order.index(Role.MIDDLE) else 0)
896 start = whole.span.start
897 parts: list[Token] = []
898 for n, text in enumerate(letters):
899 role = Role.GIVEN if n == given else Role.MIDDLE
900 parts.append(Token(text, Span(start, start + len(text)), role,
901 _INITIAL_TAGS,
902 _READINGS[(ReadingKind.INITIAL, role,
903 ReadingSource.SHAPE)]))
904 start += len(text)
905 at = tokens.index(whole)
906 reported = [a if whole not in a.tokens else Ambiguity(
907 a.kind, a.detail.replace(FIELD_READING[Role.GIVEN],
908 "the name's initials"),
909 tuple(p for t in a.tokens for p in (parts if t == whole else (t,))),
910 a.alternative) for a in ambiguities]
911 return tokens[:at] + tuple(parts) + tokens[at + 1:], reported
914_INITIAL_TAGS = frozenset({"initial"})
917def _listed_run(text: str, lexicon: Lexicon) -> bool:
918 """Whether the lexicon lists `text`, a run of single letters, as a
919 word of its own, written as it is, with its full stops: in a
920 vocabulary set, or under a case mask keyed with them that writes a
921 lower-case letter, or keyed without them and writing one (a caller's
922 honoris causa, 'h.c' or 'hc' masked 'h.c'; 'h.c' masked 'H.C' says
923 the letters are capitals, and parts). The spelling
924 without stops is how initials spell a word, so a vocabulary word or
925 a mask spelled without them says nothing about the run: 'J.R.' is
926 no 'jr', and 'K.T.' and 'B.T.' part though 'Kt' and 'Bt' are
927 credentials masked in mixed case."""
928 n = _normalize(text)
929 cap_map = lexicon._cap_map
930 mask = cap_map.get(n)
931 if mask is not None and not mask.isupper():
932 return True
933 bare = cap_map.get(n.replace(".", ""))
934 if bare is not None and "." in bare:
935 return True
936 sets = lexicon._sets
937 return any(n in getattr(sets, name) for name in _VOCAB_FIELDS)