Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Stage: post_rules.
3Consumes: tokens (roles assigned), plus pieces and structure -- the
4particle fold reads the opening piece of segment 0, or of segment 1
5under a family comma (#359). structure was always read here, for the
6rotation gate. Also comma_offsets and dropped, which R1's entry pass
7below reads to find the separators the writer typed (#436/#437), and
8extracted, one_case and original, which S1's set-apart pass reads,
9settling S13's members with its own (#810, #792), original also read
10by R1's entry pass for a comma a set-apart clause kept; and order,
11the order assign read the name under, which H1, P1's fold, P6's
12attachment, P10, O1, O2, O6, O7, O8 and W5's surname pair key on rather than
13Policy.name_order (#395).
14Produces: tokens with roles adjusted by the post rules (among them the
15TITLE and SUFFIX roles of a set-apart clause's words, rules/S1, each
16word's comma trimmed off its text and span, and the
17SUFFIX role of a delimited member its writing settles after the name,
18rules/S13, with the SUFFIX_OR_NICKNAME report on any other member left
19a nickname), the stable
20"joined" tag on a post-nominal continuing the entry before it, the
21folded-middle tag on the words O3 folds and P6's no-comma attachment
22joins to the family, with the attached tag on the latter (rules/R9
23tells the two apart by it), the second-surname tag on each word of the
24family's second unit where O6 reads two (rules/R10), the first-surname
25tag on each word of the first unit, written by O6 where it reads two
26and by O3 on the family it folds into otherwise (rules/R11), both unit
27tags where it reads a Han surname pair (rules/W5, #845), with a
28SEGMENTATION report where that pair was written undivided, and the ambiguity P6's attachment reports at the end of a name read
29family-first (#467) -- after a family comma, that attachment is
30assign's since #613, and behind an empty part before the comma too
31since #892.
32Reads: Policy.patronymic_rules, Policy.initials_as_family,
33Policy.two_family_names,
34Policy.two_given_names, Policy.name_order (O6 and O7 stand down where
35the script's order differs from the declared one),
36Policy.middle_as_family,
37Policy.extra_suffix_delimiters (R1's entry pass, for the delimiter
38cores group drops); Lexicon.given_name_titles, Lexicon.given_names
39(O6's veto), Lexicon.saint_particles (P10's saint clause; P10 runs
40under every policy and reads Policy.patronymic_rules for its Turkic
41guard), and
42Lexicon.suffix_acronyms_ambiguous (a dotted member in a clause, which
43carries no tag there, S13, through `_vocab.dotted_member`), and for S1's set-apart clause
44Lexicon.suffix_words directly and, through `_vocab.set_apart_kind` and
45`set_apart`, Lexicon.suffix_words_ambiguous, suffix_acronyms, titles,
46title_modifiers, title_styles and after_given_titles.
48Implements rules S1, S13, H1, M4, P1, P6 (its no-comma site), P10, O1, O2,
49O6, O7, O3 and R1 of docs/design/rules/; each is cited at its code below, and
50H1/P1/O1/O2's history lives in docs/design/decisions/.
51`suffix_entries` is the R1 entry pass as a state-in/state-out
52function, for Parser.revise to run over a forced-role sub-parse
53(#511); `post_rules` runs the same worker last.
54"""
55from __future__ import annotations
57import bisect
58from collections.abc import Sequence
60from nameparser._lexicon import Lexicon, _normalize, _run_addresses_by_given
61from nameparser._pipeline._assign import (
62 _name_alt, _name_pieces, _name_positions,
63)
64from nameparser._pipeline._pieces import (
65 is_lone_never_given_particle, is_lowercase_ambiguous_particle,
66 settled_by_case,
67)
68from nameparser._pipeline._state import (
69 AMBIGUOUS_ACRONYM_TAG, ATTACHED_TAG, COMMA_CHARS, COMMA_STRIP, NAME_ROLES,
70 PATRONYMIC_TAG,
71 BOUND_PHRASE_TAG, PARTICLE_GIVEN_TAG, PAST_REACH_TAG, SHARED_NAME_TAG,
72 FIRST_SURNAME_TAG, SECOND_SURNAME_TAG, SURNAME_PAIR_TAG,
73 ParseState, PendingAmbiguity, PendingReading, Structure,
74 WorkToken, _NEVER_FLIPPED, comma_bucket, copy_with, reported_suffix_fork,
75)
76from nameparser._pipeline._vocab import (
77 _HYPHENS, SetApartKind, delimiter_cores, dotted_member,
78 east_slavic_rotates, has_patronymic_ending, is_initial, is_listed_given_name,
79 is_patronymic_marker, joined_member, member_word, period_member,
80 reads_as_initials,
81 set_apart, set_apart_kind, settles_a_member, suffix_reading,
82 unit_ends,
83)
84from nameparser._policy import PatronymicRule
85from nameparser._types import (
86 FOLDED_TAG, UNJOINED_CONJUNCTION_TAG, UNJOINED_TAG, AmbiguityKind, Role,
87 Span,
88)
91#: The roles that are transparent to a run of post-nominals (R1's
92#: entry pass below). These three roles render into fields other than
93#: the name and the suffix, so a run of
94#: post-nominals is not parted by them -- 'Smith, MD Dr. PhD',
95#: 'Smith, MD "Doc" PhD' and 'Smith, MD nee Jones PhD' are each one
96#: entry. GIVEN/MIDDLE/FAMILY are name words and part it.
97_RENDERS_ELSEWHERE = frozenset({Role.TITLE, Role.NICKNAME, Role.MAIDEN})
100def _rotations_apply(state: ParseState) -> bool:
101 # Both patronymic rotations RESTORE the given-first reading a
102 # family-first listing hides, so rules/O1's scope clause holds
103 # them to the default order: a caller who declared family-first has
104 # already said what the rotation would infer, and position decides
105 # (decisions/O1, the 2026-09-07 entry on #384). `state.order`,
106 # not policy.name_order, for the reason the P1 fold gives -- a
107 # script_orders entry can override the policy, and the roles the
108 # rotations read are the ones assign actually made. None means
109 # assign positioned nothing: a family comma (which the NO_COMMA
110 # test already excludes) or an early return with no name piece to
111 # position, so there is no declaration to defer to and the
112 # rotation's own shape test decides.
113 return state.structure is Structure.NO_COMMA and (
114 state.order is None or state.order[0] is Role.GIVEN)
117def _mark_suffix_entries(tokens: list[WorkToken], state: ParseState) -> None:
118 # In place over the caller's token list, as every other post rule
119 # writes; suffix_entries() below is the state wrapper, and
120 # decisions/R1 (#511) records why the pass has two spellings.
121 # Reads dropped, comma_offsets and the policy off `state`, which
122 # post_rules does not change, so the un-rebuilt state is current.
123 #
124 # rules/R1: "a run of post-nominals written with spaces renders
125 # with spaces, and one written with commas keeps them"
126 #
127 # Two consecutive SUFFIX tokens are one entry iff they share a
128 # comma bucket and nothing between them parts the run: a name word
129 # or a dropped delimiter core parts it; a token whose role is in
130 # _RENDERS_ELSEWHERE, or a dropped token that is not a core (the
131 # marker of a bracketed maiden clause, 'Smith, MD (nee Jones)
132 # PhD'), does not. comma_bucket and delimiter_cores
133 # are segment's and group's own answers, called rather than copied
134 # (mechanisms.md#ONE-PREDICATE-PER-QUESTION). decisions/R1
135 # (#436/#437).
136 #
137 # After assign, because it keys on Role.SUFFIX; last in post_rules,
138 # after S13's pass, the one rule here that writes SUFFIX (#810), and
139 # by convention otherwise, since no other writes SUFFIX or a
140 # _RENDERS_ELSEWHERE role. Recorded as a tag because a span-less
141 # token (unpickled, or built by replace()) leaves the render
142 # nothing to scan (mechanisms.md#MARK-DONT-STRIP). Every WorkToken
143 # has a span, so `span.start` is read unguarded, and the
144 # `i not in dropped` filter is belt-and-braces: a dropped token
145 # never holds SUFFIX, and the filter keeps `suffixes` and the
146 # `parted` scan agreeing on what a dropped index is.
147 dropped = set(state.dropped)
148 # a plain loop and an early return, where a comprehension and the
149 # cores ahead of the count cost frames on every parse, and most
150 # names hold fewer than two suffixes (decisions/parse-cost, #1170)
151 suffixes: list[int] = []
152 for i, tok in enumerate(tokens):
153 if tok.role is Role.SUFFIX and i not in dropped:
154 suffixes.append(i)
155 if len(suffixes) < 2:
156 return
157 cores = delimiter_cores(state.policy.extra_suffix_delimiters)
158 for previous, current in zip(suffixes, suffixes[1:]):
159 same_part = (comma_bucket(tokens[previous].span.start,
160 state.comma_offsets)
161 == comma_bucket(tokens[current].span.start,
162 state.comma_offsets))
163 parted = any(
164 tokens[between].text in cores if between in dropped
165 else tokens[between].role not in _RENDERS_ELSEWHERE
166 for between in range(previous + 1, current)) or (
167 # rules/S1: "parts its words as a comma between them would"
168 # -- a comma a set-apart clause kept on a word, which no
169 # comma bucket holds and S1's pass trimmed off the word;
170 # decisions/S1 (#792)
171 state.original[tokens[previous].span.end:
172 tokens[previous].span.end + 1] in COMMA_CHARS
173 or state.original[tokens[previous].span.end:
174 tokens[current].span.start].replace(" ", "")
175 in COMMA_CHARS)
176 if same_part and not parted:
177 tokens[current] = copy_with(
178 tokens[current], tags=tokens[current].tags | {"joined"})
181def suffix_entries(state: ParseState) -> ParseState:
182 """rules/R1's entry pass over a whole state, wrapping
183 _mark_suffix_entries: tag a SUFFIX token "joined" when it
184 continues the entry of the SUFFIX token before it. post_rules runs
185 the worker last, over the roles assign settled; this wrapper exists
186 for Parser.revise (#511), which runs it over a SUB-PARSE of a field
187 value whose every non-dropped token it has forced to the named
188 role, so a suffix value derives its entries from its own commas by
189 the rule a whole name uses. The pass ADDS the tag and never removes
190 one; a caller forcing roles keeps what the sub-parse marked, a
191 within-piece mark being role-blind (decisions/R1, the #436
192 DECLINED list) and every between-piece mark on a suffix value
193 being one this pass sets again. Reads comma_offsets, dropped and
194 Policy.extra_suffix_delimiters; writes the "joined" tag and
195 nothing else."""
196 tokens = list(state.tokens)
197 _mark_suffix_entries(tokens, state)
198 return copy_with(state, tokens=tuple(tokens))
201#: Each role `_positions` lists, by its place in the answer.
202_POSITION_SLOT: dict[Role | None, int] = {
203 Role.TITLE: 0, Role.GIVEN: 1, Role.MIDDLE: 2, Role.FAMILY: 3}
206def _positions(tokens: list[WorkToken],
207 ) -> tuple[list[int], list[int], list[int], list[int]]:
208 """The positions of the TITLE, GIVEN, MIDDLE and FAMILY tokens, in
209 that order, each list ascending: the four roles post_rules' rules
210 read as position lists, in one walk however many of them the caller
211 wants."""
212 # one plain loop, where a comprehension per role cost a frame of its
213 # own on 3.11 beside the call's, and post_rules asks for the lists on
214 # every parse (the parse-cost band, decisions/parse-cost, #1089)
215 lists: tuple[list[int], list[int], list[int], list[int]] = (
216 [], [], [], [])
217 for i, token in enumerate(tokens):
218 slot = _POSITION_SLOT.get(token.role)
219 if slot is not None:
220 lists[slot].append(i)
221 return lists
224def _leading_name_piece(state: ParseState,
225 tokens: list[WorkToken]) -> tuple[int, ...]:
226 """The piece that OPENS the name, whatever role name_order gave it:
227 the first piece holding a GIVEN, MIDDLE or FAMILY token, in the
228 segment the positional read governs. Every piece holding none of
229 those is walked past -- title and suffix pieces, but NICKNAME and
230 MAIDEN as well, and anything assign left unroled -- and any number
231 of them, not only a single leading title. The segment is 0, except
232 under a family comma that fixed the surname, where the name
233 continues in segment 1 -- assign records no order there. A family
234 comma followed by no name word fixed nothing, and assign reads
235 segment 0 positionally and records the order (#296's bundle), so
236 the name is segment 0 again. Empty on either of two exits: that
237 segment does not exist, or none of its pieces holds a name
238 token."""
239 seg = (1 if state.structure is Structure.FAMILY_COMMA
240 and state.order is None else 0)
241 if seg >= len(state.pieces):
242 return ()
243 # nested loops, where an any() generator cost frames on every parse
244 # (decisions/parse-cost, #1170)
245 for piece in state.pieces[seg]:
246 for i in piece:
247 if tokens[i].role in NAME_ROLES:
248 return piece
249 return ()
252def _retag(tokens: list[WorkToken], i: int, role: Role) -> None:
253 tokens[i] = copy_with(tokens[i], role=role)
256# rules/P2: "a particle joins the words after it into one name
257# part, the join running until the next particle starts a group of
258# its own, a trailing suffix begins"
259# rules/P3: "the joined part is ONE name word wherever another
260# rule counts them"
261# rules/P5: "a recognized bound given-name word joins the word
262# after it into one given name"
263def _units(tokens: list[WorkToken], idx: list[int]) -> list[list[int]]:
264 """`idx` split into the units other rules COUNT: one name word
265 each, except where another rule has already made several words one
266 name. Three do -- a particle and the words it chains, a
267 conjunction-joined run, and a bound given-name word with the word
268 it completes -- so `van der Berg` and `abdul Rahman` are each one
269 unit and the fold cannot leave half of either behind.
271 Read off the TAGS rather than the pieces, for two different
272 reasons. A conjunction join grouping DID build and the prefix
273 chain then swallowed: "de la Vega y Santos Juan" reaches this as
274 [de][la Vega y Santos Juan], the ambiguous particle having chained
275 forward over the join, so the join's own boundary is gone. (The
276 leading particle keeps its piece -- it has to, or the fold's
277 lone-piece site test would not fire at all.) The bound-given join grouping
278 never built at all: P5 joins only where the bound word is the
279 first non-title piece, save a listed two-word given name P12
280 joins past the first name word (#694), and at a fold site the first piece
281 is the particle -- "ibn Awf abdul Rahman" reaches here as four
282 separate pieces. The tag is the only witness in both cases.
284 The particle chain is what keeps this partition agreeing with the
285 one `assign` reads off pieces: strip the folded run and `assign`
286 gives the same tail one role, so the fold must not hand its words
287 out separately."""
288 units: list[list[int]] = []
289 i = 0
290 for end in unit_ends([tokens[k].tags for k in idx]):
291 units.append(list(idx[i:end]))
292 i = end
293 return units
296def _fold_reach(tokens: list[WorkToken], name_idx: list[int]) -> int:
297 """How many of `name_idx` the fold takes: the particle run, plus
298 the one name word it attaches to -- one UNIT, so a conjunction
299 join goes whole ("de la Vega y Santos Juan" keeps Vega y Santos
300 together). All of them when the name is nothing but particles."""
301 i = 0
302 while i < len(name_idx) and "particle" in tokens[name_idx[i]].tags:
303 i += 1
304 if i == len(name_idx):
305 return i
306 return i + len(_units(tokens, name_idx[i:])[0])
309def _family_word_count(tokens: list[WorkToken], family: list[int],
310 hyphen_joins: bool = False) -> int:
311 """The family's name words as rules/O6 counts them: a particle run
312 counts with the word it attaches to, a connective counts as
313 nothing and a hyphen joins words without making them one, so "de
314 la Vega" is one word and "Ortega y Gasset" and "García-López" two.
315 A family of nothing but particles counts each of them, R2 reading
316 them as ordinary name words. `hyphen_joins` counts a hyphenated word
317 as one, as the word O6 takes is counted (#959). A bound word counts
318 with the word that completes it, the pair P12 joins being one name
319 ('عبد الله', #694)."""
320 # rules/P12: "the two words are one name unit past the first name word"
321 # -- so the bound word counts with the word after it
322 words = sum(1 if hyphen_joins
323 else len([part for part in _HYPHENS.split(tokens[i].text)
324 if part])
325 for i in family
326 if "particle" not in tokens[i].tags
327 and "conjunction" not in tokens[i].tags
328 and BOUND_PHRASE_TAG not in tokens[i].tags)
329 return words or len(family)
332def _hyphenated_given(text: str, lexicon: Lexicon) -> bool:
333 """A hyphenated word that writes given names, which the taken word's
334 count (a hyphenated word counting as one) would otherwise let through
335 (#959): every part of it a listed given name or an initial
336 ("María-José" under the Spanish list, "M.-J.", "M.-José")."""
337 parts = [part for part in _HYPHENS.split(text) if part]
338 return len(parts) > 1 and all(is_initial(part) or is_listed_given_name(part, lexicon)
339 for part in parts)
342def _take_trailing_initials(state: ParseState,
343 tokens: list[WorkToken]) -> None:
344 # rules/O8: "the letters ending the name are its family name" --
345 # under the given-first order, with no family comma, the middle
346 # initials standing right before a family name written as one
347 # initial join it. In place, as every post rule writes; text order
348 # already renders them first, so no tag is needed. History:
349 # decisions/O8 (#1254).
350 # A family comma and a family-first order need no test of their
351 # own: each puts the family first in text order, so no middle stands
352 # right before it for the run below to take (#1347 review).
353 _, given, middle, family = _positions(tokens)
354 if (len(family) != 1 or not middle
355 or "initial" not in tokens[family[0]].tags):
356 return
357 run: list[int] = []
358 i = family[0] - 1
359 while (i >= 0 and tokens[i].role is Role.MIDDLE
360 and "initial" in tokens[i].tags):
361 run.append(i)
362 i -= 1
363 if not run:
364 return
365 # rules/O8: "a name written as nothing but letters keeps its first
366 # letter as the given name" -- a non-initial name word must stand
367 # before the run
368 if not any("initial" not in tokens[j].tags
369 for j in given + middle if j < run[-1]):
370 return
371 for j in run:
372 tokens[j] = copy_with(tokens[j], role=Role.FAMILY)
375def _take_second_family_name(state: ParseState,
376 tokens: list[WorkToken]) -> None:
377 # rules/O6: "the family name takes the name word beside it, so
378 # the last two name words are the family" -- the first two under
379 # a family-first order. In place, as every post rule writes.
380 # History: decisions/O6 (#754).
381 # An order a name's script set (W4) other than the declared one is
382 # not the caller's declaration, and no script it keys writes two
383 # family names as separate words. Where the two agree nothing here
384 # tells them apart, and the rule reads the name (rules/O6 Accepted).
385 if (state.structure is Structure.FAMILY_COMMA or state.order is None
386 or state.order != state.policy.name_order):
387 return
388 _, given, middle, family = _positions(tokens)
389 others = sorted(given + middle)
390 if not family or _family_word_count(tokens, family) != 1:
391 return
392 # The rest in the units assign read them as, so a particle chain
393 # (P2) or a bound given-name pair (P5) moves whole or not at all.
394 units = _units(tokens, others)
395 # rules/O6: "a particle chain after the family is cut where P1's
396 # family-first fold cuts its own run" -- under a family-first order,
397 # the given name the order declares after the chain left to the
398 # order (decisions/O6, #1024); under the default order the chain
399 # ends the name and stays whole
400 if state.order[0] is not Role.GIVEN and units:
401 cut = _fold_reach(tokens, units[0])
402 if cut < len(units[0]):
403 units[:1] = [units[0][:cut], *_units(tokens, units[0][cut:])]
404 if len(units) < 2:
405 return
406 take = units[-1] if state.order[0] is Role.GIVEN else units[0]
407 # What is taken must be ONE name word, so a bound given-name pair
408 # (P5) stays put; rules/O6: "A hyphenated word counts as one when it
409 # is the word taken" (decisions/O6, #959), where the family's own
410 # hyphen counts two; and a word a family name could be: not an
411 # initial, which writes a given name (one written alone; "J.R." is
412 # taken), nor a lone dash or other mark with no letter in it.
413 if _family_word_count(tokens, take, hyphen_joins=True) != 1:
414 return
415 if len(take) == 1 and "initial" in tokens[take[0]].tags:
416 return
417 if not any(ch.isalpha() for i in take for ch in tokens[i].text):
418 return
419 # rules/O6: "a listed given name stays where it is" -- the caller's
420 # Lexicon.given_names (empty by default; locales.ES carries the
421 # Spanish list, with the switch on) names a word only a given name is, under
422 # either order. Only a unit of one token is asked: a word
423 # behind a particle ("de Miguel") is a unit of two, a surname taken
424 # as before. History: decisions/O6 (#959).
425 # A hyphenated word writing given names stays too (#959).
426 if len(take) == 1 and (
427 is_listed_given_name(tokens[take[0]].text, state.lexicon)
428 or _hyphenated_given(tokens[take[0]].text, state.lexicon)):
429 return
430 # rules/O6: "a name a community gives every member is no family
431 # name of two" -- Kaur and Singh, on either side of the join, tagged
432 # at classify from Lexicon.shared_names (decisions/O6, #704)
433 if any(SHARED_NAME_TAG in tokens[i].tags for i in take + family):
434 return
435 # rules/O6: "a given name a particle completes stays where it is" --
436 # given-first, the unit taken being the one beside a given name of
437 # one word, a particle run and a word classify tags from
438 # Lexicon.particle_given_names, in the shape rules/P10 reads; under a
439 # family-first order the unit after the family completes no given
440 # name (decisions/O6, #704)
441 if (state.order[0] is Role.GIVEN and len(units) == 2
442 and _one_given_word(tokens, units[0])
443 and all(tokens[i].role is Role.GIVEN for i in units[0])
444 and _completes_a_given_name(tokens, take, state.lexicon)):
445 return
446 if state.order[0] is Role.GIVEN:
447 # given, middles..., family: the middle unit standing right
448 # before the family joins it, and text order already renders
449 # it first, so no role around it moves. A rotation that left
450 # the name in another shape fails this test.
451 if take[-1] > family[0] or any(
452 tokens[i].role is not Role.MIDDLE for i in take):
453 return
454 _mark_surname(tokens, take, FIRST_SURNAME_TAG)
455 _mark_surname(tokens, family, SECOND_SURNAME_TAG)
456 return
457 # family first: the unit after the family joins it, and what is
458 # left is a shorter name of the same order, laid out as the P1
459 # fold above lays its remainder -- one family already placed. A
460 # particle P6 attached at the far end is family already and keeps
461 # its place.
462 if family[0] > units[0][0]:
463 return
464 # the unit taken joins the family as its second
465 _join_second_surname(tokens, family, units, state.order)
468def _join_second_surname(tokens: list[WorkToken], family: list[int],
469 units: list[list[int]],
470 order: tuple[Role, Role, Role]) -> None:
471 """The family-first join O6 and W5's pair share, in place: `family`
472 becomes the first surname and `units[0]`, the unit after it, the
473 second, and the units left are laid out as a shorter name of the
474 same order, one family already placed."""
475 _mark_surname(tokens, family, FIRST_SURNAME_TAG)
476 _mark_surname(tokens, units[0], SECOND_SURNAME_TAG)
477 for unit, role in zip(units[1:], _name_positions(
478 order, len(units))[1:]):
479 for i in unit:
480 _retag(tokens, i, role)
483def _read_surname_pair(state: ParseState,
484 tokens: list[WorkToken]) -> PendingAmbiguity | None:
485 """rules/W5: "the two surnames are the family, in two units" -- the
486 second, which script_segment tagged, joins the family assign placed
487 before it, and what is left is laid out by the order as O6's
488 family-first branch lays out its own (decisions/W5, #845). In place.
490 script_segment divides the pair only where the name is read
491 family-first, so assign placed the first surname as the family and
492 the second as the first word after it; the checks below keep the
493 tag inert anywhere else all the same.
495 The report is made here, where the pair is read, and only for a
496 name written undivided: there W1's longest listed surname first
497 reads the given name from the second surname on, and that is the
498 reading not taken, while a writer's space after the pair has
499 already said where the given name starts."""
500 if state.order is None or state.order[0] is Role.GIVEN:
501 return None
502 _, given, middle, family = _positions(tokens)
503 units = _units(tokens, sorted(given + middle))
504 if (len(family) != 1 or not units or len(units[0]) != 1
505 or units[0][0] != family[0] + 1
506 or SURNAME_PAIR_TAG not in tokens[units[0][0]].tags):
507 return None
508 first, second = family[0], units[0][0]
509 _join_second_surname(tokens, family, units, state.order)
510 after = second + 1
511 if (after == len(tokens) or tokens[after].span is None
512 or tokens[after].span.start != tokens[second].span.end):
513 return None
514 word = (tokens[first].text + tokens[second].text
515 + tokens[after].text)
516 return PendingAmbiguity(
517 AmbiguityKind.SEGMENTATION,
518 f"{word!r} splits as {tokens[first].text!r} + "
519 f"{tokens[second].text!r} + {tokens[after].text!r} on two "
520 f"surnames; {tokens[first].text!r} + "
521 f"{tokens[second].text + tokens[after].text!r} also reads",
522 (first, second, after))
525def _one_given_word(tokens: list[WorkToken], given: list[int]) -> bool:
526 """Whether the given name is ONE word, as rules/O7 and P10 count it: O6's
527 count, save that a bound given-name pair (P5) is two words though
528 its bound word is also a particle the count passes over ('Abu
529 Bakr')."""
530 return (_family_word_count(tokens, given) == 1
531 and not (len(given) > 1
532 and "vocab:bound-given" in tokens[given[0]].tags))
535def _completes_a_given_name(tokens: list[WorkToken], unit: list[int],
536 lexicon: Lexicon) -> bool:
537 """Whether `unit` is a particle run and ONE word written
538 `after_particle` ('del Carmen'), the second of a compound given name
539 as rules/P10 reads one; a run holding a saint particle ('Santa
540 María') opens a surname instead."""
541 n = 0
542 while n < len(unit) and "particle" in tokens[unit[n]].tags:
543 if _normalize(tokens[unit[n]].text) in lexicon._sets.saint_particles:
544 return False
545 n += 1
546 return (0 < n == len(unit) - 1
547 and PARTICLE_GIVEN_TAG in tokens[unit[-1]].tags)
550def _given_phrase_middle(state: ParseState,
551 tokens: list[WorkToken]) -> None:
552 # rules/P10: "the particle run and that word are a middle name" --
553 # behind a lone given name of one word, read given-first with no
554 # family comma, the family's opening particle run and the one word
555 # after it, read off the tag classify gives a word of
556 # Lexicon.particle_given_names, where a name word is left for the
557 # family (decisions/P10, #704). In place, as every post rule writes.
558 # A family comma reads the phrase as a middle already, and a
559 # family-first order puts no family behind the given name.
560 # Called only for a name holding a marked word (post_rules' loop).
561 if (state.structure is Structure.FAMILY_COMMA or state.order is None
562 or state.order[0] is not Role.GIVEN):
563 return
564 _, given, middle, family = _positions(tokens)
565 # a given name no patronymic, shared name or mark, of one word, with
566 # no middle between it and the family
567 if (not given or not family or middle
568 or family[0] < given[-1]
569 or not _one_given_word(tokens, given)
570 or any(PATRONYMIC_TAG in tokens[i].tags
571 or SHARED_NAME_TAG in tokens[i].tags for i in given)
572 or not any(ch.isalpha() for i in given for ch in tokens[i].text)):
573 return
574 cut = _fold_reach(tokens, family)
575 if cut >= len(family) or not _completes_a_given_name(
576 tokens, family[:cut], state.lexicon):
577 return
578 # a name word left for the family: one with a letter that is no
579 # connective standing alone ('María del Carmen y')
580 if not any(any(ch.isalpha() for ch in tokens[i].text)
581 and "conjunction" not in tokens[i].tags
582 for i in family[cut:]):
583 return
584 # and no Turkic marker behind the phrase, whose father's name it
585 # would be
586 if (PatronymicRule.TURKIC in state.policy.patronymic_rules
587 and is_patronymic_marker(tokens[family[cut]].text)):
588 return
589 for i in family[:cut]:
590 _retag(tokens, i, Role.MIDDLE)
593def _take_second_given_name(state: ParseState,
594 tokens: list[WorkToken]) -> None:
595 # rules/O7: "the given name takes the middle unit beside it" --
596 # O6's mirror, read after O6 has taken its word and before O3
597 # folds what is left. In place, as every post rule writes.
598 # History: decisions/O7 (#704).
599 # A family comma leaves the given part laid out given-first; without
600 # one, an order the script set (W4) is not the caller's, as for O6.
601 if state.structure is Structure.FAMILY_COMMA:
602 given_first = True
603 elif state.order is None or state.order != state.policy.name_order:
604 return
605 else:
606 given_first = state.order.index(Role.GIVEN) < state.order.index(
607 Role.MIDDLE)
608 _, given, middle, _ = _positions(tokens)
609 # a given name already holding two words keeps what it has: a
610 # hyphenated one, or a bound given-name pair (P5), counted as O6
611 # counts the family
612 if not given or not middle or not _one_given_word(tokens, given):
613 return
614 # the unit beside the given name: the positions lay a given-first
615 # part out given, middles..., and FAMILY_FIRST_GIVEN_LAST's
616 # middles..., given, so it is the first middle unit or the last
617 units = _units(tokens, middle)
618 take = units[0] if given_first else units[-1]
619 # ONE name word, counted as O6 counts one, with a letter in it; an
620 # initial is taken, since it writes a given name. A given name with
621 # no letter in it is a mark or a number, nothing to complete.
622 if (_family_word_count(tokens, take) != 1
623 or not any(ch.isalpha() for i in take for ch in tokens[i].text)
624 or not any(ch.isalpha() for i in given for ch in tokens[i].text)):
625 return
626 # a patronymic is no given name, on either side of the join: a
627 # patronymic particle's run, its reading decided at classify (R9's
628 # vocabulary arm), whether it is the unit taken or the given name
629 # itself; a word an ending rule a policy turns on reads as one; and
630 # a Turkic marker's father
631 if any(PATRONYMIC_TAG in tokens[i].tags for i in take + given):
632 return
633 # rules/O7: "a name a community gives every member is no given
634 # name" -- Kaur and Singh, on either side of the join, tagged at
635 # classify from Lexicon.shared_names (decisions/O7, #704)
636 if any(SHARED_NAME_TAG in tokens[i].tags for i in take + given):
637 return
638 rules = state.policy.patronymic_rules
639 if rules:
640 slavic = PatronymicRule.EAST_SLAVIC in rules
641 turkic = PatronymicRule.TURKIC in rules
642 after = take[-1] + 1
643 if any(slavic and has_patronymic_ending(tokens[i].text)
644 or turkic and is_patronymic_marker(tokens[i].text)
645 for i in take) or (
646 turkic and after < len(tokens)
647 and is_patronymic_marker(tokens[after].text)):
648 return
649 for i in take:
650 _retag(tokens, i, Role.GIVEN)
653# rules/R10: "the family's second name unit, the one written second,
654# makes the second-surname view" -- whichever of the two O6 took
655# (decisions/R10, #754); and rules/R11: "the family's first name unit,
656# the one written first, makes the first-surname view" -- the other one,
657# marked in the same place (decisions/R11, #1022). It moves a unit into
658# the family as it marks it, one copy per word (frame budget).
659def _mark_surname(tokens: list[WorkToken], unit: list[int],
660 tag: str) -> None:
661 for i in unit:
662 tokens[i] = copy_with(tokens[i], role=Role.FAMILY,
663 tags=tokens[i].tags | {tag})
666def _addressing_run(titles: list[int], name_word: int) -> list[int]:
667 """The title run H1 asks about: the LEADING one where one stands,
668 else the whole (trailing) run -- rules/H1 -- the run standing
669 BEFORE the one name word being the run that addresses, and a run
670 standing behind it deciding that word's field only when none
671 stands before.
673 Every title token is in the TITLE role by the time this runs, both
674 ends of `Sir John Prof.` among them, so `titles` is not a run --
675 keeping the two ends apart is what makes a trailing title
676 TRANSPARENT (rules/H5): `Sir John Prof.` is `Sir John` plus a
677 title, and reading both ends as one run keyed 'sir prof' made
678 adding the title flip the name word's field (#489, #316).
680 The split is at the NAME WORD, not at the first token of another
681 role, and the difference is a nickname or a maiden name written
682 among the titles. H1's own rationale says what stands beside the
683 name word "does not make the name any longer, so it does not
684 decide this reading", and that has to hold for WHICH run
685 addresses as well as for how many words the name has: `Dr.
686 'Smitty' Sir John` is one run written around a nickname and reads
687 given 'John' as `Dr. Sir John` does, while `'Smitty' Dr. Jones
688 Sir.` keeps family 'Jones' as `Dr. Jones Sir.` does. Splitting on
689 the first non-title token got both wrong (measured 2026-09-09).
691 Called only from inside H1's guard, after the role counts have
692 short-circuited, so a name with both a given and a family never
693 builds this list; `name_word` is the first GIVEN, or the first
694 FAMILY in the direction #720 added. The given guard has proved it
695 is the only name word there is; the family guard proves only that
696 no given or middle stands beside it, which is where a comma or
697 O5's convention set to the family leaves one name word."""
698 return [i for i in titles if i < name_word] or titles
701def _family_settled(state: ParseState, tokens: list[WorkToken],
702 families: list[int]) -> bool:
703 """Whether something other than O5's convention placed this lone family
704 word, so H1's family -> given direction leaves it (#720). After a
705 comma a particle in it is settled -- rules/C3's "the comma settles
706 P1's fork for a leading particle that could be a given name"
707 ('van Gogh, Sir'); without one, P1's fold below takes a never-given
708 particle opening the name with a word behind it ('Sir de Mesnil'),
709 or under a family-first order a piece `_family_first_lead_folds`
710 names ('Sir van Berg', rules/P11), asked by the fold's own
711 predicates on the same piece. An ambiguous particle
712 nothing settled, or a bare one with nothing to fold, is a
713 word O5's convention placed, which reaches here only with
714 `Policy.lone_name_word` set to family ('Sir Van', 'Sir Abu Bakar',
715 'Sir de', and 'Sir von und zu' under the default order, whose
716 never-given 'zu' does not open it)."""
717 if state.structure is Structure.FAMILY_COMMA:
718 return any("particle" in tokens[i].tags for i in families)
719 if len(families) < 2:
720 return False
721 lead = _leading_name_piece(state, tokens)
722 return (is_lone_never_given_particle(lead, tokens)
723 or state.order is not None and state.order[0] is Role.FAMILY
724 and _family_first_lead_folds(state, tokens, lead))
727def _family_first_lead_folds(state: ParseState, tokens: list[WorkToken],
728 lead: Sequence[int],
729 reach: int | None = None) -> bool:
730 """Whether rules/P1's fold takes `lead`, the piece opening the
731 name, beside the lone never-given particle it takes under every
732 order (asked inline by each caller first, sparing every parse a
733 frame), and asked only under a family-first order: two more -- a lone ambiguous particle written
734 in lower case in a name of mixed case (rules/P11, #670), and a
735 particle chain a title-particle in front lets P2 build (rules/P4's
736 Except, #828), cut after its particles and one name unit unless a
737 suffix-or-name report names a word past that reach ('St de Mesnil
738 Jean V'). A chain the reach already covers is left whole ('St Sir de
739 Mesnil', 'St abu Bakar'). `reach` is `_fold_reach` over `lead`
740 where the caller has it already."""
741 # rules/P1: "a particle chain behind a title-particle is cut as the
742 # run is" -- the chain clause; and rules/P11: "an ambiguous particle
743 # written in lower case, in a name of mixed case, opens the name as
744 # a never-given one does" -- the writing clause (decisions/P1 and
745 # decisions/P11, #828, #670)
746 if is_lowercase_ambiguous_particle(lead, tokens, state.one_case):
747 return True
748 if len(lead) < 2 or "particle" not in tokens[lead[0]].tags:
749 return False
750 if reach is None:
751 reach = _fold_reach(tokens, list(lead))
752 # a word past the reach a suffix-or-name report names was read
753 # against the whole chain ('St de Mesnil Jean V', whose numeral had
754 # no word to spare): the cut would strand it, so the chain stands
755 return reach < len(lead) and not reported_suffix_fork(
756 state.ambiguities, lead[reach:])
759# rules/S1: "it is no nickname and no maiden name, and the words around
760# it read as they would beside a nickname" -- the clause's words
761# take their roles here, once the name's words are placed: extract has
762# already lifted a clause wholly of unambiguous non-name words out of
763# the name, and a member of the ambiguous class joins one only after
764# the name (rules/S13), and only where its writing or another word of
765# its clause settles it; decisions/S1 (#792, #1138)
766def _set_apart_clauses(state: ParseState,
767 tokens: list[WorkToken]) -> None:
768 """Give each word of a delimited clause wholly of non-name words
769 the role its vocabulary gives it: a title a title, a credential or
770 generation the suffix, and a word that is both a title in front of
771 the name's first word or in a name with no word of its own, and the
772 suffix anywhere else. A clause extract set apart may still be
773 declined here, keeping its pair's reading: titles alone behind a
774 word read as a suffix, a credential written as initials before a
775 name word of its comma part, a member of the ambiguous class not
776 after the name, or one neither its writing nor a credential of its
777 clause settles."""
778 first_name = last_name = -1
779 for i, t in enumerate(tokens):
780 if t.role in NAME_ROLES:
781 last_name = i
782 if first_name < 0:
783 first_name = i
784 starts = [t.span.start for t in tokens]
785 lex = state.lexicon
786 # whether a name word stands after each token in its comma part,
787 # built once on first need, so a name of many clauses stays linear
788 ahead: list[bool] | None = None
789 for _, clause in state.extracted:
790 lo = bisect.bisect_left(starts, clause.start)
791 hi = bisect.bisect_left(starts, clause.end)
792 words = state.original[clause.start:clause.end].split()
793 # every word one token of the clause, as extract read them
794 if not words or hi - lo != len(words) or any(
795 tokens[i].role not in (Role.NICKNAME, Role.MAIDEN)
796 for i in range(lo, hi)):
797 continue
798 kinds: list[SetApartKind] = []
799 member = False
800 unsettled = settled = False
801 # the clause's credential words, asked `settles_a_member` only
802 # where an unsettled member needs one
803 others: list[str] = []
804 for i in range(lo, hi):
805 token = tokens[i]
806 kind = set_apart_kind(token.text, lex)
807 # a member its periods settle, or a joined word whose own
808 # chunks settle its member, settles the others, as does a
809 # credential that is no name (`settles_a_member`); one its
810 # capitals settle does not, so the clause reads as its
811 # joined spelling, whose chunks are never read by case
812 # (#1138)
813 if kind == "member":
814 kind, member, settled = "suffix", True, True
815 elif kind is None and AMBIGUOUS_ACRONYM_TAG in token.tags \
816 and settled_by_case(token, state.one_case):
817 kind, member = "suffix", True
818 elif kind is None and member_word(token.text, lex):
819 kind, member, unsettled = "suffix", True, True
820 elif kind is None:
821 break
822 else:
823 others.append(token.text)
824 kinds.append(kind)
825 else:
826 # rules/S1: "a member its writing leaves unsettled is one
827 # too where a credential of the same clause settles it" --
828 # a credential that is no name, or a member its periods
829 # settle (decisions/S1, #1138); asked only of a clause
830 # holding such a member, so one with none pays nothing
831 if unsettled and not settled:
832 for text in others:
833 if settles_a_member(text, lex):
834 break
835 else:
836 continue
837 # a member is settled after the name only (rules/S13), and
838 # a credential written as initials before a name word of its
839 # comma part stays the nickname
840 if member and not 0 <= last_name < lo:
841 continue
842 # a name word after the clause in its comma part stands
843 # behind the last name word, so a clause after the name skips
844 # the walk; a title is never initials, asked inline
845 if last_name >= hi and any(
846 kind != "title"
847 and reads_as_initials(w, kind, state.one_case, lex)
848 for w, kind in zip(words, kinds)):
849 if ahead is None:
850 ahead = _name_ahead(state, tokens)
851 if ahead[hi - 1]:
852 continue
853 # the token in front of the clause
854 if not set_apart(words, kinds, lo > 0
855 and tokens[lo - 1].role is Role.SUFFIX, lex):
856 continue
857 for i, kind in zip(range(lo, hi), kinds):
858 title = kind == "title" or (
859 kind == "dual" and (first_name < 0 or i < first_name))
860 # a comma the clause kept on its word (rules/N1) is no
861 # part of a title or a suffix: the word's span is
862 # trimmed, so its text stays the original slice, and
863 # R1's entry pass reads the comma left between two
864 # suffix words
865 token = tokens[i]
866 text = token.text.strip(COMMA_STRIP)
867 start = token.span.start + token.text.index(text)
868 tokens[i] = copy_with(
869 token, role=Role.TITLE if title else Role.SUFFIX,
870 text=text, span=Span(start, start + len(text)))
873def _name_ahead(state: ParseState, tokens: list[WorkToken]) -> list[bool]:
874 """For each token, whether a name word stands after it before the
875 next comma: one walk from the end, each comma part's answer reset
876 where the part changes."""
877 ahead = [False] * len(tokens)
878 seen = False
879 part = None
880 for i in range(len(tokens) - 1, -1, -1):
881 here = comma_bucket(tokens[i].span.start, state.comma_offsets)
882 if here != part:
883 part, seen = here, False
884 ahead[i] = seen
885 if tokens[i].role in NAME_ROLES:
886 seen = True
887 return ahead
890# rules/S13: "where the member's writing settles it as the credential,
891# the way the rules that read the writing say a credential is written,
892# it is no nickname but the suffix" -- the settling itself is
893# `_set_apart_clauses`', which reads a lone member whose writing
894# settles it after the name as a clause of one non-name word; what is
895# left here is the report on a member it left the nickname
896# (decisions/S13 (#810), decisions/S1 (#792))
897def _report_delimited_members(tokens: list[WorkToken], lexicon: Lexicon,
898 ambiguities: list[PendingAmbiguity]) -> None:
899 """Report the fork on every member of the ambiguous class left a
900 nickname: `_set_apart_clauses`, run first, has made each member its
901 writing settles after the name the suffix, so a member still a
902 nickname is one it did not settle, a dotted member or a joined word
903 holding a member ('MA/PhD') inside the name included. Reported
904 here, where the clause is decided, rather than in extract, which
905 has no token index to point at. A dotted or joined member carries
906 no tag in a clause (classify tags a shape only outside one), so
907 each is found by its text, by `dotted_member` or `joined_member` as
908 the settling finds it, whatever else the vocabulary reads it as."""
909 for i, token in enumerate(tokens):
910 if token.role is not Role.NICKNAME:
911 continue
912 # the period and the joiners asked first, at C level, so a
913 # nickname written with neither pays no frame
914 if (AMBIGUOUS_ACRONYM_TAG in token.tags
915 or "." in token.text and (
916 dotted_member(token.text, lexicon)
917 or period_member(token.text, lexicon))
918 or ("/" in token.text or "-" in token.text)
919 and joined_member(token.text, lexicon)):
920 ambiguities.append(PendingAmbiguity(
921 AmbiguityKind.SUFFIX_OR_NICKNAME,
922 f"delimited {token.text!r} is also a post-nominal; read "
923 f"as a nickname rather than a suffix",
924 (i,), alternative=suffix_reading(token)))
927def _unattached(state: ParseState, tokens: list[WorkToken],
928 word: WorkToken, start: int) -> PendingReading:
929 """The name reading of the particle run P6 attaches at the end of
930 a name read family-first, which starts at token `start`: the field
931 the positional read gave it, or the field of the name piece in
932 front where that piece would take a name word in (`_name_alt`)."""
933 role = tokens[start].role
934 assert role is not None # assign roled every name word
935 # the run ends a name read family-first, so a name word stands in
936 # front of it
937 front, first = next(
938 ((piece, k == 0) for seg_idx in range(len(state.pieces))
939 for k, piece in enumerate(_name_pieces(state, tokens, seg_idx))
940 if start - 1 in piece), (None, False))
941 return _name_alt(word, role, front, tokens, first, False)
944def post_rules(state: ParseState) -> ParseState:
945 tokens = list(state.tokens)
946 ambiguities = list(state.ambiguities)
947 if state.extracted:
948 _set_apart_clauses(state, tokens)
949 _report_delimited_members(tokens, state.lexicon, ambiguities)
950 titles, givens, middles, families = _positions(tokens)
952 # rules/H1: "a title followed by exactly one name word makes
953 # that word the family name, whatever suffix, nickname or maiden
954 # name stands beside it, unless the title is a given-name title,
955 # which keeps it the given name; a run of several titles addresses
956 # as its last title does" -- counting suffix, nickname and maiden
957 # as further name words is what emptied the family (#410)
958 # (known gap: the guard tests which roles are unoccupied, it does
959 # not count units -- decisions/H1) (v1 handle_firstnames)
960 #
961 # rules/H1: "a run of several titles addresses as its last
962 # title does, and where a run stands BEFORE the one name word it
963 # is the run that addresses, a run standing behind it deciding
964 # that word's field only when none stands before" -- #489. WHICH
965 # run that is, and why the two ends of a name are not one, are
966 # _addressing_run's; its docstring carries the history.
967 if (titles and givens and not middles and not families
968 and not _run_addresses_by_given(
969 (tokens[i].text
970 for i in _addressing_run(titles, givens[0])),
971 state.lexicon._sets.given_name_titles)):
972 for i in givens:
973 _retag(tokens, i, Role.FAMILY)
974 # every rule below reads these lists; recompute after any
975 # retag so no guard can inspect a name that has already
976 # moved -- a stale index list is the bug shape #359 fixed
977 _, givens, middles, families = _positions(tokens)
978 # rules/H1: "a comma with no given name after it does not decide
979 # the field, nor does O5's convention" -- the other direction, for
980 # the shapes assign reads as the family before H1 can see them
981 # (#720): a family comma with no name word after it and one before
982 # it, and the one name word O5's convention read as the family
983 # (`Policy.lone_name_word` set to family). The comma named no given
984 # part, and the convention places a word nothing else decided, so
985 # the title decides as it does in 'Sir John' ('John III, Sir' reads
986 # as 'Sir John III'), unless
987 # something else settled the word (_family_settled).
988 elif (titles and families and not givens and not middles
989 and not _family_settled(state, tokens, families)
990 and _run_addresses_by_given(
991 (tokens[i].text
992 for i in _addressing_run(titles, families[0])),
993 state.lexicon._sets.given_name_titles)):
994 for i in families:
995 _retag(tokens, i, Role.GIVEN)
996 # recomputed for H1's reason, stated above
997 _, givens, _, families = _positions(tokens)
999 # rules/M4: "a maiden name standing beside exactly one name
1000 # word makes that word the family name, whatever suffix or
1001 # nickname stands beside it" (#445) -- the MAIDEN role, not a
1002 # marker: M1's configured pair produces one with no marker
1003 # anywhere ("Smith (Jones)" under maiden_delimiters), and the
1004 # rationale carries over unchanged, so the rule is keyed on the
1005 # maiden name as the statement is.
1006 #
1007 # H1's sibling, and placed under it so the interaction the rule
1008 # states is decidable by reading: where H1 fired, `givens` is
1009 # empty here and this cannot fire; where H1 declined because the
1010 # title addresses by given name, `titles` is what keeps this rule
1011 # off the same word. A titled name is H1's at both outcomes.
1012 #
1013 # Sibling EXCEPT in what it counts, and that is a known gap
1014 # rather than a boundary: this guard counts GIVEN tokens where
1015 # H1 counts nothing (it tests which roles are unoccupied), so a
1016 # name word another rule joined counts as several here and the
1017 # rule declines -- `Dr. Dean of Chemistry` reads family, while
1018 # `Dean of Chemistry née Jones` keeps given 'Dean of Chemistry'.
1019 # rules/P3 says a joined part is one name word wherever
1020 # another rule counts them, so the two disagree; widening moves
1021 # no corpus name and is not this change's to make
1022 # (decisions/M4).
1023 # rules/M4: "a bound given name keeps the reading O5's convention
1024 # gives it, and so does a word read as an initial"
1025 # -- read off the tags classify already recorded rather than a
1026 # predicate of this rule's own, because this rule changes what
1027 # POSITION decided and must not reach what a word IS
1028 # (mechanisms.md#TWO-LAYER-ASSIGN).
1029 if (not titles and len(givens) == 1 and not middles and not families
1030 and any(t.role is Role.MAIDEN for t in tokens)
1031 and not (_NEVER_FLIPPED & tokens[givens[0]].tags)):
1032 _retag(tokens, givens[0], Role.FAMILY)
1033 # recomputed for H1's reason, stated at H1: a stale index list
1034 # is the bug shape #359 fixed
1035 _, givens, middles, families = _positions(tokens)
1037 # rules/P1: "the particle run and the name words it takes are
1038 # family, under every name order" -- its Then clause, for a
1039 # never-given particle opening the name and, under a family-first
1040 # order, the two pieces `_family_first_lead_folds` adds; how many words the run
1041 # takes is set by the order or a family comma, in P1's bullets and
1042 # below (v1 handle_non_first_name_prefix; decisions/P1)
1043 # How far the fold reaches depends on the order the name was READ
1044 # under (#395; decisions/P1, 2026-08-17): declaring a
1045 # family-first order asserts that what follows the family is not
1046 # more surname, which is the very question of where the run stops.
1047 # Under that declaration the run takes its own particles and ONE
1048 # name word; under the default order it keeps taking the rest of
1049 # the name, nothing having marked where the surname ends. Note what
1050 # actually holds the family-comma shape back, since it is NOT
1051 # that test -- "Smith, de Mesnil Jean" DOES fire the leading site,
1052 # segment 1 opening with the particle. It keeps the old reach
1053 # because assign records no order after a family comma, the comma
1054 # having already fixed the surname, so `order is None` here.
1055 # Anything that later gives that path an order turns the
1056 # narrowing on for it.
1057 # ONE site, the opening piece. A particle standing in the GIVEN
1058 # position was a second site until #467, and it was the parser
1059 # overriding a declaration rather than reading one: under a
1060 # family-first order that slot holds what the caller SAID is the
1061 # given name, and the never-given vocabulary supplies a default
1062 # where position leaves the question open, not a veto over an
1063 # answer position already gave. So "Menil de" under either
1064 # family-first order reports given "de" -- P6 takes the trailing
1065 # particle only where it landed in a MIDDLE, which means nothing.
1066 # Code-local: a lone PIECE is the test (`_family_first_lead_folds` names the two
1067 # pieces a family-first order adds), so a particle group already
1068 # chained forward is not a lone particle elsewhere, and rule H1
1069 # above cannot be what produces the fold's family reading -- its
1070 # given -> family branch is gated on `not families`, and its family
1071 # -> given branch leaves the word this fold takes, asking the same
1072 # predicate (_family_settled).
1073 lead = _leading_name_piece(state, tokens)
1074 family_first = state.order is not None and state.order[0] is Role.FAMILY
1075 reach = None
1076 if (family_first and len(lead) > 1
1077 and "particle" in tokens[lead[0]].tags):
1078 # the words past the fold's reach, marked for assemble's
1079 # alternatives whether or not the fold cuts there (#828)
1080 reach = _fold_reach(tokens, list(lead))
1081 for i in lead[reach:]:
1082 tokens[i] = copy_with(tokens[i],
1083 tags=tokens[i].tags | {PAST_REACH_TAG})
1084 lead_fires = (is_lone_never_given_particle(lead, tokens)
1085 or family_first
1086 and _family_first_lead_folds(state, tokens, lead, reach))
1087 if len(givens) + len(middles) + len(families) > 1 and lead_fires:
1088 order = state.order
1089 if order is not None and order[0] is Role.FAMILY:
1090 # `state.order`, not policy.name_order: a script_orders
1091 # entry can put the family first under a given-first
1092 # policy, and the roles below have to match the read
1093 # assign actually made.
1094 name_idx = sorted(givens + middles + families)
1095 cut = _fold_reach(tokens, name_idx)
1096 for i in name_idx[:cut]:
1097 _retag(tokens, i, Role.FAMILY)
1098 # What is left is a shorter name of the same order: one
1099 # family already placed, so drop that slot and lay the
1100 # rest out as _name_positions would for n + 1 pieces.
1101 rest = _units(tokens, name_idx[cut:])
1102 for unit, role in zip(rest, _name_positions(
1103 order, len(rest) + 1)[1:]):
1104 for i in unit:
1105 _retag(tokens, i, role)
1106 else:
1107 for i in givens + middles:
1108 _retag(tokens, i, Role.FAMILY)
1109 # downstream rules key on the role counts: recompute
1110 _, givens, middles, families = _positions(tokens)
1112 # v1 gates both rotations on `not self._had_comma`; the
1113 # middle_as_family fold below runs comma or not (v1 order:
1114 # patronymics first, then handle_middle_name_as_last)
1115 rules = state.policy.patronymic_rules
1116 # gated on the rules, which a default parse leaves empty, so it
1117 # asks nothing (decisions/parse-cost, #1170)
1118 rotations_apply = bool(rules) and _rotations_apply(state)
1119 # rules/O1: "a name of exactly three name words — titles,
1120 # suffixes and nicknames aside — whose last name word carries a
1121 # patronymic ending and whose middle name word does not reads as
1122 # family-first" (history: decisions/O1)
1123 if rotations_apply and PatronymicRule.EAST_SLAVIC in rules and \
1124 len(givens) == 1 and len(middles) == 1 and len(families) == 1:
1125 tail = tokens[families[0]].text
1126 mid = tokens[middles[0]].text
1127 # rules/O1: "T5 wins, and T4 in a name written wholly in one
1128 # case" -- an initial first or in the middle keeps the order,
1129 # read off classify's tag, which carries both rules' reading
1130 # (so a one-case name's two spellings read alike, a letter P8
1131 # reads as a connective rotating in either; decisions/O1,
1132 # #1185, #1217)
1133 if (east_slavic_rotates(mid, tail)
1134 and "initial" not in tokens[givens[0]].tags
1135 and "initial" not in tokens[middles[0]].tags):
1136 g, m, f = givens[0], middles[0], families[0]
1137 _retag(tokens, m, Role.GIVEN)
1138 _retag(tokens, f, Role.MIDDLE)
1139 _retag(tokens, g, Role.FAMILY)
1140 # recomputed for H1's reason, stated at H1: a stale index
1141 # list is the bug shape #359 fixed
1142 _, givens, middles, families = _positions(tokens)
1143 # rules/O2: "a name of exactly four name words — titles,
1144 # suffixes and nicknames aside — ending in a standalone
1145 # patronymic marker reads family-first: the first name word is
1146 # the family name" (history: decisions/O2)
1147 if rotations_apply and PatronymicRule.TURKIC in rules and \
1148 len(givens) == 1 and len(middles) == 2 and len(families) == 1:
1149 tail = tokens[families[0]].text
1150 if is_patronymic_marker(tail):
1151 g, m1, m2, f = givens[0], middles[0], middles[1], families[0]
1152 _retag(tokens, m1, Role.GIVEN)
1153 _retag(tokens, m2, Role.MIDDLE)
1154 _retag(tokens, f, Role.MIDDLE)
1155 _retag(tokens, g, Role.FAMILY)
1156 # recomputed for H1's reason, stated at H1: a stale index
1157 # list is the bug shape #359 fixed
1158 _, givens, middles, families = _positions(tokens)
1159 # rules/P6: "a particle ending the name attaches to that family
1160 # name and is written before it" -- the no-comma site (#467).
1161 # A comma is not the only thing that names the family; a declared
1162 # family-first order does too, and the same listing is written
1163 # both ways ("Jong, Anke de" and "Jong Anke de" under
1164 # FAMILY_FIRST both give family "de Jong").
1165 #
1166 # Keyed on the SLOT the run landed in, not on its vocabulary.
1167 # MIDDLE is the one position that means nothing here: middles are
1168 # further given names, and a particle is not one. Only
1169 # FAMILY_FIRST puts a trailing piece there --
1170 # FAMILY_FIRST_GIVEN_LAST puts it in GIVEN, where the caller's own
1171 # declaration says it IS the given name, and P1's site above no
1172 # longer overrides that either.
1173 #
1174 # Which is why no vocabulary test appears here and none is wanted.
1175 # The comma path needs one because a comma cannot separate the
1176 # Dutch reading from the Vietnamese; the declared ORDER can, when
1177 # the caller declares the right one for the name: "Beethoven
1178 # Ludwig van" under FAMILY_FIRST gives family "van Beethoven"
1179 # though `van` is ambiguous vocabulary, and "Nguyen Thi Van" under
1180 # FAMILY_FIRST_GIVEN_LAST keeps given "Van" though the same word
1181 # is in the same set. (Under the WRONG order each loses: the
1182 # Vietnamese name read as FAMILY_FIRST gives family "Van Nguyen".
1183 # rules/P6 records that as accepted -- it is one order, not
1184 # both.) A never-given test here would have excluded every
1185 # ambiguous particle -- von, di, da, del, le and `van` itself,
1186 # the flagship word of the listing this rule is named for.
1187 #
1188 # NO RE-LAYOUT, and that is a property of ONE order rather than a
1189 # simplification: under FAMILY_FIRST the roles run family, given,
1190 # middle, middle..., so dropping a trailing MIDDLE leaves every
1191 # other piece's role untouched. It is not true of
1192 # FAMILY_FIRST_GIVEN_LAST (family, middle..., given) and not true
1193 # of the default order, which is why this site tests the order
1194 # rather than trusting the slot to imply it -- review found the
1195 # first draft firing on 52 default-order names, where a
1196 # conjunction stops a particle's forward chain and leaves it
1197 # standing in a middle ("Maria Luisa y de la Cruz"). An earlier
1198 # draft still (#466) removed a piece and re-laid the leftover out,
1199 # which lost a given name outright on "van Berg Jan de" and
1200 # promoted a post-nominal into the given slot on
1201 # "Berg Jan Jr. de"; neither is reachable without a re-layout.
1202 #
1203 # The family must still hold a base of its own -- R2's invariant,
1204 # that a non-empty family has a non-empty base, not a ban on
1205 # particle-before-particle. Without it "van Berg Jan de" reports
1206 # family 'de van', whose words R2 reads as ordinary name words,
1207 # and no rule reorders those. It does NOT stop a second particle
1208 # joining a family that merely OPENS with one: "de Mesnil Jan de"
1209 # gives 'de de Mesnil', which keeps its base and is accepted.
1210 # every retagging block above recomputes the lists, so they are
1211 # current here (a stale index list is the bug shape #359 fixed)
1212 names = givens + middles + families
1213 if middles and families and state.order is not None:
1214 run: list[int] = []
1215 i = len(middles)
1216 while i and "particle" in tokens[middles[i - 1]].tags:
1217 i -= 1
1218 run.append(middles[i])
1219 # ENDING the name is the rule's own word, and it is the whole
1220 # test: the trailing MIDDLE is not always the trailing NAME
1221 # word. Under FAMILY_FIRST_GIVEN_LAST the given name stands
1222 # behind the middles, and under the default order the family
1223 # does, so in both a middle that ends the name is impossible
1224 # and this declines without asking which order was declared.
1225 # Only FAMILY_FIRST can put a name's last word in a middle.
1226 #
1227 # A draft tested the SLOT alone and fired on 52 default-order
1228 # names, where a conjunction stops a particle's forward chain
1229 # and leaves it standing in a middle ("Maria Luisa y de la
1230 # Cruz"), and on 366 FAMILY_FIRST_GIVEN_LAST middles with the
1231 # given name still behind them. A later draft tested the order
1232 # as well; measured over 542,592 generated parses, that second
1233 # test never decides anything this one has not already decided,
1234 # so it is not here.
1235 trailing = bool(run) and max(run) == max(names)
1236 if trailing and any("particle" not in tokens[j].tags
1237 for j in families):
1238 # mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE: "Emit at
1239 # the site that takes the branch, not where an ambiguous
1240 # tag sits". decisions/P6 (#405) settled that this
1241 # attachment reports the fork it decides, and the state it
1242 # fixed is the one this site would otherwise recreate --
1243 # `Beethoven, Ludwig van` reporting while `Beethoven Ludwig
1244 # van` under FAMILY_FIRST decides the identical fork in
1245 # silence. ONE arm here where the comma path has two: the
1246 # suffix reading stands without a comma, so no post-nominal
1247 # reading is ever overridden for this site to report.
1248 ambiguous = [j for j in run
1249 if "vocab:particle-ambiguous" in tokens[j].tags]
1250 if ambiguous:
1251 word = tokens[ambiguous[0]].text
1252 text = " ".join(tokens[j].text for j in sorted(run))
1253 ambiguities.append(PendingAmbiguity(
1254 AmbiguityKind.PARTICLE_OR_GIVEN,
1255 f"{word!r} is both a family-name particle and an "
1256 f"ordinary given name; ending a name read "
1257 f"family-first, {text!r} joins the family that "
1258 f"order named rather than standing as a name word "
1259 f"of its own",
1260 tuple(sorted(run)),
1261 # the field the positional read gave it, which this
1262 # attachment moves, unless the piece in front would
1263 # take a name word in
1264 alternative=_unattached(state, tokens,
1265 tokens[ambiguous[0]],
1266 min(run))))
1267 for j in run:
1268 tokens[j] = copy_with(
1269 tokens[j], role=Role.FAMILY,
1270 tags=tokens[j].tags | {FOLDED_TAG, ATTACHED_TAG})
1271 # recomputed for H1's reason, stated at H1: a stale index
1272 # list is the bug shape #359 fixed
1273 _, givens, middles, families = _positions(tokens)
1275 # every parse asks these, so a name holding no marked word pays one
1276 # plain loop and no call (the parse-cost band, decisions/parse-cost);
1277 # P10 runs first, and adds no surname pair's tag
1278 phrase = pair = False
1279 for token in tokens:
1280 if PARTICLE_GIVEN_TAG in token.tags:
1281 phrase = True
1282 if SURNAME_PAIR_TAG in token.tags:
1283 pair = True
1284 if phrase:
1285 _given_phrase_middle(state, tokens)
1286 if pair and (report := _read_surname_pair(state, tokens)) is not None:
1287 ambiguities.append(report)
1288 if state.policy.initials_as_family:
1289 _take_trailing_initials(state, tokens)
1290 if state.policy.two_family_names:
1291 _take_second_family_name(state, tokens)
1292 if state.policy.two_given_names:
1293 _take_second_given_name(state, tokens)
1295 # rules/O3: "every middle word joins the family name and is
1296 # rendered before it" (v1 handle_middle_name_as_last). v1
1297 # PREPENDED middle_list to last_list; mechanisms.md#FOLDED_TAG:
1298 # "tokens never move: a rule that needs different rendering order
1299 # tags the token, and the rendering views consult the tag"
1300 #
1301 # The lists walked again, for this fold and R2 below: P10, W5's
1302 # pair, O6 and O7 above retag without updating them.
1303 _, givens, middles, families = _positions(tokens)
1304 if state.policy.middle_as_family:
1305 # rules/R11: "middle words folded into the family belong to
1306 # neither unit" -- where O6 marked no units the family before the
1307 # fold is the first, marked here so the views can leave the
1308 # folded words out; only a fold that moves a word pays for it
1309 # (decisions/R11, #1022). A family O6 split carries its marks
1310 # already; a loop, not a generator, which costs a frame
1311 if middles:
1312 for i in families:
1313 if FIRST_SURNAME_TAG in tokens[i].tags:
1314 break
1315 else:
1316 _mark_surname(tokens, families, FIRST_SURNAME_TAG)
1317 for i in middles:
1318 tokens[i] = copy_with(
1319 tokens[i], role=Role.FAMILY,
1320 tags=tokens[i].tags | {FOLDED_TAG})
1321 # the lists as a walk would now find them, for R2
1322 families = sorted(families + middles)
1323 middles = []
1324 # rules/R2: "a name part whose every word is particle
1325 # vocabulary is a part where none of them is doing a particle's
1326 # work — nothing joins them to a name — so they read as ordinary
1327 # name words"
1328 #
1329 # Last of the rules that MOVE a token, because every rule above
1330 # can still move one between parts:
1331 # P1's fold, P6's attachment and O3's fold all rewrite roles, and
1332 # this reads the roles they settle on.
1333 #
1334 # Marked, not untagged: `particle` is stable API and says the word
1335 # IS particle vocabulary wherever it lands, which stays true.
1336 #
1337 # All three roles for uniformity with the rule, not because all
1338 # three are observable: no view filters tags on GIVEN (initials
1339 # exempt that role outright), so restricting this loop to MIDDLE
1340 # and FAMILY moves 0 of 4,506 parses. The GIVEN arm is marked so a
1341 # future view reading the mark gets a consistent answer.
1342 #
1343 # rules/R3: "A part holding nothing else is a part where it is
1344 # joining nothing, and there it initials like any other name word"
1345 # -- #461's mark, decided HERE for the same reason R2's is: the
1346 # question is about the PART, and a view that re-derives a
1347 # part-level fact is how initials() and family_base came apart.
1348 #
1349 # ONE walk decides both marks -- a plain loop, not two generator
1350 # expressions: this runs per role on every parse and a genexp
1351 # costs a frame of its own (frame budget). An EMPTY role needs no
1352 # `continue` ahead of the walk, which is inert on one: measured
1353 # 2026-09-20, py3.11, one survived the whole suite and moved
1354 # neither frame count (parse=406.00 facade=443.00 either way).
1355 # `bool(part)` says the same thing about the R2 arm as
1356 # `_types._remarked`'s twin of this walk does.
1357 #
1358 # rules/R2: "A connective that is not also particle vocabulary,
1359 # standing between two words of the part, is set aside when the
1360 # part is asked whether every word is particle vocabulary" --
1361 # `between` is the part's own first and last word excluded, and a
1362 # set-aside connective is left unmarked, so it stays in the base,
1363 # initials nothing and keeps its case (decisions/R2, #939). A set,
1364 # since the marking loop asks it per word: a list made a part
1365 # alternating particle and connective quadratic (test_benchmark's
1366 # `connective_between_particles` row).
1367 for part in (givens, middles, families):
1368 all_particle = bool(part)
1369 conj: list[int] = []
1370 between: set[int] = set()
1371 others = 0
1372 last = len(part) - 1
1373 for n, i in enumerate(part):
1374 tags = tokens[i].tags
1375 is_conj = "conjunction" in tags
1376 if is_conj:
1377 conj.append(i)
1378 if "particle" not in tags:
1379 if is_conj and 0 < n < last:
1380 between.add(i)
1381 continue
1382 all_particle = False
1383 if not is_conj:
1384 others += 1
1385 if all_particle:
1386 for i in part:
1387 if i not in between:
1388 tokens[i] = copy_with(
1389 tokens[i], tags=tokens[i].tags | {UNJOINED_TAG})
1390 elif conj and not others:
1391 # #461: nothing here for the connective to join. The `elif`
1392 # is deliberate -- where the part is all-particle R2's mark
1393 # already readmits every word but a connective it set aside,
1394 # a word that is BOTH particle and connective included
1395 # (never set aside, being a particle), which is what keeps
1396 # a caller's `add("y", kind="particle")` readings unchanged;
1397 # and it keeps this mark off a connective R2 set aside, which
1398 # joins the particles around it and initials nothing.
1399 for i in conj:
1400 tokens[i] = copy_with(
1401 tokens[i],
1402 tags=tokens[i].tags | {UNJOINED_CONJUNCTION_TAG})
1403 _mark_suffix_entries(tokens, state)
1404 return copy_with(state, tokens=tuple(tokens),
1405 ambiguities=tuple(ambiguities))