Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Stage: assign.
3Consumes: pieces + piece_tags (grouped), segments, structure, tokens.
4Produces: tokens with roles set on every main-stream token.
5Reads: Policy.name_order (#270), is_suffix_lenient on the trailing
6piece of a two-part comma name, and Policy.script_orders (#271, which
7overrides it when every name piece is written wholly in one script, or
8in the Han/Hiragana/Katakana repertoire the #272 kana license shares
9across pieces); token/piece tags; Lexicon only through tags already
10applied by classify (plus the leading-title period rule).
12Implements rules H2, H4, H5, N3, O4, O5 and W4 of docs/design/rules.md,
13each cited at its code below. Ports v1's assignment loops.
14NO_COMMA (per name_order):
15leading title pieces chain while no given-position name has been seen
16(a title needs a following piece, unless the whole name is one title);
17then positional assignment per name_order with the trailing-suffix
18rule: the piece from which everything after is a strict suffix is the
19last name-position piece, the rest are suffixes. Behind that peel a
20trailing run of period-marked title words chains into the title from
21the end, leaving one name piece standing; where the run TAKES
22something the peel was only provisional and runs again over the pieces
23with the titled ones spliced out, the two alternating until the run
24takes nothing, so a trailing title is transparent to the suffix
25reading however many titles are written (_pieces.tail_reading). Where
26the run takes nothing -- almost every name -- the first peel is the
27only one and its answer stands.
28The v1 single-name+nickname rule lives here (decisions.md#N3): a
29nonempty nickname beside exactly one piece in total puts that piece
30in FAMILY.
31FAMILY_COMMA: segment 0 wholly FAMILY (v1 parity) UNLESS segment 1
32holds no name word (titles and suffixes only), which fixed no family
33boundary -- there segment 0 takes the NO_COMMA positional read instead,
34order and all ('John Smith, Dr.', 'John Smith, Mr. Jr.'); segment
351 is wholly SUFFIX when it is nothing but suffix pieces ('Smith, Jr.',
36'Smith, Ph. D. Jr.' -- the credential run C1 describes, in the listing
37form), else gets leading titles, then the same trailing title run
38over the pieces this segment does not read as suffixes ('Smith, John
39Prof.'), then given, then middles with those suffix-reading pieces to
40suffix -- one predicate for both; segments 2+ are suffixes (lenient --
41segment already flagged non-suffixy ones COMMA_STRUCTURE).
42SUFFIX_COMMA: segment 0 as NO_COMMA; segments 1+ wholly SUFFIX.
43Emits PARTICLE_OR_GIVEN when the leading name piece is a lone
44particles_ambiguous token with more pieces following ("Van Johnson",
45and since #367 "Dr. Van Johnson" too, a title no longer displacing the
46particle out of that position) -- whatever role name_order assigns.
47Emits SUFFIX_OR_NAME at three sites: the trailing roman numeral, each
48ambiguous acronym the trailing peel had to resolve, and the bare-suffix
49carve-out where an input that is nothing but post-nominal vocabulary
50gets its first word made into the name (H4's suffix half, #491). And
51at the one site that places a LONE name word, GIVEN_OR_FAMILY for the
52field the convention picked (O5, #449) and TITLE_OR_NAME for the two
53shapes where the doubt is whether a word is a title instead (H4,
54#491): the peel leaving one title-vocabulary word standing, and a
55joined unit carrying title vocabulary.
56"""
57from __future__ import annotations
59import dataclasses
60from collections.abc import Sequence, Set
61from typing import NamedTuple
63from nameparser._lexicon import Lexicon
64from nameparser._pipeline._vocab import (
65 effective_script, is_suffix_lenient, resolve_script_set,
66)
67from nameparser._pipeline._pieces import (
68 is_suffix_piece, leading_titles, peel_walk, segment_suffix_reading,
69 tail_reading, trailing_titles,
70)
71from nameparser._pipeline._state import (
72 ParseState, PendingAmbiguity, Structure, WorkToken, _NEVER_FLIPPED,
73)
74from nameparser._policy import Policy, Script
75from nameparser._types import AmbiguityKind, Role
77def _set_roles(tokens: list[WorkToken], piece: tuple[int, ...],
78 role: Role) -> None:
79 for i in piece:
80 tokens[i] = dataclasses.replace(tokens[i], role=role)
83#: Tags that say the word's own reading was claimed before position
84#: could speak, so O5's convention decided nothing. Built from M4's
85#: `_NEVER_FLIPPED` pair rather than respelling it: the two sets answer
86#: different questions -- may M4 retag the word, and did anything
87#: decide the field -- and share the pair because a word vocabulary
88#: claimed as a given name, or wrote as an initial, answers both. The
89#: third, `particle`, is here alone because a lone particle's reading
90#: is P4's. None is a predicate this emitter owns: they are read off
91#: the tags classify already recorded (mechanisms.md#TWO-LAYER-ASSIGN).
92_WORD_ALREADY_CLAIMED = _NEVER_FLIPPED | frozenset({"particle"})
95# rules.md#H2: "an abbreviation opening the part of the name that
96# carries the given name — the whole name, or the part after a
97# family comma — reads as a title even when unlisted" -- the count is
98# _pieces.leading_titles since #424 (its test, is_leading_title, is
99# the leading-particle scan's too); the roles are set here.
100def _peel_leading_titles(pieces: tuple[tuple[int, ...], ...],
101 ptags: tuple[frozenset[str], ...],
102 tokens: list[WorkToken]) -> int:
103 """Assign TITLE to the leading title pieces and return the first
104 non-title index."""
105 n = leading_titles(pieces, ptags, tokens)
106 for k in range(n):
107 _set_roles(tokens, pieces[k], Role.TITLE)
108 return n
111class EffectiveOrder(NamedTuple):
112 """What _effective_order made of a name's scripts. `order` is the
113 order the positional read uses; `by_script` says a script_orders
114 entry RESOLVED it, which is not the same as `order` happening to
115 equal the declared name_order -- under a declared family-first
116 order a Han name's W4 entry and the declaration agree, and only
117 this flag distinguishes the script rule DECIDING the reading from
118 the caller's declaration standing unopposed. O5's convention
119 report reads it (#449)."""
121 order: tuple[Role, Role, Role]
122 by_script: bool
125# rules.md#W4: "a name written wholly in one East Asian script, or in
126# the kana-licensed Japanese repertoire, reads family-first whatever
127# order the caller declared; a wholly-katakana name keeps the declared
128# order" (history: decisions.md#W4)
129def _effective_order(policy: Policy,
130 pieces: Sequence[tuple[int, ...]],
131 name_pieces: Sequence[int],
132 tokens: list[WorkToken],
133 *, dot_divided: bool) -> EffectiveOrder:
134 """script_orders resolution (#271): when every name piece is
135 written wholly in ONE script that has an entry, that script's
136 order governs the positional read; anything else -- Latin, mixed
137 scripts, no entry -- falls back to name_order. A 间隔号-divided
138 name (`dot_divided`, #298) suppresses the whole lookup first: the
139 dot marks a transcription -- playing the role pure katakana plays
140 in the kana license, orthography naming the convention -- so the
141 license yields to name_order. Piece-level, after
142 title/suffix peeling: 'Dr. 毛泽东' is a wholly-Han NAME under a
143 Latin title. Kana-licensed tokens (高橋みなみ, #272) resolve to
144 HIRAGANA the same way a wholly-Han or wholly-Hangul token resolves
145 to its own script -- and so does a kana-licensed NAME split across
146 separately single-script PIECES ('高橋 みなみ', Han piece plus
147 Hiragana piece): resolve_script_set generalizes the license from
148 one token's characters to the whole found-script set below, which
149 is why Han+Hangul ('毛 김') still declines even though both
150 individually read family-first -- the license is specific to the
151 Han/Hiragana/Katakana repertoire, not "the entries happen to
152 agree".
154 Naming note, since the two are easy to conflate: THIS function
155 resolves the ORDER for a whole name; `_vocab.effective_script`
156 resolves the SCRIPT for a single token. This function calls that
157 one per token below.
159 Takes the segment's pieces and WHICH of them the name kept, the
160 shape every piece-layer predicate takes: a caller holding those
161 indices had to build a second list of the same pieces to hand
162 over otherwise, which on 3.11 is a comprehension frame on every
163 parse (decisions.md#parse-cost).
165 Returns an EffectiveOrder: the order triple, and `by_script` set only on
166 the one path where an entry answered. Every fallback below is a
167 script rule DECLINING, and reports it as such.
168 """
169 declared = EffectiveOrder(policy.name_order, by_script=False)
170 # #298 transcription marker -- see the docstring; codepoint-scoped
171 # (only U+00B7 records; decisions.md#T3)
172 if dot_divided:
173 return declared
174 if not policy.script_orders:
175 return declared
176 # Collect every token's script rather than comparing pairwise as
177 # tokens are seen: the kana license needs the WHOLE set (a Han
178 # piece and a Hiragana piece only license together, never one at a
179 # time), so resolution is deferred to resolve_script_set below.
180 found: set[Script] = set()
181 for piece_idx in name_pieces:
182 for i in pieces[piece_idx]:
183 script = effective_script(tokens[i].text)
184 if script is None:
185 # Latin, mixed, or a script with no entry: never a key
186 return declared
187 found.add(script)
188 resolved = resolve_script_set(found)
189 if resolved is None:
190 # e.g. Han+Hangul: two scripts, neither the kana license's
191 # Han/Hiragana/Katakana repertoire -- no single tradition
192 return declared
193 for script, order in policy.script_orders:
194 if script is resolved:
195 return EffectiveOrder(order, by_script=True)
196 # the resolved script has no entry: the declaration stands, and
197 # nothing about the writing system decided the reading
198 return declared
201# rules.md#O4: "words no vocabulary has claimed read by position. In
202# the default given-first order the first name word is the given name,
203# the last is the family name, and everything between is middle names"
204def _name_positions(order: tuple[Role, Role, Role],
205 count: int) -> list[Role]:
206 """Roles for `count` name pieces (titles/suffixes already peeled),
207 per name_order. GIVEN_FIRST: given, middles..., family.
208 FAMILY_FIRST: family, given, middles... FAMILY_FIRST_GIVEN_LAST:
209 family, middles..., given. One piece takes order[0]'s role; two
210 pieces take order[0] and the other primary."""
211 first, second = order[0], order[1]
212 # rules.md#O5: "a name of one name word that nothing else has
213 # decided reads that word as the given name under the default
214 # given-first order, and as the family name under a declared
215 # family-first one" -- a convention, not a determination: O4 has
216 # no positions to compare at one word, so this line is where the
217 # library picks one of two equally consistent readings and picks
218 # it the same way every time. The rules that DO decide such a
219 # name (H1, N3, M4) run after this and retag.
220 if count == 1:
221 return [first]
222 if first is Role.GIVEN: # GIVEN_FIRST
223 return ([Role.GIVEN] + [Role.MIDDLE] * (count - 2)
224 + [Role.FAMILY])
225 if second is Role.GIVEN: # FAMILY_FIRST
226 return ([Role.FAMILY, Role.GIVEN]
227 + [Role.MIDDLE] * (count - 2))
228 return ([Role.FAMILY] + [Role.MIDDLE] * (count - 2) # F_F_GIVEN_LAST
229 + [Role.GIVEN])
232def _assign_main(seg_idx: int, state: ParseState,
233 tokens: list[WorkToken],
234 ambiguities: list[PendingAmbiguity],
235 ) -> tuple[Role, Role, Role] | None:
236 """Returns the order the positional read used, for ParseState.order
237 -- None on every path that returns before resolving one."""
238 pieces = state.pieces[seg_idx]
239 ptags = state.piece_tags[seg_idx]
240 n = _peel_leading_titles(pieces, ptags, tokens)
241 if n == len(pieces):
242 return None
243 # group-flagged suffix pieces (the ph-d merge) are suffixes at ANY
244 # position -- v1's fix_phd extracted the credential from the string
245 # before parsing, so position never mattered (PR review I3).
246 # Walked rather than collected first: the list was never read
247 # again, and on 3.11 a comprehension is a frame of its own, which
248 # is one frame back against the one the H5 reading below costs. A
249 # 3.11 FACT and not a portable one: PEP 709 inlines comprehensions
250 # from 3.12, where this tree's reference parse costs 395 written
251 # either way (measured 2026-09-09, against 416 and 417 on 3.11).
252 # 3.11 is the interpreter the band is quoted for, so 3.11 is what
253 # the shape is chosen on (decisions.md#parse-cost).
254 for k in range(n, len(pieces)):
255 if "suffix" in ptags[k]:
256 _set_roles(tokens, pieces[k], Role.SUFFIX)
257 rest = peel_walk(n, ptags)
258 if not rest:
259 return None
260 # rules.md#N3: "a name that is only a nickname and one name word
261 # reads that word as the family name" (history: decisions.md#N3)
262 # -- v1's p_len == 1 counted
263 # the WHOLE segment before any title peeling -- 'Xyz. (Bud) Smith'
264 # has two pieces, so the title peel wins and Smith stays the given
265 # name (pinned live 2026-07-17)
266 # The nickname scan sits LAST in the test: it is a generator, which
267 # every interpreter resumes once per token (seven call events on
268 # the reference name), and only a one-piece segment ever reads its
269 # answer. Hoisting it to the top of the read, where it once stood,
270 # is what put 3.12-3.15 one call over the band (decisions.md#parse-cost).
271 if (len(pieces) == 1 and len(rest) == 1
272 and any(t.role is Role.NICKNAME for t in tokens)):
273 _set_roles(tokens, pieces[rest[0]], Role.FAMILY)
274 return None
275 # rules.md#S2's trailing peel and rules.md#H5's title chain, read
276 # together to their fixed point by _pieces.tail_reading -- one
277 # function since the /simplify round, shared with the bound-given
278 # reserve (P5), which must count the name words this leaves.
279 #
280 # rules.md#H5: "successive single words that wear the abbreviation
281 # shape and are title vocabulary chain into the title from the end,
282 # leaving one name word standing" -- the titles are set BEFORE
283 # _name_positions, so the shortened list is what the positional
284 # read and the script test both see (a trailing Latin title must
285 # not make a wholly-CJK name look mixed-script, the same reason the
286 # leading peel runs first). Read ahead of the bare-suffix carve-out
287 # because with no name piece left there is nothing for the chain to
288 # read: its floor keeps 0 pieces of an empty list, so the branch
289 # below is reached exactly as before.
290 #
291 # Every bare ambiguous acronym the FINAL peel had to resolve is one
292 # coin-flip each, in either direction, so the report collects
293 # rather than overwrites. Deferred to after assignment because the
294 # wording reads the role back, and which role "not peeled" means
295 # depends on name_order. (The roman-numeral fork needs no such
296 # deferral and is reported here.)
297 rest, titled_tail, peeled = tail_reading(rest, pieces, ptags, tokens)
298 for piece_idx in titled_tail:
299 _set_roles(tokens, pieces[piece_idx], Role.TITLE)
300 if peeled.numeral is not None:
301 # a trailing single letter is a name part unless it happens
302 # to be a roman numeral -- and V/X/I are ordinary middle
303 # initials, so taking it as a suffix is a call, not a fact
304 ambiguities.append(PendingAmbiguity(
305 AmbiguityKind.SUFFIX_OR_NAME,
306 f"{tokens[peeled.numeral[0]].text!r} is a roman numeral, so "
307 f"it reads as a generational suffix; any other single "
308 f"letter there would be a middle initial",
309 peeled.numeral))
310 name_pieces, suffix_pieces = rest[:peeled.names], rest[peeled.names:]
311 if peeled.names == 0:
312 # everything suffix-shaped after titles: first one is the name
313 name_pieces, suffix_pieces = suffix_pieces[:1], suffix_pieces[1:]
314 # AFTER the whole tail reading, and load-bearing: the script test
315 # sees the NAME pieces only, so a Latin title or suffix ('Dr. 毛
316 # 泽东', '毛 泽东, PhD') cannot make a wholly-CJK name look
317 # mixed-script.
318 resolved = _effective_order(state.policy, pieces, name_pieces, tokens,
319 dot_divided=bool(state.interpunct_offsets))
320 order = resolved.order
321 roles = _name_positions(order, len(name_pieces))
322 for pos, piece_idx in enumerate(name_pieces):
323 _set_roles(tokens, pieces[piece_idx], roles[pos])
324 for piece_idx in suffix_pieces:
325 _set_roles(tokens, pieces[piece_idx], Role.SUFFIX)
326 # Both emitters below turn on the PEEL's count, so neither can
327 # reach a name that kept two name pieces, and the two scans are
328 # nested under the count rather than run beside it: they cost the
329 # call budget on every parse otherwise (decisions.md#parse-cost).
330 if peeled.names <= 1:
331 head = pieces[name_pieces[0]]
332 token = tokens[head[0]]
333 assert token.role is not None
334 # Both conventions here turn on a lone name word, so both
335 # report at the site that places one
336 # (mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE), and one
337 # predicate carries what says nothing else decided the FIELD:
338 # a title (segment 0's own peel, or one a family comma left in
339 # the next segment -- either is a Role.TITLE by now, and
340 # either makes the reading H1's), a maiden name (M4's), or a
341 # script order convention. `resolved.by_script` rather than a
342 # comparison against name_order: a declared family-first order
343 # agreeing with a Han name's entry is agreement, not
344 # authorship, and for a lone CJK honorific ('さん', '씨') it
345 # scopes W2's reading out rather than excusing it (#271/#308).
346 titled = any(t.role is Role.TITLE for t in tokens)
347 field_undecided = (
348 not resolved.by_script and not titled
349 and not any(t.role is Role.MAIDEN for t in tokens))
350 # rules.md#H4: "an input whose every word is post-nominal
351 # vocabulary reads its first word as a name and reports
352 # `suffix-or-name`" (history: decisions.md#H4) -- only the
353 # word made into a name reports, and no name piece survived
354 # the peel, so the carve-out above made the first post-nominal
355 # the name. The role comes off the token for the reason stated
356 # at the particle emitter below.
357 if peeled.names == 0 and field_undecided:
358 text = " ".join(tokens[i].text for i in head)
359 ambiguities.append(PendingAmbiguity(
360 AmbiguityKind.SUFFIX_OR_NAME,
361 f"{text!r} is post-nominal vocabulary with no name word "
362 f"beside it; read as a {token.role.value} name rather than "
363 f"a post-nominal, nothing else being left to be the name",
364 tuple(head)))
365 # One name piece off the peel: the convention placed a lone
366 # name word. A suffix beside it is not a decision -- 'Smith
367 # Jr.' and "'Smitty' Jones Jr." ARE this convention -- and the
368 # count comes off the peel rather than off name_pieces, which
369 # is what keeps the carve-out above out of these branches.
370 # Title vocabulary in the unit makes the doubt H4's (is the
371 # WORD a title), anything else O5's (which FIELD it took), so
372 # only the second reads `field_undecided`.
373 if peeled.names == 1 and not resolved.by_script:
374 # rules.md#H4: "an input whose only remaining name word
375 # after the title peel is itself title vocabulary reads
376 # that word as the name by convention and reports
377 # `title-or-name`" (history: decisions.md#H4), and H4's
378 # join clause, stated at rules.md#O5 as its exception --
379 # one branch, its detail naming a field only in the join
380 # shape ('John of Prince', 'Attorney General of
381 # Minnesota'), where the unit is more than the title word.
382 # A LONE title word never reaches here ('Dr.', 'Prince of
383 # Wales'): the leading-title peel took the whole name.
384 if any("vocab:title" in tokens[i].tags for i in head):
385 text = " ".join(tokens[i].text for i in head)
386 ambiguities.append(PendingAmbiguity(
387 AmbiguityKind.TITLE_OR_NAME,
388 f"{text!r} is title vocabulary and the only name word "
389 f"the title peel left standing; read as the name by "
390 f"convention rather than as more title"
391 if len(head) == 1 else
392 f"{text!r} is the only name unit and joins title "
393 f"vocabulary to a name word; read as a "
394 f"{token.role.value} name by convention",
395 tuple(head)))
396 # rules.md#O5: "a name of one name word that nothing else has
397 # decided reads that word as the given name under the default
398 # given-first order, and as the family name under a declared
399 # family-first one" (history: decisions.md#O5). Past
400 # `field_undecided`, two clauses of O5's own: the word's
401 # own reading may have claimed it, and A2's content test
402 # -- a piece with no alphanumeric character is no name
403 # word, so parse("(") keeps its unbalanced-delimiter
404 # report and gains nothing here.
405 elif (field_undecided
406 and all(tokens[i].tags.isdisjoint(_WORD_ALREADY_CLAIMED)
407 for i in head)
408 and any(c.isalnum() for i in head
409 for c in tokens[i].text)):
410 text = " ".join(tokens[i].text for i in head)
411 ambiguities.append(PendingAmbiguity(
412 AmbiguityKind.GIVEN_OR_FAMILY,
413 f"{text!r} is the only name word and nothing else "
414 f"decides it; read as a {token.role.value} name by "
415 f"convention, which follows the read order",
416 tuple(head)))
417 for piece in peeled.picks:
418 # every pick is in rest, so the loops above just gave it a role
419 token = tokens[piece[0]]
420 assert token.role is not None
421 taken, declined = (
422 ("a suffix", "a name part") if token.role is Role.SUFFIX
423 else (f"a {token.role.value} name", "a post-nominal"))
424 ambiguities.append(PendingAmbiguity(
425 AmbiguityKind.SUFFIX_OR_NAME,
426 f"{token.text!r} written without periods is both a "
427 f"post-nominal and an ordinary name; read as {taken} "
428 f"rather than {declined}",
429 piece))
430 # leading ambiguous particle read as a name (#121 surfaced)
431 if name_pieces:
432 head = pieces[name_pieces[0]]
433 if (len(head) == 1 and len(name_pieces) > 1
434 and "vocab:particle-ambiguous" in tokens[head[0]].tags):
435 # the loops above gave the head piece its role from
436 # `order`, which is _effective_order's answer and not
437 # necessarily name_order's -- a script_orders entry
438 # overrides it. So read the role off the token rather than
439 # assume given, or re-derive it here; same reason as
440 # SUFFIX_OR_NAME just above.
441 token = tokens[head[0]]
442 assert token.role is not None
443 ambiguities.append(PendingAmbiguity(
444 AmbiguityKind.PARTICLE_OR_GIVEN,
445 f"leading {token.text!r} may be a family-name "
446 f"particle; read as a {token.role.value} name",
447 tuple(head)))
448 return order
451def _reads_as_a_trailing_suffix(piece: Sequence[int],
452 prev_piece: Sequence[int],
453 prev_ptags: Set[str],
454 tokens: Sequence[WorkToken],
455 lexicon: Lexicon) -> bool:
456 """The lenient tail test for a trailing one-token piece after a
457 family comma, and #432's carve-out from it.
459 A word that could be a middle initial, written with the period that
460 marks an abbreviation, is name material -- so 'Smith, John V.' is
461 middle 'V.' where 'Smith, John V' stays suffix 'V' (v1 parity,
462 #144). Only behind a NAME word: behind a suffix the credential run
463 owns it, and 'Smith, John PhD I.' keeps suffix 'PhD, I.'.
465 The period is the whole carve-out. is_initial_shaped would be
466 redundant beside it: the caller only consults this where
467 is_suffix_piece said no, and a lenient single token it refuses is
468 one carrying the `initial` tag, so the shape is already implied.
470 NOT is_trailing_numeral_suffix, though it answers the period half:
471 it also refuses a numeral behind an initial-shaped piece, which is
472 a no-comma rule and the opposite of this path's v1 parity --
473 'Chang, Andy C I' is first Andy, middle C, suffix I, and asking
474 that predicate here made the numeral a middle. The #401/#421 entry
475 under decisions.md#P5 records the same fork declining to transfer
476 to this walk under LENIENT.
477 """
478 text = tokens[piece[0]].text
479 if text.endswith(".") and not is_suffix_piece(
480 prev_piece, prev_ptags, tokens):
481 return False
482 return is_suffix_lenient(text, lexicon)
485def assign(state: ParseState) -> ParseState:
486 tokens = list(state.tokens)
487 ambiguities = list(state.ambiguities)
488 if not state.segments:
489 return state
490 order: tuple[Role, Role, Role] | None = None
491 if state.structure is Structure.NO_COMMA:
492 order = _assign_main(0, state, tokens, ambiguities)
493 tail = len(state.segments)
494 elif state.structure is Structure.SUFFIX_COMMA:
495 order = _assign_main(0, state, tokens, ambiguities)
496 tail = 1
497 else: # FAMILY_COMMA
498 # PARTICLE_OR_GIVEN is deliberately not emitted on the
499 # wholly-family read: after a comma that fixed the family, a
500 # leading given-position particle is not meaningfully
501 # ambiguous, and script_orders is not consulted for the parallel
502 # reason. Scoped, not silent: the comma fixed WHICH PIECE is the
503 # family and said nothing about a particle trailing the given
504 # name, so P6's attachment in post_rules decides that fork and
505 # reports it there, at the site that takes the branch (#405).
506 # The positional read below (a comma followed by no
507 # name word) emits and consults both, being the no-comma read
508 # of segment 0; group's chain emitter still does not, since
509 # group runs before assign decides which read applies (recorded
510 # at decisions.md#C1).
511 # v1: "lastname part may have suffixes in it" -- the first
512 # piece is always the family even if suffix-shaped; any later
513 # strict-suffix piece goes to SUFFIX per piece ('Smith Jr.,
514 # John' -> family=Smith, suffix=Jr.)
515 fam_pieces = state.pieces[0]
516 fam_tags = state.piece_tags[0]
517 # A comma followed by no name word fixed nothing, so segment 0
518 # keeps its positional read -- including script_orders, the
519 # particle fork, and the ORDER, which post_rules' family-first
520 # fold (P1) and its leading-piece scan key on: "assign records
521 # no order after a family comma" is the invariant those rules
522 # rest on, and this is the path that gives one, so they read
523 # segment 0 as the name it is ('de Mesnil Jean, Dr.' under a
524 # family-first order keeps family 'de Mesnil'; the test review
525 # found the fold missing it). The wholly-family branch below
526 # suppresses all three precisely because the comma HAD fixed
527 # the family. Needs two NAME pieces: with one, the positional
528 # read would make it a lone GIVEN, which is worse than what it
529 # replaces -- and the count is of name pieces, since the
530 # positional read peels a trailing suffix first: 'Smith Jr.,
531 # Mr.' has two pieces and one name, and read positionally lost
532 # its family (the code review).
533 reading = segment_suffix_reading(
534 state.pieces[1], state.piece_tags[1], tokens,
535 state.policy.lenient_comma_suffixes)
536 # Segment 1 is read FIRST, ahead of either branch below. It
537 # consumes `reading`, piece tags and text only -- nothing
538 # segment 0's read writes -- and running it first is what puts
539 # a TITLE role on the title standing after the comma ('John V,
540 # Dr.') before _assign_main scans for one, so the positional
541 # read below sees that title the way it sees its own peeled
542 # ones (#449 review round; it replaced a `titled` keyword).
543 if len(state.segments) > 1:
544 pieces = state.pieces[1]
545 ptags = state.piece_tags[1]
546 # Both are the walk's, and both are empty on the gate's
547 # path below, which reads the whole segment as a
548 # credential run and leaves no piece for the walk to
549 # place.
550 titled_idx: tuple[int, ...] = ()
551 walkable: list[int] = []
553 def previous_kept(m: int, titled: tuple[int, ...]) -> int:
554 """The piece before `m` that the H5 chain did NOT
555 take. Both readings this segment needs are that one:
556 the piece the lenient tail test measures against, and
557 -- asked of one past the end -- where the segment's
558 name ENDS, since a title the chain took is not where a
559 name ends (#144). One walk for both, so 'as if the
560 titled pieces were absent' cannot come to mean two
561 things.
563 DEFENSIVE, and measured inert: over 191,146 generated
564 inputs the skip fired on 153 of the 140,227 walks the
565 lenient test's `prev` asked for, and deleting it
566 changed no parse among them (2026-09-09, on the shape
567 this replaced, where the name's END walked past the
568 same pieces in a copy of this loop). Kept because "as
569 if the titled pieces were absent" is the rule the
570 predicate below implements, and a caller's vocabulary
571 reaches shapes the sweep's word list does not -- an
572 inert branch is cheaper than a rule with a hole in it.
573 """
574 m -= 1
575 while m in titled:
576 m -= 1
577 return m
579 def reads_as_a_suffix(m: int, titled: tuple[int, ...]) -> bool:
580 """Does this segment's walk read piece `m` as a suffix?
582 Asked twice, and by one predicate rather than by two
583 conditions written to match
584 (mechanisms.md#ONE-PREDICATE-PER-QUESTION): once to
585 find the pieces the H5 title chain must not reach
586 past, and once by the walk order below, which is the
587 site that places them.
589 `titled` is a PARAMETER because the two passes hand it
590 different values -- () on the first, the chain's own
591 pieces on the second, which is what makes that second
592 reading the one 'as if the titled pieces were absent'.
593 A closure over the caller's local said the same thing,
594 but only by WHEN it was rebound.
595 """
596 if is_suffix_piece(pieces[m], ptags[m], tokens):
597 return True
598 prev = previous_kept(m, titled)
599 # trailing piece of a two-part name is unambiguously
600 # positioned: v1 accepts the lenient test there
601 # ('Smith, John V' -> suffix='V', #144); with a third
602 # comma part the trailing token is more likely a middle
603 # initial, so strict only
604 return (m == previous_kept(len(pieces), titled)
605 and len(state.segments) == 2
606 and len(pieces[m]) == 1
607 and _reads_as_a_trailing_suffix(
608 pieces[m], pieces[prev], ptags[prev],
609 tokens, state.lexicon))
611 # rules.md#C1: "a credential run after the comma means the
612 # name is in natural order with suffixes appended" -- and
613 # with one word before the comma the listing form holds,
614 # the family is that word, and the run is still the
615 # credential run. A no-name segment is read piece by
616 # piece, the suffix vocabulary's verdict BEFORE the title
617 # reading of the same word: the slot after a family comma
618 # is postnominal position, so a segment of nothing but
619 # suffix pieces is the credential run, whole (#296:
620 # 'Smith, Jr.' read title 'Jr.' through the period-
621 # abbreviation inference; #325: 'Smith, Ph. D. Jr.' put
622 # the split credential in the given name, the lone-piece
623 # route not applying), and a mixed run is a title and a
624 # postnominal, each where it stands ('Smith, Mr. Jr.').
625 # Vocabulary decides which words qualify -- 'Smith, Dr.'
626 # reads the title, 'dr' not being suffix vocabulary since
627 # the audit -- and position breaks the tie for the genuine
628 # duals ('Smith, Sr.' is Senior, 'Sr. Garcia' Señor). A
629 # name word in the segment makes it v1's walk ('Smith,
630 # John Jr.').
631 if reading is not None:
632 # the gate's own reading (#430)
633 for k, piece in enumerate(pieces):
634 _set_roles(tokens, piece,
635 Role.SUFFIX if reading[k] else Role.TITLE)
636 n = len(pieces)
637 else:
638 n = _peel_leading_titles(pieces, ptags, tokens)
639 # rules.md#H5: "the title is TRANSPARENT to the suffix
640 # reading: where two or more name words stand, what
641 # stands once the chain is taken reads exactly as it
642 # would read written without the title, plus the title"
643 # -- the trailing title run, on this walk
644 # too. A name word in segment 1 is what keeps the gate
645 # above from reading the segment as a credential run,
646 # so 'Smith, John Prof.' had no route to title at all
647 # and read middle 'Prof.' at every baseline; the
648 # 'Smith, Dr.' family of rows that already route are
649 # the gate's doing, a different mechanism.
650 #
651 # The candidates are the pieces this walk would NOT
652 # read as a suffix, which is this path's answer to the
653 # peel the no-comma path runs first -- 'Smith, John
654 # Prof. Jr.' must reach `Prof.` past the postnominal
655 # behind it, and 'Smith, John Prof. V' past the
656 # numeral the lenient tail test claims (#144), which
657 # is why the filter is the walk's own predicate and
658 # not the strict suffix test alone. Piece `n` is
659 # always the given below, whatever that predicate
660 # would say of it, so it is always a candidate. That
661 # `k == n` is LOAD-BEARING, not defensive: it is what
662 # the walk's floor stands on when the given piece
663 # itself reads as a suffix, and dropping it leaves
664 # 'Smith, II Mr. V' a middle 'Mr.' where the title is
665 # (24 inputs of that shape move, of 191,146 generated,
666 # measured 2026-09-09).
667 walkable = [k for k in range(n, len(pieces))
668 if k == n or not reads_as_a_suffix(k, ())]
669 kept = trailing_titles(walkable, pieces, ptags, tokens)
670 titled_idx = tuple(walkable[kept:])
671 for k in titled_idx:
672 _set_roles(tokens, pieces[k], Role.TITLE)
673 # v1 walk order: the first non-title piece is ALWAYS the
674 # given, before any suffix check -- 'Hardman, RN - CRNA'
675 # keeps first='RN'. The one deliberate 2.0 deviation,
676 # classified fix(comma-family) -- a last piece that is
677 # unambiguously suffix-shaped is a suffix, where v1 made
678 # it the given ('Andrews, M.D.', 'Smith, Dr. Jr.') -- is
679 # the no-name read above now: a segment whose only
680 # non-title piece is a suffix piece holds no name word,
681 # so the walk here never meets the case.
682 if n < len(pieces):
683 _set_roles(tokens, pieces[n], Role.GIVEN)
684 # The chain's floor leaves a name piece standing, so the
685 # given above is never one of the pieces it took. What the
686 # chain DOES move is where this segment's name ends, which
687 # the lenient tail test turns on (#144) -- so where it took
688 # something the question is re-asked with the pieces it
689 # left. Where it took nothing, `walkable` already IS this
690 # predicate's answer for every piece: the first pass's own
691 # memo, not a second spelling of the question, and it
692 # cannot have gone stale because nothing the chain does
693 # moved the end of the name.
694 for m in range(n + 1, len(pieces)):
695 if m in titled_idx:
696 continue
697 suffix_here = (reads_as_a_suffix(m, titled_idx)
698 if titled_idx else m not in walkable)
699 _set_roles(tokens, pieces[m],
700 Role.SUFFIX if suffix_here else Role.MIDDLE)
701 if reading is not None and sum(
702 1 for k, piece in enumerate(fam_pieces)
703 if not is_suffix_piece(piece, fam_tags[k], tokens)) > 1:
704 order = _assign_main(0, state, tokens, ambiguities)
705 else:
706 for k, piece in enumerate(fam_pieces):
707 if k > 0 and is_suffix_piece(piece, fam_tags[k], tokens):
708 _set_roles(tokens, piece, Role.SUFFIX)
709 else:
710 _set_roles(tokens, piece, Role.FAMILY)
711 tail = 2
712 # segments past the structure's name segments are wholly suffixes
713 for seg_idx in range(tail, len(state.segments)):
714 for piece in state.pieces[seg_idx]:
715 _set_roles(tokens, piece, Role.SUFFIX)
716 return dataclasses.replace(state, tokens=tuple(tokens),
717 order=order,
718 ambiguities=tuple(ambiguities))