Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Stage: segment.
3Consumes: tokens (role-None main stream), comma_offsets.
4Produces: segments (runs of main-token indices; interior segments may
5be EMPTY -- doubled commas keep their structural position), structure,
6COMMA_STRUCTURE ambiguities for unrecognized extra segments.
7Reads: Lexicon suffix vocabulary and Policy, both through
8_vocab.is_wholly_suffix -- the suffix-comma decision is definitionally
9vocabulary-dependent (decisions.md#C1), and the predicate
10owns the rest (Policy.lenient_comma_suffixes picks the lenient or
11strict token test; Policy.extra_suffix_delimiters gives v1
12suffix_delimiter parity, a delimiter-core token being transparent).
14Implements rules C1 and C2 of docs/design/rules.md, cited at the
15decision site below; history in decisions.md#C1.
16"""
17from __future__ import annotations
19import dataclasses
21from nameparser._pipeline._state import (
22 ParseState, PendingAmbiguity, Structure, comma_bucket,
23)
24from nameparser._pipeline._vocab import is_wholly_suffix
25from nameparser._types import AmbiguityKind
31def segment(state: ParseState) -> ParseState:
32 main = [i for i, t in enumerate(state.tokens) if t.role is None]
33 if not main:
34 return dataclasses.replace(state, segments=(),
35 structure=Structure.NO_COMMA)
36 if not state.comma_offsets:
37 return dataclasses.replace(state, segments=(tuple(main),),
38 structure=Structure.NO_COMMA)
39 buckets: list[list[int]] = [[] for _ in range(len(state.comma_offsets) + 1)]
40 for i in main:
41 # _state.comma_bucket, not a local bisect: classify asks the
42 # same question of the same offsets to keep a marker run inside
43 # one segment, and the two must be one expression rather than
44 # two that agree
45 buckets[comma_bucket(state.tokens[i].span.start,
46 state.comma_offsets)].append(i)
47 groups = [tuple(b) for b in buckets]
48 # v1 strips exactly ONE trailing comma as cosmetic (parser.py's
49 # collapse_whitespace); every other empty bucket is STRUCTURAL and
50 # keeps its position -- in 'Doe,, Jr.' the given segment is empty,
51 # so 'Jr.' stays a tail suffix instead of masquerading as a lone
52 # post-comma title (v1 parity, pinned live 2026-07-16)
53 if len(groups) > 1 and not groups[-1]:
54 groups.pop()
55 if len(groups) <= 1:
56 segs = tuple(groups) if groups and groups[0] else (tuple(main),)
57 return dataclasses.replace(state, segments=segs,
58 structure=Structure.NO_COMMA)
60 def suffixy(seg: tuple[int, ...]) -> bool:
61 return is_wholly_suffix([state.tokens[i].text for i in seg],
62 state.lexicon, state.policy)
64 # rules.md#C1: "the name reads as trailing suffixes when the part
65 # after the first comma is entirely suffix words and more than one
66 # word precedes the comma; otherwise it reads as the listing form"
67 # (v1 parity: only parts[1] decides, parser.py:1318; history:
68 # decisions.md#C1)
69 # rules.md#C2: "a non-empty extra part that is not entirely suffix
70 # words is flagged as a structural ambiguity rather than rejected"
71 # -- parts[2:] are consumed as suffixes unconditionally either
72 # way, so a non-suffix tail segment gets the COMMA_STRUCTURE
73 # flag, not a structure veto
74 structure = (Structure.SUFFIX_COMMA
75 if suffixy(groups[1]) and len(groups[0]) > 1
76 else Structure.FAMILY_COMMA)
77 ambiguities = list(state.ambiguities)
78 for seg in groups[2:]:
79 # empty segments are consumed silently (v1 skips them without
80 # comment); only non-empty non-suffix tails get flagged
81 if seg and not suffixy(seg):
82 texts = " ".join(state.tokens[i].text for i in seg)
83 ambiguities.append(PendingAmbiguity(
84 AmbiguityKind.COMMA_STRUCTURE,
85 f"segment {texts!r} beyond the recognized comma "
86 f"structures; consumed as suffix best-effort",
87 tuple(seg)))
88 return dataclasses.replace(state, segments=tuple(groups),
89 structure=structure,
90 ambiguities=tuple(ambiguities))