Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Internal pipeline state: WorkToken and ParseState.
3WorkTokens are pipeline-internal (no validation -- the tokenizer is the
4only producer) and are addressed BY INDEX in every stage: pieces and
5segments are runs of token indices, never joined strings, so value-based
6lookup (v1's #100 family) is structurally impossible.
8Layering: imports _types, _lexicon, _policy only (enforced by
9tests/v2/test_layering.py).
10"""
11from __future__ import annotations
13import bisect
14from collections.abc import Sequence
15from dataclasses import dataclass
16from enum import Enum, auto
18from nameparser._lexicon import Lexicon
19from nameparser._policy import Policy
20from nameparser._types import AmbiguityKind, Role, Segmenter, Span
23# The comma characters (ASCII/Arabic/fullwidth, #265). Shared here so
24# tokenize (separators/segmentation) and extract (close-quote
25# boundaries) cannot drift apart.
26COMMA_CHARS = frozenset({",", "\u060c", "\uff0c"})
29def comma_bucket(start: int, comma_offsets: Sequence[int]) -> int:
30 """Which comma-delimited part of the name a token starting at
31 `start` falls in: the number of commas before it.
33 Shared here for the reason COMMA_CHARS is, and the sharing is
34 load-bearing in the same way. segment BUILDS the segments with this
35 (comma_offsets is sorted and no offset ever equals a token start,
36 so bisect_left counts the commas before the token); classify asks
37 it to decide whether two tokens could be in one segment, which is
38 half of what stops a maiden marker run from spanning a boundary no
39 segment holds. Two tokens agree here iff segment would put them in
40 one segment, and that is an identity rather than a resemblance
41 only while both sides ask this function.
42 """
43 return bisect.bisect_left(comma_offsets, start)
45@dataclass(frozen=True, slots=True)
46class WorkToken:
47 """One tokenized word. role stays None until assign; extracted
48 nickname/maiden tokens arrive with their role pre-set. text is
49 always the exact original slice (tokenize is the sole producer;
50 the anti-#100 invariant depends on it)."""
52 text: str
53 span: Span
54 tags: frozenset[str] = frozenset()
55 role: Role | None = None
58#: M4's two carve-outs, as the tags classify recorded them: a bound
59#: given-name word is vocabulary claiming the word as a given name,
60#: and `initial` is the initial reading. Neither is a predicate M4 owns.
61#: Shared here beside WorkToken.tags for the reason COMMA_CHARS is:
62#: assign's `_WORD_ALREADY_CLAIMED` is built from this pair, and the
63#: two stages must not drift (post_rules imports _assign, so _assign
64#: cannot reach the other way).
65_NEVER_FLIPPED = frozenset({"vocab:bound-given", "initial"})
68class Structure(Enum):
69 """segment's comma-structure decision."""
71 NO_COMMA = auto()
72 FAMILY_COMMA = auto() # "Family, Given ..." (v1 lastname-comma)
73 SUFFIX_COMMA = auto() # "Given Family, Suffix ..."
76@dataclass(frozen=True, slots=True)
77# rules.md#A1: "parsing never fails on any input: where the text's
78# structure or a word's reading is genuinely uncertain, the parse
79# completes on the best reading and carries an ambiguity report
80# naming the doubt"
81class PendingAmbiguity:
82 """An ambiguity recorded mid-pipeline by token INDEX; assemble
83 materializes real Ambiguity objects over the final tokens.
85 ``origin`` is for the one stage that runs BEFORE tokens exist:
86 extract_delimited knows only a character offset, so it records that
87 and tokenize resolves it to the containing token's index. Stages
88 after tokenize set ``indices`` directly and leave ``origin`` None.
89 """
91 kind: AmbiguityKind
92 detail: str
93 indices: tuple[int, ...] = ()
94 origin: int | None = None
97@dataclass(frozen=True, slots=True)
98class ParseState:
99 """Carried through the stage fold. Frozen; stages return copies via
100 dataclasses.replace. Fields are filled progressively:
101 extract_delimited -> extracted/masked; tokenize -> tokens (span-
102 sorted)/comma_offsets/interpunct_offsets (the 间隔号 offsets the
103 order and segmentation decisions consult, #298; the nakaguro
104 separators record NOTHING); segment -> segments/structure;
105 script_segment -> tokens and segments again (the one stage that
106 changes the token COUNT: an unspaced CJK token splits into n+1
107 pieces, still as sub-slices of the original, and every later index
108 in the segment runs shifts by n); classify -> token tags; group ->
109 pieces/piece_tags/dropped AND maiden token roles;
110 assign -> the remaining token roles AND `order`, the effective
111 order it read them under; post_rules -> roles again, and the
112 ambiguity P6's attachment reports.
113 Ambiguities are recorded by every stage that DECIDES one --
114 extract (resolved to a token index by tokenize), segment,
115 script_segment, classify, group, assign, and post_rules -- since a
116 fork whose branches are taken in different stages needs an emitter
117 in each. Post-group, segments may retain indices of dropped tokens
118 -- assign iterates pieces, never segments. This ownership map is
119 pinned by tests/v2/pipeline/test_state.py.
121 segmenter belongs to no stage: like original/lexicon/policy it is
122 passed in at construction by Parser.parse and only ever READ (by
123 script_segment, for a token the vocabulary declined)."""
125 original: str
126 lexicon: Lexicon
127 policy: Policy
128 #: The optional Parser(segmenter=...) hook; None = not configured.
129 segmenter: Segmenter | None = None
130 extracted: tuple[tuple[Role, Span], ...] = ()
131 masked: tuple[Span, ...] = ()
132 tokens: tuple[WorkToken, ...] = ()
133 comma_offsets: tuple[int, ...] = ()
134 interpunct_offsets: tuple[int, ...] = ()
135 segments: tuple[tuple[int, ...], ...] = ()
136 structure: Structure = Structure.NO_COMMA
137 # pieces[s][p] = run of token indices: piece p of segment s.
138 # piece_tags[s][p] = derived flags for that piece ("title", "prefix",
139 # "suffix", "conjunction") set by group's joins.
140 pieces: tuple[tuple[tuple[int, ...], ...], ...] = ()
141 piece_tags: tuple[tuple[frozenset[str], ...], ...] = ()
142 dropped: tuple[int, ...] = () # structural tokens (maiden markers)
143 #: The order `assign` actually READ the name under -- name_order,
144 #: or the script_orders entry that overrode it. None wherever no
145 #: positional read happened: after a family comma (which fixes the
146 #: family, so assign consults no order at all), and on every early
147 #: return in `_assign_main`, where a segment holds no name piece
148 #: to position. Recorded rather than recomputed downstream,
149 #: because the two can differ and a post_rules rule keyed on
150 #: `policy.name_order` would then disagree with the roles assign
151 #: already wrote (#395). Reaching that divergence needs a custom
152 #: lexicon -- every shipped particle is Latin, and Latin has no
153 #: script_orders entry -- which is why the test for it builds its
154 #: own (test_post_rules.py).
155 order: tuple[Role, Role, Role] | None = None
156 ambiguities: tuple[PendingAmbiguity, ...] = ()