Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Internal pipeline state: WorkToken and ParseState.
3WorkTokens are pipeline-internal (no validation -- the tokenizer is the
4only producer) and are addressed BY INDEX in every stage: pieces and
5segments are runs of token indices, never joined strings, so value-based
6lookup (v1's #100 family) is structurally impossible.
8Layering: imports _types, _lexicon, _policy only (enforced by
9tests/v2/test_layering.py).
10"""
11from __future__ import annotations
13import bisect
14import dataclasses
15from collections.abc import Iterable, Mapping, Sequence
16from dataclasses import dataclass
17from enum import Enum, auto
18from types import MappingProxyType
19from typing import TYPE_CHECKING, TypeVar
21from nameparser._lexicon import Lexicon
22from nameparser._policy import Policy
23from nameparser._types import AmbiguityKind, Segmenter, Span
24# re-exported for _vocab, which layering keeps off _types
25from nameparser._types import ReadingKind as ReadingKind
26from nameparser._types import ReadingSource as ReadingSource
27from nameparser._types import Role as Role
28from nameparser._types import SHAPE_ACRONYM_TAG as SHAPE_ACRONYM_TAG
29from nameparser._types import SPLIT_CREDENTIAL_TAG as SPLIT_CREDENTIAL_TAG
30from nameparser._types import SPLIT_MEMBER_TAG as SPLIT_MEMBER_TAG
32if TYPE_CHECKING:
33 from nameparser._pipeline._pieces import TailRead
36# The comma characters (ASCII/Arabic/fullwidth, #265; Ethiopic and
37# Burmese, #1202; N'Ko, Mongolian, Manchu, Lisu, Vai and Bamum, #1202's
38# follow-up -- not the Armenian U+055D, which is no comma, decisions/C1).
39# Shared here so tokenize (separators/segmentation), extract
40# (close-quote boundaries, a listed particle's word end) and _vocab
41# cannot drift apart.
42COMMA_CHARS = frozenset({",", "\u060c", "\uff0c", "\u1363", "\u104a",
43 "\u07f8", "\u1802", "\u1808", "\ua4fe",
44 "\ua60d", "\ua6f5"})
45#: The same characters as one string, for str.strip: a word of a
46#: set-apart clause keeps the comma written against it (rules/N1), and
47#: every site reading that word without it strips this, so a clause
48#: reads alike whichever script's comma divides it (decisions/C1, #1202).
49COMMA_STRIP = "".join(sorted(COMMA_CHARS))
52def comma_bucket(start: int, comma_offsets: Sequence[int]) -> int:
53 """Which comma-delimited part of the name a token starting at
54 `start` falls in: the number of commas before it.
56 Shared here for the reason COMMA_CHARS is, and the sharing is
57 load-bearing in the same way. segment BUILDS the segments with this
58 (comma_offsets is sorted and no offset ever equals a token start,
59 so bisect_left counts the commas before the token); classify asks
60 it to decide whether two tokens could be in one segment, which is
61 half of what stops a maiden marker run from spanning a boundary no
62 segment holds. Two tokens agree here iff segment would put them in
63 one segment, and that is an identity rather than a resemblance
64 only while both sides ask this function.
65 """
66 return bisect.bisect_left(comma_offsets, start)
68@dataclass(frozen=True, slots=True)
69class WorkToken:
70 """One tokenized word. role stays None until assign; extracted
71 nickname/maiden tokens arrive with their role pre-set. text is
72 always the exact original slice (tokenize is the sole producer;
73 the anti-#100 invariant depends on it)."""
75 text: str
76 span: Span
77 tags: frozenset[str] = frozenset()
78 role: Role | None = None
81#: M4's two carve-outs, as the tags classify recorded them: a bound
82#: given-name word is vocabulary claiming the word as a given name,
83#: and `initial` is the initial reading. Neither is a predicate M4 owns.
84#: Shared here beside WorkToken.tags for the reason COMMA_CHARS is:
85#: assign's `_WORD_ALREADY_CLAIMED` is built from this pair, and the
86#: two stages must not drift (post_rules imports _assign, so _assign
87#: cannot reach the other way).
88_NEVER_FLIPPED = frozenset({"vocab:bound-given", "initial"})
90# SHAPE_ACRONYM_TAG is defined and explained in _types (its #: block
91# above the constant, not restated here -- defined there so _render
92# can read it too); re-exported here beside the membership half below
93# because the stages read both halves from one module.
95#: The MEMBERSHIP half of the same class: classify's tag for a token
96#: the ambiguous credential vocabulary claims, by listing
97#: (`Lexicon.suffix_acronyms_ambiguous`) or -- where a Policy switch
98#: admits the by-shape half -- beside `SHAPE_ACRONYM_TAG`. Beside that
99#: constant and for its reason: the string was spelled out at eight
100#: sites across four modules, each of them a place for a typo to pass
101#: silently, since a tag that is never written is simply a tag no
102#: reader ever finds.
103AMBIGUOUS_ACRONYM_TAG = "vocab:suffix-ambiguous"
105#: Beside `AMBIGUOUS_ACRONYM_TAG` on a suffix WORD that is also a name
106#: (`Lexicon.suffix_words_ambiguous`, rules/S11): the class is the
107#: same, and so is every site that weighs it, but the writing leans the
108#: other way -- lower case in a mixed-case name is the suffix, where an
109#: acronym's capitals are (`_vocab.ambiguous_lean`), but only on a
110#: word written `title_case_is_name` (`fils`, `kc`), which
111#: `TITLE_CASE_NAME_TAG` marks; any other leans nowhere (#1039).
112#: `AGNOMEN_TAG` rides beside both on an agnomen, for its reading's kind.
113AMBIGUOUS_WORD_TAG = "vocab:suffix-word-ambiguous"
114AGNOMEN_TAG = "vocab:agnomen"
115TITLE_CASE_NAME_TAG = "vocab:title-case-name"
117#: Beside "vocab:suffix-word" on a word whose shipped reading index
118#: kind is a generation (`jr`), a credential written as a word
119#: (`博士`) or an honorific (`さん`, #326), which the compiled sets do
120#: not record (rules/R8, 3.0's 4b):
121#: written by classify from the word as it reads it, a delimiter glued
122#: on removed (`Jr.;`, rules/C10), and read by `_vocab.suffix_kind`.
123GENERATION_WORD_TAG = "vocab:generation-word"
124CREDENTIAL_WORD_TAG = "vocab:credential-word"
125HONORIFIC_WORD_TAG = "vocab:honorific-word"
127#: Beside "particle" on a word the shipped reading index holds as a
128#: patronymic particle (`bin`, `ibn`, `bat`), which the compiled sets
129#: do not record (rules/R9, 3.0's 4c): written by classify, read by
130#: assemble for `Token.reading` and by assign for the reading a report
131#: names, and dropped from the published token like the kind tags above.
132PATRONYMIC_TAG = "vocab:patronymic"
134#: On each word of a particle chain opening a family-first name past
135#: the reach of rules/P1's fold (#828), written by post_rules whether or
136#: not the fold cuts the chain there, so assemble can tell where a
137#: report's other reading would land; dropped from the published token
138#: like the kind tags above.
139PAST_REACH_TAG = "chain:past-reach"
142def reported_suffix_fork(ambiguities: Sequence[PendingAmbiguity],
143 indices: Iterable[int],
144 skip: PendingAmbiguity | None = None) -> bool:
145 """rules/P1's question about a chain opening a family-first name
146 that the fold would cut (#828): whether a suffix-or-name report (the
147 trailing read's, or group's on a word the chain took) names a word
148 of `indices`, the words past the fold's reach. Such a word was read
149 against the whole chain, so the chain stands. Asked by the fold in post_rules and, with `skip`
150 the report whose other reading it names, by assemble."""
151 past = set(indices)
152 return any(a is not skip and a.kind is AmbiguityKind.SUFFIX_OR_NAME
153 and not past.isdisjoint(a.indices) for a in ambiguities)
156#: Beside FOLDED_TAG on the particles rules/P6 attaches to the family
157#: written before them, at either of its sites, so rules/R9 can tell
158#: the attachment from O3's middle fold, which writes FOLDED_TAG alone;
159#: dropped from the published token like the kind tags above.
160ATTACHED_TAG = "vocab:attached"
162#: On each word of the second name unit rules/O6 reads a family as,
163#: written where O6 takes its word, and dropped from the published
164#: token like the kind tags above: `Reading.second_surname` carries it
165#: out (rules/R10, 3.0's 4d).
166SECOND_SURNAME_TAG = "vocab:second-surname"
168#: Its twin on each word of the FIRST name unit: written where O6 reads
169#: two, on the family O6 found under a family-first order (a particle
170#: P6 attached included) and on the unit it took under the default one,
171#: and where O6 marked none, by O3 on the family it folds middles into,
172#: so a folded middle is in no unit; carried out on
173#: `Reading.first_surname` (rules/R11, #1022).
174FIRST_SURNAME_TAG = "vocab:first-surname"
176#: On the second of two listed single surnames that script_segment
177#: divides a Han name's family into (rules/W5, #845), so post_rules
178#: reads it as the family's second unit and marks both units with the
179#: two tags above; dropped from the published token like them.
180SURNAME_PAIR_TAG = "vocab:surname-pair"
182#: On a word of `Lexicon.titles_ambiguous` whichever way classify read
183#: it (rules/H9): read as a title it carries "vocab:title" too, read
184#: out of the title it carries this alone, which is how assign finds
185#: the fork to report where the word opens the name.
186TITLE_AMBIGUOUS_TAG = "vocab:title-ambiguous"
188#: Written by classify on every occurrence of a word of
189#: `Lexicon.titles_ambiguous` whose title reading is written short
190#: (`Lexicon.title_abbreviations`, `maj`), whichever way the phrase
191#: test read it; read by classify's CHAIN_TITLE_TAG below, rules/H5's
192#: trailing chain taking the word where it wears the abbreviation
193#: shape, and by group's P5 decline. Dropped from the published token like
194#: the kind tags above; the published token keeps TITLE_AMBIGUOUS_TAG.
195TRAILING_TITLE_TAG = "vocab:trailing-title"
197#: Written by classify on a word of `Lexicon.titles` (not
198#: `titles_ambiguous`), listed whole, whose letters a period joins, a
199#: hyphen beside it or not (`_vocab._JOINED_WORD`: 'Dott.ssa',
200#: 'Priv.-Doz'), and on a word with interior periods every chunk of
201#: which is a title ('Lt.Gov', #1131), never on one read as a title
202#: through some chunk only ('Dr.PhD', 'St.John') nor on one a hyphen
203#: alone joins ('Co-Founder'); read by classify's CHAIN_TITLE_TAG
204#: below, rules/H5's trailing chain taking it without a final period
205#: (#1104). Dropped from the published token like the kind tags above.
206JOINED_TITLE_TAG = "vocab:joined-title"
208#: Written by classify, after rules/H9's phrase test has settled which
209#: words keep "vocab:title", on every word rules/H5's trailing chain
210#: takes: a word carrying JOINED_TITLE_TAG, or one wearing the trailing
211#: abbreviation shape (`_vocab._TRAILING_TITLE_SHAPE`) that carries
212#: "vocab:title" or TRAILING_TITLE_TAG. The one statement of that test,
213#: read as a set membership by the chain (`_pieces.trailing_titles`),
214#: by its one-word form (`_pieces.is_trailing_title_word`, which the
215#: maiden take and P13's walk back ask) and by `_pieces._weighed`
216#: (#1171). Whether the word stands where the chain reaches is the
217#: reader's question. Dropped from the published token like the kind
218#: tags above.
219CHAIN_TITLE_TAG = "vocab:chain-title"
221#: On a run of two or more single letters joined by full stops whose
222#: spelling without them is a title written `dotted`
223#: (`Lexicon.dotted_titles`: 'U.S.', 'P.D.', #490), a word the
224#: vocabulary does not list as a title as written: rules/H10's two
225#: title runs at the head of a part, the given part's
226#: (`_pieces.leading_titles`) and H6's before a family comma (assign),
227#: take it as a title only where a title the run keeps stands behind it,
228#: so 'U.S. Grant' stays a name; classify's H9 fold and assemble's
229#: reading source read it too, and C4's count (`_vocab.name_word_count`)
230#: asks the same question without it.
231DOTTED_TITLE_TAG = "vocab:dotted-title"
233#: On a word of `Lexicon.given_name_abbreviations`, a given name written
234#: short ('Chas.', 'Ma.'): rules/H2's shape test declines a word
235#: carrying it, so the given reading outranks the shape-derived title
236#: (`_pieces.is_leading_title`, `_vocab.is_title_shaped`).
237GIVEN_ABBREVIATION_TAG = "vocab:given-abbreviation"
239#: On a word of `Lexicon.shared_names`, a name a community gives every
240#: member (Sikh 'Kaur', 'Singh'): rules/O6 and O7 decline a join that
241#: takes or completes one, and P10 completes none
242#: (`_post_rules._take_second_family_name`, `_take_second_given_name`,
243#: `_given_phrase_middle`).
244#: Dropped from the published token like the kind tags above (#704).
245SHARED_NAME_TAG = "vocab:shared-name"
247#: On a word of `Lexicon.particle_given_names`, a given name written
248#: behind a particle as the second of a compound given name ('del
249#: Carmen'): rules/P10 reads the particle run and the word as a
250#: middle name behind a one-word given name, and O6 leaves them there.
251#: Dropped from the published token like the kind tags above (#704).
252PARTICLE_GIVEN_TAG = "vocab:particle-given"
254#: On a bound given name whose word and the word after it are a phrase
255#: of `Lexicon.bound_phrases` ('عبد' before 'الله'), which rules/P12
256#: joins past the first name word (`_group._join_bound_phrases`), and
257#: group leaves it only on a pair it joined. Dropped
258#: from the published token like the kind tags above (#694).
259BOUND_PHRASE_TAG = "vocab:bound-phrase"
261#: EITHER way a token joins the ambiguous credential class -- the
262#: vocabulary's claim and the writing's. The two emitters that report
263#: a fork the peel called and declined ask exactly this: `_group`'s
264#: prefix chain (a listed member the case lean read as a name, 'John
265#: van der Berg Ma'; a by-shape member the count left standing,
266#: 'Freiherr von Berg X.Y.I.') and `_assign`'s family-comma slot. One
267#: constant beside the two above and for their reason -- the pair was
268#: spelled two ways, a frozenset here and an `or` of two `in` tests
269#: there -- and a frozenset so each test is one `isdisjoint`, a C call
270#: with no Python frame, on a branch every chained name reaches.
271_AMBIGUOUS_CREDENTIAL_TAGS = frozenset(
272 {AMBIGUOUS_ACRONYM_TAG, SHAPE_ACRONYM_TAG})
274# SPLIT_CREDENTIAL_TAG and SPLIT_MEMBER_TAG are defined and explained in
275# _types beside SHAPE_ACRONYM_TAG, case repair reading them too.
277#: The roles that make a piece a name word (rules/P6's walk), and
278#: the roles of a word not yet read as one: a suffix, or nothing yet
279#: -- what `_pieces.particle_tail` reads, asked by assign's given-part
280#: walk when it reaches the credential run, whose words are unread.
281NAME_ROLES = (Role.GIVEN, Role.MIDDLE, Role.FAMILY)
282SUFFIX_OR_UNREAD = (Role.SUFFIX, None)
285class Structure(Enum):
286 """The comma structure: segment writes NO_COMMA or FAMILY_COMMA,
287 and group's head decides a comma form once classify has tagged the
288 words, which may make it SUFFIX_COMMA (#613, `_comma.decide`)."""
290 NO_COMMA = auto()
291 FAMILY_COMMA = auto() # "Family, Given ..." (v1 lastname-comma)
292 SUFFIX_COMMA = auto() # "Given Family, Suffix ..."
295@dataclass(frozen=True, slots=True)
296# rules/A1: "the parse completes on the best reading and carries an
297# ambiguity report naming the doubt, as parsing never fails on any
298# input" -- its Then clause: this is the report's mid-pipeline record,
299# by token index (decisions/A1)
300class PendingAmbiguity:
301 """An ambiguity recorded mid-pipeline by token INDEX; assemble
302 materializes real Ambiguity objects over the final tokens.
304 ``origin`` is for the one stage that runs BEFORE tokens exist:
305 extract_delimited knows only a character offset, so it records that
306 and tokenize resolves it to the containing token's index. Stages
307 after tokenize set ``indices`` directly and leave ``origin`` None.
309 ``field_tail`` is for a report that names the field its word was
310 read into. post_rules may still move the word (H1's move behind a
311 title, P1's family-first fold, P6's attachment, and under opt-in
312 policies the patronymic rotations, O6's two family names and
313 `middle_as_family`'s fold),
314 so the emitter
315 writes ``detail`` up to the field and the rest of its sentence
316 here -- ``""`` where the field ends the sentence, which is why
317 assemble tests ``is not None`` -- and assemble joins them around
318 the FIELD_READING of the first referent's final role, the one
319 place that knows it (#626). Until then ``detail`` is a fragment.
320 """
322 kind: AmbiguityKind
323 detail: str
324 indices: tuple[int, ...] = ()
325 origin: int | None = None
326 field_tail: str | None = None
327 alternative: PendingReading | None = None
330class Deferred(Enum):
331 """A PendingReading field its emitter cannot name yet."""
333 #: O5's: a lone name word read as one of the two primary fields
334 #: would be read as the other, named from its FINAL role by
335 #: assemble, since H1 or a rotation may still move it after the
336 #: emitter (#626's reason for ``field_tail``).
337 OTHER_PRIMARY = auto()
338 #: classify's connective: the letter read as a connective would
339 #: join the name words on either side into one, whose field assign
340 #: names once it has positions -- the positional read's, where the
341 #: join would merge two pieces, and the letter's own piece's
342 #: (`Beside`), where both words are already in it.
343 JOINED = auto()
344 #: group's particle chain: the particle it chained would stand as a
345 #: name word of its own in front of the piece, named by assign.
346 UNCHAINED = auto()
349@dataclass(frozen=True, slots=True)
350class Beside:
351 """A PendingReading field named from another token's: the reading
352 lands where the token at `index` lands, named by assemble from that
353 token's final role -- a word the name piece in front would take in
354 (a particle chain, which runs to the end of the name words, or a
355 bound given name), or a connective whose join stays inside its own
356 piece. Where P6 then attached that token to the family
357 (`FOLDED_TAG`), which it would not have done with a word behind
358 it, `unfolded`, the field the word held when the report was made,
359 stands instead. Where the piece in front is a chain, `last` is its
360 last word: rules/P10 reads a chain's opening phrase as a middle name
361 and leaves its last word the family, where the word would join, so
362 a `last` still in the family names the field (#704)."""
364 index: int
365 unfolded: Role | None = None
366 last: int | None = None
369@dataclass(frozen=True, slots=True)
370class PendingReading:
371 """The reading a report's decision did not take, as its emitter
372 knows it; assemble turns it into the public Reading.
374 ``role`` is the field the first referent would hold under that
375 reading, with every other word of the name read as the parse read
376 it, a `Deferred` form for a stage that runs before the field is
377 known, or `Beside` for a field another token's final role names.
378 A JOINED or UNCHAINED form assign leaves unresolved -- a referent
379 that ends outside every name position -- surfaces as no
380 alternative rather than a guess. Assemble also folds a MIDDLE into
381 the family under ``middle_as_family``, the one later rule that
382 moves a field whatever word stands in it."""
384 kind: ReadingKind
385 source: ReadingSource
386 role: Role | Deferred | Beside
389#: How a report names the field a word was read into, by final role.
390FIELD_READING: Mapping[Role, str] = MappingProxyType({
391 Role.TITLE: "a title", Role.GIVEN: "a given name",
392 Role.MIDDLE: "a middle name", Role.FAMILY: "a family name",
393 Role.SUFFIX: "a suffix", Role.NICKNAME: "a nickname",
394 Role.MAIDEN: "a maiden name"})
395# every role a referent can hold, or a new one is a KeyError at the
396# first parse whose report lands in it
397assert FIELD_READING.keys() == set(Role)
400@dataclass(frozen=True, slots=True)
401class ParseState:
402 """Carried through the stage fold. Frozen; stages return copies via
403 copy_with. Fields are filled progressively:
404 extract_delimited -> extracted/masked; tokenize -> tokens (span-
405 sorted)/comma_offsets/interpunct_offsets (the 间隔号 and fullwidth-stop offsets the
406 order and segmentation decisions consult, #298; the nakaguro
407 separators record NOTHING); segment -> segments/structure/one_case
408 (lazily, only where C2's flag asks for it -- #289/#516, #613);
409 script_segment -> tokens and segments again (the one stage that
410 changes the token COUNT: an unspaced CJK token, or a word glued at
411 its full stop before any comma (rules/T8), splits into n+1 pieces,
412 still as sub-slices of the original, and every later index in the
413 segment runs shifts by n); classify -> token tags AND
414 one_case; group -> structure (its head decides a comma form, #613)
415 AND the SUFFIX/TITLE roles of a postnominal part after the comma,
416 whose segment it empties so no join reaches it,
417 then pieces/piece_tags/dropped AND maiden token roles, plus the
418 SUFFIX/TITLE roles of the run a maiden take gives up (#601), AND
419 tail_reads, S2's trailing run read once before its joins (#614);
420 assign -> the remaining token roles AND `order`, the effective
421 order it read them under, AND the folded-middle and attached tags
422 on the particles P6's attachment joins after a family comma (#613);
423 post_rules -> roles again (the TITLE and SUFFIX roles of a clause
424 S1 sets apart, each word's comma trimmed off its text and span,
425 #792; a delimited member S13 settles becomes the suffix, #810),
426 the folded-middle tag on what O3 folds and the
427 no-comma attachment joins, with the attached tag on the latter,
428 the second-surname tag on the words of the family's second unit
429 where O6 reads two or W5 reads a Han surname pair (rules/R10), the
430 first-surname tag on the words of its first unit (rules/R11), and the ambiguity P6's no-comma
431 attachment
432 reports (#467) and SUFFIX_OR_NICKNAME on a member it leaves a
433 nickname.
434 Ambiguities are recorded by every stage that DECIDES one --
435 extract (resolved to a token index by tokenize), segment,
436 script_segment, classify, group, assign, and post_rules -- since a
437 fork whose branches are taken in different stages needs an emitter
438 in each. Post-group, segments may retain indices of dropped tokens
439 -- assign iterates pieces, never segments. This ownership map is
440 pinned by tests/v2/pipeline/test_state.py.
442 segmenter belongs to no stage: like original/lexicon/policy it is
443 passed in at construction by Parser.parse and only ever READ (by
444 script_segment, for a token the vocabulary declined)."""
446 original: str
447 lexicon: Lexicon
448 policy: Policy
449 #: The optional Parser(segmenter=...) hook; None = not configured.
450 segmenter: Segmenter | None = None
451 extracted: tuple[tuple[Role, Span], ...] = ()
452 masked: tuple[Span, ...] = ()
453 tokens: tuple[WorkToken, ...] = ()
454 comma_offsets: tuple[int, ...] = ()
455 interpunct_offsets: tuple[int, ...] = ()
456 segments: tuple[tuple[int, ...], ...] = ()
457 structure: Structure = Structure.NO_COMMA
458 # pieces[s][p] = run of token indices: piece p of segment s.
459 # piece_tags[s][p] = derived flags for that piece ("title", "prefix",
460 # "suffix", "conjunction") set by group's joins.
461 pieces: tuple[tuple[tuple[int, ...], ...], ...] = ()
462 piece_tags: tuple[tuple[frozenset[str], ...], ...] = ()
463 dropped: tuple[int, ...] = () # structural tokens (maiden markers)
464 #: rules/S2's trailing run per segment, as group read it ONCE
465 #: after P3's joins and before the particle chain (#614), for
466 #: assign to place and report
467 #: rather than peel again; None where group read none -- a family
468 #: comma's parts, a part past a comma, a segment of fewer than
469 #: three pieces, a run with a name piece inside it -- and assign
470 #: reads those for itself, as it does a name group's joins left
471 #: without a name word. Aligned with `pieces`.
472 tail_reads: tuple[TailRead | None, ...] = ()
473 #: The order `assign` actually READ the name under -- name_order,
474 #: or the script_orders entry that overrode it. None wherever no
475 #: positional read happened: after a family comma (which fixes the
476 #: family, so assign consults no order at all), and on every early
477 #: return in `_assign_main`, where a segment holds no name piece
478 #: to position. Recorded rather than recomputed downstream,
479 #: because the two can differ and a post_rules rule keyed on
480 #: `policy.name_order` would then disagree with the roles assign
481 #: already wrote (#395). Reaching that divergence needs a custom
482 #: lexicon -- every shipped particle is Latin, and Latin has no
483 #: script_orders entry -- which is why the test for it builds its
484 #: own (test_post_rules.py).
485 order: tuple[Role, Role, Role] | None = None
486 #: Whether the name's OWN words are written wholly in one case --
487 #: all upper or all lower alike -- and so carry no case EVIDENCE
488 #: about any word in them (rules/P3's own-words span,
489 #: _pieces.own_words). None means NOT ASKED YET: the fact is
490 #: computed by whichever of segment and classify needs it first,
491 #: segment only when C2's flag on a part past the second comma asks
492 #: for it, so a reader between the two stages sees None and must
493 #: not guess -- script_segment asks for itself there (#613).
494 #: Recorded rather than recomputed, the way `order` above is: the
495 #: trailing suffix slot, the post-comma slot, the tail-segment
496 #: reading, the prefix chain's own tail measure (_group) and the
497 #: glued-honorific peel's decline of a post-comma run
498 #: (_script_segment) all consult it and must not disagree
499 #: (#289/#516, decisions/S6). Five stages read it in all --
500 #: segment, script_segment, classify, group and assign -- plus the
501 #: two predicate layers they read it through (_pieces, _vocab).
502 #: A fact segment records SURVIVES script_segment, the one stage
503 #: that changes the token count, and survives it unrecomputed
504 #: because splitting a token cannot change the answer: the verdict
505 #: is `is_one_case` over the name's own words JOINED, and a split
506 #: only moves a space into a string whose upper/lower comparison
507 #: ignores spaces entirely -- '김민준씨' and '김민준 씨' fold alike.
508 #: So the field is carried through rather than invalidated, and
509 #: script_segment reads it where segment recorded it (its suffix-run
510 #: predicate takes the lean), asking for itself only where segment
511 #: did not, without recording the answer (`_one_case`).
512 one_case: bool | None = None
513 ambiguities: tuple[PendingAmbiguity, ...] = ()
516def _copy_refusals(cls: type) -> list[str]:
517 """Why a field copy of `cls` might not build what
518 `dataclasses.replace` builds, one reason per unmet condition.
520 `replace` calls `obj.__class__(...)`, and every hook that call can
521 run -- a metaclass `__call__`, a `__new__`, the `__init__` and any
522 `__setattr__` it goes through, a `__post_init__`, an `InitVar` it
523 demands for want of a default -- is one a field copy skips. These conditions are a
524 TRIPWIRE for a realistic edit to the pipeline's three classes, not
525 a proof against any class at all: a forged `__qualname__` on a
526 borrowed `__init__` combined with its own decoration, or a
527 `__class__` property, is out of scope, and `copy_with` copies
528 nothing but those three anyway."""
529 reasons = []
530 params = cls.__dict__.get("__dataclass_params__")
531 if params is None:
532 reasons.append("not decorated as a dataclass itself")
533 elif not params.frozen:
534 reasons.append("not frozen")
535 init = cls.__dict__.get("__init__")
536 # dataclasses compiles the __init__ it generates from a string and
537 # names it after the class; one written in a class body carries its
538 # source file, and one borrowed from another class keeps that
539 # class's name.
540 if (init is None or init.__code__.co_filename != "<string>"
541 or init.__qualname__ != f"{cls.__qualname__}.__init__"):
542 reasons.append("__init__ not generated for it")
543 init = None
544 fields = dataclasses.fields(cls)
545 if init is not None:
546 # an InitVar is an __init__ parameter that is not a field
547 code = init.__code__
548 taken = code.co_argcount + code.co_kwonlyargcount - 1
549 if taken != sum(f.init for f in fields):
550 reasons.append("__init__ does not take exactly its init fields")
551 if type(cls) is not type:
552 reasons.append("built by a metaclass")
553 if any("__new__" in vars(k) for k in cls.__mro__[:-1]):
554 reasons.append("defines __new__")
555 if hasattr(cls, "__post_init__"):
556 reasons.append("defines __post_init__")
557 if not all(f.init for f in fields):
558 reasons.append("has an init=False field")
559 return reasons
562def _copyable_fields(cls: type) -> tuple[str, ...]:
563 """The fields `copy_with` carries for `cls`, or TypeError naming
564 every condition of `_copy_refusals` it fails."""
565 reasons = _copy_refusals(cls)
566 if reasons:
567 raise TypeError(
568 f"copy_with cannot copy {cls.__name__}: it needs a frozen "
569 "dataclass whose generated __init__ only assigns its fields "
570 f"({'; '.join(reasons)})")
571 return tuple(f.name for f in dataclasses.fields(cls))
574#: Built once, at import: a pipeline class that stops qualifying fails
575#: here rather than in a parse, and `copy_with` copies nothing else.
576_COPY_FIELDS: Mapping[type, tuple[str, ...]] = MappingProxyType({
577 cls: _copyable_fields(cls)
578 for cls in (WorkToken, PendingAmbiguity, PendingReading, ParseState)})
580_T = TypeVar("_T")
582if TYPE_CHECKING:
583 # mypy's dataclass plugin checks replace's keywords against the
584 # class, which a `**changes: object` signature would not.
585 from dataclasses import replace as copy_with
586else:
587 def copy_with(obj: _T, /, **changes: object) -> _T:
588 """`dataclasses.replace` for the pipeline's own dataclasses,
589 without its per-call cost: a direct field copy, which for the
590 classes in `_COPY_FIELDS` builds the same object
591 (decisions/parse-cost has the measurement)."""
592 cls = type(obj)
593 names = _COPY_FIELDS.get(cls)
594 if names is None:
595 raise TypeError(
596 f"copy_with copies only the pipeline's own dataclasses, "
597 f"not {cls.__name__}")
598 new = object.__new__(cls)
599 for name in names:
600 value = changes.pop(name) if name in changes else getattr(obj, name)
601 object.__setattr__(new, name, value)
602 if changes:
603 raise TypeError(f"{cls.__name__} has no field named "
604 f"{', '.join(sorted(changes))}")
605 return new