// This file is part of ICU4X. For terms of use, please see the file

// called LICENSE at the top level of the ICU4X source tree

// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).

use crate::complex::*;

use crate::indices::*;

use crate::provider::*;

use crate::SegmenterError;

use alloc::string::String;

use alloc::vec;

use alloc::vec::Vec;

use core::char;

use core::str::CharIndices;

use icu_provider::prelude::*;

use utf8_iter::Utf8CharIndices;

// TODO(#1637): These constants should be data driven.

#[allow(dead_code)]

const UNKNOWN: u8 = 0;

#[allow(dead_code)]

const AI: u8 = 1;

#[allow(dead_code)]

const AL: u8 = 2;

#[allow(dead_code)]

const B2: u8 = 3;

#[allow(dead_code)]

const BA: u8 = 4;

#[allow(dead_code)]

const BB: u8 = 5;

#[allow(dead_code)]

const BK: u8 = 6;

#[allow(dead_code)]

const CB: u8 = 7;

#[allow(dead_code)]

const CJ: u8 = 8;

#[allow(dead_code)]

const CL: u8 = 9;

#[allow(dead_code)]

const CM: u8 = 10;

#[allow(dead_code)]

const CP: u8 = 11;

#[allow(dead_code)]

const CR: u8 = 12;

#[allow(dead_code)]

const EB: u8 = 13;

#[allow(dead_code)]

const EM: u8 = 14;

#[allow(dead_code)]

const EX: u8 = 15;

#[allow(dead_code)]

const GL: u8 = 16;

#[allow(dead_code)]

const H2: u8 = 17;

#[allow(dead_code)]

const H3: u8 = 18;

#[allow(dead_code)]

const HL: u8 = 19;

#[allow(dead_code)]

const HY: u8 = 20;

#[allow(dead_code)]

const ID: u8 = 21;

#[allow(dead_code)]

const ID_CN: u8 = 22;

#[allow(dead_code)]

const IN: u8 = 23;

#[allow(dead_code)]

const IS: u8 = 24;

#[allow(dead_code)]

const JL: u8 = 25;

#[allow(dead_code)]

const JT: u8 = 26;

#[allow(dead_code)]

const JV: u8 = 27;

#[allow(dead_code)]

const LF: u8 = 28;

#[allow(dead_code)]

const NL: u8 = 29;

#[allow(dead_code)]

const NS: u8 = 30;

#[allow(dead_code)]

const NU: u8 = 31;

#[allow(dead_code)]

const OP_EA: u8 = 32;

#[allow(dead_code)]

const OP_OP30: u8 = 33;

#[allow(dead_code)]

const PO: u8 = 34;

#[allow(dead_code)]

const PO_EAW: u8 = 35;

#[allow(dead_code)]

const PR: u8 = 36;

#[allow(dead_code)]

const PR_EAW: u8 = 37;

#[allow(dead_code)]

const QU: u8 = 38;

#[allow(dead_code)]

const RI: u8 = 39;

#[allow(dead_code)]

const SA: u8 = 40;

#[allow(dead_code)]

const SG: u8 = 41;

#[allow(dead_code)]

const SP: u8 = 42;

#[allow(dead_code)]

const SY: u8 = 43;

#[allow(dead_code)]

const WJ: u8 = 44;

#[allow(dead_code)]

const XX: u8 = 45;

#[allow(dead_code)]

const ZW: u8 = 46;

#[allow(dead_code)]

const ZWJ: u8 = 47;

/// An enum specifies the strictness of line-breaking rules. It can be passed as

/// an argument when creating a line segmenter.

///

/// Each enum value has the same meaning with respect to the `line-break`

/// property values in the CSS Text spec. See the details in

/// <https://drafts.csswg.org/css-text-3/#line-break-property>.

#[non_exhaustive]

#[derive(Copy, Clone, PartialEq, Eq, Debug)]

pub enum LineBreakStrictness {

    /// Breaks text using the least restrictive set of line-breaking rules.

    /// Typically used for short lines, such as in newspapers.

    /// <https://drafts.csswg.org/css-text-3/#valdef-line-break-loose>

    Loose,

    /// Breaks text using the most common set of line-breaking rules.

    /// <https://drafts.csswg.org/css-text-3/#valdef-line-break-normal>

    Normal,

    /// Breaks text using the most stringent set of line-breaking rules.

    /// <https://drafts.csswg.org/css-text-3/#valdef-line-break-strict>

    ///

    /// This is the default behaviour of the Unicode Line Breaking Algorithm,

    /// resolving class [CJ](https://www.unicode.org/reports/tr14/#CJ) to

    /// [NS](https://www.unicode.org/reports/tr14/#NS);

    /// see rule [LB1](https://www.unicode.org/reports/tr14/#LB1).

    Strict,

    /// Breaks text assuming there is a soft wrap opportunity around every

    /// typographic character unit, disregarding any prohibition against line

    /// breaks. See more details in

    /// <https://drafts.csswg.org/css-text-3/#valdef-line-break-anywhere>.

    Anywhere,

}

/// An enum specifies the line break opportunities between letters. It can be

/// passed as an argument when creating a line segmenter.

///

/// Each enum value has the same meaning with respect to the `word-break`

/// property values in the CSS Text spec. See the details in

/// <https://drafts.csswg.org/css-text-3/#word-break-property>

#[non_exhaustive]

#[derive(Copy, Clone, PartialEq, Eq, Debug)]

pub enum LineBreakWordOption {

    /// Words break according to their customary rules. See the details in

    /// <https://drafts.csswg.org/css-text-3/#valdef-word-break-normal>.

    Normal,

    /// Breaking is allowed within "words".

    /// <https://drafts.csswg.org/css-text-3/#valdef-word-break-break-all>

    BreakAll,

    /// Breaking is forbidden within "word".

    /// <https://drafts.csswg.org/css-text-3/#valdef-word-break-keep-all>

    KeepAll,

}

/// Options to tailor line-breaking behavior.

#[non_exhaustive]

#[derive(Copy, Clone, PartialEq, Eq, Debug)]

pub struct LineBreakOptions {

    /// Strictness of line-breaking rules. See [`LineBreakStrictness`].

    pub strictness: LineBreakStrictness,

    /// Line break opportunities between letters. See [`LineBreakWordOption`].

    pub word_option: LineBreakWordOption,

    /// Use `true` as a hint to the line segmenter that the writing

    /// system is Chinese or Japanese. This allows more break opportunities when

    /// `LineBreakStrictness` is `Normal` or `Loose`. See

    /// <https://drafts.csswg.org/css-text-3/#line-break-property> for details.

    ///

    /// This option has no effect in Latin-1 mode.

    pub ja_zh: bool,

}

impl Default for LineBreakOptions {

    fn default() -> Self {

        Self {

            strictness: LineBreakStrictness::Strict,

            word_option: LineBreakWordOption::Normal,

            ja_zh: false,

        }

    }

}

/// Line break iterator for an `str` (a UTF-8 string).

///

/// For examples of use, see [`LineSegmenter`].

pub type LineBreakIteratorUtf8<'l, 's> = LineBreakIterator<'l, 's, LineBreakTypeUtf8>;

/// Line break iterator for a potentially invalid UTF-8 string.

///

/// For examples of use, see [`LineSegmenter`].

pub type LineBreakIteratorPotentiallyIllFormedUtf8<'l, 's> =

    LineBreakIterator<'l, 's, LineBreakTypePotentiallyIllFormedUtf8>;

/// Line break iterator for a Latin-1 (8-bit) string.

///

/// For examples of use, see [`LineSegmenter`].

pub type LineBreakIteratorLatin1<'l, 's> = LineBreakIterator<'l, 's, LineBreakTypeLatin1>;

/// Line break iterator for a UTF-16 string.

///

/// For examples of use, see [`LineSegmenter`].

pub type LineBreakIteratorUtf16<'l, 's> = LineBreakIterator<'l, 's, LineBreakTypeUtf16>;

/// Supports loading line break data, and creating line break iterators for different string

/// encodings.

///

/// The segmenter returns mandatory breaks (as defined by [definition LD7][LD7] of

/// Unicode Standard Annex #14, _Unicode Line Breaking Algorithm_) as well as

/// line break opportunities ([definition LD3][LD3]).

/// It does not distinguish them.  Callers requiring that distinction can check

/// the Line_Break property of the code point preceding the break against those

/// listed in rules [LB4][LB4] and [LB5][LB5], special-casing the end of text

/// according to [LB3][LB3].

///

/// For consistency with the grapheme, word, and sentence segmenters, there is

/// always a breakpoint returned at index 0, but this breakpoint is not a

/// meaningful line break opportunity.

///

/// [LD3]: https://www.unicode.org/reports/tr14/#LD3

/// [LD7]: https://www.unicode.org/reports/tr14/#LD7

/// [LB3]: https://www.unicode.org/reports/tr14/#LB3

/// [LB4]: https://www.unicode.org/reports/tr14/#LB4

/// [LB5]: https://www.unicode.org/reports/tr14/#LB5

///

/// ```rust

/// # use icu::segmenter::LineSegmenter;

/// #

/// # let segmenter = LineSegmenter::new_auto();

/// #

/// let text = "Summary\r\nThis annex…";

/// let breakpoints: Vec<usize> = segmenter.segment_str(text).collect();

/// // 9 and 22 are mandatory breaks, 14 is a line break opportunity.

/// assert_eq!(&breakpoints, &[0, 9, 14, 22]);

///

/// // There is a break opportunity between emoji, but not within the ZWJ sequence 🏳️‍🌈.

/// let flag_equation = "🏳️➕🌈🟰🏳️\u{200D}🌈";

/// let possible_first_lines: Vec<&str> =

///     segmenter.segment_str(flag_equation).skip(1).map(|i| &flag_equation[..i]).collect();

/// assert_eq!(

///     &possible_first_lines,

///     &[

///         "🏳️",

///         "🏳️➕",

///         "🏳️➕🌈",

///         "🏳️➕🌈🟰",

///         "🏳️➕🌈🟰🏳️‍🌈"

///     ]

/// );

/// ```

///

/// # Examples

///

/// Segment a string with default options:

///

/// ```rust

/// use icu::segmenter::LineSegmenter;

///

/// let segmenter = LineSegmenter::new_auto();

///

/// let breakpoints: Vec<usize> =

///     segmenter.segment_str("Hello World").collect();

/// assert_eq!(&breakpoints, &[0, 6, 11]);

/// ```

///

/// Segment a string with CSS option overrides:

///

/// ```rust

/// use icu::segmenter::{

///     LineBreakOptions, LineBreakStrictness, LineBreakWordOption,

///     LineSegmenter,

/// };

///

/// let mut options = LineBreakOptions::default();

/// options.strictness = LineBreakStrictness::Strict;

/// options.word_option = LineBreakWordOption::BreakAll;

/// options.ja_zh = false;

/// let segmenter = LineSegmenter::new_auto_with_options(options);

///

/// let breakpoints: Vec<usize> =

///     segmenter.segment_str("Hello World").collect();

/// assert_eq!(&breakpoints, &[0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11]);

/// ```

///

/// Segment a Latin1 byte string:

///

/// ```rust

/// use icu::segmenter::LineSegmenter;

///

/// let segmenter = LineSegmenter::new_auto();

///

/// let breakpoints: Vec<usize> =

///     segmenter.segment_latin1(b"Hello World").collect();

/// assert_eq!(&breakpoints, &[0, 6, 11]);

/// ```

///

/// Separate mandatory breaks from the break opportunities:

///

/// ```rust

/// use icu::properties::{maps, LineBreak};

/// use icu::segmenter::LineSegmenter;

///

/// # let segmenter = LineSegmenter::new_auto();

/// #

/// let text = "Summary\r\nThis annex…";

///

/// let mandatory_breaks: Vec<usize> = segmenter

///     .segment_str(text)

///     .into_iter()

///     .filter(|&i| {

///         text[..i].chars().next_back().map_or(false, |c| {

///             matches!(

///                 maps::line_break().get(c),

///                 LineBreak::MandatoryBreak

///                     | LineBreak::CarriageReturn

///                     | LineBreak::LineFeed

///                     | LineBreak::NextLine

///             ) || i == text.len()

///         })

///     })

///     .collect();

/// assert_eq!(&mandatory_breaks, &[9, 22]);

/// ```

#[derive(Debug)]

pub struct LineSegmenter {

    options: LineBreakOptions,

    payload: DataPayload<LineBreakDataV1Marker>,

    complex: ComplexPayloads,

}

impl LineSegmenter {

    /// Constructs a [`LineSegmenter`] with an invariant locale and the best available compiled data for

    /// complex scripts (Khmer, Lao, Myanmar, and Thai).

    ///

    /// The current behavior, which is subject to change, is to use the LSTM model when available.

    ///

    /// See also [`Self::new_auto_with_options`].

    ///

    /// ✨ *Enabled with the `compiled_data` and `auto` Cargo features.*

    ///

    /// [📚 Help choosing a constructor](icu_provider::constructors)

    #[cfg(feature = "compiled_data")]

    #[cfg(feature = "auto")]

    pub fn new_auto() -> Self {

        Self::new_auto_with_options(Default::default())

    }

    #[cfg(feature = "auto")]

    icu_provider::gen_any_buffer_data_constructors!(

        locale: skip,

        options: skip,

        error: SegmenterError,

        #[cfg(skip)]

        functions: [

            new_auto,

            try_new_auto_with_any_provider,

            try_new_auto_with_buffer_provider,

            try_new_auto_unstable,

            Self,

        ]

    );

    #[doc = icu_provider::gen_any_buffer_unstable_docs!(UNSTABLE, Self::new_auto)]

    #[cfg(feature = "auto")]

    pub fn try_new_auto_unstable<D>(provider: &D) -> Result<Self, SegmenterError>

    where

        D: DataProvider<LineBreakDataV1Marker>

            + DataProvider<LstmForWordLineAutoV1Marker>

            + DataProvider<GraphemeClusterBreakDataV1Marker>

            + ?Sized,

    {

        Self::try_new_auto_with_options_unstable(provider, Default::default())

    }

Unexecuted instantiation: <icu_segmenter::line::LineSegmenter>::try_new_auto_unstable::<icu_provider::any::DowncastingAnyProvider<icu_provider_adapters::empty::EmptyDataProvider>>

Unexecuted instantiation: <icu_segmenter::line::LineSegmenter>::try_new_auto_unstable::<_>

    /// Constructs a [`LineSegmenter`] with an invariant locale and compiled LSTM data for

    /// complex scripts (Khmer, Lao, Myanmar, and Thai).

    ///

    /// The LSTM, or Long Term Short Memory, is a machine learning model. It is smaller than

    /// the full dictionary but more expensive during segmentation (inference).

    ///

    /// See also [`Self::new_lstm_with_options`].

    ///

    /// ✨ *Enabled with the `compiled_data` and `lstm` Cargo features.*

    ///

    /// [📚 Help choosing a constructor](icu_provider::constructors)

    #[cfg(feature = "compiled_data")]

    #[cfg(feature = "lstm")]

    pub fn new_lstm() -> Self {

        Self::new_lstm_with_options(Default::default())

    }

    #[cfg(feature = "lstm")]

    icu_provider::gen_any_buffer_data_constructors!(

        locale: skip,

        options: skip,

        error: SegmenterError,

        #[cfg(skip)]

        functions: [

            new_lstm,

            try_new_lstm_with_any_provider,

            try_new_lstm_with_buffer_provider,

            try_new_lstm_unstable,

            Self,

        ]

    );

    #[cfg(feature = "lstm")]

    #[doc = icu_provider::gen_any_buffer_unstable_docs!(UNSTABLE, Self::new_lstm)]

    pub fn try_new_lstm_unstable<D>(provider: &D) -> Result<Self, SegmenterError>

    where

        D: DataProvider<LineBreakDataV1Marker>

            + DataProvider<LstmForWordLineAutoV1Marker>

            + DataProvider<GraphemeClusterBreakDataV1Marker>

            + ?Sized,

    {

        Self::try_new_lstm_with_options_unstable(provider, Default::default())

    }

Unexecuted instantiation: <icu_segmenter::line::LineSegmenter>::try_new_lstm_unstable::<icu_provider::any::DowncastingAnyProvider<icu_provider_adapters::empty::EmptyDataProvider>>

Unexecuted instantiation: <icu_segmenter::line::LineSegmenter>::try_new_lstm_unstable::<_>

    /// Constructs a [`LineSegmenter`] with an invariant locale and compiled dictionary data for

    /// complex scripts (Khmer, Lao, Myanmar, and Thai).

    ///

    /// The dictionary model uses a list of words to determine appropriate breakpoints. It is

    /// faster than the LSTM model but requires more data.

    ///

    /// See also [`Self::new_dictionary_with_options`].

    ///

    /// ✨ *Enabled with the `compiled_data` Cargo feature.*

    ///

    /// [📚 Help choosing a constructor](icu_provider::constructors)

    #[cfg(feature = "compiled_data")]

    pub fn new_dictionary() -> Self {

        Self::new_dictionary_with_options(Default::default())

    }

    icu_provider::gen_any_buffer_data_constructors!(

        locale: skip,

        options: skip,

        error: SegmenterError,

        #[cfg(skip)]

        functions: [

            new_dictionary,

            try_new_dictionary_with_any_provider,

            try_new_dictionary_with_buffer_provider,

            try_new_dictionary_unstable,

            Self,

        ]

    );

    #[doc = icu_provider::gen_any_buffer_unstable_docs!(UNSTABLE, Self::new_dictionary)]

    pub fn try_new_dictionary_unstable<D>(provider: &D) -> Result<Self, SegmenterError>

    where

        D: DataProvider<LineBreakDataV1Marker>

            + DataProvider<DictionaryForWordLineExtendedV1Marker>

            + DataProvider<GraphemeClusterBreakDataV1Marker>

            + ?Sized,

    {

        Self::try_new_dictionary_with_options_unstable(provider, Default::default())

    }

Unexecuted instantiation: <icu_segmenter::line::LineSegmenter>::try_new_dictionary_unstable::<icu_provider::any::DowncastingAnyProvider<icu_provider_adapters::empty::EmptyDataProvider>>

Unexecuted instantiation: <icu_segmenter::line::LineSegmenter>::try_new_dictionary_unstable::<_>

    /// Constructs a [`LineSegmenter`] with an invariant locale, custom [`LineBreakOptions`], and

    /// the best available compiled data for complex scripts (Khmer, Lao, Myanmar, and Thai).

    ///

    /// The current behavior, which is subject to change, is to use the LSTM model when available.

    ///

    /// See also [`Self::new_auto`].

    ///

    /// ✨ *Enabled with the `compiled_data` and `auto` Cargo features.*

    ///

    /// [📚 Help choosing a constructor](icu_provider::constructors)

    #[cfg(feature = "auto")]

    #[cfg(feature = "compiled_data")]

    pub fn new_auto_with_options(options: LineBreakOptions) -> Self {

        Self::new_lstm_with_options(options)

    }

    #[cfg(feature = "auto")]

    icu_provider::gen_any_buffer_data_constructors!(

        locale: skip,

        options: LineBreakOptions,

        error: SegmenterError,

        #[cfg(skip)]

        functions: [

            new_auto_with_options,

            try_new_auto_with_options_with_any_provider,

            try_new_auto_with_options_with_buffer_provider,

            try_new_auto_with_options_unstable,

            Self,

        ]

    );

    #[cfg(feature = "auto")]

    #[doc = icu_provider::gen_any_buffer_unstable_docs!(UNSTABLE, Self::new_auto_with_options)]

    pub fn try_new_auto_with_options_unstable<D>(

        provider: &D,

        options: LineBreakOptions,

    ) -> Result<Self, SegmenterError>

    where

        D: DataProvider<LineBreakDataV1Marker>

            + DataProvider<LstmForWordLineAutoV1Marker>

            + DataProvider<GraphemeClusterBreakDataV1Marker>

            + ?Sized,

    {

        Self::try_new_lstm_with_options_unstable(provider, options)

    }

Unexecuted instantiation: <icu_segmenter::line::LineSegmenter>::try_new_auto_with_options_unstable::<icu_provider::any::DowncastingAnyProvider<icu_provider_adapters::empty::EmptyDataProvider>>

Unexecuted instantiation: <icu_segmenter::line::LineSegmenter>::try_new_auto_with_options_unstable::<_>

    /// Constructs a [`LineSegmenter`] with an invariant locale, custom [`LineBreakOptions`], and

    /// compiled LSTM data for complex scripts (Khmer, Lao, Myanmar, and Thai).

    ///

    /// The LSTM, or Long Term Short Memory, is a machine learning model. It is smaller than

    /// the full dictionary but more expensive during segmentation (inference).

    ///

    /// See also [`Self::new_dictionary`].

    ///

    /// ✨ *Enabled with the `compiled_data` and `lstm` Cargo features.*

    ///

    /// [📚 Help choosing a constructor](icu_provider::constructors)

    #[cfg(feature = "lstm")]

    #[cfg(feature = "compiled_data")]

    pub fn new_lstm_with_options(options: LineBreakOptions) -> Self {

        Self {

            options,

            payload: DataPayload::from_static_ref(

                crate::provider::Baked::SINGLETON_SEGMENTER_LINE_V1,

            ),

            complex: ComplexPayloads::new_lstm(),

        }

    }

    #[cfg(feature = "lstm")]

    icu_provider::gen_any_buffer_data_constructors!(

        locale: skip,

        options: LineBreakOptions,

        error: SegmenterError,

        #[cfg(skip)]

        functions: [

            try_new_lstm_with_options,

            try_new_lstm_with_options_with_any_provider,

            try_new_lstm_with_options_with_buffer_provider,

            try_new_lstm_with_options_unstable,

            Self,

        ]

    );

    #[cfg(feature = "lstm")]

    #[doc = icu_provider::gen_any_buffer_unstable_docs!(UNSTABLE, Self::new_lstm_with_options)]

    pub fn try_new_lstm_with_options_unstable<D>(

        provider: &D,

        options: LineBreakOptions,

    ) -> Result<Self, SegmenterError>

    where

        D: DataProvider<LineBreakDataV1Marker>

            + DataProvider<LstmForWordLineAutoV1Marker>

            + DataProvider<GraphemeClusterBreakDataV1Marker>

            + ?Sized,

    {

        Ok(Self {

            options,

            payload: provider.load(Default::default())?.take_payload()?,

            complex: ComplexPayloads::try_new_lstm(provider)?,

        })

    }

Unexecuted instantiation: <icu_segmenter::line::LineSegmenter>::try_new_lstm_with_options_unstable::<icu_provider::any::DowncastingAnyProvider<icu_provider_adapters::empty::EmptyDataProvider>>

Unexecuted instantiation: <icu_segmenter::line::LineSegmenter>::try_new_lstm_with_options_unstable::<_>

    /// Constructs a [`LineSegmenter`] with an invariant locale, custom [`LineBreakOptions`], and

    /// compiled dictionary data for complex scripts (Khmer, Lao, Myanmar, and Thai).

    ///

    /// The dictionary model uses a list of words to determine appropriate breakpoints. It is

    /// faster than the LSTM model but requires more data.

    ///

    /// See also [`Self::new_dictionary`].

    ///

    /// ✨ *Enabled with the `compiled_data` Cargo feature.*

    ///

    /// [📚 Help choosing a constructor](icu_provider::constructors)

    #[cfg(feature = "compiled_data")]

    pub fn new_dictionary_with_options(options: LineBreakOptions) -> Self {

        Self {

            options,

            payload: DataPayload::from_static_ref(

                crate::provider::Baked::SINGLETON_SEGMENTER_LINE_V1,

            ),

            // Line segmenter doesn't need to load CJ dictionary because UAX 14 rules handles CJK

            // characters [1]. Southeast Asian languages however require complex context analysis

            // [2].

            //

            // [1]: https://www.unicode.org/reports/tr14/#ID

            // [2]: https://www.unicode.org/reports/tr14/#SA

            complex: ComplexPayloads::new_southeast_asian(),

        }

    }

    icu_provider::gen_any_buffer_data_constructors!(

        locale: skip,

        options: LineBreakOptions,

        error: SegmenterError,

        #[cfg(skip)]

        functions: [

            new_dictionary_with_options,

            try_new_dictionary_with_options_with_any_provider,

            try_new_dictionary_with_options_with_buffer_provider,

            try_new_dictionary_with_options_unstable,

            Self,

        ]

    );

    #[doc = icu_provider::gen_any_buffer_unstable_docs!(UNSTABLE, Self::new_dictionary_with_options)]

    pub fn try_new_dictionary_with_options_unstable<D>(

        provider: &D,

        options: LineBreakOptions,

    ) -> Result<Self, SegmenterError>

    where

        D: DataProvider<LineBreakDataV1Marker>

            + DataProvider<DictionaryForWordLineExtendedV1Marker>

            + DataProvider<GraphemeClusterBreakDataV1Marker>

            + ?Sized,

    {

        Ok(Self {

            options,

            payload: provider.load(Default::default())?.take_payload()?,

            // Line segmenter doesn't need to load CJ dictionary because UAX 14 rules handles CJK

            // characters [1]. Southeast Asian languages however require complex context analysis

            // [2].

            //

            // [1]: https://www.unicode.org/reports/tr14/#ID

            // [2]: https://www.unicode.org/reports/tr14/#SA

            complex: ComplexPayloads::try_new_southeast_asian(provider)?,

        })

    }

Unexecuted instantiation: <icu_segmenter::line::LineSegmenter>::try_new_dictionary_with_options_unstable::<icu_provider::any::DowncastingAnyProvider<icu_provider_adapters::empty::EmptyDataProvider>>

Unexecuted instantiation: <icu_segmenter::line::LineSegmenter>::try_new_dictionary_with_options_unstable::<_>

    /// Creates a line break iterator for an `str` (a UTF-8 string).

    ///

    /// There are always breakpoints at 0 and the string length, or only at 0 for the empty string.

    pub fn segment_str<'l, 's>(&'l self, input: &'s str) -> LineBreakIteratorUtf8<'l, 's> {

        LineBreakIterator {

            iter: input.char_indices(),

            len: input.len(),

            current_pos_data: None,

            result_cache: Vec::new(),

            data: self.payload.get(),

            options: &self.options,

            complex: &self.complex,

        }

    }

    /// Creates a line break iterator for a potentially ill-formed UTF8 string

    ///

    /// Invalid characters are treated as REPLACEMENT CHARACTER

    ///

    /// There are always breakpoints at 0 and the string length, or only at 0 for the empty string.

    pub fn segment_utf8<'l, 's>(

        &'l self,

        input: &'s [u8],

    ) -> LineBreakIteratorPotentiallyIllFormedUtf8<'l, 's> {

        LineBreakIterator {

            iter: Utf8CharIndices::new(input),

            len: input.len(),

            current_pos_data: None,

            result_cache: Vec::new(),

            data: self.payload.get(),

            options: &self.options,

            complex: &self.complex,

        }

    }

    /// Creates a line break iterator for a Latin-1 (8-bit) string.

    ///

    /// There are always breakpoints at 0 and the string length, or only at 0 for the empty string.

    pub fn segment_latin1<'l, 's>(&'l self, input: &'s [u8]) -> LineBreakIteratorLatin1<'l, 's> {

        LineBreakIterator {

            iter: Latin1Indices::new(input),

            len: input.len(),

            current_pos_data: None,

            result_cache: Vec::new(),

            data: self.payload.get(),

            options: &self.options,

            complex: &self.complex,

        }

    }

    /// Creates a line break iterator for a UTF-16 string.

    ///

    /// There are always breakpoints at 0 and the string length, or only at 0 for the empty string.

    pub fn segment_utf16<'l, 's>(&'l self, input: &'s [u16]) -> LineBreakIteratorUtf16<'l, 's> {

        LineBreakIterator {

            iter: Utf16Indices::new(input),

            len: input.len(),

            current_pos_data: None,

            result_cache: Vec::new(),

            data: self.payload.get(),

            options: &self.options,

            complex: &self.complex,

        }

    }

}

impl RuleBreakDataV1<'_> {

    fn get_linebreak_property_utf32_with_rule(

        &self,

        codepoint: u32,

        strictness: LineBreakStrictness,

        word_option: LineBreakWordOption,

    ) -> u8 {

        // Note: Default value is 0 == UNKNOWN

        let prop = self.property_table.get32(codepoint);

        if word_option == LineBreakWordOption::BreakAll

            || strictness == LineBreakStrictness::Loose

            || strictness == LineBreakStrictness::Normal

        {

            return match prop {

                CJ => ID, // All CJ's General_Category is Other_Letter (Lo).

                _ => prop,

            };

        }

        // CJ is treated as NS by default, yielding strict line breaking.

        // https://www.unicode.org/reports/tr14/#CJ

        prop

    }

    #[inline]

    fn get_break_state_from_table(&self, left: u8, right: u8) -> BreakState {

        let idx = (left as usize) * (self.property_count as usize) + (right as usize);

        // We use unwrap_or to fall back to the base case and prevent panics on bad data.

        self.break_state_table.get(idx).unwrap_or(BreakState::Keep)

    }

Unexecuted instantiation: <icu_segmenter::provider::RuleBreakDataV1>::get_break_state_from_table

Unexecuted instantiation: <icu_segmenter::provider::RuleBreakDataV1>::get_break_state_from_table

    #[inline]

    fn use_complex_breaking_utf32(&self, codepoint: u32) -> bool {

        let line_break_property = self.get_linebreak_property_utf32_with_rule(

            codepoint,

            LineBreakStrictness::Strict,

            LineBreakWordOption::Normal,

        );

        line_break_property == SA

    }

Unexecuted instantiation: <icu_segmenter::provider::RuleBreakDataV1>::use_complex_breaking_utf32

Unexecuted instantiation: <icu_segmenter::provider::RuleBreakDataV1>::use_complex_breaking_utf32

}

#[inline]

fn is_break_utf32_by_loose(

    right_codepoint: u32,

    left_prop: u8,

    right_prop: u8,

    ja_zh: bool,

) -> Option<bool> {

    // breaks before hyphens

    if right_prop == BA {

        if left_prop == ID && (right_codepoint == 0x2010 || right_codepoint == 0x2013) {

            return Some(true);

        }

    } else if right_prop == NS {

        // breaks before certain CJK hyphen-like characters

        if right_codepoint == 0x301C || right_codepoint == 0x30A0 {

            return Some(ja_zh);

        }

        // breaks before iteration marks

        if right_codepoint == 0x3005

            || right_codepoint == 0x303B

            || right_codepoint == 0x309D

            || right_codepoint == 0x309E

            || right_codepoint == 0x30FD

            || right_codepoint == 0x30FE

        {

            return Some(true);

        }

        // breaks before certain centered punctuation marks:

        if right_codepoint == 0x30FB

            || right_codepoint == 0xFF1A

            || right_codepoint == 0xFF1B

            || right_codepoint == 0xFF65

            || right_codepoint == 0x203C

            || (0x2047..=0x2049).contains(&right_codepoint)

        {

            return Some(ja_zh);

        }

    } else if right_prop == IN {

        // breaks between inseparable characters such as U+2025, U+2026 i.e. characters with the Unicode Line Break property IN

        return Some(true);

    } else if right_prop == EX {

        // breaks before certain centered punctuation marks:

        if right_codepoint == 0xFF01 || right_codepoint == 0xFF1F {

            return Some(ja_zh);

        }

    }

    // breaks before suffixes:

    // Characters with the Unicode Line Break property PO and the East Asian Width property

    if right_prop == PO_EAW {

        return Some(ja_zh);

    }

    // breaks after prefixes:

    // Characters with the Unicode Line Break property PR and the East Asian Width property

    if left_prop == PR_EAW {

        return Some(ja_zh);

    }

    None

}

Unexecuted instantiation: icu_segmenter::line::is_break_utf32_by_loose

Unexecuted instantiation: icu_segmenter::line::is_break_utf32_by_loose

/// A trait allowing for LineBreakIterator to be generalized to multiple string iteration methods.

///

/// This is implemented by ICU4X for several common string types.

pub trait LineBreakType<'l, 's> {

    /// The iterator over characters.

    type IterAttr: Iterator<Item = (usize, Self::CharType)> + Clone;

    /// The character type.

    type CharType: Copy + Into<u32>;

    fn use_complex_breaking(iterator: &LineBreakIterator<'l, 's, Self>, c: Self::CharType) -> bool;

    fn get_linebreak_property_with_rule(

        iterator: &LineBreakIterator<'l, 's, Self>,

        c: Self::CharType,

    ) -> u8;

    fn get_current_position_character_len(iterator: &LineBreakIterator<'l, 's, Self>) -> usize;

    fn handle_complex_language(

        iterator: &mut LineBreakIterator<'l, 's, Self>,

        left_codepoint: Self::CharType,

    ) -> Option<usize>;

}

/// Implements the [`Iterator`] trait over the line break opportunities of the given string.

///

/// Lifetimes:

///

/// - `'l` = lifetime of the [`LineSegmenter`] object from which this iterator was created

/// - `'s` = lifetime of the string being segmented

///

/// The [`Iterator::Item`] is an [`usize`] representing index of a code unit

/// _after_ the break (for a break at the end of text, this index is the length

/// of the [`str`] or array of code units).

///

/// For examples of use, see [`LineSegmenter`].

#[derive(Debug)]

pub struct LineBreakIterator<'l, 's, Y: LineBreakType<'l, 's> + ?Sized> {

    iter: Y::IterAttr,

    len: usize,

    current_pos_data: Option<(usize, Y::CharType)>,

    result_cache: Vec<usize>,

    data: &'l RuleBreakDataV1<'l>,

    options: &'l LineBreakOptions,

    complex: &'l ComplexPayloads,

}

impl<'l, 's, Y: LineBreakType<'l, 's>> Iterator for LineBreakIterator<'l, 's, Y> {

    type Item = usize;

    fn next(&mut self) -> Option<Self::Item> {

        match self.check_eof() {

            StringBoundaryPosType::Start => return Some(0),

            StringBoundaryPosType::End => return None,

            _ => (),

        }

        // If we have break point cache by previous run, return this result

        if let Some(&first_pos) = self.result_cache.first() {

            let mut i = 0;

            loop {

                if i == first_pos {

                    self.result_cache = self.result_cache.iter().skip(1).map(|r| r - i).collect();

Unexecuted instantiation: <icu_segmenter::line::LineBreakIterator<icu_segmenter::line::LineBreakTypeUtf16> as core::iter::traits::iterator::Iterator>::next::{closure#0}

Unexecuted instantiation: <icu_segmenter::line::LineBreakIterator<icu_segmenter::line::LineBreakTypeLatin1> as core::iter::traits::iterator::Iterator>::next::{closure#0}

Unexecuted instantiation: <icu_segmenter::line::LineBreakIterator<icu_segmenter::line::LineBreakTypePotentiallyIllFormedUtf8> as core::iter::traits::iterator::Iterator>::next::{closure#0}

Unexecuted instantiation: <icu_segmenter::line::LineBreakIterator<_> as core::iter::traits::iterator::Iterator>::next::{closure#0}

                    return self.get_current_position();

                }

                i += Y::get_current_position_character_len(self);

                self.advance_iter();

                if self.is_eof() {

                    self.result_cache.clear();

                    return Some(self.len);

                }

            }

        }

        'a: loop {

            debug_assert!(!self.is_eof());

            let left_codepoint = self.get_current_codepoint()?;

            let mut left_prop = self.get_linebreak_property(left_codepoint);

            self.advance_iter();

            let Some(right_codepoint) = self.get_current_codepoint() else {

                return Some(self.len);

            };

            let right_prop = self.get_linebreak_property(right_codepoint);

            // CSS word-break property handling

            match (self.options.word_option, left_prop, right_prop) {

                (LineBreakWordOption::BreakAll, AL | NU | SA, _) => {

                    left_prop = ID;

                }

                //  typographic letter units shouldn't be break

                (

                    LineBreakWordOption::KeepAll,

                    AI | AL | ID | NU | HY | H2 | H3 | JL | JV | JT | CJ,

                    AI | AL | ID | NU | HY | H2 | H3 | JL | JV | JT | CJ,

                ) => {

                    continue;

                }

                _ => (),

            }

            // CSS line-break property handling

            match self.options.strictness {

                LineBreakStrictness::Normal => {

                    if self.is_break_by_normal(right_codepoint) {

                        return self.get_current_position();

                    }

                }

                LineBreakStrictness::Loose => {

                    if let Some(breakable) = is_break_utf32_by_loose(

                        right_codepoint.into(),

                        left_prop,

                        right_prop,

                        self.options.ja_zh,

                    ) {

                        if breakable {

                            return self.get_current_position();

                        }

                        continue;

                    }

                }

                LineBreakStrictness::Anywhere => {

                    return self.get_current_position();

                }

                _ => (),

            };

            // UAX14 doesn't have Thai etc, so use another way.

            if self.options.word_option != LineBreakWordOption::BreakAll

                && Y::use_complex_breaking(self, left_codepoint)

                && Y::use_complex_breaking(self, right_codepoint)

            {

                let result = Y::handle_complex_language(self, left_codepoint);

                if result.is_some() {

                    return result;

                }

                // I may have to fetch text until non-SA character?.

            }

            // If break_state is equals or grater than 0, it is alias of property.

            let mut index = match self.data.get_break_state_from_table(left_prop, right_prop) {

                BreakState::Index(index) => index,

                // Line break uses more that 64 states, so they spill over into the intermediate range,

                // and we cannot change that at the moment

                BreakState::Intermediate(index) => index + 64,

                BreakState::Break | BreakState::NoMatch => return self.get_current_position(),

                BreakState::Keep => continue,

            };

            let mut previous_iter = self.iter.clone();

            let mut previous_pos_data = self.current_pos_data;

            loop {

                self.advance_iter();

                let Some(prop) = self.get_current_linebreak_property() else {

                    // Reached EOF. But we are analyzing multiple characters now, so next break may be previous point.

                    let break_state = self

                        .data

                        .get_break_state_from_table(index, self.data.eot_property);

                    if break_state == BreakState::NoMatch {

                        self.iter = previous_iter;

                        self.current_pos_data = previous_pos_data;

                        return self.get_current_position();

                    }

                    // EOF

                    return Some(self.len);

                };

                match self.data.get_break_state_from_table(index, prop) {

                    BreakState::Keep => continue 'a,

                    BreakState::NoMatch => {

                        self.iter = previous_iter;

                        self.current_pos_data = previous_pos_data;

                        return self.get_current_position();

                    }

                    BreakState::Break => return self.get_current_position(),

                    BreakState::Index(i) => {

                        index = i;

                        previous_iter = self.iter.clone();

                        previous_pos_data = self.current_pos_data;

                    }

                    BreakState::Intermediate(i) => {

                        index = i + 64;

                        previous_iter = self.iter.clone();

                        previous_pos_data = self.current_pos_data;

                    }

                }

            }

        }

    }

Unexecuted instantiation: <icu_segmenter::line::LineBreakIterator<icu_segmenter::line::LineBreakTypeUtf16> as core::iter::traits::iterator::Iterator>::next

Unexecuted instantiation: <icu_segmenter::line::LineBreakIterator<icu_segmenter::line::LineBreakTypeLatin1> as core::iter::traits::iterator::Iterator>::next

Unexecuted instantiation: <icu_segmenter::line::LineBreakIterator<icu_segmenter::line::LineBreakTypePotentiallyIllFormedUtf8> as core::iter::traits::iterator::Iterator>::next

Unexecuted instantiation: <icu_segmenter::line::LineBreakIterator<_> as core::iter::traits::iterator::Iterator>::next

}

enum StringBoundaryPosType {

    Start,

    Middle,

    End,

}

impl<'l, 's, Y: LineBreakType<'l, 's>> LineBreakIterator<'l, 's, Y> {

    fn advance_iter(&mut self) {

        self.current_pos_data = self.iter.next();

    }

Unexecuted instantiation: <icu_segmenter::line::LineBreakIterator<icu_segmenter::line::LineBreakTypeLatin1>>::advance_iter

Unexecuted instantiation: <icu_segmenter::line::LineBreakIterator<icu_segmenter::line::LineBreakTypeUtf8>>::advance_iter

Unexecuted instantiation: <icu_segmenter::line::LineBreakIterator<icu_segmenter::line::LineBreakTypeUtf16>>::advance_iter

Unexecuted instantiation: <icu_segmenter::line::LineBreakIterator<icu_segmenter::line::LineBreakTypePotentiallyIllFormedUtf8>>::advance_iter

    fn is_eof(&self) -> bool {

        self.current_pos_data.is_none()

    }

Unexecuted instantiation: <icu_segmenter::line::LineBreakIterator<icu_segmenter::line::LineBreakTypeLatin1>>::is_eof

Unexecuted instantiation: <icu_segmenter::line::LineBreakIterator<icu_segmenter::line::LineBreakTypeUtf8>>::is_eof

Unexecuted instantiation: <icu_segmenter::line::LineBreakIterator<icu_segmenter::line::LineBreakTypeUtf16>>::is_eof

Unexecuted instantiation: <icu_segmenter::line::LineBreakIterator<icu_segmenter::line::LineBreakTypePotentiallyIllFormedUtf8>>::is_eof

    #[inline]