/src/serenity/Userland/Libraries/LibLocale/Segmenter.cpp

Source
/*
 * Copyright (c) 2024, Tim Flynn <trflynn89@serenityos.org>
 *
 * SPDX-License-Identifier: BSD-2-Clause
 */

#include <AK/BinarySearch.h>
#include <AK/Utf16View.h>
#include <AK/Utf32View.h>
#include <LibLocale/Locale.h>
#include <LibLocale/Segmenter.h>
#include <LibUnicode/Segmentation.h>

namespace Locale {

SegmenterGranularity segmenter_granularity_from_string(StringView segmenter_granularity)
{
    if (segmenter_granularity == "grapheme"sv)
        return SegmenterGranularity::Grapheme;
    if (segmenter_granularity == "sentence"sv)
        return SegmenterGranularity::Sentence;
    if (segmenter_granularity == "word"sv)
        return SegmenterGranularity::Word;
    VERIFY_NOT_REACHED();
}

StringView segmenter_granularity_to_string(SegmenterGranularity segmenter_granularity)
{
    switch (segmenter_granularity) {
    case SegmenterGranularity::Grapheme:
        return "grapheme"sv;
    case SegmenterGranularity::Sentence:
        return "sentence"sv;
    case SegmenterGranularity::Word:
        return "word"sv;
    }
    VERIFY_NOT_REACHED();
}

class SegmenterImpl : public Segmenter {
public:
    SegmenterImpl(SegmenterGranularity segmenter_granularity)
        : Segmenter(segmenter_granularity)
    {
    }

    virtual ~SegmenterImpl() override = default;

    virtual NonnullOwnPtr<Segmenter> clone() const override
    {
        return make<SegmenterImpl>(m_segmenter_granularity);
    }

    virtual void set_segmented_text(String text) override
    {
        m_string_storage = move(text);
        set_text(m_string_storage.code_points());
    }

    virtual void set_segmented_text(Utf16View const& text) override
    {
        set_text(text);
    }

    void set_segmented_text(Utf32View const& text)
    {
        set_text(text);
    }

    virtual size_t current_boundary() override
    {
        return m_current_boundary;
    }

    virtual Optional<size_t> previous_boundary(size_t boundary, Inclusive inclusive) override
    {
        recompute_boundaries_if_necessary();

        if (inclusive == Inclusive::Yes)
            ++boundary;

        auto lower_bound_index = strict_lower_bound(m_boundaries.span(), boundary);

        if (!lower_bound_index.has_value())
            return OptionalNone {};
        m_current_boundary = m_boundaries[*lower_bound_index];
        return m_current_boundary;
    }

    virtual Optional<size_t> next_boundary(size_t boundary, Inclusive inclusive) override
    {
        recompute_boundaries_if_necessary();

        if (inclusive == Inclusive::Yes)
            --boundary;

        auto upper_bound_index = upper_bound(m_boundaries.span(), boundary);

        if (!upper_bound_index.has_value())
            return OptionalNone {};
        m_current_boundary = m_boundaries[*upper_bound_index];
        return m_current_boundary;
    }

    virtual void for_each_boundary(String text, SegmentationCallback callback) override
    {
        for_each_segmentation_boundary(text.code_points(), move(callback));
    }

    virtual void for_each_boundary(Utf16View const& text, SegmentationCallback callback) override
    {
        for_each_segmentation_boundary(text, move(callback));
    }

    virtual void for_each_boundary(Utf32View const& text, SegmentationCallback callback) override
    {
        for_each_segmentation_boundary(text, move(callback));
    }

    virtual bool is_current_boundary_word_like() const override
    {
        // FIXME: Implement one day.
        return false;
    }

private:
    void set_text(Variant<Utf8View, Utf16View, Utf32View> text)
    {
        m_segmented_text = text;
        m_must_recompute_boundaries = true;
    }

    void recompute_boundaries_if_necessary()
    {
        if (!m_must_recompute_boundaries)
            return;

        m_boundaries.clear();
        auto callback = [&](size_t boundary) {
            m_boundaries.append(boundary);
            return IterationDecision::Continue;
        };
        m_segmented_text.visit([&](auto const& text) { return for_each_segmentation_boundary(text, move(callback)); });
        m_must_recompute_boundaries = false;
    }

    template<class T>
    void for_each_segmentation_boundary(T const& text, SegmentationCallback callback)
    {
        switch (segmenter_granularity()) {
        case SegmenterGranularity::Grapheme:
            Unicode::for_each_grapheme_segmentation_boundary(text, move(callback));
            break;
        case SegmenterGranularity::Sentence:
            Unicode::for_each_sentence_segmentation_boundary(text, move(callback));
            break;
        case SegmenterGranularity::Word:
            Unicode::for_each_word_segmentation_boundary(text, move(callback));
            break;
        }
    }

    bool m_must_recompute_boundaries { true };
    Vector<size_t> m_boundaries;
    size_t m_current_boundary { 0 };
    String m_string_storage;
    Variant<Utf8View, Utf16View, Utf32View> m_segmented_text { Utf8View {} };
};

NonnullOwnPtr<Segmenter> Segmenter::create(SegmenterGranularity segmenter_granularity)
{
    return Segmenter::create(default_locale(), segmenter_granularity);
}

NonnullOwnPtr<Segmenter> Segmenter::create(StringView locale, SegmenterGranularity segmenter_granularity)
{
    // FIXME: Implement locale-specific segmentation.
    (void)locale;
    return make<SegmenterImpl>(segmenter_granularity);
}

}

Line	Count	Source
1		/*
2		* Copyright (c) 2024, Tim Flynn <trflynn89@serenityos.org>
3		*
4		* SPDX-License-Identifier: BSD-2-Clause
5		*/
6
7		#include <AK/BinarySearch.h>
8		#include <AK/Utf16View.h>
9		#include <AK/Utf32View.h>
10		#include <LibLocale/Locale.h>
11		#include <LibLocale/Segmenter.h>
12		#include <LibUnicode/Segmentation.h>
13
14		namespace Locale {
15
16		SegmenterGranularity segmenter_granularity_from_string(StringView segmenter_granularity)
17	0	{
18	0	if (segmenter_granularity == "grapheme"sv)
19	0	return SegmenterGranularity::Grapheme;
20	0	if (segmenter_granularity == "sentence"sv)
21	0	return SegmenterGranularity::Sentence;
22	0	if (segmenter_granularity == "word"sv)
23	0	return SegmenterGranularity::Word;
24	0	VERIFY_NOT_REACHED();
25	0	}
26
27		StringView segmenter_granularity_to_string(SegmenterGranularity segmenter_granularity)
28	0	{
29	0	switch (segmenter_granularity) {
30	0	case SegmenterGranularity::Grapheme:
31	0	return "grapheme"sv;
32	0	case SegmenterGranularity::Sentence:
33	0	return "sentence"sv;
34	0	case SegmenterGranularity::Word:
35	0	return "word"sv;
36	0	}
37	0	VERIFY_NOT_REACHED();
38	0	}
39
40		class SegmenterImpl : public Segmenter {
41		public:
42		SegmenterImpl(SegmenterGranularity segmenter_granularity)
43	0	: Segmenter(segmenter_granularity)
44	0	{
45	0	}
46
47	0	virtual ~SegmenterImpl() override = default;
48
49		virtual NonnullOwnPtr<Segmenter> clone() const override
50	0	{
51	0	return make<SegmenterImpl>(m_segmenter_granularity);
52	0	}
53
54		virtual void set_segmented_text(String text) override
55	0	{
56	0	m_string_storage = move(text);
57	0	set_text(m_string_storage.code_points());
58	0	}
59
60		virtual void set_segmented_text(Utf16View const& text) override
61	0	{
62	0	set_text(text);
63	0	}
64
65		void set_segmented_text(Utf32View const& text)
66	0	{
67	0	set_text(text);
68	0	}
69
70		virtual size_t current_boundary() override
71	0	{
72	0	return m_current_boundary;
73	0	}
74
75		virtual Optional<size_t> previous_boundary(size_t boundary, Inclusive inclusive) override
76	0	{
77	0	recompute_boundaries_if_necessary();
78
79	0	if (inclusive == Inclusive::Yes)
80	0	++boundary;
81
82	0	auto lower_bound_index = strict_lower_bound(m_boundaries.span(), boundary);
83
84	0	if (!lower_bound_index.has_value())
85	0	return OptionalNone {};
86	0	m_current_boundary = m_boundaries[*lower_bound_index];
87	0	return m_current_boundary;
88	0	}
89
90		virtual Optional<size_t> next_boundary(size_t boundary, Inclusive inclusive) override
91	0	{
92	0	recompute_boundaries_if_necessary();
93
94	0	if (inclusive == Inclusive::Yes)
95	0	--boundary;
96
97	0	auto upper_bound_index = upper_bound(m_boundaries.span(), boundary);
98
99	0	if (!upper_bound_index.has_value())
100	0	return OptionalNone {};
101	0	m_current_boundary = m_boundaries[*upper_bound_index];
102	0	return m_current_boundary;
103	0	}
104
105		virtual void for_each_boundary(String text, SegmentationCallback callback) override
106	0	{
107	0	for_each_segmentation_boundary(text.code_points(), move(callback));
108	0	}
109
110		virtual void for_each_boundary(Utf16View const& text, SegmentationCallback callback) override
111	0	{
112	0	for_each_segmentation_boundary(text, move(callback));
113	0	}
114
115		virtual void for_each_boundary(Utf32View const& text, SegmentationCallback callback) override
116	0	{
117	0	for_each_segmentation_boundary(text, move(callback));
118	0	}
119
120		virtual bool is_current_boundary_word_like() const override
121	0	{
122		// FIXME: Implement one day.
123	0	return false;
124	0	}
125
126		private:
127		void set_text(Variant<Utf8View, Utf16View, Utf32View> text)
128	0	{
129	0	m_segmented_text = text;
130	0	m_must_recompute_boundaries = true;
131	0	}
132
133		void recompute_boundaries_if_necessary()
134	0	{
135	0	if (!m_must_recompute_boundaries)
136	0	return;
137
138	0	m_boundaries.clear();
139	0	auto callback = [&](size_t boundary) {
140	0	m_boundaries.append(boundary);
141	0	return IterationDecision::Continue;
142	0	};
143	0	m_segmented_text.visit([&](auto const& text) { return for_each_segmentation_boundary(text, move(callback)); }); Unexecuted instantiation: auto Locale::SegmenterImpl::recompute_boundaries_if_necessary()::{lambda(auto:1 const&)#1}::operator()<AK::Utf8View>(AK::Utf8View const&) const Unexecuted instantiation: auto Locale::SegmenterImpl::recompute_boundaries_if_necessary()::{lambda(auto:1 const&)#1}::operator()<AK::Utf16View>(AK::Utf16View const&) const Unexecuted instantiation: auto Locale::SegmenterImpl::recompute_boundaries_if_necessary()::{lambda(auto:1 const&)#1}::operator()<AK::Utf32View>(AK::Utf32View const&) const
144	0	m_must_recompute_boundaries = false;
145	0	}
146
147		template<class T>
148		void for_each_segmentation_boundary(T const& text, SegmentationCallback callback)
149	0	{
150	0	switch (segmenter_granularity()) {
151	0	case SegmenterGranularity::Grapheme:
152	0	Unicode::for_each_grapheme_segmentation_boundary(text, move(callback));
153	0	break;
154	0	case SegmenterGranularity::Sentence:
155	0	Unicode::for_each_sentence_segmentation_boundary(text, move(callback));
156	0	break;
157	0	case SegmenterGranularity::Word:
158	0	Unicode::for_each_word_segmentation_boundary(text, move(callback));
159	0	break;
160	0	}
161	0	} Unexecuted instantiation: void Locale::SegmenterImpl::for_each_segmentation_boundary<AK::Utf8View>(AK::Utf8View const&, AK::Function<AK::IterationDecision (unsigned long)>) Unexecuted instantiation: void Locale::SegmenterImpl::for_each_segmentation_boundary<AK::Utf16View>(AK::Utf16View const&, AK::Function<AK::IterationDecision (unsigned long)>) Unexecuted instantiation: void Locale::SegmenterImpl::for_each_segmentation_boundary<AK::Utf32View>(AK::Utf32View const&, AK::Function<AK::IterationDecision (unsigned long)>)
162
163		bool m_must_recompute_boundaries { true };
164		Vector<size_t> m_boundaries;
165		size_t m_current_boundary { 0 };
166		String m_string_storage;
167		Variant<Utf8View, Utf16View, Utf32View> m_segmented_text { Utf8View {} };
168		};
169
170		NonnullOwnPtr<Segmenter> Segmenter::create(SegmenterGranularity segmenter_granularity)
171	0	{
172	0	return Segmenter::create(default_locale(), segmenter_granularity);
173	0	}
174
175		NonnullOwnPtr<Segmenter> Segmenter::create(StringView locale, SegmenterGranularity segmenter_granularity)
176	0	{
177		// FIXME: Implement locale-specific segmentation.
178	0	(void)locale;
179	0	return make<SegmenterImpl>(segmenter_granularity);
180	0	}
181
182		}

Coverage Report

Created: 2026-09-26 06:49