/src/icu/icu4c/source/i18n/collationdata.h

Source (jump to first uncovered line)
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2010-2015, International Business Machines
* Corporation and others.  All Rights Reserved.
*******************************************************************************
* collationdata.h
*
* created on: 2010oct27
* created by: Markus W. Scherer
*/

#ifndef __COLLATIONDATA_H__
#define __COLLATIONDATA_H__

#include "unicode/utypes.h"

#if !UCONFIG_NO_COLLATION

#include "unicode/ucol.h"
#include "unicode/uniset.h"
#include "collation.h"
#include "normalizer2impl.h"
#include "utrie2.h"

struct UDataMemory;

U_NAMESPACE_BEGIN

class UVector32;

/**
 * Collation data container.
 * Immutable data created by a CollationDataBuilder, or loaded from a file,
 * or deserialized from API-provided binary data.
 *
 * Includes data for the collation base (root/default), aliased if this is not the base.
 */
struct U_I18N_API CollationData : public UMemory {
    // Note: The ucadata.icu loader could discover the reserved ranges by setting an array
    // parallel with the ranges, and resetting ranges that are indexed.
    // The reordering builder code could clone the resulting template array.
    static constexpr int32_t REORDER_RESERVED_BEFORE_LATIN = UCOL_REORDER_CODE_FIRST + 14;
    static constexpr int32_t REORDER_RESERVED_AFTER_LATIN = REORDER_RESERVED_BEFORE_LATIN + 1;

    static constexpr int32_t MAX_NUM_SPECIAL_REORDER_CODES = 8;
    /** C++ only, data reader check scriptStartsLength. */
    static constexpr int32_t MAX_NUM_SCRIPT_RANGES = 256;

    CollationData(const Normalizer2Impl &nfc)
            : trie(nullptr),
              ce32s(nullptr), ces(nullptr), contexts(nullptr), base(nullptr),
              jamoCE32s(nullptr),
              nfcImpl(nfc),
              numericPrimary(0x12000000),
              ce32sLength(0), cesLength(0), contextsLength(0),
              compressibleBytes(nullptr),
              unsafeBackwardSet(nullptr),
              fastLatinTable(nullptr), fastLatinTableLength(0),
              numScripts(0), scriptsIndex(nullptr), scriptStarts(nullptr), scriptStartsLength(0),
              rootElements(nullptr), rootElementsLength(0) {}

    uint32_t getCE32(UChar32 c) const {
        return UTRIE2_GET32(trie, c);
    }

    uint32_t getCE32FromSupplementary(UChar32 c) const {
        return UTRIE2_GET32_FROM_SUPP(trie, c);
    }

    UBool isDigit(UChar32 c) const {
        return c < 0x660 ? c <= 0x39 && 0x30 <= c :
                Collation::hasCE32Tag(getCE32(c), Collation::DIGIT_TAG);
    }

    UBool isUnsafeBackward(UChar32 c, UBool numeric) const {
        return unsafeBackwardSet->contains(c) || (numeric && isDigit(c));
    }

    UBool isCompressibleLeadByte(uint32_t b) const {
        return compressibleBytes[b];
    }

    inline UBool isCompressiblePrimary(uint32_t p) const {
        return isCompressibleLeadByte(p >> 24);
    }

    /**
     * Returns the CE32 from two contexts words.
     * Access to the defaultCE32 for contraction and prefix matching.
     */
    static uint32_t readCE32(const char16_t *p) {
        return ((uint32_t)p[0] << 16) | p[1];
    }

    /**
     * Returns the CE32 for an indirect special CE32 (e.g., with DIGIT_TAG).
     * Requires that ce32 is special.
     */
    uint32_t getIndirectCE32(uint32_t ce32) const;
    /**
     * Returns the CE32 for an indirect special CE32 (e.g., with DIGIT_TAG),
     * if ce32 is special.
     */
    uint32_t getFinalCE32(uint32_t ce32) const;

    /**
     * Computes a CE from c's ce32 which has the OFFSET_TAG.
     */
    int64_t getCEFromOffsetCE32(UChar32 c, uint32_t ce32) const {
        int64_t dataCE = ces[Collation::indexFromCE32(ce32)];
        return Collation::makeCE(Collation::getThreeBytePrimaryForOffsetData(c, dataCE));
    }

    /**
     * Returns the single CE that c maps to.
     * Sets U_UNSUPPORTED_ERROR if c does not map to a single CE.
     */
    int64_t getSingleCE(UChar32 c, UErrorCode &errorCode) const;

    /**
     * Returns the FCD16 value for code point c. c must be >= 0.
     */
    uint16_t getFCD16(UChar32 c) const {
        return nfcImpl.getFCD16(c);
    }

    /**
     * Returns the first primary for the script's reordering group.
     * @return the primary with only the first primary lead byte of the group
     *         (not necessarily an actual root collator primary weight),
     *         or 0 if the script is unknown
     */
    uint32_t getFirstPrimaryForGroup(int32_t script) const;

    /**
     * Returns the last primary for the script's reordering group.
     * @return the last primary of the group
     *         (not an actual root collator primary weight),
     *         or 0 if the script is unknown
     */
    uint32_t getLastPrimaryForGroup(int32_t script) const;

    /**
     * Finds the reordering group which contains the primary weight.
     * @return the first script of the group, or -1 if the weight is beyond the last group
     */
    int32_t getGroupForPrimary(uint32_t p) const;

    int32_t getEquivalentScripts(int32_t script,
                                 int32_t dest[], int32_t capacity, UErrorCode &errorCode) const;

    /**
     * Writes the permutation of primary-weight ranges
     * for the given reordering of scripts and groups.
     * The caller checks for illegal arguments and
     * takes care of [DEFAULT] and memory allocation.
     *
     * Each list element will be a (limit, offset) pair as described
     * for the CollationSettings::reorderRanges.
     * The list will be empty if no ranges are reordered.
     */
    void makeReorderRanges(const int32_t *reorder, int32_t length,
                           UVector32 &ranges, UErrorCode &errorCode) const;

    /** @see jamoCE32s */
    static const int32_t JAMO_CE32S_LENGTH = 19 + 21 + 27;

    /** Main lookup trie. */
    const UTrie2 *trie;
    /**
     * Array of CE32 values.
     * At index 0 there must be CE32(U+0000)
     * to support U+0000's special-tag for NUL-termination handling.
     */
    const uint32_t *ce32s;
    /** Array of CE values for expansions and OFFSET_TAG. */
    const int64_t *ces;
    /** Array of prefix and contraction-suffix matching data. */
    const char16_t *contexts;
    /** Base collation data, or nullptr if this data itself is a base. */
    const CollationData *base;
    /**
     * Simple array of JAMO_CE32S_LENGTH=19+21+27 CE32s, one per canonical Jamo L/V/T.
     * They are normally simple CE32s, rarely expansions.
     * For fast handling of HANGUL_TAG.
     */
    const uint32_t *jamoCE32s;
    const Normalizer2Impl &nfcImpl;
    /** The single-byte primary weight (xx000000) for numeric collation. */
    uint32_t numericPrimary;

    int32_t ce32sLength;
    int32_t cesLength;
    int32_t contextsLength;

    /** 256 flags for which primary-weight lead bytes are compressible. */
    const UBool *compressibleBytes;
    /**
     * Set of code points that are unsafe for starting string comparison after an identical prefix,
     * or in backwards CE iteration.
     */
    const UnicodeSet *unsafeBackwardSet;

    /**
     * Fast Latin table for common-Latin-text string comparisons.
     * Data structure see class CollationFastLatin.
     */
    const uint16_t *fastLatinTable;
    int32_t fastLatinTableLength;

    /**
     * Data for scripts and reordering groups.
     * Uses include building a reordering permutation table and
     * providing script boundaries to AlphabeticIndex.
     */
    int32_t numScripts;
    /**
     * The length of scriptsIndex is numScripts+16.
     * It maps from a UScriptCode or a special reorder code to an entry in scriptStarts.
     * 16 special reorder codes (not all used) are mapped starting at numScripts.
     * Up to MAX_NUM_SPECIAL_REORDER_CODES are codes for special groups like space/punct/digit.
     * There are special codes at the end for reorder-reserved primary ranges.
     *
     * Multiple scripts may share a range and index, for example Hira & Kana.
     */
    const uint16_t *scriptsIndex;
    /**
     * Start primary weight (top 16 bits only) for a group/script/reserved range
     * indexed by scriptsIndex.
     * The first range (separators & terminators) and the last range (trailing weights)
     * are not reorderable, and no scriptsIndex entry points to them.
     */
    const uint16_t *scriptStarts;
    int32_t scriptStartsLength;

    /**
     * Collation elements in the root collator.
     * Used by the CollationRootElements class. The data structure is described there.
     * nullptr in a tailoring.
     */
    const uint32_t *rootElements;
    int32_t rootElementsLength;

private:
    int32_t getScriptIndex(int32_t script) const;
    void makeReorderRanges(const int32_t *reorder, int32_t length,
                           UBool latinMustMove,
                           UVector32 &ranges, UErrorCode &errorCode) const;
    int32_t addLowScriptRange(uint8_t table[], int32_t index, int32_t lowStart) const;
    int32_t addHighScriptRange(uint8_t table[], int32_t index, int32_t highLimit) const;
};

U_NAMESPACE_END

#endif  // !UCONFIG_NO_COLLATION
#endif  // __COLLATIONDATA_H__

Coverage Report

Created: 2023-03-04 07:00

Line	Count	Source (jump to first uncovered line)
1		// © 2016 and later: Unicode, Inc. and others.
2		// License & terms of use: http://www.unicode.org/copyright.html
3		/*
4		*******************************************************************************
5		* Copyright (C) 2010-2015, International Business Machines
6		* Corporation and others. All Rights Reserved.
7		*******************************************************************************
8		* collationdata.h
9		*
10		* created on: 2010oct27
11		* created by: Markus W. Scherer
12		*/
13
14		#ifndef __COLLATIONDATA_H__
15		#define __COLLATIONDATA_H__
16
17		#include "unicode/utypes.h"
18
19		#if !UCONFIG_NO_COLLATION
20
21		#include "unicode/ucol.h"
22		#include "unicode/uniset.h"
23		#include "collation.h"
24		#include "normalizer2impl.h"
25		#include "utrie2.h"
26
27		struct UDataMemory;
28
29		U_NAMESPACE_BEGIN
30
31		class UVector32;
32
33		/**
34		* Collation data container.
35		* Immutable data created by a CollationDataBuilder, or loaded from a file,
36		* or deserialized from API-provided binary data.
37		*
38		* Includes data for the collation base (root/default), aliased if this is not the base.
39		*/
40		struct U_I18N_API CollationData : public UMemory {
41		// Note: The ucadata.icu loader could discover the reserved ranges by setting an array
42		// parallel with the ranges, and resetting ranges that are indexed.
43		// The reordering builder code could clone the resulting template array.
44		static constexpr int32_t REORDER_RESERVED_BEFORE_LATIN = UCOL_REORDER_CODE_FIRST + 14;
45		static constexpr int32_t REORDER_RESERVED_AFTER_LATIN = REORDER_RESERVED_BEFORE_LATIN + 1;
46
47		static constexpr int32_t MAX_NUM_SPECIAL_REORDER_CODES = 8;
48		/** C++ only, data reader check scriptStartsLength. */
49		static constexpr int32_t MAX_NUM_SCRIPT_RANGES = 256;
50
51		CollationData(const Normalizer2Impl &nfc)
52		: trie(nullptr),
53		ce32s(nullptr), ces(nullptr), contexts(nullptr), base(nullptr),
54		jamoCE32s(nullptr),
55		nfcImpl(nfc),
56		numericPrimary(0x12000000),
57		ce32sLength(0), cesLength(0), contextsLength(0),
58		compressibleBytes(nullptr),
59		unsafeBackwardSet(nullptr),
60		fastLatinTable(nullptr), fastLatinTableLength(0),
61		numScripts(0), scriptsIndex(nullptr), scriptStarts(nullptr), scriptStartsLength(0),
62	1.80k	rootElements(nullptr), rootElementsLength(0) {}
63
64	100M	uint32_t getCE32(UChar32 c) const {
65	100M	return UTRIE2_GET32(trie, c);
66	100M	}
67
68	3.24k	uint32_t getCE32FromSupplementary(UChar32 c) const {
69	3.24k	return UTRIE2_GET32_FROM_SUPP(trie, c);
70	3.24k	}
71
72	0	UBool isDigit(UChar32 c) const {
73	0	return c < 0x660 ? c <= 0x39 && 0x30 <= c :
74	0	Collation::hasCE32Tag(getCE32(c), Collation::DIGIT_TAG);
75	0	}
76
77	337k	UBool isUnsafeBackward(UChar32 c, UBool numeric) const {
78	337k	return unsafeBackwardSet->contains(c) \|\| (numeric && isDigit(c));
79	337k	}
80
81	4.27k	UBool isCompressibleLeadByte(uint32_t b) const {
82	4.27k	return compressibleBytes[b];
83	4.27k	}
84
85	4.27k	inline UBool isCompressiblePrimary(uint32_t p) const {
86	4.27k	return isCompressibleLeadByte(p >> 24);
87	4.27k	}
88
89		/**
90		* Returns the CE32 from two contexts words.
91		* Access to the defaultCE32 for contraction and prefix matching.
92		*/
93	10.4M	static uint32_t readCE32(const char16_t *p) {
94	10.4M	return ((uint32_t)p[0] << 16) \| p[1];
95	10.4M	}
96
97		/**
98		* Returns the CE32 for an indirect special CE32 (e.g., with DIGIT_TAG).
99		* Requires that ce32 is special.
100		*/
101		uint32_t getIndirectCE32(uint32_t ce32) const;
102		/**
103		* Returns the CE32 for an indirect special CE32 (e.g., with DIGIT_TAG),
104		* if ce32 is special.
105		*/
106		uint32_t getFinalCE32(uint32_t ce32) const;
107
108		/**
109		* Computes a CE from c's ce32 which has the OFFSET_TAG.
110		*/
111	30.3M	int64_t getCEFromOffsetCE32(UChar32 c, uint32_t ce32) const {
112	30.3M	int64_t dataCE = ces[Collation::indexFromCE32(ce32)];
113	30.3M	return Collation::makeCE(Collation::getThreeBytePrimaryForOffsetData(c, dataCE));
114	30.3M	}
115
116		/**
117		* Returns the single CE that c maps to.
118		* Sets U_UNSUPPORTED_ERROR if c does not map to a single CE.
119		*/
120		int64_t getSingleCE(UChar32 c, UErrorCode &errorCode) const;
121
122		/**
123		* Returns the FCD16 value for code point c. c must be >= 0.
124		*/
125	412M	uint16_t getFCD16(UChar32 c) const {
126	412M	return nfcImpl.getFCD16(c);
127	412M	}
128
129		/**
130		* Returns the first primary for the script's reordering group.
131		* @return the primary with only the first primary lead byte of the group
132		* (not necessarily an actual root collator primary weight),
133		* or 0 if the script is unknown
134		*/
135		uint32_t getFirstPrimaryForGroup(int32_t script) const;
136
137		/**
138		* Returns the last primary for the script's reordering group.
139		* @return the last primary of the group
140		* (not an actual root collator primary weight),
141		* or 0 if the script is unknown
142		*/
143		uint32_t getLastPrimaryForGroup(int32_t script) const;
144
145		/**
146		* Finds the reordering group which contains the primary weight.
147		* @return the first script of the group, or -1 if the weight is beyond the last group
148		*/
149		int32_t getGroupForPrimary(uint32_t p) const;
150
151		int32_t getEquivalentScripts(int32_t script,
152		int32_t dest[], int32_t capacity, UErrorCode &errorCode) const;
153
154		/**
155		* Writes the permutation of primary-weight ranges
156		* for the given reordering of scripts and groups.
157		* The caller checks for illegal arguments and
158		* takes care of [DEFAULT] and memory allocation.
159		*
160		* Each list element will be a (limit, offset) pair as described
161		* for the CollationSettings::reorderRanges.
162		* The list will be empty if no ranges are reordered.
163		*/
164		void makeReorderRanges(const int32_t *reorder, int32_t length,
165		UVector32 &ranges, UErrorCode &errorCode) const;
166
167		/** @see jamoCE32s */
168		static const int32_t JAMO_CE32S_LENGTH = 19 + 21 + 27;
169
170		/** Main lookup trie. */
171		const UTrie2 *trie;
172		/**
173		* Array of CE32 values.
174		* At index 0 there must be CE32(U+0000)
175		* to support U+0000's special-tag for NUL-termination handling.
176		*/
177		const uint32_t *ce32s;
178		/** Array of CE values for expansions and OFFSET_TAG. */
179		const int64_t *ces;
180		/** Array of prefix and contraction-suffix matching data. */
181		const char16_t *contexts;
182		/** Base collation data, or nullptr if this data itself is a base. */
183		const CollationData *base;
184		/**
185		* Simple array of JAMO_CE32S_LENGTH=19+21+27 CE32s, one per canonical Jamo L/V/T.
186		* They are normally simple CE32s, rarely expansions.
187		* For fast handling of HANGUL_TAG.
188		*/
189		const uint32_t *jamoCE32s;
190		const Normalizer2Impl &nfcImpl;
191		/** The single-byte primary weight (xx000000) for numeric collation. */
192		uint32_t numericPrimary;
193
194		int32_t ce32sLength;
195		int32_t cesLength;
196		int32_t contextsLength;
197
198		/** 256 flags for which primary-weight lead bytes are compressible. */
199		const UBool *compressibleBytes;
200		/**
201		* Set of code points that are unsafe for starting string comparison after an identical prefix,
202		* or in backwards CE iteration.
203		*/
204		const UnicodeSet *unsafeBackwardSet;
205
206		/**
207		* Fast Latin table for common-Latin-text string comparisons.
208		* Data structure see class CollationFastLatin.
209		*/
210		const uint16_t *fastLatinTable;
211		int32_t fastLatinTableLength;
212
213		/**
214		* Data for scripts and reordering groups.
215		* Uses include building a reordering permutation table and
216		* providing script boundaries to AlphabeticIndex.
217		*/
218		int32_t numScripts;
219		/**
220		* The length of scriptsIndex is numScripts+16.
221		* It maps from a UScriptCode or a special reorder code to an entry in scriptStarts.
222		* 16 special reorder codes (not all used) are mapped starting at numScripts.
223		* Up to MAX_NUM_SPECIAL_REORDER_CODES are codes for special groups like space/punct/digit.
224		* There are special codes at the end for reorder-reserved primary ranges.
225		*
226		* Multiple scripts may share a range and index, for example Hira & Kana.
227		*/
228		const uint16_t *scriptsIndex;
229		/**
230		* Start primary weight (top 16 bits only) for a group/script/reserved range
231		* indexed by scriptsIndex.
232		* The first range (separators & terminators) and the last range (trailing weights)
233		* are not reorderable, and no scriptsIndex entry points to them.
234		*/
235		const uint16_t *scriptStarts;
236		int32_t scriptStartsLength;
237
238		/**
239		* Collation elements in the root collator.
240		* Used by the CollationRootElements class. The data structure is described there.
241		* nullptr in a tailoring.
242		*/
243		const uint32_t *rootElements;
244		int32_t rootElementsLength;
245
246		private:
247		int32_t getScriptIndex(int32_t script) const;
248		void makeReorderRanges(const int32_t *reorder, int32_t length,
249		UBool latinMustMove,
250		UVector32 &ranges, UErrorCode &errorCode) const;
251		int32_t addLowScriptRange(uint8_t table[], int32_t index, int32_t lowStart) const;
252		int32_t addHighScriptRange(uint8_t table[], int32_t index, int32_t highLimit) const;
253		};
254
255		U_NAMESPACE_END
256
257		#endif // !UCONFIG_NO_COLLATION
258		#endif // __COLLATIONDATA_H__