/src/icu/source/i18n/csrmbcs.cpp

Source
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
 **********************************************************************
 *   Copyright (C) 2005-2016, International Business Machines
 *   Corporation and others.  All Rights Reserved.
 **********************************************************************
 */

#include "unicode/utypes.h"

#if !UCONFIG_NO_CONVERSION

#include "cmemory.h"
#include "csmatch.h"
#include "csrmbcs.h"

#include <math.h>

U_NAMESPACE_BEGIN

#define min(x,y) (((x)<(y))?(x):(y))

static const uint16_t commonChars_sjis [] = {
// TODO:  This set of data comes from the character frequency-
//        of-occurrence analysis tool.  The data needs to be moved
//        into a resource and loaded from there.
0x8140, 0x8141, 0x8142, 0x8145, 0x815b, 0x8169, 0x816a, 0x8175, 0x8176, 0x82a0,
0x82a2, 0x82a4, 0x82a9, 0x82aa, 0x82ab, 0x82ad, 0x82af, 0x82b1, 0x82b3, 0x82b5,
0x82b7, 0x82bd, 0x82be, 0x82c1, 0x82c4, 0x82c5, 0x82c6, 0x82c8, 0x82c9, 0x82cc,
0x82cd, 0x82dc, 0x82e0, 0x82e7, 0x82e8, 0x82e9, 0x82ea, 0x82f0, 0x82f1, 0x8341,
0x8343, 0x834e, 0x834f, 0x8358, 0x835e, 0x8362, 0x8367, 0x8375, 0x8376, 0x8389,
0x838a, 0x838b, 0x838d, 0x8393, 0x8e96, 0x93fa, 0x95aa};

static const uint16_t commonChars_euc_jp[] = {
// TODO:  This set of data comes from the character frequency-
//        of-occurrence analysis tool.  The data needs to be moved
//        into a resource and loaded from there.
0xa1a1, 0xa1a2, 0xa1a3, 0xa1a6, 0xa1bc, 0xa1ca, 0xa1cb, 0xa1d6, 0xa1d7, 0xa4a2,
0xa4a4, 0xa4a6, 0xa4a8, 0xa4aa, 0xa4ab, 0xa4ac, 0xa4ad, 0xa4af, 0xa4b1, 0xa4b3,
0xa4b5, 0xa4b7, 0xa4b9, 0xa4bb, 0xa4bd, 0xa4bf, 0xa4c0, 0xa4c1, 0xa4c3, 0xa4c4,
0xa4c6, 0xa4c7, 0xa4c8, 0xa4c9, 0xa4ca, 0xa4cb, 0xa4ce, 0xa4cf, 0xa4d0, 0xa4de,
0xa4df, 0xa4e1, 0xa4e2, 0xa4e4, 0xa4e8, 0xa4e9, 0xa4ea, 0xa4eb, 0xa4ec, 0xa4ef,
0xa4f2, 0xa4f3, 0xa5a2, 0xa5a3, 0xa5a4, 0xa5a6, 0xa5a7, 0xa5aa, 0xa5ad, 0xa5af,
0xa5b0, 0xa5b3, 0xa5b5, 0xa5b7, 0xa5b8, 0xa5b9, 0xa5bf, 0xa5c3, 0xa5c6, 0xa5c7,
0xa5c8, 0xa5c9, 0xa5cb, 0xa5d0, 0xa5d5, 0xa5d6, 0xa5d7, 0xa5de, 0xa5e0, 0xa5e1,
0xa5e5, 0xa5e9, 0xa5ea, 0xa5eb, 0xa5ec, 0xa5ed, 0xa5f3, 0xb8a9, 0xb9d4, 0xbaee,
0xbbc8, 0xbef0, 0xbfb7, 0xc4ea, 0xc6fc, 0xc7bd, 0xcab8, 0xcaf3, 0xcbdc, 0xcdd1};

static const uint16_t commonChars_euc_kr[] = {
// TODO:  This set of data comes from the character frequency-
//        of-occurrence analysis tool.  The data needs to be moved
//        into a resource and loaded from there.
0xb0a1, 0xb0b3, 0xb0c5, 0xb0cd, 0xb0d4, 0xb0e6, 0xb0ed, 0xb0f8, 0xb0fa, 0xb0fc,
0xb1b8, 0xb1b9, 0xb1c7, 0xb1d7, 0xb1e2, 0xb3aa, 0xb3bb, 0xb4c2, 0xb4cf, 0xb4d9,
0xb4eb, 0xb5a5, 0xb5b5, 0xb5bf, 0xb5c7, 0xb5e9, 0xb6f3, 0xb7af, 0xb7c2, 0xb7ce,
0xb8a6, 0xb8ae, 0xb8b6, 0xb8b8, 0xb8bb, 0xb8e9, 0xb9ab, 0xb9ae, 0xb9cc, 0xb9ce,
0xb9fd, 0xbab8, 0xbace, 0xbad0, 0xbaf1, 0xbbe7, 0xbbf3, 0xbbfd, 0xbcad, 0xbcba,
0xbcd2, 0xbcf6, 0xbdba, 0xbdc0, 0xbdc3, 0xbdc5, 0xbec6, 0xbec8, 0xbedf, 0xbeee,
0xbef8, 0xbefa, 0xbfa1, 0xbfa9, 0xbfc0, 0xbfe4, 0xbfeb, 0xbfec, 0xbff8, 0xc0a7,
0xc0af, 0xc0b8, 0xc0ba, 0xc0bb, 0xc0bd, 0xc0c7, 0xc0cc, 0xc0ce, 0xc0cf, 0xc0d6,
0xc0da, 0xc0e5, 0xc0fb, 0xc0fc, 0xc1a4, 0xc1a6, 0xc1b6, 0xc1d6, 0xc1df, 0xc1f6,
0xc1f8, 0xc4a1, 0xc5cd, 0xc6ae, 0xc7cf, 0xc7d1, 0xc7d2, 0xc7d8, 0xc7e5, 0xc8ad};

static const uint16_t commonChars_big5[] = {
// TODO:  This set of data comes from the character frequency-
//        of-occurrence analysis tool.  The data needs to be moved
//        into a resource and loaded from there.
0xa140, 0xa141, 0xa142, 0xa143, 0xa147, 0xa149, 0xa175, 0xa176, 0xa440, 0xa446,
0xa447, 0xa448, 0xa451, 0xa454, 0xa457, 0xa464, 0xa46a, 0xa46c, 0xa477, 0xa4a3,
0xa4a4, 0xa4a7, 0xa4c1, 0xa4ce, 0xa4d1, 0xa4df, 0xa4e8, 0xa4fd, 0xa540, 0xa548,
0xa558, 0xa569, 0xa5cd, 0xa5e7, 0xa657, 0xa661, 0xa662, 0xa668, 0xa670, 0xa6a8,
0xa6b3, 0xa6b9, 0xa6d3, 0xa6db, 0xa6e6, 0xa6f2, 0xa740, 0xa751, 0xa759, 0xa7da,
0xa8a3, 0xa8a5, 0xa8ad, 0xa8d1, 0xa8d3, 0xa8e4, 0xa8fc, 0xa9c0, 0xa9d2, 0xa9f3,
0xaa6b, 0xaaba, 0xaabe, 0xaacc, 0xaafc, 0xac47, 0xac4f, 0xacb0, 0xacd2, 0xad59,
0xaec9, 0xafe0, 0xb0ea, 0xb16f, 0xb2b3, 0xb2c4, 0xb36f, 0xb44c, 0xb44e, 0xb54c,
0xb5a5, 0xb5bd, 0xb5d0, 0xb5d8, 0xb671, 0xb7ed, 0xb867, 0xb944, 0xbad8, 0xbb44,
0xbba1, 0xbdd1, 0xc2c4, 0xc3b9, 0xc440, 0xc45f};

static const uint16_t commonChars_gb_18030[] = {
// TODO:  This set of data comes from the character frequency-
//        of-occurrence analysis tool.  The data needs to be moved
//        into a resource and loaded from there.
0xa1a1, 0xa1a2, 0xa1a3, 0xa1a4, 0xa1b0, 0xa1b1, 0xa1f1, 0xa1f3, 0xa3a1, 0xa3ac,
0xa3ba, 0xb1a8, 0xb1b8, 0xb1be, 0xb2bb, 0xb3c9, 0xb3f6, 0xb4f3, 0xb5bd, 0xb5c4,
0xb5e3, 0xb6af, 0xb6d4, 0xb6e0, 0xb7a2, 0xb7a8, 0xb7bd, 0xb7d6, 0xb7dd, 0xb8b4,
0xb8df, 0xb8f6, 0xb9ab, 0xb9c9, 0xb9d8, 0xb9fa, 0xb9fd, 0xbacd, 0xbba7, 0xbbd6,
0xbbe1, 0xbbfa, 0xbcbc, 0xbcdb, 0xbcfe, 0xbdcc, 0xbecd, 0xbedd, 0xbfb4, 0xbfc6,
0xbfc9, 0xc0b4, 0xc0ed, 0xc1cb, 0xc2db, 0xc3c7, 0xc4dc, 0xc4ea, 0xc5cc, 0xc6f7,
0xc7f8, 0xc8ab, 0xc8cb, 0xc8d5, 0xc8e7, 0xc9cf, 0xc9fa, 0xcab1, 0xcab5, 0xcac7,
0xcad0, 0xcad6, 0xcaf5, 0xcafd, 0xccec, 0xcdf8, 0xceaa, 0xcec4, 0xced2, 0xcee5,
0xcfb5, 0xcfc2, 0xcfd6, 0xd0c2, 0xd0c5, 0xd0d0, 0xd0d4, 0xd1a7, 0xd2aa, 0xd2b2,
0xd2b5, 0xd2bb, 0xd2d4, 0xd3c3, 0xd3d0, 0xd3fd, 0xd4c2, 0xd4da, 0xd5e2, 0xd6d0};

static int32_t binarySearch(const uint16_t *array, int32_t len, uint16_t value)
{
    int32_t start = 0, end = len-1;
    int32_t mid = (start+end)/2;

    while(start <= end) {
        if(array[mid] == value) {
            return mid;
        }

        if(array[mid] < value){
            start = mid+1;
        } else {
            end = mid-1;
        }

        mid = (start+end)/2;
    }

    return -1;
}

IteratedChar::IteratedChar() : 
charValue(0), index(-1), nextIndex(0), error(false), done(false)
{
    // nothing else to do.
}

/*void IteratedChar::reset()
{
    charValue = 0;
    index     = -1;
    nextIndex = 0;
    error     = false;
    done      = false;
}*/

int32_t IteratedChar::nextByte(InputText *det)
{
    if (nextIndex >= det->fRawLength) {
        done = true;

        return -1;
    }

    return det->fRawInput[nextIndex++];
}

CharsetRecog_mbcs::~CharsetRecog_mbcs()
{
    // nothing to do.
}

int32_t CharsetRecog_mbcs::match_mbcs(InputText *det, const uint16_t commonChars[], int32_t commonCharsLen) const {
    int32_t doubleByteCharCount = 0;
    int32_t commonCharCount     = 0;
    int32_t badCharCount        = 0;
    int32_t totalCharCount      = 0;
    int32_t confidence          = 0;
    IteratedChar iter;

    while (nextChar(&iter, det)) {
        totalCharCount++;

        if (iter.error) {
            badCharCount++;
        } else {
            if (iter.charValue > 0xFF) {
                doubleByteCharCount++;

                if (commonChars != nullptr) {
                    if (binarySearch(commonChars, commonCharsLen, static_cast<uint16_t>(iter.charValue)) >= 0){
                        commonCharCount += 1;
                    }
                }
            }
        }


        if (badCharCount >= 2 && badCharCount*5 >= doubleByteCharCount) {
            // Bail out early if the byte data is not matching the encoding scheme.
            // break detectBlock;
            return confidence;
        }
    }

    if (doubleByteCharCount <= 10 && badCharCount == 0) {
        // Not many multi-byte chars.
        if (doubleByteCharCount == 0 && totalCharCount < 10) {
            // There weren't any multibyte sequences, and there was a low density of non-ASCII single bytes.
            // We don't have enough data to have any confidence.
            // Statistical analysis of single byte non-ASCII characters would probably help here.
            confidence = 0;
        }
        else {
            //   ASCII or ISO file?  It's probably not our encoding,
            //   but is not incompatible with our encoding, so don't give it a zero.
            confidence = 10;
        }

        return confidence;
    }

    //
    //  No match if there are too many characters that don't fit the encoding scheme.
    //    (should we have zero tolerance for these?)
    //
    if (doubleByteCharCount < 20*badCharCount) {
        confidence = 0;

        return confidence;
    }

    if (commonChars == nullptr) {
        // We have no statistics on frequently occurring characters.
        //  Assess confidence purely on having a reasonable number of
        //  multi-byte characters (the more the better)
        confidence = 30 + doubleByteCharCount - 20*badCharCount;

        if (confidence > 100) {
            confidence = 100;
        }
    } else {
        //
        // Frequency of occurrence statistics exist.
        //

        double maxVal = log(static_cast<double>(doubleByteCharCount) / 4); /*(float)?*/
        double scaleFactor = 90.0 / maxVal;
        confidence = static_cast<int32_t>(log(static_cast<double>(commonCharCount) + 1) * scaleFactor + 10.0);

        confidence = min(confidence, 100);
    }

    if (confidence < 0) {
        confidence = 0;
    }

    return confidence;
}

CharsetRecog_sjis::~CharsetRecog_sjis()
{
    // nothing to do
}

UBool CharsetRecog_sjis::nextChar(IteratedChar* it, InputText* det) const {
    it->index = it->nextIndex;
    it->error = false;

    int32_t firstByte = it->charValue = it->nextByte(det);

    if (firstByte < 0) {
        return false;
    }

    if (firstByte <= 0x7F || (firstByte > 0xA0 && firstByte <= 0xDF)) {
        return true;
    }

    int32_t secondByte = it->nextByte(det);
    if (secondByte >= 0) {
        it->charValue = (firstByte << 8) | secondByte;
    }
    // else we'll handle the error later.

    if (! ((secondByte >= 0x40 && secondByte <= 0x7F) || (secondByte >= 0x80 && secondByte <= 0xFE))) {
        // Illegal second byte value.
        it->error = true;
    }

    return true;
}

UBool CharsetRecog_sjis::match(InputText* det, CharsetMatch *results) const {
    int32_t confidence = match_mbcs(det, commonChars_sjis, UPRV_LENGTHOF(commonChars_sjis));
    results->set(det, this, confidence);
    return (confidence > 0);
}

const char *CharsetRecog_sjis::getName() const
{
    return "Shift_JIS";
}

const char *CharsetRecog_sjis::getLanguage() const
{
    return "ja";
}

CharsetRecog_euc::~CharsetRecog_euc()
{
    // nothing to do
}

UBool CharsetRecog_euc::nextChar(IteratedChar* it, InputText* det) const {
    int32_t firstByte  = 0;
    int32_t secondByte = 0;
    int32_t thirdByte  = 0;

    it->index = it->nextIndex;
    it->error = false;
    firstByte = it->charValue = it->nextByte(det);

    if (firstByte < 0) {
        // Ran off the end of the input data
        return false;
    }

    if (firstByte <= 0x8D) {
        // single byte char
        return true;
    }

    secondByte = it->nextByte(det);
    if (secondByte >= 0) {
        it->charValue = (it->charValue << 8) | secondByte;
    }
    // else we'll handle the error later.

    if (firstByte >= 0xA1 && firstByte <= 0xFE) {
        // Two byte Char
        if (secondByte < 0xA1) {
            it->error = true;
        }

        return true;
    }

    if (firstByte == 0x8E) {
        // Code Set 2.
        //   In EUC-JP, total char size is 2 bytes, only one byte of actual char value.
        //   In EUC-TW, total char size is 4 bytes, three bytes contribute to char value.
        // We don't know which we've got.
        // Treat it like EUC-JP.  If the data really was EUC-TW, the following two
        //   bytes will look like a well formed 2 byte char.
        if (secondByte < 0xA1) {
            it->error = true;
        }

        return true;
    }

    if (firstByte == 0x8F) {
        // Code set 3.
        // Three byte total char size, two bytes of actual char value.
        thirdByte    = it->nextByte(det);
        it->charValue = (it->charValue << 8) | thirdByte;

        if (thirdByte < 0xa1) {
            // Bad second byte or ran off the end of the input data with a non-ASCII first byte.
            it->error = true;
        }
    }

    return true;

}

CharsetRecog_euc_jp::~CharsetRecog_euc_jp()
{
    // nothing to do
}

const char *CharsetRecog_euc_jp::getName() const
{
    return "EUC-JP";
}

const char *CharsetRecog_euc_jp::getLanguage() const
{
    return "ja";
}

UBool CharsetRecog_euc_jp::match(InputText *det, CharsetMatch *results) const
{
    int32_t confidence = match_mbcs(det, commonChars_euc_jp, UPRV_LENGTHOF(commonChars_euc_jp));
    results->set(det, this, confidence);
    return (confidence > 0);
}

CharsetRecog_euc_kr::~CharsetRecog_euc_kr()
{
    // nothing to do
}

const char *CharsetRecog_euc_kr::getName() const
{
    return "EUC-KR";
}

const char *CharsetRecog_euc_kr::getLanguage() const
{
    return "ko";
}

UBool CharsetRecog_euc_kr::match(InputText *det, CharsetMatch *results) const
{
    int32_t confidence =  match_mbcs(det, commonChars_euc_kr, UPRV_LENGTHOF(commonChars_euc_kr));
    results->set(det, this, confidence);
    return (confidence > 0);
}

CharsetRecog_big5::~CharsetRecog_big5()
{
    // nothing to do
}

UBool CharsetRecog_big5::nextChar(IteratedChar* it, InputText* det) const
{
    int32_t firstByte;

    it->index = it->nextIndex;
    it->error = false;
    firstByte = it->charValue = it->nextByte(det);

    if (firstByte < 0) {
        return false;
    }

    if (firstByte <= 0x7F || firstByte == 0xFF) {
        // single byte character.
        return true;
    }

    int32_t secondByte = it->nextByte(det);
    if (secondByte >= 0)  {
        it->charValue = (it->charValue << 8) | secondByte;
    }
    // else we'll handle the error later.

    if (secondByte < 0x40 || secondByte == 0x7F || secondByte == 0xFF) {
        it->error = true;
    }

    return true;
}

const char *CharsetRecog_big5::getName() const
{
    return "Big5";
}

const char *CharsetRecog_big5::getLanguage() const
{
    return "zh";
}

UBool CharsetRecog_big5::match(InputText *det, CharsetMatch *results) const
{
    int32_t confidence = match_mbcs(det, commonChars_big5, UPRV_LENGTHOF(commonChars_big5));
    results->set(det, this, confidence);
    return (confidence > 0);
}

CharsetRecog_gb_18030::~CharsetRecog_gb_18030()
{
    // nothing to do
}

UBool CharsetRecog_gb_18030::nextChar(IteratedChar* it, InputText* det) const {
    int32_t firstByte  = 0;
    int32_t secondByte = 0;
    int32_t thirdByte  = 0;
    int32_t fourthByte = 0;

    it->index = it->nextIndex;
    it->error = false;
    firstByte = it->charValue = it->nextByte(det);

    if (firstByte < 0) {
        // Ran off the end of the input data
        return false;
    }

    if (firstByte <= 0x80) {
        // single byte char
        return true;
    }

    secondByte = it->nextByte(det);
    if (secondByte >= 0) {
        it->charValue = (it->charValue << 8) | secondByte;
    }
    // else we'll handle the error later.

    if (firstByte >= 0x81 && firstByte <= 0xFE) {
        // Two byte Char
        if ((secondByte >= 0x40 && secondByte <= 0x7E) || (secondByte >=80 && secondByte <= 0xFE)) {
            return true;
        }

        // Four byte char
        if (secondByte >= 0x30 && secondByte <= 0x39) {
            thirdByte = it->nextByte(det);

            if (thirdByte >= 0x81 && thirdByte <= 0xFE) {
                fourthByte = it->nextByte(det);

                if (fourthByte >= 0x30 && fourthByte <= 0x39) {
                    it->charValue = (it->charValue << 16) | (thirdByte << 8) | fourthByte;

                    return true;
                }
            }
        }

        // Something wasn't valid, or we ran out of data (-1).
        it->error = true;
    }

    return true;
}

const char *CharsetRecog_gb_18030::getName() const
{
    return "GB18030";
}

const char *CharsetRecog_gb_18030::getLanguage() const
{
    return "zh";
}

UBool CharsetRecog_gb_18030::match(InputText *det, CharsetMatch *results) const
{
    int32_t confidence = match_mbcs(det, commonChars_gb_18030, UPRV_LENGTHOF(commonChars_gb_18030));
    results->set(det, this, confidence);
    return (confidence > 0);
}

U_NAMESPACE_END
#endif

Coverage Report

Created: 2026-09-14 06:37

Line	Count	Source
1		// © 2016 and later: Unicode, Inc. and others.
2		// License & terms of use: http://www.unicode.org/copyright.html
3		/*
4		**********************************************************************
5		* Copyright (C) 2005-2016, International Business Machines
6		* Corporation and others. All Rights Reserved.
7		**********************************************************************
8		*/
9
10		#include "unicode/utypes.h"
11
12		#if !UCONFIG_NO_CONVERSION
13
14		#include "cmemory.h"
15		#include "csmatch.h"
16		#include "csrmbcs.h"
17
18		#include <math.h>
19
20		U_NAMESPACE_BEGIN
21
22	4.47k	#define min(x,y) (((x)<(y))?(x):(y))
23
24		static const uint16_t commonChars_sjis [] = {
25		// TODO: This set of data comes from the character frequency-
26		// of-occurrence analysis tool. The data needs to be moved
27		// into a resource and loaded from there.
28		0x8140, 0x8141, 0x8142, 0x8145, 0x815b, 0x8169, 0x816a, 0x8175, 0x8176, 0x82a0,
29		0x82a2, 0x82a4, 0x82a9, 0x82aa, 0x82ab, 0x82ad, 0x82af, 0x82b1, 0x82b3, 0x82b5,
30		0x82b7, 0x82bd, 0x82be, 0x82c1, 0x82c4, 0x82c5, 0x82c6, 0x82c8, 0x82c9, 0x82cc,
31		0x82cd, 0x82dc, 0x82e0, 0x82e7, 0x82e8, 0x82e9, 0x82ea, 0x82f0, 0x82f1, 0x8341,
32		0x8343, 0x834e, 0x834f, 0x8358, 0x835e, 0x8362, 0x8367, 0x8375, 0x8376, 0x8389,
33		0x838a, 0x838b, 0x838d, 0x8393, 0x8e96, 0x93fa, 0x95aa};
34
35		static const uint16_t commonChars_euc_jp[] = {
36		// TODO: This set of data comes from the character frequency-
37		// of-occurrence analysis tool. The data needs to be moved
38		// into a resource and loaded from there.
39		0xa1a1, 0xa1a2, 0xa1a3, 0xa1a6, 0xa1bc, 0xa1ca, 0xa1cb, 0xa1d6, 0xa1d7, 0xa4a2,
40		0xa4a4, 0xa4a6, 0xa4a8, 0xa4aa, 0xa4ab, 0xa4ac, 0xa4ad, 0xa4af, 0xa4b1, 0xa4b3,
41		0xa4b5, 0xa4b7, 0xa4b9, 0xa4bb, 0xa4bd, 0xa4bf, 0xa4c0, 0xa4c1, 0xa4c3, 0xa4c4,
42		0xa4c6, 0xa4c7, 0xa4c8, 0xa4c9, 0xa4ca, 0xa4cb, 0xa4ce, 0xa4cf, 0xa4d0, 0xa4de,
43		0xa4df, 0xa4e1, 0xa4e2, 0xa4e4, 0xa4e8, 0xa4e9, 0xa4ea, 0xa4eb, 0xa4ec, 0xa4ef,
44		0xa4f2, 0xa4f3, 0xa5a2, 0xa5a3, 0xa5a4, 0xa5a6, 0xa5a7, 0xa5aa, 0xa5ad, 0xa5af,
45		0xa5b0, 0xa5b3, 0xa5b5, 0xa5b7, 0xa5b8, 0xa5b9, 0xa5bf, 0xa5c3, 0xa5c6, 0xa5c7,
46		0xa5c8, 0xa5c9, 0xa5cb, 0xa5d0, 0xa5d5, 0xa5d6, 0xa5d7, 0xa5de, 0xa5e0, 0xa5e1,
47		0xa5e5, 0xa5e9, 0xa5ea, 0xa5eb, 0xa5ec, 0xa5ed, 0xa5f3, 0xb8a9, 0xb9d4, 0xbaee,
48		0xbbc8, 0xbef0, 0xbfb7, 0xc4ea, 0xc6fc, 0xc7bd, 0xcab8, 0xcaf3, 0xcbdc, 0xcdd1};
49
50		static const uint16_t commonChars_euc_kr[] = {
51		// TODO: This set of data comes from the character frequency-
52		// of-occurrence analysis tool. The data needs to be moved
53		// into a resource and loaded from there.
54		0xb0a1, 0xb0b3, 0xb0c5, 0xb0cd, 0xb0d4, 0xb0e6, 0xb0ed, 0xb0f8, 0xb0fa, 0xb0fc,
55		0xb1b8, 0xb1b9, 0xb1c7, 0xb1d7, 0xb1e2, 0xb3aa, 0xb3bb, 0xb4c2, 0xb4cf, 0xb4d9,
56		0xb4eb, 0xb5a5, 0xb5b5, 0xb5bf, 0xb5c7, 0xb5e9, 0xb6f3, 0xb7af, 0xb7c2, 0xb7ce,
57		0xb8a6, 0xb8ae, 0xb8b6, 0xb8b8, 0xb8bb, 0xb8e9, 0xb9ab, 0xb9ae, 0xb9cc, 0xb9ce,
58		0xb9fd, 0xbab8, 0xbace, 0xbad0, 0xbaf1, 0xbbe7, 0xbbf3, 0xbbfd, 0xbcad, 0xbcba,
59		0xbcd2, 0xbcf6, 0xbdba, 0xbdc0, 0xbdc3, 0xbdc5, 0xbec6, 0xbec8, 0xbedf, 0xbeee,
60		0xbef8, 0xbefa, 0xbfa1, 0xbfa9, 0xbfc0, 0xbfe4, 0xbfeb, 0xbfec, 0xbff8, 0xc0a7,
61		0xc0af, 0xc0b8, 0xc0ba, 0xc0bb, 0xc0bd, 0xc0c7, 0xc0cc, 0xc0ce, 0xc0cf, 0xc0d6,
62		0xc0da, 0xc0e5, 0xc0fb, 0xc0fc, 0xc1a4, 0xc1a6, 0xc1b6, 0xc1d6, 0xc1df, 0xc1f6,
63		0xc1f8, 0xc4a1, 0xc5cd, 0xc6ae, 0xc7cf, 0xc7d1, 0xc7d2, 0xc7d8, 0xc7e5, 0xc8ad};
64
65		static const uint16_t commonChars_big5[] = {
66		// TODO: This set of data comes from the character frequency-
67		// of-occurrence analysis tool. The data needs to be moved
68		// into a resource and loaded from there.
69		0xa140, 0xa141, 0xa142, 0xa143, 0xa147, 0xa149, 0xa175, 0xa176, 0xa440, 0xa446,
70		0xa447, 0xa448, 0xa451, 0xa454, 0xa457, 0xa464, 0xa46a, 0xa46c, 0xa477, 0xa4a3,
71		0xa4a4, 0xa4a7, 0xa4c1, 0xa4ce, 0xa4d1, 0xa4df, 0xa4e8, 0xa4fd, 0xa540, 0xa548,
72		0xa558, 0xa569, 0xa5cd, 0xa5e7, 0xa657, 0xa661, 0xa662, 0xa668, 0xa670, 0xa6a8,
73		0xa6b3, 0xa6b9, 0xa6d3, 0xa6db, 0xa6e6, 0xa6f2, 0xa740, 0xa751, 0xa759, 0xa7da,
74		0xa8a3, 0xa8a5, 0xa8ad, 0xa8d1, 0xa8d3, 0xa8e4, 0xa8fc, 0xa9c0, 0xa9d2, 0xa9f3,
75		0xaa6b, 0xaaba, 0xaabe, 0xaacc, 0xaafc, 0xac47, 0xac4f, 0xacb0, 0xacd2, 0xad59,
76		0xaec9, 0xafe0, 0xb0ea, 0xb16f, 0xb2b3, 0xb2c4, 0xb36f, 0xb44c, 0xb44e, 0xb54c,
77		0xb5a5, 0xb5bd, 0xb5d0, 0xb5d8, 0xb671, 0xb7ed, 0xb867, 0xb944, 0xbad8, 0xbb44,
78		0xbba1, 0xbdd1, 0xc2c4, 0xc3b9, 0xc440, 0xc45f};
79
80		static const uint16_t commonChars_gb_18030[] = {
81		// TODO: This set of data comes from the character frequency-
82		// of-occurrence analysis tool. The data needs to be moved
83		// into a resource and loaded from there.
84		0xa1a1, 0xa1a2, 0xa1a3, 0xa1a4, 0xa1b0, 0xa1b1, 0xa1f1, 0xa1f3, 0xa3a1, 0xa3ac,
85		0xa3ba, 0xb1a8, 0xb1b8, 0xb1be, 0xb2bb, 0xb3c9, 0xb3f6, 0xb4f3, 0xb5bd, 0xb5c4,
86		0xb5e3, 0xb6af, 0xb6d4, 0xb6e0, 0xb7a2, 0xb7a8, 0xb7bd, 0xb7d6, 0xb7dd, 0xb8b4,
87		0xb8df, 0xb8f6, 0xb9ab, 0xb9c9, 0xb9d8, 0xb9fa, 0xb9fd, 0xbacd, 0xbba7, 0xbbd6,
88		0xbbe1, 0xbbfa, 0xbcbc, 0xbcdb, 0xbcfe, 0xbdcc, 0xbecd, 0xbedd, 0xbfb4, 0xbfc6,
89		0xbfc9, 0xc0b4, 0xc0ed, 0xc1cb, 0xc2db, 0xc3c7, 0xc4dc, 0xc4ea, 0xc5cc, 0xc6f7,
90		0xc7f8, 0xc8ab, 0xc8cb, 0xc8d5, 0xc8e7, 0xc9cf, 0xc9fa, 0xcab1, 0xcab5, 0xcac7,
91		0xcad0, 0xcad6, 0xcaf5, 0xcafd, 0xccec, 0xcdf8, 0xceaa, 0xcec4, 0xced2, 0xcee5,
92		0xcfb5, 0xcfc2, 0xcfd6, 0xd0c2, 0xd0c5, 0xd0d0, 0xd0d4, 0xd1a7, 0xd2aa, 0xd2b2,
93		0xd2b5, 0xd2bb, 0xd2d4, 0xd3c3, 0xd3d0, 0xd3fd, 0xd4c2, 0xd4da, 0xd5e2, 0xd6d0};
94
95		static int32_t binarySearch(const uint16_t *array, int32_t len, uint16_t value)
96	13.5M	{
97	13.5M	int32_t start = 0, end = len-1;
98	13.5M	int32_t mid = (start+end)/2;
99
100	98.9M	while(start <= end) {
101	85.7M	if(array[mid] == value) {
102	329k	return mid;
103	329k	}
104
105	85.4M	if(array[mid] < value){
106	41.2M	start = mid+1;
107	44.1M	} else {
108	44.1M	end = mid-1;
109	44.1M	}
110
111	85.4M	mid = (start+end)/2;
112	85.4M	}
113
114	13.2M	return -1;
115	13.5M	}
116
117		IteratedChar::IteratedChar() :
118	3.60M	charValue(0), index(-1), nextIndex(0), error(false), done(false)
119	3.60M	{
120		// nothing else to do.
121	3.60M	}
122
123		/*void IteratedChar::reset()
124		{
125		charValue = 0;
126		index = -1;
127		nextIndex = 0;
128		error = false;
129		done = false;
130		}*/
131
132		int32_t IteratedChar::nextByte(InputText *det)
133	87.2M	{
134	87.2M	if (nextIndex >= det->fRawLength) {
135	3.79M	done = true;
136
137	3.79M	return -1;
138	3.79M	}
139
140	83.4M	return det->fRawInput[nextIndex++];
141	87.2M	}
142
143		CharsetRecog_mbcs::~CharsetRecog_mbcs()
144	0	{
145		// nothing to do.
146	0	}
147
148	3.60M	int32_t CharsetRecog_mbcs::match_mbcs(InputText *det, const uint16_t commonChars[], int32_t commonCharsLen) const {
149	3.60M	int32_t doubleByteCharCount = 0;
150	3.60M	int32_t commonCharCount = 0;
151	3.60M	int32_t badCharCount = 0;
152	3.60M	int32_t totalCharCount = 0;
153	3.60M	int32_t confidence = 0;
154	3.60M	IteratedChar iter;
155
156	72.2M	while (nextChar(&iter, det)) {
157	68.7M	totalCharCount++;
158
159	68.7M	if (iter.error) {
160	1.17M	badCharCount++;
161	67.6M	} else {
162	67.6M	if (iter.charValue > 0xFF) {
163	13.5M	doubleByteCharCount++;
164
165	13.5M	if (commonChars != nullptr) {
166	13.5M	if (binarySearch(commonChars, commonCharsLen, static_cast<uint16_t>(iter.charValue)) >= 0){
167	329k	commonCharCount += 1;
168	329k	}
169	13.5M	}
170	13.5M	}
171	67.6M	}
172
173
174	68.7M	if (badCharCount >= 2 && badCharCount*5 >= doubleByteCharCount) {
175		// Bail out early if the byte data is not matching the encoding scheme.
176		// break detectBlock;
177	151k	return confidence;
178	151k	}
179	68.7M	}
180
181	3.45M	if (doubleByteCharCount <= 10 && badCharCount == 0) {
182		// Not many multi-byte chars.
183	3.09M	if (doubleByteCharCount == 0 && totalCharCount < 10) {
184		// There weren't any multibyte sequences, and there was a low density of non-ASCII single bytes.
185		// We don't have enough data to have any confidence.
186		// Statistical analysis of single byte non-ASCII characters would probably help here.
187	2.69M	confidence = 0;
188	2.69M	}
189	400k	else {
190		// ASCII or ISO file? It's probably not our encoding,
191		// but is not incompatible with our encoding, so don't give it a zero.
192	400k	confidence = 10;
193	400k	}
194
195	3.09M	return confidence;
196	3.09M	}
197
198		//
199		// No match if there are too many characters that don't fit the encoding scheme.
200		// (should we have zero tolerance for these?)
201		//
202	358k	if (doubleByteCharCount < 20*badCharCount) {
203	354k	confidence = 0;
204
205	354k	return confidence;
206	354k	}
207
208	4.47k	if (commonChars == nullptr) {
209		// We have no statistics on frequently occurring characters.
210		// Assess confidence purely on having a reasonable number of
211		// multi-byte characters (the more the better)
212	0	confidence = 30 + doubleByteCharCount - 20*badCharCount;
213
214	0	if (confidence > 100) {
215	0	confidence = 100;
216	0	}
217	4.47k	} else {
218		//
219		// Frequency of occurrence statistics exist.
220		//
221
222	4.47k	double maxVal = log(static_cast<double>(doubleByteCharCount) / 4); /(float)?/
223	4.47k	double scaleFactor = 90.0 / maxVal;
224	4.47k	confidence = static_cast<int32_t>(log(static_cast<double>(commonCharCount) + 1) * scaleFactor + 10.0);
225
226	4.47k	confidence = min(confidence, 100);
227	4.47k	}
228
229	4.47k	if (confidence < 0) {
230	0	confidence = 0;
231	0	}
232
233	4.47k	return confidence;
234	358k	}
235
236		CharsetRecog_sjis::~CharsetRecog_sjis()
237		{
238		// nothing to do
239		}
240
241	12.5M	UBool CharsetRecog_sjis::nextChar(IteratedChar* it, InputText* det) const {
242	12.5M	it->index = it->nextIndex;
243	12.5M	it->error = false;
244
245	12.5M	int32_t firstByte = it->charValue = it->nextByte(det);
246
247	12.5M	if (firstByte < 0) {
248	680k	return false;
249	680k	}
250
251	11.8M	if (firstByte <= 0x7F \|\| (firstByte > 0xA0 && firstByte <= 0xDF)) {
252	10.1M	return true;
253	10.1M	}
254
255	1.74M	int32_t secondByte = it->nextByte(det);
256	1.74M	if (secondByte >= 0) {
257	1.68M	it->charValue = (firstByte << 8) \| secondByte;
258	1.68M	}
259		// else we'll handle the error later.
260
261	1.74M	if (! ((secondByte >= 0x40 && secondByte <= 0x7F) \|\| (secondByte >= 0x80 && secondByte <= 0xFE))) {
262		// Illegal second byte value.
263	260k	it->error = true;
264	260k	}
265
266	1.74M	return true;
267	11.8M	}
268
269	721k	UBool CharsetRecog_sjis::match(InputText* det, CharsetMatch *results) const {
270	721k	int32_t confidence = match_mbcs(det, commonChars_sjis, UPRV_LENGTHOF(commonChars_sjis));
271	721k	results->set(det, this, confidence);
272	721k	return (confidence > 0);
273	721k	}
274
275		const char *CharsetRecog_sjis::getName() const
276	721k	{
277	721k	return "Shift_JIS";
278	721k	}
279
280		const char *CharsetRecog_sjis::getLanguage() const
281	721k	{
282	721k	return "ja";
283	721k	}
284
285		CharsetRecog_euc::~CharsetRecog_euc()
286		{
287		// nothing to do
288		}
289
290	25.5M	UBool CharsetRecog_euc::nextChar(IteratedChar* it, InputText* det) const {
291	25.5M	int32_t firstByte = 0;
292	25.5M	int32_t secondByte = 0;
293	25.5M	int32_t thirdByte = 0;
294
295	25.5M	it->index = it->nextIndex;
296	25.5M	it->error = false;
297	25.5M	firstByte = it->charValue = it->nextByte(det);
298
299	25.5M	if (firstByte < 0) {
300		// Ran off the end of the input data
301	1.38M	return false;
302	1.38M	}
303
304	24.2M	if (firstByte <= 0x8D) {
305		// single byte char
306	18.0M	return true;
307	18.0M	}
308
309	6.19M	secondByte = it->nextByte(det);
310	6.19M	if (secondByte >= 0) {
311	6.04M	it->charValue = (it->charValue << 8) \| secondByte;
312	6.04M	}
313		// else we'll handle the error later.
314
315	6.19M	if (firstByte >= 0xA1 && firstByte <= 0xFE) {
316		// Two byte Char
317	1.54M	if (secondByte < 0xA1) {
318	317k	it->error = true;
319	317k	}
320
321	1.54M	return true;
322	1.54M	}
323
324	4.64M	if (firstByte == 0x8E) {
325		// Code Set 2.
326		// In EUC-JP, total char size is 2 bytes, only one byte of actual char value.
327		// In EUC-TW, total char size is 4 bytes, three bytes contribute to char value.
328		// We don't know which we've got.
329		// Treat it like EUC-JP. If the data really was EUC-TW, the following two
330		// bytes will look like a well formed 2 byte char.
331	6.60k	if (secondByte < 0xA1) {
332	3.85k	it->error = true;
333	3.85k	}
334
335	6.60k	return true;
336	6.60k	}
337
338	4.63M	if (firstByte == 0x8F) {
339		// Code set 3.
340		// Three byte total char size, two bytes of actual char value.
341	17.1k	thirdByte = it->nextByte(det);
342	17.1k	it->charValue = (it->charValue << 8) \| thirdByte;
343
344	17.1k	if (thirdByte < 0xa1) {
345		// Bad second byte or ran off the end of the input data with a non-ASCII first byte.
346	6.22k	it->error = true;
347	6.22k	}
348	17.1k	}
349
350	4.63M	return true;
351
352	4.64M	}
353
354		CharsetRecog_euc_jp::~CharsetRecog_euc_jp()
355		{
356		// nothing to do
357		}
358
359		const char *CharsetRecog_euc_jp::getName() const
360	721k	{
361	721k	return "EUC-JP";
362	721k	}
363
364		const char *CharsetRecog_euc_jp::getLanguage() const
365	721k	{
366	721k	return "ja";
367	721k	}
368
369		UBool CharsetRecog_euc_jp::match(InputText det, CharsetMatch results) const
370	721k	{
371	721k	int32_t confidence = match_mbcs(det, commonChars_euc_jp, UPRV_LENGTHOF(commonChars_euc_jp));
372	721k	results->set(det, this, confidence);
373	721k	return (confidence > 0);
374	721k	}
375
376		CharsetRecog_euc_kr::~CharsetRecog_euc_kr()
377		{
378		// nothing to do
379		}
380
381		const char *CharsetRecog_euc_kr::getName() const
382	721k	{
383	721k	return "EUC-KR";
384	721k	}
385
386		const char *CharsetRecog_euc_kr::getLanguage() const
387	721k	{
388	721k	return "ko";
389	721k	}
390
391		UBool CharsetRecog_euc_kr::match(InputText det, CharsetMatch results) const
392	721k	{
393	721k	int32_t confidence = match_mbcs(det, commonChars_euc_kr, UPRV_LENGTHOF(commonChars_euc_kr));
394	721k	results->set(det, this, confidence);
395	721k	return (confidence > 0);
396	721k	}
397
398		CharsetRecog_big5::~CharsetRecog_big5()
399		{
400		// nothing to do
401		}
402
403		UBool CharsetRecog_big5::nextChar(IteratedChar* it, InputText* det) const
404	14.1M	{
405	14.1M	int32_t firstByte;
406
407	14.1M	it->index = it->nextIndex;
408	14.1M	it->error = false;
409	14.1M	firstByte = it->charValue = it->nextByte(det);
410
411	14.1M	if (firstByte < 0) {
412	692k	return false;
413	692k	}
414
415	13.4M	if (firstByte <= 0x7F \|\| firstByte == 0xFF) {
416		// single byte character.
417	11.0M	return true;
418	11.0M	}
419
420	2.41M	int32_t secondByte = it->nextByte(det);
421	2.41M	if (secondByte >= 0) {
422	2.37M	it->charValue = (it->charValue << 8) \| secondByte;
423	2.37M	}
424		// else we'll handle the error later.
425
426	2.41M	if (secondByte < 0x40 \|\| secondByte == 0x7F \|\| secondByte == 0xFF) {
427	251k	it->error = true;
428	251k	}
429
430	2.41M	return true;
431	13.4M	}
432
433		const char *CharsetRecog_big5::getName() const
434	721k	{
435	721k	return "Big5";
436	721k	}
437
438		const char *CharsetRecog_big5::getLanguage() const
439	721k	{
440	721k	return "zh";
441	721k	}
442
443		UBool CharsetRecog_big5::match(InputText det, CharsetMatch results) const
444	721k	{
445	721k	int32_t confidence = match_mbcs(det, commonChars_big5, UPRV_LENGTHOF(commonChars_big5));
446	721k	results->set(det, this, confidence);
447	721k	return (confidence > 0);
448	721k	}
449
450		CharsetRecog_gb_18030::~CharsetRecog_gb_18030()
451		{
452		// nothing to do
453		}
454
455	19.9M	UBool CharsetRecog_gb_18030::nextChar(IteratedChar* it, InputText* det) const {
456	19.9M	int32_t firstByte = 0;
457	19.9M	int32_t secondByte = 0;
458	19.9M	int32_t thirdByte = 0;
459	19.9M	int32_t fourthByte = 0;
460
461	19.9M	it->index = it->nextIndex;
462	19.9M	it->error = false;
463	19.9M	firstByte = it->charValue = it->nextByte(det);
464
465	19.9M	if (firstByte < 0) {
466		// Ran off the end of the input data
467	694k	return false;
468	694k	}
469
470	19.2M	if (firstByte <= 0x80) {
471		// single byte char
472	14.6M	return true;
473	14.6M	}
474
475	4.51M	secondByte = it->nextByte(det);
476	4.51M	if (secondByte >= 0) {
477	4.43M	it->charValue = (it->charValue << 8) \| secondByte;
478	4.43M	}
479		// else we'll handle the error later.
480
481	4.51M	if (firstByte >= 0x81 && firstByte <= 0xFE) {
482		// Two byte Char
483	3.04M	if ((secondByte >= 0x40 && secondByte <= 0x7E) \|\| (secondByte >=80 && secondByte <= 0xFE)) {
484	2.70M	return true;
485	2.70M	}
486
487		// Four byte char
488	337k	if (secondByte >= 0x30 && secondByte <= 0x39) {
489	79.1k	thirdByte = it->nextByte(det);
490
491	79.1k	if (thirdByte >= 0x81 && thirdByte <= 0xFE) {
492	22.2k	fourthByte = it->nextByte(det);
493
494	22.2k	if (fourthByte >= 0x30 && fourthByte <= 0x39) {
495	2.98k	it->charValue = (it->charValue << 16) \| (thirdByte << 8) \| fourthByte;
496
497	2.98k	return true;
498	2.98k	}
499	22.2k	}
500	79.1k	}
501
502		// Something wasn't valid, or we ran out of data (-1).
503	334k	it->error = true;
504	334k	}
505
506	1.80M	return true;
507	4.51M	}
508
509		const char *CharsetRecog_gb_18030::getName() const
510	721k	{
511	721k	return "GB18030";
512	721k	}
513
514		const char *CharsetRecog_gb_18030::getLanguage() const
515	721k	{
516	721k	return "zh";
517	721k	}
518
519		UBool CharsetRecog_gb_18030::match(InputText det, CharsetMatch results) const
520	721k	{
521	721k	int32_t confidence = match_mbcs(det, commonChars_gb_18030, UPRV_LENGTHOF(commonChars_gb_18030));
522	721k	results->set(det, this, confidence);
523	721k	return (confidence > 0);
524	721k	}
525
526		U_NAMESPACE_END
527		#endif