/src/kcodecs/src/probers/JpCntx.h
Line | Count | Source |
1 | | /* -*- C++ -*- |
2 | | SPDX-FileCopyrightText: 1998 Netscape Communications Corporation <developer@mozilla.org> |
3 | | |
4 | | SPDX-License-Identifier: MIT |
5 | | */ |
6 | | |
7 | | #ifndef __JPCNTX_H__ |
8 | | #define __JPCNTX_H__ |
9 | | |
10 | | #include "kcodecs_export.h" |
11 | | |
12 | | #define NUM_OF_CATEGORY 6 |
13 | | |
14 | 3.85k | #define ENOUGH_REL_THRESHOLD 100 |
15 | 4.90M | #define MAX_REL_THRESHOLD 1000 |
16 | | namespace kencodingprober |
17 | | { |
18 | | // hiragana frequency category table |
19 | | extern const char jp2CharContext[83][83]; |
20 | | |
21 | | class KCODECS_NO_EXPORT JapaneseContextAnalysis |
22 | | { |
23 | | public: |
24 | 8.78k | JapaneseContextAnalysis() = default; |
25 | 8.78k | virtual ~JapaneseContextAnalysis() = default; |
26 | | |
27 | | void HandleOneChar(const char *aStr, unsigned int aCharLen) |
28 | 4.90M | { |
29 | | // if we received enough data, stop here |
30 | 4.90M | if (mTotalRel > MAX_REL_THRESHOLD) { |
31 | 111k | mDone = true; |
32 | 111k | } |
33 | 4.90M | if (mDone) { |
34 | 111k | return; |
35 | 111k | } |
36 | | |
37 | | // Only 2-bytes characters are of our interest |
38 | 4.79M | const int code = (aCharLen == 2) ? GetCode(aStr) : -1; |
39 | 4.79M | if (code != -1 && mLastCharCode != -1) { |
40 | 80.1k | mTotalRel++; |
41 | | // count this sequence to its category counter |
42 | 80.1k | mRelSample[(int)jp2CharContext[mLastCharCode][code]]++; |
43 | 80.1k | } |
44 | 4.79M | mLastCharCode = code; |
45 | 4.79M | } |
46 | | |
47 | | float GetConfidence(); |
48 | | bool GotEnoughData() |
49 | 3.85k | { |
50 | 3.85k | return mTotalRel > ENOUGH_REL_THRESHOLD; |
51 | 3.85k | } |
52 | | |
53 | | protected: |
54 | | // Get cell code in Hiragana row, or -1 for non-Hiragana |
55 | | virtual int GetCode(const char *str) = 0; |
56 | | |
57 | | // category counters, each integer counts sequence in its category |
58 | | unsigned int mRelSample[NUM_OF_CATEGORY] = {0}; |
59 | | |
60 | | // total sequence received |
61 | | unsigned int mTotalRel = 0; |
62 | | |
63 | | // The code of previous char |
64 | | int mLastCharCode = -1; |
65 | | |
66 | | // If this flag is set to true, detection is done and conclusion has been made |
67 | | bool mDone = false; |
68 | | }; |
69 | | |
70 | | class KCODECS_NO_EXPORT SJISContextAnalysis : public JapaneseContextAnalysis |
71 | | { |
72 | | protected: |
73 | | int GetCode(const char *str) override |
74 | 74.2k | { |
75 | | // We only interested in Hiragana, so first byte is '\202' |
76 | 74.2k | if (*str == '\202' && (unsigned char)*(str + 1) >= (unsigned char)0x9f && (unsigned char)*(str + 1) <= (unsigned char)0xf1) { |
77 | 26.4k | return (unsigned char)*(str + 1) - (unsigned char)0x9f; |
78 | 26.4k | } |
79 | 47.7k | return -1; |
80 | 74.2k | } |
81 | | }; |
82 | | |
83 | | class KCODECS_NO_EXPORT EUCJPContextAnalysis : public JapaneseContextAnalysis |
84 | | { |
85 | | protected: |
86 | | int GetCode(const char *str) override |
87 | | // We only interested in Hiragana, so first byte is '\244' |
88 | 1.72M | { |
89 | 1.72M | if (*str == '\244' // |
90 | 61.6k | && (unsigned char)*(str + 1) >= (unsigned char)0xa1 // |
91 | 61.6k | && (unsigned char)*(str + 1) <= (unsigned char)0xf3) { |
92 | 61.2k | return (unsigned char)*(str + 1) - (unsigned char)0xa1; |
93 | 61.2k | } |
94 | 1.66M | return -1; |
95 | 1.72M | } |
96 | | }; |
97 | | } |
98 | | #endif /* __JPCNTX_H__ */ |