/src/kcodecs/src/probers/nsSBCharSetProber.cpp
Line | Count | Source |
1 | | /* -*- C++ -*- |
2 | | SPDX-FileCopyrightText: 1998 Netscape Communications Corporation <developer@mozilla.org> |
3 | | |
4 | | SPDX-License-Identifier: MIT |
5 | | */ |
6 | | |
7 | | #include "nsSBCharSetProber.h" |
8 | | |
9 | | #include <format> |
10 | | #include <numeric> |
11 | | |
12 | | namespace kencodingprober |
13 | | { |
14 | | template class nsSingleByteCharSetProber<false>; |
15 | | template class nsSingleByteCharSetProber<true>; |
16 | | |
17 | | template<bool Reversed> |
18 | | nsProbingState nsSingleByteCharSetProber<Reversed>::HandleData(const char *aBuf, unsigned int aLen) |
19 | 32.6k | { |
20 | 871M | for (unsigned int i = 0; i < aLen; i++) { |
21 | 871M | const unsigned char order = mModel.charToOrderMap[(unsigned char)aBuf[i]]; |
22 | | |
23 | 871M | if (order < SYMBOL_CAT_ORDER) { |
24 | 706M | mTotalChar++; |
25 | 706M | } |
26 | 871M | if (order < SAMPLE_SIZE) { |
27 | 186M | mFreqChar++; |
28 | | |
29 | 186M | if (mLastOrder < SAMPLE_SIZE) { |
30 | 161M | unsigned int index = Reversed ? mLastOrder + (SAMPLE_SIZE * order) : (mLastOrder * SAMPLE_SIZE) + order; |
31 | 161M | ++(mSeqCounters[mModel.precedenceMatrix[index]]); |
32 | 161M | } |
33 | 186M | } |
34 | 871M | mLastOrder = order; |
35 | 871M | } |
36 | | |
37 | 32.6k | mTotalSeqs = std::accumulate(mSeqCounters.begin(), mSeqCounters.end(), 0); |
38 | 32.6k | if (mState == eDetecting) { |
39 | 32.6k | if (mTotalSeqs > 1024) { |
40 | 3.00k | float cf = GetConfidence(); |
41 | 3.00k | if (cf > 0.95f) { |
42 | 136 | mState = eFoundIt; |
43 | 2.87k | } else if (cf < 0.05) { |
44 | 2.09k | mState = eNotMe; |
45 | 2.09k | } |
46 | 3.00k | } |
47 | 32.6k | } |
48 | | |
49 | 32.6k | return mState; |
50 | 32.6k | } kencodingprober::nsSingleByteCharSetProber<false>::HandleData(char const*, unsigned int) Line | Count | Source | 19 | 29.9k | { | 20 | 799M | for (unsigned int i = 0; i < aLen; i++) { | 21 | 799M | const unsigned char order = mModel.charToOrderMap[(unsigned char)aBuf[i]]; | 22 | | | 23 | 799M | if (order < SYMBOL_CAT_ORDER) { | 24 | 639M | mTotalChar++; | 25 | 639M | } | 26 | 799M | if (order < SAMPLE_SIZE) { | 27 | 177M | mFreqChar++; | 28 | | | 29 | 177M | if (mLastOrder < SAMPLE_SIZE) { | 30 | 154M | unsigned int index = Reversed ? mLastOrder + (SAMPLE_SIZE * order) : (mLastOrder * SAMPLE_SIZE) + order; | 31 | 154M | ++(mSeqCounters[mModel.precedenceMatrix[index]]); | 32 | 154M | } | 33 | 177M | } | 34 | 799M | mLastOrder = order; | 35 | 799M | } | 36 | | | 37 | 29.9k | mTotalSeqs = std::accumulate(mSeqCounters.begin(), mSeqCounters.end(), 0); | 38 | 29.9k | if (mState == eDetecting) { | 39 | 29.9k | if (mTotalSeqs > 1024) { | 40 | 2.75k | float cf = GetConfidence(); | 41 | 2.75k | if (cf > 0.95f) { | 42 | 68 | mState = eFoundIt; | 43 | 2.68k | } else if (cf < 0.05) { | 44 | 2.00k | mState = eNotMe; | 45 | 2.00k | } | 46 | 2.75k | } | 47 | 29.9k | } | 48 | | | 49 | 29.9k | return mState; | 50 | 29.9k | } |
kencodingprober::nsSingleByteCharSetProber<true>::HandleData(char const*, unsigned int) Line | Count | Source | 19 | 2.71k | { | 20 | 72.6M | for (unsigned int i = 0; i < aLen; i++) { | 21 | 72.6M | const unsigned char order = mModel.charToOrderMap[(unsigned char)aBuf[i]]; | 22 | | | 23 | 72.6M | if (order < SYMBOL_CAT_ORDER) { | 24 | 67.0M | mTotalChar++; | 25 | 67.0M | } | 26 | 72.6M | if (order < SAMPLE_SIZE) { | 27 | 9.42M | mFreqChar++; | 28 | | | 29 | 9.42M | if (mLastOrder < SAMPLE_SIZE) { | 30 | 7.47M | unsigned int index = Reversed ? mLastOrder + (SAMPLE_SIZE * order) : (mLastOrder * SAMPLE_SIZE) + order; | 31 | 7.47M | ++(mSeqCounters[mModel.precedenceMatrix[index]]); | 32 | 7.47M | } | 33 | 9.42M | } | 34 | 72.6M | mLastOrder = order; | 35 | 72.6M | } | 36 | | | 37 | 2.71k | mTotalSeqs = std::accumulate(mSeqCounters.begin(), mSeqCounters.end(), 0); | 38 | 2.71k | if (mState == eDetecting) { | 39 | 2.71k | if (mTotalSeqs > 1024) { | 40 | 252 | float cf = GetConfidence(); | 41 | 252 | if (cf > 0.95f) { | 42 | 68 | mState = eFoundIt; | 43 | 184 | } else if (cf < 0.05) { | 44 | 86 | mState = eNotMe; | 45 | 86 | } | 46 | 252 | } | 47 | 2.71k | } | 48 | | | 49 | 2.71k | return mState; | 50 | 2.71k | } |
|
51 | | |
52 | | //#define NEGATIVE_APPROACH 1 |
53 | | |
54 | | template<bool Reversed> |
55 | | float nsSingleByteCharSetProber<Reversed>::GetConfidence(void) |
56 | 3.00k | { |
57 | | #ifdef NEGATIVE_APPROACH |
58 | | if (mTotalSeqs > 0) |
59 | | if (mTotalSeqs > mSeqCounters.front() * 10) { |
60 | | return (mTotalSeqs - mSeqCounters.front() * 10.f) / mTotalSeqs * mFreqChar / mTotalChar; |
61 | | } |
62 | | return 0.01f; |
63 | | #else // POSITIVE_APPROACH |
64 | | |
65 | 3.00k | if (mTotalSeqs > 0) { |
66 | 3.00k | float r = 1.0f * mSeqCounters.back() / mTotalSeqs / mModel.mTypicalPositiveRatio; |
67 | 3.00k | r = r * mFreqChar / mTotalChar; |
68 | 3.00k | if (r >= 0.99f) { |
69 | 102 | r = 0.99f; |
70 | 102 | } |
71 | 3.00k | return r; |
72 | 3.00k | } |
73 | 0 | return 0.01f; |
74 | 3.00k | #endif |
75 | 3.00k | } kencodingprober::nsSingleByteCharSetProber<false>::GetConfidence() Line | Count | Source | 56 | 2.75k | { | 57 | | #ifdef NEGATIVE_APPROACH | 58 | | if (mTotalSeqs > 0) | 59 | | if (mTotalSeqs > mSeqCounters.front() * 10) { | 60 | | return (mTotalSeqs - mSeqCounters.front() * 10.f) / mTotalSeqs * mFreqChar / mTotalChar; | 61 | | } | 62 | | return 0.01f; | 63 | | #else // POSITIVE_APPROACH | 64 | | | 65 | 2.75k | if (mTotalSeqs > 0) { | 66 | 2.75k | float r = 1.0f * mSeqCounters.back() / mTotalSeqs / mModel.mTypicalPositiveRatio; | 67 | 2.75k | r = r * mFreqChar / mTotalChar; | 68 | 2.75k | if (r >= 0.99f) { | 69 | 52 | r = 0.99f; | 70 | 52 | } | 71 | 2.75k | return r; | 72 | 2.75k | } | 73 | 0 | return 0.01f; | 74 | 2.75k | #endif | 75 | 2.75k | } |
kencodingprober::nsSingleByteCharSetProber<true>::GetConfidence() Line | Count | Source | 56 | 252 | { | 57 | | #ifdef NEGATIVE_APPROACH | 58 | | if (mTotalSeqs > 0) | 59 | | if (mTotalSeqs > mSeqCounters.front() * 10) { | 60 | | return (mTotalSeqs - mSeqCounters.front() * 10.f) / mTotalSeqs * mFreqChar / mTotalChar; | 61 | | } | 62 | | return 0.01f; | 63 | | #else // POSITIVE_APPROACH | 64 | | | 65 | 252 | if (mTotalSeqs > 0) { | 66 | 252 | float r = 1.0f * mSeqCounters.back() / mTotalSeqs / mModel.mTypicalPositiveRatio; | 67 | 252 | r = r * mFreqChar / mTotalChar; | 68 | 252 | if (r >= 0.99f) { | 69 | 50 | r = 0.99f; | 70 | 50 | } | 71 | 252 | return r; | 72 | 252 | } | 73 | 0 | return 0.01f; | 74 | 252 | #endif | 75 | 252 | } |
|
76 | | |
77 | | template<bool Reversed> |
78 | | const char *nsSingleByteCharSetProber<Reversed>::GetCharSetName() |
79 | 14 | { |
80 | 14 | return mModel.charsetName; |
81 | 14 | } kencodingprober::nsSingleByteCharSetProber<false>::GetCharSetName() Line | Count | Source | 79 | 14 | { | 80 | 14 | return mModel.charsetName; | 81 | 14 | } |
Unexecuted instantiation: kencodingprober::nsSingleByteCharSetProber<true>::GetCharSetName() |
82 | | |
83 | | template<bool Reversed> |
84 | | std::string nsSingleByteCharSetProber<Reversed>::StatusOutput(uint8_t /* indent */) |
85 | 0 | { |
86 | 0 | return std::format( // |
87 | 0 | "{:1.3f} [{}] [{} {} {} {} | {}] [{} / {}]", |
88 | 0 | GetConfidence(), |
89 | 0 | GetCharSetName(), |
90 | 0 | mSeqCounters[0], |
91 | 0 | mSeqCounters[1], |
92 | 0 | mSeqCounters[2], |
93 | 0 | mSeqCounters[3], |
94 | 0 | mTotalSeqs, |
95 | 0 | mFreqChar, |
96 | 0 | mTotalChar); |
97 | 0 | } Unexecuted instantiation: kencodingprober::nsSingleByteCharSetProber<false>::StatusOutput(unsigned char) Unexecuted instantiation: kencodingprober::nsSingleByteCharSetProber<true>::StatusOutput(unsigned char) |
98 | | } |