TextQualityFeaturesTest.java
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.tika.ml.junkdetect;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.Test;
import org.apache.tika.ml.junkdetect.TextQualityFeatures.StripMode;
class TextQualityFeaturesTest {
@Test
void alphabeticRatio_polishCorrectVsMojibake() {
// Correct decode: every codepoint is a letter
assertEquals(1.0, TextQualityFeatures.alphabeticRatio("ci��nienia"), 1e-9);
// Wrong decode: pilcrow replaces ��, so 8/9 letters
assertEquals(8.0 / 9.0, TextQualityFeatures.alphabeticRatio("ci��nienia"), 1e-9);
}
@Test
void letterPairDensity_polishCorrectVsMojibake() {
// ci��nienia: 8 adjacent pairs, all (letter, letter, same-cluster)
assertEquals(1.0, TextQualityFeatures.letterPairDensity("ci��nienia"), 1e-9);
// ci��nienia: pairs (i,��) and (��,n) fail ��� 6/8 = 0.75
assertEquals(0.75, TextQualityFeatures.letterPairDensity("ci��nienia"), 1e-9);
}
@Test
void letterPairDensity_mixedScriptToken() {
// Latin + Cyrillic + Greek in one "word" ��� none of the adjacent
// letter pairs are same-cluster.
// h(Latin) e(Latin) l(Latin) l(Latin) ��(Cyr) ��(Greek)
// pairs: (h,e) Latin-Latin same; (e,l) same; (l,l) same;
// (l,��) Latin-Cyr different; (��,��) Cyr-Greek different.
// 3/5 same cluster.
assertEquals(3.0 / 5.0, TextQualityFeatures.letterPairDensity("hell����"), 1e-9);
}
@Test
void letterPairDensity_cjkClusterGroupsKana() {
// ������������������ ��� HAN, HIRAGANA, HAN, HAN, HIRAGANA, HIRAGANA.
// All in the CJK cluster, all letters ��� 1.0
assertEquals(1.0, TextQualityFeatures.letterPairDensity("������������������"), 1e-9);
}
@Test
void replacementCount_countsUFFFD() {
assertEquals(0, TextQualityFeatures.replacementCount("hello"));
assertEquals(2, TextQualityFeatures.replacementCount("he���ll���o"));
}
@Test
void highByteEntropy_zeroWhenAllAscii() {
assertEquals(0.0, TextQualityFeatures.highByteEntropy("hello world"), 1e-9);
}
@Test
void highByteEntropy_higherForFannedOutMojibake() {
// Realistic CJK-as-Latin1 mojibake fans out across many high bytes
String mojibake = "���� ���� ���� ���� ���� ���� ���� ����";
// Polish text uses a small set of high-byte letters repeatedly
String polish = "ci��nienia ci��nienia ci��nienia ci��nienia";
assertTrue(TextQualityFeatures.highByteEntropy(mojibake)
> TextQualityFeatures.highByteEntropy(polish),
"mojibake should have higher high-byte entropy than repeated Polish word");
}
@Test
void perWordScriptPurity_mixedScriptTokenScoresLow() {
// Two clean words + one mixed word ��� 2/3 pure
assertEquals(2.0 / 3.0,
TextQualityFeatures.perWordScriptPurity("hello world hell����"),
1e-9);
}
@Test
void perWordScriptPurity_allCleanWords() {
assertEquals(1.0,
TextQualityFeatures.perWordScriptPurity("hello world foo bar"),
1e-9);
}
@Test
void strip_noneIsIdentity() {
String s = "hello world �� ��";
assertEquals(s, TextQualityFeatures.strip(s, StripMode.NONE));
}
@Test
void strip_whitespaceKeepsPunctuation() {
assertEquals("hello��world",
TextQualityFeatures.strip("hello �� world", StripMode.WHITESPACE));
assertEquals("a��!b",
TextQualityFeatures.strip("a\t��!\nb", StripMode.WHITESPACE));
}
@Test
void strip_whitespaceControlAlsoRemovesControls() {
// is a CONTROL char; �� should survive
assertEquals("hello��world",
TextQualityFeatures.strip("hello ��world",
StripMode.WHITESPACE_CONTROL));
}
@Test
void combiningMarkRatio_vietnameseVsMojibake() {
// Vietnamese "V���" written as V + e + combining-hook (U+0309) ��� 2/3 letters, 1/3 mark
String vietnamese = "Ve��";
assertEquals(1.0 / 3.0,
TextQualityFeatures.combiningMarkRatio(vietnamese), 1e-9);
// Latin-1 mojibake form "Ve��" has no combining marks
assertEquals(0.0,
TextQualityFeatures.combiningMarkRatio("Ve��"), 1e-9);
}
@Test
void letterAdjacentToMarkRatio_vietnameseDecoration() {
// V + e + �� ��� pairs (V,e) no, (e,mark) yes ��� 1/2
assertEquals(0.5,
TextQualityFeatures.letterAdjacentToMarkRatio("Ve��"),
1e-9);
// No marks ��� 0
assertEquals(0.0,
TextQualityFeatures.letterAdjacentToMarkRatio("hello world"),
1e-9);
}
@Test
void scriptDensity_allCommonScoresZero() {
assertEquals(0.0, TextQualityFeatures.scriptDensity(" \t\n"), 1e-9);
assertEquals(0.0, TextQualityFeatures.scriptDensity("12345 67890"), 1e-9);
assertEquals(0.0, TextQualityFeatures.scriptDensity("!@#$%^&*()"), 1e-9);
}
@Test
void scriptDensity_pureScriptScoresOne() {
assertEquals(1.0, TextQualityFeatures.scriptDensity("hello"), 1e-9);
assertEquals(1.0, TextQualityFeatures.scriptDensity("ci��nienia"), 1e-9);
assertEquals(1.0, TextQualityFeatures.scriptDensity("������������������"), 1e-9);
}
@Test
void scriptDensity_mixedTextScoresPartial() {
// "hi 5" ��� h,i (LATIN), ' ' (COMMON), 5 (COMMON) ��� 2/4
assertEquals(0.5, TextQualityFeatures.scriptDensity("hi 5"), 1e-9);
}
@Test
void scriptFragmentation_singleScriptScoresZero() {
assertEquals(0.0, TextQualityFeatures.scriptFragmentation("hello"), 1e-9);
assertEquals(0.0, TextQualityFeatures.scriptFragmentation("ci��nienia"), 1e-9);
// COMMON codepoints don't count
assertEquals(0.0, TextQualityFeatures.scriptFragmentation("hello world"), 1e-9);
}
@Test
void scriptFragmentation_noScriptedContentScoresZero() {
assertEquals(0.0, TextQualityFeatures.scriptFragmentation(" 12345"), 1e-9);
}
@Test
void scriptFragmentation_scriptSaladHigh() {
// Mixed LATIN + CYRILLIC + GREEK + HEBREW, each single codepoint
// ��� 4 scripted codepoints, longest run = 1, fragmentation = 0.75
assertEquals(0.75,
TextQualityFeatures.scriptFragmentation("a������"), 1e-9);
}
@Test
void scriptFragmentation_mostlyOneScriptLowFragmentation() {
// "hello����" ��� 5 LATIN + 2 CYR. longest_run=5, total=7 ��� 1 - 5/7 ��� 0.286
assertEquals(1.0 - 5.0 / 7.0,
TextQualityFeatures.scriptFragmentation("hello����"), 1e-9);
}
@Test
void strip_allCommonMatchesProductionBehaviour() {
// ALL_COMMON should drop �� (which is COMMON-script punctuation), space, and !
assertEquals("helloworld",
TextQualityFeatures.strip("hello �� world!", StripMode.ALL_COMMON));
// But should keep script-bearing letters like �� (Latin)
assertEquals("hel��loworld",
TextQualityFeatures.strip("hel�� lo �� world", StripMode.ALL_COMMON));
}
}