LatinSiblingComparisonTest.java
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.tika.ml.junkdetect;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.apache.tika.quality.TextQualityComparison;
/**
* Latin SBCS sibling discrimination. Mojibuster correctly emits windows-1252
* at high confidence for these inputs; the chain's tournament still picks the
* wrong sibling (IBM850 / IBM852 / x-MacRoman). This test exercises the
* {@link JunkDetector#compare} math in isolation, so a failure here means the
* model lacks signal for the sibling case, and a pass means the bug is in
* {@link JunkFilterEncodingDetector}'s arbitration rather than in JunkDetector
* itself.
*
* <p>Probe strings are synthesized Western European prose using phrases the
* production-loss inspection ({@code 20260518-junk-charsets.md}) verified are
* present in the failing corpus files; no corpus content is checked in.
*/
public class LatinSiblingComparisonTest {
private static JunkDetector detector;
@BeforeAll
static void loadModel() throws Exception {
detector = JunkDetector.loadFromClasspath();
}
private static final String[] WRONG_CHARSETS = {"IBM850", "IBM852", "x-MacRoman"};
/** Synthesized probes ��� each is real Western European prose that survives
* a windows-1252 round-trip (no 0x81/0x8D/0x8F/0x90/0x9D unassigned-byte
* hazards) and contains the discriminating letters from the loss table. */
private static final Probe[] PROBES = {
new Probe("finnish",
"Talven ensimm��inen pakkasaamu Hausj��rven j����ll�� oli h��ik��isev��n kirkas. "
+ "Lapset ihmettelev��t, miksi vesi muuttuu yht��kki�� kovaksi, ja ��iti selitt���� "
+ "miten l��mp��tila vaikuttaa j��rven pintaan koko talven ajan."),
new Probe("french-euro",
"Le sp��cialiste propose une consultation �� 115����� pour les nouveaux clients. "
+ "Apr��s un premier ��change, il pr��pare un devis d��taill�� qui pr��cise les "
+ "diff��rentes ��tapes du projet et les d��lais associ��s."),
new Probe("german-umlauts",
"F��r Anf��nger empfehlen wir den Grundkurs, in dem die wichtigsten Regeln erkl��rt werden. "
+ "Sie k��nnen jederzeit Fragen stellen, und unsere Lehrkr��fte gehen gerne auf alle "
+ "Schwierigkeiten ein. Die n��chste Stunde beginnt am Montag um neun Uhr."),
new Probe("portuguese",
"As sprites f��meas do novo jogo trazem habilidades especiais que n��o existiam na vers��o "
+ "anterior. Cada personagem possui uma hist��ria pr��pria e o jogador pode escolher "
+ "o estilo de combate que prefere antes de iniciar a primeira miss��o."),
new Probe("spanish-acutes",
"Para obtener m��s informaci��n sobre el tama��o del archivo, consulte la secci��n "
+ "correspondiente en la direcci��n indicada y ��ntrela en el formulario. "
+ "Si tiene dudas, env��e un correo electr��nico al equipo de soporte t��cnico."),
new Probe("spanish-names",
"Jos�� Canalda escribi�� una novela de ficci��n y fantas��a ambientada en Alcal��, "
+ "donde un grupo de j��venes investiga una serie de fen��menos extra��os. "
+ "La obra recibi�� cr��ticas muy favorables en revistas especializadas."),
new Probe("spanish-guillemets",
"Garcilaso de la Vega, conocido como ��El Inca��, escribi�� sobre la historia de los "
+ "pueblos andinos antes y despu��s de la conquista. Sus textos combinan testimonio "
+ "personal con relatos transmitidos por la tradici��n oral."),
};
@Test
void junkDetectorPicksWindows1252OverLatinSiblings() {
runMatrix("clean prose", probe -> probe.text);
}
private void runMatrix(String label, java.util.function.Function<Probe, String> shaper) {
List<String> failures = new ArrayList<>();
List<String> passes = new ArrayList<>();
Charset win1252 = Charset.forName("windows-1252");
for (Probe probe : PROBES) {
String shaped = shaper.apply(probe);
byte[] bytes = shaped.getBytes(win1252);
String asWin1252 = new String(bytes, win1252);
for (String wrong : WRONG_CHARSETS) {
Charset wrongCs = Charset.forName(wrong);
String asWrong = new String(bytes, wrongCs);
TextQualityComparison cmp = detector.compare(
"windows-1252", asWin1252, wrong, asWrong);
String tag = String.format("%-20s vs %-12s", probe.name, wrong);
if ("windows-1252".equals(cmp.winner())) {
passes.add(String.format("PASS %s delta=%.3f", tag, cmp.delta()));
} else {
failures.add(String.format("FAIL %s winner=%-12s delta=%.3f",
tag, cmp.winner(), cmp.delta()));
}
}
}
System.out.println("\n=== Latin SBCS sibling comparison: " + label + " ===");
passes.forEach(System.out::println);
failures.forEach(System.out::println);
System.out.printf("%d pass, %d fail (of %d cells)%n",
passes.size(), failures.size(), passes.size() + failures.size());
assertEquals(0, failures.size(),
"JunkDetector should pick windows-1252 over every Latin SBCS sibling "
+ "for " + label + " input. Failures above indicate the model lacks "
+ "signal for the sibling case under this input shape.");
}
private static final class Probe {
final String name;
final String text;
Probe(String name, String text) {
this.name = name;
this.text = text;
}
}
}