CjkDecodeValidatorTest.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.tika.ml.chardetect;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.io.ByteArrayOutputStream;
import java.nio.charset.Charset;
import java.util.Arrays;

import org.junit.jupiter.api.Test;

public class CjkDecodeValidatorTest {

    @Test
    public void realJapaneseFarBelowVetoThreshold() throws Exception {
        byte[] b = ("������������������������������������������������������������������������������������������"
                + "���������������������������������").getBytes("Shift_JIS");
        double rate = CjkDecodeValidator.strippedFailureRate(b, Charset.forName("Shift_JIS"));
        assertTrue(rate >= 0.0 && rate < 0.025, "real JP should be near-zero failure, got " + rate);
    }

    @Test
    public void realKoreanFarBelowVetoThreshold() throws Exception {
        byte[] b = ("��������������� ��������� ������ ��������� ��������������� ��������������� ��������� ������������ ������ "
                + "������ ��������� ������������").getBytes("EUC-KR");
        double rate = CjkDecodeValidator.strippedFailureRate(b, Charset.forName("EUC-KR"));
        assertTrue(rate >= 0.0 && rate < 0.025, "real KR should be near-zero failure, got " + rate);
    }

    /** Mixed-encoding: legacy CJK body + an embedded UTF-8 run. Stripping the UTF-8
     *  run de-confounds, so the rate stays low (the WS2 breakthrough). */
    @Test
    public void mixedLegacyPlusUtf8NotVetoed() throws Exception {
        ByteArrayOutputStream bo = new ByteArrayOutputStream();
        bo.writeBytes("������������������������������������������������������������������������������������������".getBytes("Shift_JIS"));
        bo.writeBytes("���������UTF-8���������������������������������".getBytes("UTF-8")); // embedded UTF-8
        double rate = CjkDecodeValidator.strippedFailureRate(bo.toByteArray(),
                Charset.forName("Shift_JIS"));
        assertTrue(rate >= 0.0 && rate < 0.025, "mixed real CJK should stay low post-strip, got " + rate);
    }

    @Test
    public void garbageHighBytesVetoed() {
        byte[] b = new byte[60];
        Arrays.fill(b, (byte) 0xFF); // 0xFF is not a valid GB18030 lead ��� all malformed
        double rate = CjkDecodeValidator.strippedFailureRate(b, Charset.forName("GB18030"));
        assertTrue(rate >= 0.025, "garbage high bytes should be vetoed, got " + rate);
    }

    @Test
    public void insufficientHighBytesReturnsMinusOne() {
        byte[] b = "mostly ascii with a couple high bytes".getBytes(java.nio.charset.StandardCharsets.ISO_8859_1);
        assertEquals(-1.0, CjkDecodeValidator.strippedFailureRate(b, Charset.forName("GB18030")));
    }

    /**
     * Pure UTF-8 file (zero legacy CJK bytes, many UTF-8 multi-byte sequences):
     * strippedFailureRate must return 1.0 so the CJK veto fires for all CJK charsets.
     * This covers the regression where Shift_JIS / Big5-HKSCS / GB18030 were wrongly
     * chosen over UTF-8 STRUCTURAL for pure-UTF-8 Latin/Cyrillic/etc. files.
     */
    @Test
    public void pureUtf8ReturnsCjkVeto() throws Exception {
        // Croatian text encoded as UTF-8 ��� all high bytes are valid UTF-8 sequences,
        // none are legacy CJK lead bytes.
        byte[] b = ("Ovo je ��ist UTF-8 tekst s hrvatskim slovima: "
                + "���������� ����������.  Ponavljamo dovoljno puta da prema��imo prag od "
                + "trideset UTF-8 sekvenci: �������������������� �������������������� ��������������������.")
                .getBytes("UTF-8");
        for (String cs : new String[]{"Shift_JIS", "Big5-HKSCS", "GB18030", "EUC-JP"}) {
            double rate = CjkDecodeValidator.strippedFailureRate(b, Charset.forName(cs));
            assertEquals(1.0, rate, 0.0,
                    "pure UTF-8 must return 1.0 (veto) for " + cs + ", got " + rate);
        }
    }

    @Test
    public void appliesToLegacyCjkButNotIso2022OrLatin() {
        assertTrue(CjkDecodeValidator.appliesTo("GB18030"));
        assertTrue(CjkDecodeValidator.appliesTo("Shift_JIS"));
        assertTrue(CjkDecodeValidator.appliesTo("Big5-HKSCS"));
        assertTrue(CjkDecodeValidator.appliesTo("x-windows-949"));
        assertEquals(false, CjkDecodeValidator.appliesTo("ISO-2022-JP"));
        assertEquals(false, CjkDecodeValidator.appliesTo("windows-1252"));
    }
}