TikaCLITest.java
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.tika.cli;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assumptions.assumeTrue;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.io.Reader;
import java.net.URI;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.BasicFileAttributes;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.HexFormat;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.zip.GZIPOutputStream;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.serialization.JsonMetadataList;
import org.apache.tika.utils.StringUtils;
/**
* Tests the Tika's cli
*/
public class TikaCLITest {
private static final Logger LOG = LoggerFactory.getLogger(TikaCLITest.class);
static final File TEST_DATA_FILE = new File("src/test/resources/test-data");
static final File CONFIGS_DIR = new File("src/test/resources/configs");
// OCR off for the image-heavy fork runs (CI installs tesseract). Only the fork path merges
// a --config onto its defaults; in-process, any --config replaces the CLI defaults wholesale.
static final String NO_OCR = "--config=" + new File(CONFIGS_DIR, "tika-config-no-ocr.json").getAbsolutePath();
private final URI testDataURI = TEST_DATA_FILE.toURI();
@TempDir
private Path extractDir;
/* Test members */
private ByteArrayOutputStream outContent = null;
private ByteArrayOutputStream errContent = null;
private PrintStream stdout = null;
private PrintStream stderr = null;
private String resourcePrefix;
// -z inputs share one directory-mode run: one pipes startup instead of one per file
private static final String[] SHALLOW_INPUTS = {"coffee.xls", "testZip_absolutePath.zip",
"testZip_relative.zip", "testZip_overlappingNames.zip", "testZip_zeroByte.zip",
"test-documents.tgz", "testWithSubdirs.zip", "testPDF_childAttachments.pdf"};
@TempDir
private static Path shallowInputDir;
@TempDir
private static Path shallowOutputDir;
private static Set<String> shallowOutput;
@BeforeAll
public static void extractShallowOnce() throws Exception {
for (String f : SHALLOW_INPUTS) {
Files.copy(TEST_DATA_FILE.toPath().resolve(f), shallowInputDir.resolve(f));
}
TikaCLI.main(new String[]{"-z", NO_OCR, "-p", Paths.get("target/plugins").toAbsolutePath().toString(),
shallowInputDir.toAbsolutePath().toString(), shallowOutputDir.toAbsolutePath().toString()});
shallowOutput = getFileNames(shallowOutputDir);
}
protected static void assertExtracted(Path p, String allFiles) throws IOException {
assertTrue(Files.exists(p), "File " + p.getFileName() + " not found in " + allFiles);
assertFalse(Files.isDirectory(p), "File " + p.getFileName() + " is a directory!");
assertTrue(Files.size(p) > 0, "File " + p.getFileName() + " wasn't extracted with " + "contents");
}
/**
* reset resourcePrefix
* save original System.out and System.err
* clear outContent and errContent if they are not empty
* set outContent and errContent as System.out and System.err
*/
@BeforeEach
public void setUp() throws Exception {
resourcePrefix = testDataURI.toString();
stdout = System.out;
stderr = System.err;
resetContent();
}
/**
* Tears down the test. Returns the System.out and System.err
*/
@AfterEach
public void tearDown() {
System.setOut(stdout);
System.setErr(stderr);
}
/**
* clear outContent and errContent if they are not empty by create a new one.
* set outContent and errContent as System.out and System.err
*/
private void resetContent() throws Exception {
if (outContent == null || outContent.size() > 0) {
outContent = new ByteArrayOutputStream();
System.setOut(new PrintStream(outContent, true, UTF_8.name()));
}
if (errContent == null || errContent.size() > 0) {
errContent = new ByteArrayOutputStream();
System.setErr(new PrintStream(errContent, true, UTF_8.name()));
}
}
/**
* Tests --list-parser-detail option of the cli
* Tests --list-parser-details option of the cli
*
* @throws Exception
*/
@Test
public void testListParserDetail() throws Exception {
String content = getParamOutContent("--list-parser-detail");
assertTrue(content.contains("application/vnd.oasis.opendocument.text-web"));
content = getParamOutContent("--list-parser-details");
assertTrue(content.contains("application/vnd.oasis.opendocument.text-web"));
}
/**
* Tests --list-parser option of the cli
*
* @throws Exception
*/
@Test
public void testListParsers() throws Exception {
String content = getParamOutContent("--list-parser");
assertTrue(content.contains("org.apache.tika.parser.iwork.IWorkPackageParser"));
}
/**
* Tests -x option of the cli
*
* @throws Exception
*/
@Test
public void testXMLOutput() throws Exception {
String content = getParamOutContent("-x", resourcePrefix + "alice.cli.test");
assertTrue(content.contains("?xml version=\"1.0\" encoding=\"UTF-8\"?"));
content = getParamOutContent("-x", "--digest=sha256", resourcePrefix + "alice.cli.test");
assertTrue(content.contains("<meta name=\"tk:digest:SHA-256\" content=\"e90779adbac09c4ee"));
}
/**
* Tests a -h option of the cli
*
* @throws Exception
*/
@Test
public void testHTMLOutput() throws Exception {
String content = getParamOutContent("-h", resourcePrefix + "alice.cli.test");
assertTrue(content.contains("html xmlns=\"http://www.w3.org/1999/xhtml"));
assertTrue(content.contains("<title></title>"), "Expanded <title></title> element should be present");
content = getParamOutContent("-h", "--digest=sha384", resourcePrefix + "alice.cli.test");
assertTrue(content.contains("<meta name=\"tk:digest:SHA-384\" content=\"c69ea023f5da95a026"));
}
/**
* Tests -t option of the cli
*
* @throws Exception
*/
@Test
public void testTextOutput() throws Exception {
String content = getParamOutContent("-t", resourcePrefix + "alice.cli.test");
assertTrue(content.contains("finished off the cake"));
}
/**
* TIKA-4813 follow-up: --fork-timeout was removed in 4.0 (its meaning was ambiguous
* between the total per-file budget and the stall detector) in favor of
* --task-timeout/--progress-timeout. It must fail loudly with a migration hint rather
* than silently falling through to an "unknown option" / file-not-found error.
*/
/**
* The inline short forms -eX/-pX/-c<uri> were removed in 4.0.0. They were the only
* short flags matched by prefix, which made a single-dash long name silently become
* their value: -config=tika.json set the client URI to "onfig=tika.json" and loaded
* no config. Each must now reach the unknown-option error.
*/
@Test
public void testSingleDashLongNamesAreRejected() {
for (String arg : new String[]{"-config=tika-config.json", "-encoding=UTF-8",
"-password=secret", "-pretty-print", "-client=http://localhost:9998/"}) {
IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
() -> TikaCLI.main(new String[]{arg, resourcePrefix + "alice.cli.test"}), arg);
assertTrue(e.getMessage().contains("two dashes"),
arg + " -> " + e.getMessage());
}
}
@Test
public void testForkTimeoutFlagRemoved() {
IllegalArgumentException e = assertThrows(
IllegalArgumentException.class,
() -> TikaCLI.main(new String[]{"--fork-timeout=60000", resourcePrefix + "alice.cli.test"}));
assertTrue(e.getMessage().contains("--task-timeout"));
assertTrue(e.getMessage().contains("--progress-timeout"));
}
/**
* TIKA-4813 follow-up: --task-timeout and --progress-timeout must be recognized
* arguments (not fall through to the generic "Unknown option" error) and not disturb
* a normal parse. Exercised without --fork here since fork mode needs a built
* plugins directory that this test module doesn't set up (no other test in this
* class runs fork mode either); the values' actual effect on fork mode's
* TimeoutLimits is covered directly at the pipes layer.
*/
@Test
public void testTaskAndProgressTimeoutFlagsAreRecognized() throws Exception {
String content = getParamOutContent("-t", "--task-timeout=30000",
"--progress-timeout=10000", resourcePrefix + "alice.cli.test");
assertTrue(content.contains("finished off the cake"));
}
/**
* Tests -A option of the cli
*
* @throws Exception
*/
@Test
public void testContentAllOutput() throws Exception {
String[] params = {"-A", resourcePrefix + "testJsonMultipleInts.html"};
TikaCLI.main(params);
String out = outContent.toString(UTF_8.name());
assertTrue(out.contains("this is a title"));
assertTrue(out.contains("body"));
}
/**
* Tests -m option of the cli
*
* @throws Exception
*/
@Test
public void testMetadataOutput() throws Exception {
String content = getParamOutContent("-m", resourcePrefix + "alice.cli.test");
assertTrue(content.contains("text/plain"));
content = getParamOutContent("-m", "--digest=SHA512", resourcePrefix + "alice.cli.test");
assertTrue(content.contains("text/plain"));
assertTrue(content.contains("tk:digest:SHA-512: dd459d99bc19ff78fd31fbae46e0"));
}
/**
* Basic tests for -json option
*
* @throws Exception
*/
@Test
public void testJsonMetadataOutput() throws Exception {
String json = getParamOutContent("--json", "--digest=MD2", resourcePrefix + "testJsonMultipleInts.html");
//TIKA-1310
assertTrue(json.contains("\"html:fb:admins\":\"1,2,3,4\","));
assertTrue(json.contains("\"tk:digest:MD2\":"));
}
/**
* Keys a parser sets after its SAX document ends (the PDF parser's document totals in
* its finally block, tk:parsed-by-full-set from the composite) must still reach -m and
* --json, which used to write at endDocument.
*/
@Test
public void testMetadataSetAfterEndDocumentIsOutput() throws Exception {
String content = getParamOutContent("-m", resourcePrefix + "testPDF_childAttachments.pdf");
assertTrue(content.contains("pdf:ocr-page-count: 0"), content);
assertTrue(content.contains("tk:parsed-by-full-set:"), content);
String json = getParamOutContent("--json", resourcePrefix + "testPDF_childAttachments.pdf");
assertTrue(json.contains("\"pdf:ocr-page-count\":"), json);
assertTrue(json.contains("\"tk:parsed-by-full-set\":"), json);
}
/**
* Test for -json with prettyprint option
*
* @throws Exception
*/
@Test
public void testJsonMetadataPrettyPrintOutput() throws Exception {
String json = getParamOutContent("--json", "-r", resourcePrefix + "testJsonMultipleInts.html");
assertTrue(json.contains("org.apache.tika.parser.DefaultParser\", \"org.apache.tika.parser.html.JSoupParser"));
//test pretty-print alphabetic sort of keys
int enc = json.indexOf("\"Content-Encoding\"");
int fb = json.indexOf("fb:admins");
int title = json.indexOf("\"dc:title\"");
assertTrue(enc > -1 && fb > -1 && enc < fb);
assertTrue(fb > -1 && title > -1 && fb > title);
}
@Test
public void testJsonRecursiveWritesOnContainerException(@TempDir Path tmp) throws Exception {
// 2.6M chars from a few KB of gzip trips a 100:1 zip-bomb ratio
Path config = tmp.resolve("ratio-100.json");
Files.writeString(config, "{\"parse-context\":{\"output-limits\":{\"zipBombRatio\":100}}}");
Path gz = tmp.resolve("repetitive.txt.gz");
try (OutputStream os = new GZIPOutputStream(Files.newOutputStream(gz))) {
byte[] line = "the same log line, over and over\n".getBytes(UTF_8);
for (int i = 0; i < 80_000; i++) {
os.write(line);
}
}
resetContent();
assertThrows(TikaException.class, () -> TikaCLI.main(new String[]{"--config=" + config, "-J", gz.toString()}));
String json = outContent.toString(UTF_8);
assertTrue(json.contains("tk:exception:container-exception"), json);
assertTrue(json.contains("Suspected zip bomb"), json);
assertTrue(json.contains("the same log line"), json);
}
@Test
public void testDefaultPDFIncrementalUpdateSettings() throws Exception {
String json = getParamOutContent("-J",
resourcePrefix + "testPDF_incrementalUpdates.pdf");
assertTrue(json.contains("pdf:incremental-update-count\":\"2\""));
assertTrue(json.contains("tk:embedded-resource-type\":\"VERSION\""));
}
@Test
public void testExtractJavascript() throws Exception {
String json = getParamOutContent("-J", "-x", resourcePrefix + "testPDFPackage.pdf");
assertTrue(json.contains("type=\\\"PDActionJavaScript\\\""));
assertTrue(json.contains("MACRO"));
assertTrue(json.contains("NAMES_TREE"));
}
@Test
public void testMacros() throws Exception {
String json = getParamOutContent("-J", resourcePrefix + "testPPT_macros.ppt");
assertTrue(json.contains("MACRO"));
assertTrue(json.contains("Module1"));
}
@Test
public void testRUnpack() throws Exception {
//TODO -- rework this to use two separate emitters
//one for bytes and one for json
// TODO: 1.bin extension may be wrong - see ~/Desktop/unpack-discussion/mime-todo.txt
// a single input file: the output dir is the package
String[] expectedChildren = new String[]{
"datapackage.json",
"rmeta.json",
"unpacked/1.bin",
"unpacked/2.jpg",
"unpacked/3.pdf",
"unpacked/4.pdf"};
testRecursiveUnpack("testPDFPackage.pdf", expectedChildren, 3);
}
@Test
public void testPSTRUnpack() throws Exception {
// TODO: The .bin extensions for embedded .msg files are wrong - they should be .msg
// CONTENT_TYPE is not being set for embedded documents - see ~/Desktop/unpack-discussion/mime-todo.txt
String[] expectedChildren = new String[]{"rmeta.json",
"unpacked/7.bin",
"unpacked/1.bin",
"unpacked/8.bin",
"unpacked/4.bin",
"unpacked/3.bin",
"unpacked/2.bin",
"unpacked/5.bin",
"unpacked/9.docx",
"unpacked/6.bin"};
testRecursiveUnpack("testPST.pst", expectedChildren, 3);
try (Reader reader = Files.newBufferedReader(extractDir.resolve("rmeta.json"))) {
List<Metadata> metadataList = JsonMetadataList.fromJson(reader);
for (Metadata m : metadataList) {
String content = m.get(TikaCoreProperties.TIKA_CONTENT);
assertFalse(StringUtils.isBlank(content));
}
}
}
/**
* Tests -l option of the cli
*
* @throws Exception
*/
@Test
public void testLanguageOutput() throws Exception {
String content = getParamOutContent("-l", resourcePrefix + "alice.cli.test");
assertTrue(content.contains("en"));
}
/**
* Tests -d option of the cli
*
* @throws Exception
*/
@Test
public void testDetectOutput() throws Exception {
String content = getParamOutContent("-d", resourcePrefix + "alice.cli.test");
assertTrue(content.contains("text/plain"));
}
/**
* Tests --list-met-models option of the cli
*
* @throws Exception
*/
@Test
public void testListMetModels() throws Exception {
String content = getParamOutContent("--list-met-models", "-x", resourcePrefix + "alice.cli.test");
assertTrue(content.contains("text/plain"));
}
/**
* Tests --list-supported-types option of the cli
*
* @throws Exception
*/
@Test
public void testListSupportedTypes() throws Exception {
String content = getParamOutContent("--list-supported-types", resourcePrefix + "alice.cli.test");
assertTrue(content.contains("supertype: application/octet-stream"));
}
@Test
public void testExtractSimple() throws Exception {
assertShallow("coffee.xls", 10, "coffee.xls/rmeta.json", "coffee.xls/unpacked/1.emf",
"coffee.xls/unpacked/6.cdx", "coffee.xls/unpacked/5.png");
}
@Test
public void testExtractAbsolute() throws Exception {
assertShallow("testZip_absolutePath.zip", 4, "testZip_absolutePath.zip/rmeta.json",
"testZip_absolutePath.zip/unpacked/1.bin");
}
@Test
public void testExtractRelative() throws Exception {
assertShallow("testZip_relative.zip", 3, "testZip_relative.zip/rmeta.json");
}
@Test
public void testExtractOverlapping() throws Exception {
// overlapping names are handled by numbering
assertShallow("testZip_overlappingNames.zip", 4, "testZip_overlappingNames.zip/rmeta.json");
}
@Test
public void testExtract0x00() throws Exception {
assertShallow("testZip_zeroByte.zip", 3, "testZip_zeroByte.zip/rmeta.json");
}
private void testRecursiveUnpack(String targetFile, String[] expectedChildrenFileNames) throws Exception {
testRecursiveUnpack(targetFile, expectedChildrenFileNames, expectedChildrenFileNames.length);
}
private void testRecursiveUnpack(String targetFile, String[] expectedChildrenFileNames, int expectedLength) throws Exception {
Path input = Paths.get(new URI(resourcePrefix + "/" + targetFile));
Path pluginsDir = Paths.get("target/plugins");
String[] params = {"-Z", NO_OCR,
"-p", pluginsDir.toAbsolutePath().toString(),
input.toAbsolutePath().toString(),
extractDir.toAbsolutePath().toString()};
TikaCLI.main(params);
Set<String> fileNames = getFileNames(extractDir);
String[] jsonFile = extractDir
.toFile()
.list();
assertNotNull(jsonFile);
// Debug: log actual files found
LOG.info("=== Actual files found ===");
for (String name : fileNames) {
LOG.info(" {}", name);
}
LOG.info("=== End actual files ===");
assertEquals(expectedLength, jsonFile.length);
for (String expectedChildName : expectedChildrenFileNames) {
assertContainsFile(fileNames, expectedChildName);
}
}
private static Set<String> getFileNames(Path extractDir) throws IOException {
final Set<String> names = new HashSet<>();
Files.walkFileTree(extractDir, new FileVisitor<Path>() {
@Override
public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes basicFileAttributes) throws IOException {
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path path, BasicFileAttributes basicFileAttributes) throws IOException {
names.add(extractDir.relativize(path).toString().replace('\\', '/'));
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(Path path, IOException e) throws IOException {
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path path, IOException e) throws IOException {
return FileVisitResult.CONTINUE;
}
});
return names;
}
private void assertContainsFile(Set<String> fileNames, String expected) {
assertTrue(fileNames.contains(expected), "Expected " + expected + " in " + fileNames);
}
@Test
public void testExtractTgz() throws Exception {
//TIKA-2564
assertFalse(shallowOutputOf("test-documents.tgz").isEmpty(), "Should have extracted some files");
}
// TIKA-920
@Test
public void testMultiValuedMetadata() throws Exception {
String content = getParamOutContent("-m", resourcePrefix + "testMultipleSheets.numbers");
assertTrue(content.contains("sheet-names: Checking"));
assertTrue(content.contains("sheet-names: Secon sheet"));
assertTrue(content.contains("sheet-names: Logical Sheet 3"));
assertTrue(content.contains("sheet-names: Sheet 4"));
}
// TIKA-1031
@Test
public void testZipWithSubdirs() throws Exception {
Set<String> fileNames = shallowOutputOf("testWithSubdirs.zip");
assertTrue(fileNames.stream().anyMatch(f -> f.endsWith(".json")),
"Should have a .json metadata file, got: " + fileNames);
assertTrue(fileNames.stream().anyMatch(f -> f.contains("/unpacked/")),
"Should have extracted embedded files under unpacked/, got: " + fileNames);
}
@Test
public void testExtractInlineImages() throws Exception {
Set<String> fileNames = shallowOutputOf("testPDF_childAttachments.pdf");
assertTrue(fileNames.stream().anyMatch(f -> f.endsWith(".json")),
"Should have a .json metadata file in " + fileNames);
assertTrue(fileNames.size() >= 2,
"Should have at least 2 files (json + embedded), got " + fileNames.size() + ": " + fileNames);
}
private static Set<String> shallowOutputOf(String inputName) {
Set<String> mine = new HashSet<>();
for (String f : shallowOutput) {
if (f.equals(inputName + ".json") || f.startsWith(inputName + "/")) {
mine.add(f);
}
}
return mine;
}
/** Every file the shared -z run wrote belongs to some input; a stray would hide from the per-input filters. */
@Test
public void testShallowOutputHasNoStrays() {
Set<String> claimed = new HashSet<>();
for (String input : SHALLOW_INPUTS) {
claimed.addAll(shallowOutputOf(input));
}
assertEquals(shallowOutput, claimed);
}
private static void assertShallow(String inputName, int expectedCount, String... expected) {
Set<String> fileNames = shallowOutputOf(inputName);
assertEquals(expectedCount, fileNames.size(), fileNames.toString());
for (String e : expected) {
assertTrue(fileNames.contains(e), "Expected " + e + " in " + fileNames);
}
}
/**
* Test that --extract-dir option correctly sets the output directory
* for both -z (shallow) and -Z (recursive) extraction modes.
*/
@Test
public void testExtractDirOption() throws Exception {
Path input = Paths.get(new URI(resourcePrefix + "/test_recursive_embedded.docx"));
Path pluginsDir = Paths.get("target/plugins");
// Test with -z (shallow extraction)
String[] params = {"-z",
"--extract-dir=" + extractDir.toAbsolutePath(),
"-p", pluginsDir.toAbsolutePath().toString(),
input.toAbsolutePath().toString()};
TikaCLI.main(params);
Set<String> fileNames = getFileNames(extractDir);
// Should have extracted files in the specified directory, not current dir
assertTrue(fileNames.contains("rmeta.json"),
"Should have rmeta.json in extractDir, got: " + fileNames);
assertTrue(fileNames.stream().anyMatch(f -> f.startsWith("unpacked/")),
"Should have extracted embedded files in extractDir, got: " + fileNames);
}
/**
* TIKA-4681: the default is a Frictionless package laid out as loose files; for a single
* input file the output dir is the package (TIKA-4971).
*/
@Test
public void testDefaultIsALooseFrictionlessPackage() throws Exception {
Set<String> fileNames = unpack("-Z");
assertTrue(fileNames.contains("datapackage.json"),
"package should be laid out at the output dir's root, got: " + fileNames);
assertTrue(fileNames.contains("rmeta.json"),
"a Frictionless package carries rmeta.json by default: " + fileNames);
assertTrue(fileNames.stream().anyMatch(f -> f.matches("unpacked/1\\.[^/]+")),
"embedded files are unpadded and under unpacked/: " + fileNames);
assertFalse(fileNames.stream().anyMatch(f -> f.endsWith(".zip") && !f.startsWith("unpacked/")),
"no zip when no mode was asked for: " + fileNames);
assertFalse(fileNames.contains("test_recursive_embedded.docx.json"),
"rmeta.json already carries the metadata list: " + fileNames);
}
/** TIKA-4971: rmeta.json is a resource of the package, with its size and hash. */
@Test
public void testManifestListsRmetaJson() throws Exception {
unpack("-Z");
byte[] metadataJson = Files.readAllBytes(extractDir.resolve("rmeta.json"));
JsonNode resources = new ObjectMapper().readTree(
extractDir.resolve("datapackage.json").toFile()).get("resources");
JsonNode resource = null;
for (JsonNode r : resources) {
if ("rmeta.json".equals(r.get("path").asText())) {
resource = r;
}
}
assertNotNull(resource, "rmeta.json not in " + resources);
assertEquals("application/json", resource.get("mediatype").asText());
assertEquals(metadataJson.length, resource.get("bytes").asLong());
assertEquals("sha256:" + HexFormat.of().formatHex(
MessageDigest.getInstance("SHA-256").digest(metadataJson)), resource.get("hash").asText());
}
/**
* TIKA-4971: the rules frictionless validate enforces on resource names -- present, unique,
* lowercase [a-z0-9._-] -- and tk:embedded-id resolves to the file's row in rmeta.json.
*/
@Test
public void testManifestIsValidFrictionless() throws Exception {
unpack("-Z");
ObjectMapper mapper = new ObjectMapper();
JsonNode resources = mapper.readTree(extractDir.resolve("datapackage.json").toFile())
.get("resources");
Map<String, JsonNode> rowsById = new HashMap<>();
for (JsonNode row : mapper.readTree(extractDir.resolve("rmeta.json").toFile())) {
if (row.has("tk:embedded-id")) {
rowsById.put(row.get("tk:embedded-id").asText(), row);
}
}
Set<String> names = new HashSet<>();
int unpacked = 0;
for (JsonNode r : resources) {
String name = r.path("name").asText(null);
assertNotNull(name, "every resource needs a name: " + r);
assertTrue(name.matches("[a-z0-9._-]+"), "invalid Frictionless name: " + r);
assertTrue(names.add(name), "duplicate name: " + r);
if (!r.get("path").asText().startsWith("unpacked/")) {
continue;
}
unpacked++;
JsonNode row = rowsById.get(r.get("tk:embedded-id").asText());
assertNotNull(row, "tk:embedded-id has no rmeta.json row: " + r);
assertEquals(row.path("tk:resource-name").asText(null),
r.path("tk:resource-name").asText(null), r.toString());
}
assertTrue(unpacked > 0, resources.toString());
}
@Test
public void testZippedModeWritesOnePackageZip() throws Exception {
Set<String> fileNames = unpack("-Z", "--unpack-mode=ZIPPED");
assertTrue(fileNames.contains("test_recursive_embedded.docx-frictionless.zip"), fileNames.toString());
assertFalse(fileNames.stream().anyMatch(f -> f.contains("/unpacked/")),
"ZIPPED must not also write loose files: " + fileNames);
}
@Test
public void testRegularFormatIsTheFlatLayout() throws Exception {
Set<String> fileNames = unpack("-Z", "--unpack-format=REGULAR");
assertTrue(fileNames.stream().anyMatch(f -> f.matches("test_recursive_embedded\\.docx-embed/1\\.[^/]+")),
fileNames.toString());
assertFalse(fileNames.stream().anyMatch(f -> f.endsWith("datapackage.json") || f.contains("/unpacked/")),
"REGULAR has no manifest and no unpacked/: " + fileNames);
}
/** --unpack-mode is the CLI's packaging knob in both formats. */
@Test
public void testRegularZippedWritesOneFlatZip() throws Exception {
Set<String> fileNames = unpack("-Z", "--unpack-format=REGULAR", "--unpack-mode=ZIPPED");
assertTrue(fileNames.contains("test_recursive_embedded.docx-embedded.zip"), fileNames.toString());
assertFalse(fileNames.stream().anyMatch(f -> f.contains("-embed/")),
"ZIPPED must not also write loose files: " + fileNames);
}
@Test
public void testFrictionlessIncludeMetadata() throws Exception {
Set<String> fileNames = unpack("-Z", "--unpack-include-metadata");
assertTrue(fileNames.contains("rmeta.json"), "no rmeta.json in " + fileNames);
String content = Files.readString(extractDir.resolve("rmeta.json"));
assertTrue(content.contains("\"tk:embedded-depth\" : \"0\""),
"rmeta.json should carry the container row: " + content);
assertTrue(content.contains("\"tk:content\""), content);
}
/** A -c file's unpack-config is honoured; it is the only way to get the original back. */
@Test
public void testConfigFileUnpackConfigIsHonoured() throws Exception {
Set<String> fileNames = unpack("-Z", "--config=" + CONFIGS_DIR + "/tika-config-unpack-original.json");
assertTrue(fileNames.stream().anyMatch(f -> f.matches("unpacked/0\\.[^/]+")),
"includeOriginal from -c should add the container as 0.<ext>: " + fileNames);
assertTrue(fileNames.stream().anyMatch(f -> f.matches("unpacked/1\\.[^/]+")),
"-c must not change the CLI's file naming or packaging: " + fileNames);
}
private Set<String> unpack(String... flags) throws Exception {
Path input = Paths.get(new URI(resourcePrefix + "/test_recursive_embedded.docx"));
List<String> params = new ArrayList<>(Arrays.asList(flags));
if (params.stream().noneMatch(f -> f.startsWith("--config="))) {
params.add(NO_OCR);
}
params.add("-p");
params.add(Paths.get("target/plugins").toAbsolutePath().toString());
params.add(input.toAbsolutePath().toString());
params.add(extractDir.toAbsolutePath().toString());
TikaCLI.main(params.toArray(new String[0]));
return getFileNames(extractDir);
}
@Test
public void testDefaultConfigException() throws Exception {
//default xml parser will throw TikaException
//this and TestConfig() are broken into separate tests so that
//setUp and tearDown() are called each time
String[] params = {resourcePrefix + "bad_xml.xml"};
boolean tikaEx = false;
try {
TikaCLI.main(params);
} catch (TikaException e) {
tikaEx = true;
}
assertTrue(tikaEx);
}
@Test
public void testConfig() throws Exception {
String content = getParamOutContent("--config=" + CONFIGS_DIR.toString() + "/tika-config1.json", "-x", resourcePrefix + "bad_xml.xml");
assertTrue(content.contains("apple"));
assertTrue(content.contains("org.apache.tika.parser.html.JSoupParser"));
}
@Test
public void testJsonRecursiveMetadataParserMetadataOnly() throws Exception {
String content = getParamOutContent("-m", "-J", "-r", resourcePrefix + "test_recursive_embedded.docx");
assertTrue(content.contains("\"extended-properties:AppVersion\" : \"15.0000\","));
assertTrue(content.contains("\"extended-properties:Application\" : \"Microsoft Office Word\","));
assertTrue(content.contains("\"tk:embedded-resource-path\" : \"/embed1.zip\""));
assertFalse(content.contains("\"tk:content\""));
}
@Test
public void testJsonRecursiveMetadataParserDefault() throws Exception {
// TIKA-4663: default handler is markdown, so recursive content is markdown, not XHTML.
String content = getParamOutContent("-J", "-r", resourcePrefix + "test_recursive_embedded.docx");
assertFalse(content.contains("<html xmlns=\\\"http://www.w3.org/1999/xhtml"),
"default recursive content should be markdown, not XHTML");
assertTrue(content.contains("# embed1.zip"),
"default recursive content should be markdown (heading syntax)");
}
@Test
public void testJsonRecursiveMetadataParserText() throws Exception {
String content = getParamOutContent("-J", "-r", "-t", resourcePrefix + "test_recursive_embedded.docx");
assertTrue(content.contains("\\n\\nembed_4\\n"));
assertTrue(content.contains("\\n\\nembed_0"));
}
@Test
public void testDigestInJson() throws Exception {
String content = getParamOutContent("-J", "-r", "-t", "--digest=md5", resourcePrefix + "test_recursive_embedded.docx");
assertTrue(content.contains("\"tk:digest:MD5\" : \"59f626e09a8c16ab6dbc2800c685f772\","));
assertTrue(content.contains("\"tk:digest:MD5\" : \"f9627095ef86c482e61d99f0cc1cf87d\""));
}
@Test
@Disabled("until we re-implement serialization")
public void testConfigSerializationStaticAndCurrent() throws Exception {
String content = getParamOutContent("--dump-static-config");
//make sure at least one detector is there
assertTrue(content.contains("<detector class=\"org.apache.tika.detect.microsoft.POIFSContainerDetector\"/>"));
//make sure Executable is there because follow on tests of custom config
//test that it has been turned off.
assertTrue(content.contains("<parser class=\"org.apache.tika.parser.executable.ExecutableParser\"/>"));
content = getParamOutContent("--dump-current-config");
//make sure at least one detector is there
assertTrue(content.contains("<detector class=\"org.apache.tika.detect.DefaultDetector\"/>"));
//and at least one parser
assertTrue(content.contains("<parser class=\"org.apache.tika.parser.DefaultParser\"/>"));
}
@Test
@Disabled("until we re-implement serialization")
public void testConfigSerializationCustomMinimal() throws Exception {
String content = getParamOutContent("--config=" + CONFIGS_DIR.toString() + "/tika-config2.json", "--dump-minimal-config").replaceAll("[\r\n\t ]+", " ");
String expected =
"<parser class=\"org.apache.tika.parser.DefaultParser\">" + " <mime-exclude>application/pdf</mime-exclude>" + " <mime-exclude>image/jpeg</mime-exclude> " +
"</parser> " + "<parser class=\"org.apache.tika.parser.EmptyParser\">" + " <mime>application/pdf</mime> " + "</parser>";
assertTrue(content.contains(expected));
}
@Test
@Disabled("until we re-implement serialization")
public void testConfigSerializationCustomStatic() throws Exception {
String content = getParamOutContent("--config=" + TEST_DATA_FILE.toString() + "/tika-config2.json", "--dump-static-config");
assertFalse(content.contains("org.apache.tika.parser.executable.Executable"));
}
/**
* Tests --list-detector option of the cli
* Tests --list-detectors option of the cli
*
* @throws Exception
*/
@Test
public void testListDetectors() throws Exception {
String content = getParamOutContent("--list-detector");
assertTrue(content.contains("org.apache.tika.detect.DefaultDetector"));
content = getParamOutContent("--list-detectors");
assertTrue(content.contains("org.apache.tika.detect.DefaultDetector"));
}
/**
* Tests --list-parser-detail-apt option of the cli
* Tests --list-parser-details-apt option of the cli
*
* @throws Exception
*/
@Test
public void testListParserDetailApt() throws Exception {
String content = getParamOutContent("--list-parser-detail-apt");
assertTrue(content.contains("application/vnd.oasis.opendocument.text-web"));
content = getParamOutContent("--list-parser-details-apt");
assertTrue(content.contains("application/vnd.oasis.opendocument.text-web"));
}
/**
* The docs' Supported Formats page includes a checked-in copy of the adoc
* listing. Fail when it drifts from the parsers actually on the classpath.
*/
@Test
public void testSupportedFormatsPartialIsCurrent() throws Exception {
String content = getParamOutContent("--list-parser-details-adoc");
assertTrue(content.startsWith(SupportedFormatsAdoc.HEADER));
assertTrue(content.contains(
"link:{tika-javadoc-url}/org/apache/tika/parser/pdf/PDFParser.html[PDFParser]"));
assertTrue(content.contains("** `application/vnd.oasis.opendocument.text-web`"));
// Tesseract is hidden so the listing is the same with or without the binary.
assertFalse(content.contains("TesseractOCRParser"));
Path partial = Paths.get("..", "docs", "modules", "ROOT", "partials", "supported-formats.adoc");
assumeTrue(Files.isRegularFile(partial), "docs partial not present in this checkout");
String expected = content.replace("\r\n", "\n");
String actual = Files.readString(partial, UTF_8).replace("\r\n", "\n");
assertEquals(expected, actual, "docs/modules/ROOT/partials/supported-formats.adoc is stale; regenerate with:\n" +
" java -jar tika-app/target/tika-app-<version>.jar --list-parser-details-adoc " +
"> docs/modules/ROOT/partials/supported-formats.adoc");
}
/**
* Tests --convert-config-xml-to-json with no separate config file.
* Regression test for TIKA-4734: the flag used to be misrouted to async
* mode (the input arg ended in ".json"), failing with a TikaConfigException
* unless a --config was also passed. It must now run standalone and write
* the converted JSON to stdout.
*/
@Test
public void testConvertConfigXmlToJson() throws Exception {
String xmlPath = Paths.get(getClass().getResource("/xml-configs/tika-config-simple.xml").toURI()).toString();
String content = getParamOutContent("--convert-config-xml-to-json=" + xmlPath);
// stdout should be pure JSON; parse and assert on structure, not formatting
JsonNode root = new ObjectMapper().readTree(content.trim());
JsonNode parsers = root.get("parsers");
assertNotNull(parsers, "Expected parsers section, got: " + content);
assertTrue(parsers.isArray() && parsers.size() > 0, "Expected non-empty parsers array, got: " + content);
JsonNode pdfEntry = null;
for (JsonNode entry : parsers) {
if (entry.has("pdf-parser")) {
pdfEntry = entry.get("pdf-parser");
break;
}
}
assertNotNull(pdfEntry, "Expected pdf-parser entry, got: " + content);
JsonNode sortByPosition = pdfEntry.findValue("sortByPosition");
assertNotNull(sortByPosition, "Expected sortByPosition under pdf-parser, got: " + content);
assertTrue(sortByPosition.asBoolean(), "Expected sortByPosition=true, got: " + sortByPosition);
}
/**
* reset outContent and errContent if they are not empty
* run given params in TikaCLI and return outContent String with UTF-8
*/
String getParamOutContent(String... params) throws Exception {
resetContent();
TikaCLI.main(params);
return outContent.toString("UTF-8");
}
}