MimeDetectionTest.java
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.tika.mime;
import static java.nio.charset.StandardCharsets.US_ASCII;
import static java.nio.charset.StandardCharsets.UTF_16BE;
import static java.nio.charset.StandardCharsets.UTF_16LE;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import org.apache.commons.io.ByteOrderMark;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
public class MimeDetectionTest {
private static MimeTypes MIME_TYPES;
private static MediaTypeRegistry REGISTRY;
/**
* @inheritDoc
*/
@BeforeAll
public static void setUp() {
MIME_TYPES = MimeTypes.getDefaultMimeTypes();
REGISTRY = MIME_TYPES.getMediaTypeRegistry();
}
@Test
public void testDetection() throws Exception {
testFile("image/svg+xml", "circles.svg");
testFile("image/svg+xml", "circles-with-prefix.svg");
testFile("image/png", "datamatrix.png");
testFile("text/html", "test.html");
testFile("application/xml", "test-iso-8859-1.xml");
testFile("application/xml", "test-utf8.xml");
testFile("application/xml", "test-utf8-bom.xml");
testFile("application/xml", "test-utf16le.xml");
testFile("application/xml", "test-utf16be.xml");
testFile("application/xml", "test-long-comment.xml");
testFile("application/xslt+xml", "stylesheet.xsl");
testUrl("application/rdf+xml", "http://www.ai.sri.com/daml/services/owl-s/1.2/Process.owl",
"test-difficult-rdf1.xml");
testUrl("application/rdf+xml", "http://www.w3.org/2002/07/owl#", "test-difficult-rdf2.xml");
// add evil test from TIKA-327
testFile("text/html", "test-tika-327.html");
// add another evil html test from TIKA-357
testFile("text/html", "testlargerbuffer.html");
// test fragment of HTML with <div> (TIKA-1102)
testFile("text/html", "htmlfragment");
// test fragment of HTML with <iframe> and potentially misleading file suffix
testFile("text/html", "test-html-snippet-iframe.jsp");
// test binary CGM detection (TIKA-1170)
testFile("image/cgm", "plotutils-bin-cgm-v3.cgm");
// test HTML detection of malformed file, previously identified as image/cgm (TIKA-1170)
testFile("text/html", "test-malformed-header.html.bin");
//test GCMD Directory Interchange Format (.dif) TIKA-1561
testFile("application/dif+xml", "brwNIMS_2014.dif");
// truncated xml should still be detected as xml, See TIKA-3596
testFile("application/xml", "truncated-utf16-xml.xyz");
// Android Binary XML (compiled AndroidManifest.xml / res/*.xml inside an APK).
// Carries a .xml extension, so magic must win over the *.xml glob and it must
// NOT be routed to application/xml / the XML parser. See TIKA-4747.
testFile("application/vnd.android.axml", "test-android-binary.xml");
}
/** The html magic needs the tag to end: an XML element such as htmlPath is not html. */
@Test
public void testHtmlMagicNeedsTagEnd() throws Exception {
String pad = " ".repeat(200);
String[] notHtml = {
"<htmlPath><![CDATA[TmplData\\tmpl.html]]></htmlPath>",
"<flashTemplate><info>" + pad + "<htmlPath>x</htmlPath></info></flashTemplate>",
"<root>" + pad + "<htmlContent>x</htmlContent></root>"
};
for (String xml : notHtml) {
assertDetected(MediaType.TEXT_PLAIN, xml.getBytes(UTF_8));
Metadata metadata = new Metadata();
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "flashTemplate.xml");
try (TikaInputStream tis = TikaInputStream.get(xml.getBytes(UTF_8))) {
assertEquals(MediaType.APPLICATION_XML, MIME_TYPES.detect(tis, metadata, new ParseContext()), xml);
}
}
String[] html = {
"<html><body>x</body></html>",
"<HTML>\n<body>x</body></HTML>",
"<html\nlang=\"en\"><body>x</body></html>",
"garbage\n".repeat(30) + "<html><body>x</body></html>"
};
for (String h : html) {
assertDetected(MediaType.TEXT_HTML, h.getBytes(UTF_8));
}
}
/** An OS/2 bitmap array's first entry header (size 40, next at 46) is not an icon. */
@Test
public void testOs2BitmapArrayIsNotAnIcon() throws Exception {
for (String tag : new String[]{"BM", "IC", "CI", "CP", "PT"}) {
byte[] bitmapArray = new byte[64];
bitmapArray[0] = 'B';
bitmapArray[1] = 'A';
bitmapArray[2] = 0x28;
bitmapArray[6] = 0x2e;
bitmapArray[14] = (byte) tag.charAt(0);
bitmapArray[15] = (byte) tag.charAt(1);
try (TikaInputStream tis = TikaInputStream.get(bitmapArray)) {
assertEquals(MediaType.parse("image/x-os2-graphics; charset=binary"),
MIME_TYPES.detect(tis, new Metadata(), new ParseContext()), tag);
}
}
}
/**
* TGA has no signature. The rules key on the colour map type and image
* type, the colour map spec and the pixel depth.
*/
@Test
public void testTgaDetection() throws Exception {
MediaType tga = MediaType.image("x-tga");
MediaType cursor = MediaType.image("x-win-bitmap");
// true colour, no id, no colour map, 24 bpp
assertDetected(tga, tgaHeader(0, 0, 2, 0, 24));
// the same with an id field and 32 bpp, and the RLE form
assertDetected(tga, tgaHeader(5, 0, 2, 0, 32));
assertDetected(tga, tgaHeader(0, 0, 10, 0, 32));
// 15 bpp, with and without an id
assertDetected(tga, tgaHeader(0, 0, 2, 0, 15));
assertDetected(tga, tgaHeader(3, 0, 2, 0, 15));
// greyscale
assertDetected(tga, tgaHeader(0, 0, 3, 0, 8));
assertDetected(tga, tgaHeader(0, 0, 11, 0, 8));
// colour mapped, 24 bit entries, 8 bit indices, and the RLE form
assertDetected(tga, tgaHeader(0, 1, 1, 24, 8));
assertDetected(tga, tgaHeader(0, 1, 9, 24, 8));
// no colour map but an entry size in the spec, as some writers leave it
byte[] strayEntrySize = tgaHeader(0, 0, 2, 0, 24);
strayEntrySize[7] = 24;
assertDetected(tga, strayEntrySize);
// a pixel depth no TGA has
assertDetected(MediaType.OCTET_STREAM, tgaHeader(0, 0, 2, 0, 7));
assertDetected(MediaType.OCTET_STREAM, tgaHeader(1, 0, 2, 0, 7));
// a colour map entry size and an index depth no TGA has
assertDetected(MediaType.OCTET_STREAM, tgaHeader(0, 1, 1, 7, 8));
assertDetected(MediaType.OCTET_STREAM, tgaHeader(0, 1, 1, 24, 99));
// a Windows cursor with one image starts like a true colour TGA
byte[] oneImage = tgaHeader(0, 0, 2, 0, 24);
oneImage[4] = 1;
assertDetected(cursor, oneImage);
// a cursor with no images is nothing; 256 images is a TGA with a colour map length
byte[] noImages = tgaHeader(0, 0, 2, 0, 24);
noImages[16] = 0;
assertDetected(MediaType.OCTET_STREAM, noImages);
byte[] images256 = tgaHeader(0, 0, 2, 0, 24);
images256[5] = 1;
assertDetected(MediaType.OCTET_STREAM, images256);
// Lotus 1-2-3 shares the first four bytes
byte[] lotus = new byte[64];
lotus[2] = 2;
lotus[4] = 4;
lotus[5] = 4;
assertDetected(MediaType.parse("application/vnd.lotus-1-2-3;version=1"), lotus);
}
@Test
public void testAnimatedCursorDetection() throws Exception {
MediaType ani = MediaType.application("x-navi-animation");
byte[] acon = new byte[64];
System.arraycopy("RIFF".getBytes(US_ASCII), 0, acon, 0, 4);
acon[4] = 56;
System.arraycopy("ACONanih".getBytes(US_ASCII), 0, acon, 8, 8);
assertDetected(ani, acon);
// the content beats a *.cur name
assertEquals(ani, detect(acon, "pointer.cur"));
}
private static byte[] tgaHeader(int idLength, int colourMapType, int imageType,
int colourMapEntrySize, int pixelDepth) {
byte[] header = new byte[64];
header[0] = (byte) idLength;
header[1] = (byte) colourMapType;
header[2] = (byte) imageType;
if (colourMapType == 1) {
// 256 entries
header[6] = 1;
header[7] = (byte) colourMapEntrySize;
}
header[12] = 32;
header[14] = 32;
header[16] = (byte) pixelDepth;
return header;
}
private static void assertDetected(MediaType expected, byte[] bytes) throws IOException {
try (TikaInputStream tis = TikaInputStream.get(bytes)) {
assertEquals(expected, MIME_TYPES.detect(tis, new Metadata(), new ParseContext()));
}
}
private static MediaType detect(byte[] bytes, String name) throws IOException {
Metadata metadata = new Metadata();
if (name != null) {
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, name);
}
try (TikaInputStream tis = TikaInputStream.get(bytes)) {
return MIME_TYPES.detect(tis, metadata, new ParseContext());
}
}
@Test
public void testXulByContent() throws IOException {
MediaType xul = MediaType.application("vnd.mozilla.xul+xml");
String ns = "http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul";
byte[] plain = ("<?xml version=\"1.0\"?><window xmlns=\"" + ns + "\"/>").getBytes(UTF_8);
// the root's attribute entity stops the root-element sniff
byte[] entityInRoot = ("<?xml version=\"1.0\"?>\n<!DOCTYPE wizard [<!ENTITY % d SYSTEM"
+ " \"chrome://a/b.dtd\"> %d;]>\n<wizard title=\"&wizard.title;\" xmlns=\"" + ns
+ "\"/>").getBytes(UTF_8);
byte[] noDeclaration = ("<window title=\"t\" xmlns=\"" + ns + "\"/>").getBytes(UTF_8);
for (byte[] doc : new byte[][]{plain, entityInRoot, noDeclaration}) {
assertEquals(xul, detect(doc, null));
assertEquals(xul, detect(doc, "attachment.xml"));
}
// mentioning the namespace is not enough
assertEquals(MediaType.TEXT_PLAIN,
detect(("document.createElementNS(\"" + ns + "\", \"label\");").getBytes(UTF_8), null));
assertEquals(MediaType.APPLICATION_XML,
detect(("<?xml version=\"1.0\"?><bindings xmlns=\"http://www.mozilla.org/xbl\""
+ " xmlns:xul=\"" + ns + "\"/>").getBytes(UTF_8), null));
}
@Test
public void testRustVersusRlsServices() throws IOException {
MediaType rust = MediaType.text("x-rust");
MediaType rls = MediaType.application("rls-services+xml");
assertEquals(rust, detect("fn main() {}\n".getBytes(UTF_8), "main.rs"));
assertEquals(rust, detect("// Licensed to the ASF\nfn main() {}\n".getBytes(UTF_8),
"main.rs"));
byte[] rlsXml = ("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ "<rls-services xmlns=\"urn:ietf:params:xml:ns:rls-services\">"
+ "<service uri=\"sip:friends@example.com\"/></rls-services>").getBytes(UTF_8);
assertEquals(rls, detect(rlsXml, "services.rs"));
assertEquals(rls, detect(rlsXml, null));
byte[] png = {(byte) 0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n', 0, 0, 0, 0};
assertEquals(MediaType.image("png"), detect(png, "main.rs"));
}
@Test
public void testDetectionWithoutContent() throws IOException {
testUrlWithoutContent("text/html", "test.html");
testUrlWithoutContent("text/html", "http://test.com/test.html");
testUrlWithoutContent("text/plain", "http://test.com/test.txt");
// In case the url contains a filename referencing a server-side scripting language,
// it gives us no clue concerning the actual mime type of the response
testUrlWithoutContent("application/octet-stream", "http://test.com/test.php");
testUrlWithoutContent("application/octet-stream", "http://test.com/test.cgi");
testUrlWithoutContent("application/octet-stream", "http://test.com/test.jsp");
// But in case the protocol is not http or https, the script is probably not interpreted
testUrlWithoutContent("text/x-php", "ftp://test.com/test.php");
}
@Test
public void testByteOrderMark() throws Exception {
try (TikaInputStream tis = TikaInputStream.get("\ufefftest".getBytes(UTF_16LE))) {
assertEquals(MediaType.TEXT_PLAIN, MIME_TYPES.detect(tis, new Metadata(), new ParseContext()));
}
try (TikaInputStream tis = TikaInputStream.get("\ufefftest".getBytes(UTF_16BE))) {
assertEquals(MediaType.TEXT_PLAIN, MIME_TYPES.detect(tis, new Metadata(), new ParseContext()));
}
try (TikaInputStream tis = TikaInputStream.get("\ufefftest".getBytes(UTF_8))) {
assertEquals(MediaType.TEXT_PLAIN, MIME_TYPES.detect(tis, new Metadata(), new ParseContext()));
}
}
@Test
public void testRFC822WithBOM() throws Exception {
String header = "From: blah <blah@blah.com>\r\n" + "Received: Friday, January 24, 2020 3:24 PM\r\n" +
"To: someone@somewhere.com\r\n" + "Cc: someone-else@other.com\r\n" +
"Subject: Received\r\n";
MediaType rfc822 = MediaType.parse("message/rfc822");
try (TikaInputStream tis = TikaInputStream.get(header.getBytes(UTF_8))) {
assertEquals(rfc822, MIME_TYPES.detect(tis, new Metadata(), new ParseContext()));
}
int utfLength = ByteOrderMark.UTF_8.length();
byte[] bytes = new byte[header.getBytes(UTF_8).length + utfLength];
System.arraycopy(ByteOrderMark.UTF_8.getBytes(), 0, bytes, 0, utfLength);
System.arraycopy(header.getBytes(UTF_8), 0, bytes, 3, header.getBytes(UTF_8).length);
try (TikaInputStream tis = TikaInputStream.get(bytes)) {
assertEquals(rfc822, MIME_TYPES.detect(tis, new Metadata(), new ParseContext()));
}
}
@Test
public void testSuperTypes() {
assertTrue(REGISTRY.isSpecializationOf(MediaType.parse("text/something; charset=UTF-8"),
MediaType.parse("text/something")));
assertTrue(REGISTRY.isSpecializationOf(MediaType.parse("text/something; charset=UTF-8"),
MediaType.TEXT_PLAIN));
assertTrue(REGISTRY.isSpecializationOf(MediaType.parse("text/something; charset=UTF-8"),
MediaType.OCTET_STREAM));
assertTrue(REGISTRY.isSpecializationOf(MediaType.parse("text/something"),
MediaType.TEXT_PLAIN));
assertTrue(REGISTRY.isSpecializationOf(MediaType.parse("application/something+xml"),
MediaType.APPLICATION_XML));
assertTrue(REGISTRY.isSpecializationOf(MediaType.parse("application/something+zip"),
MediaType.APPLICATION_ZIP));
assertTrue(REGISTRY.isSpecializationOf(MediaType.APPLICATION_XML, MediaType.TEXT_PLAIN));
assertTrue(REGISTRY.isSpecializationOf(MediaType.parse("application/vnd.apple.iwork"),
MediaType.APPLICATION_ZIP));
assertTrue(REGISTRY.isSpecializationOf(MediaType.parse("application/json"),
MediaType.TEXT_PLAIN));
}
@SuppressWarnings("unused")
private void testUrlOnly(String expected, String url) throws IOException {
InputStream in = new URL(url).openStream();
testStream(expected, url, in);
}
private void testUrlWithoutContent(String expected, String url) throws IOException {
Metadata metadata = new Metadata();
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, url);
String mime = this.MIME_TYPES.detect(null, metadata, new ParseContext()).toString();
assertEquals(expected, mime,
url + " is not properly detected using only resource name");
}
private void testUrl(String expected, String url, String file) throws IOException {
try (InputStream in = getClass().getResourceAsStream(file)) {
testStream(expected, url, in);
}
}
private void testFile(String expected, String filename) throws IOException {
try (InputStream in = getClass().getResourceAsStream(filename)) {
testStream(expected, filename, in);
}
}
private void testStream(String expected, String urlOrFileName, InputStream in)
throws IOException {
assertNotNull(in, "Test stream: [" + urlOrFileName + "] is null!");
try (TikaInputStream tis = TikaInputStream.get(in)) {
Metadata metadata = new Metadata();
String mime = this.MIME_TYPES.detect(tis, metadata, new ParseContext()).toString();
assertEquals(expected, mime,
urlOrFileName + " is not properly detected: detected.");
//Add resource name and test again
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, urlOrFileName);
mime = this.MIME_TYPES.detect(tis, metadata, new ParseContext()).toString();
assertEquals(expected, mime,
urlOrFileName + " is not properly detected after adding resource name.");
}
}
/**
* Test for type detection of empty documents.
*
* @see <a href="https://issues.apache.org/jira/browse/TIKA-483">TIKA-483</a>
*/
@Test
public void testEmptyDocument() throws IOException {
try (TikaInputStream tis = TikaInputStream.get(new byte[0])) {
assertEquals(MediaType.OCTET_STREAM,
MIME_TYPES.detect(tis, new Metadata(), new ParseContext()));
}
Metadata namehint = new Metadata();
namehint.set(TikaCoreProperties.RESOURCE_NAME_KEY, "test.txt");
try (TikaInputStream tis = TikaInputStream.get(new byte[0])) {
assertEquals(MediaType.TEXT_PLAIN,
MIME_TYPES.detect(tis, namehint, new ParseContext()));
}
Metadata typehint = new Metadata();
typehint.set(HttpHeaders.CONTENT_TYPE, "text/plain");
try (TikaInputStream tis = TikaInputStream.get(new byte[0])) {
assertEquals(MediaType.TEXT_PLAIN,
MIME_TYPES.detect(tis, typehint, new ParseContext()));
}
}
/**
* Test for things like javascript files whose content is enclosed in XML
* comment delimiters, but that aren't actually XML.
*
* @see <a href="https://issues.apache.org/jira/browse/TIKA-426">TIKA-426</a>
*/
@Test
public void testNotXML() throws IOException {
try (TikaInputStream tis = TikaInputStream.get("<!-- test -->".getBytes(UTF_8))) {
assertEquals(MediaType.TEXT_PLAIN, MIME_TYPES.detect(tis, new Metadata(), new ParseContext()));
}
}
/**
* Tests that when we repeatedly test the detection of a document
* that can be detected with Mime Magic, that we consistently
* detect it correctly. See TIKA-391 for more details.
*/
@Test
public void testMimeMagicStability() throws IOException {
for (int i = 0; i < 100; i++) {
testFile("application/vnd.ms-excel", "test.xls");
}
}
/**
* Tests that when two magic matches both apply, and both
* have the same priority, we use the name to pick the
* right one based on the glob, or the first one we
* come across if not. See TIKA-1292 for more details.
*/
@Test
public void testMimeMagicClashSamePriority() throws IOException {
byte[] helloWorld = "Hello, World!".getBytes(UTF_8);
MediaType helloType = MediaType.parse("hello/world-file");
MediaType helloXType = MediaType.parse("hello/x-world-hello");
Metadata metadata;
// With a filename, picks the right one
metadata = new Metadata();
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "test.hello.world");
try (TikaInputStream tis = TikaInputStream.get(helloWorld)) {
assertEquals(helloType, MIME_TYPES.detect(tis, metadata, new ParseContext()));
}
metadata = new Metadata();
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "test.x-hello-world");
try (TikaInputStream tis = TikaInputStream.get(helloWorld)) {
assertEquals(helloXType, MIME_TYPES.detect(tis, metadata, new ParseContext()));
}
// Without, goes for the one that sorts last
metadata = new Metadata();
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "testingTESTINGtesting");
try (TikaInputStream tis = TikaInputStream.get(helloWorld)) {
assertEquals(helloXType, MIME_TYPES.detect(tis, metadata, new ParseContext()));
}
}
/**
* Test for TIKA-3771.
*/
@Test
public void testPNGWithSomeEmlHeaders() throws IOException {
testFile("image/png", "test-pngNotEml.bin");
}
@Test
public void testLyingDeclaredLengthDoesNotTruncateMagic() throws Exception {
byte[] pdf = "%PDF-1.4\nsome pdf content".getBytes(UTF_8);
Metadata metadata = new Metadata();
// an under-declared Content-Length must not shrink the magic read
metadata.set(HttpHeaders.CONTENT_LENGTH, "2");
try (TikaInputStream tis = TikaInputStream.get(
new java.io.ByteArrayInputStream(pdf),
new org.apache.tika.io.TemporaryResources(), metadata)) {
assertEquals(MediaType.application("pdf"),
MIME_TYPES.detect(tis, metadata, new ParseContext()));
}
}
}