PDFMarkedContent2XHTMLTest.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.tika.parser.pdf;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;

import java.util.List;

import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;

import org.apache.tika.TikaTest;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.PDF;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;

/** The tagged writer on the real tagged test PDFs (Word, Acrobat output). */
public class PDFMarkedContent2XHTMLTest extends TikaTest {

    static ParseContext MARKUP_CONTEXT = new ParseContext();

    @BeforeAll
    public static void setUp() {
        PDFParserConfig config = new PDFParserConfig();
        // the deprecated flag still selects TAGS
        config.setExtractMarkedContent(true);

        MARKUP_CONTEXT.set(PDFParserConfig.class, config);
    }

    @Test
    public void testJournal() throws Exception {
        XMLResult r = getXML("testJournalParser.pdf", MARKUP_CONTEXT);
        String xml = r.xml;
        assertContains("<h1>I. INTRODUCTION", xml);
        assertContains("<h1>V. CONCLUSION", xml);
        // one cell per TD, even though the row is drawn as one run of text
        assertContains("<table><tr>\t<td><p />", xml);
        assertContains("<td><p>NHG</p>", xml);
        assertContains("<td><p>STRING</p>", xml);
        assertContainsCount("<div class=\"page\">", xml, 10);
        assertEquals(10, r.metadata.getInt(PDF.MARKED_CONTENT_PAGES_TAGGED));
        assertEquals(0, r.metadata.getInt(PDF.MARKED_CONTENT_PAGES_FALLBACK));
        assertNull(r.metadata.get(PDF.MARKED_CONTENT_REJECTIONS));
        // the stripper's per-page counters still run under the tagged writer
        assertEquals(10, r.metadata.getValues(PDF.CHARACTERS_PER_PAGE).length);
    }

    @Test
    public void testVarious() throws Exception {
        String xml = getXML("testPDFVarious.pdf", MARKUP_CONTEXT).xml;
        assertContains("<div class=\"textbox\"><p>Here is a text box", xml);
        assertContains("<div class=\"footnote\"><p>1\n This is a footnote.", xml);
        assertContains("<p>Bold italic underline superscript subscript", xml);
        assertContains("<ul>\t<li>Bullet 1", xml);
        assertContains("<table><tr>\t<td><p>Row 1 Col 1</p>", xml);
        assertContains("</td>\t<td><p>Row 1 Col 2</p>", xml);
        assertContains("<p>Here is a citation:", xml);
        assertContains("<p><a href=\"http://tika.apache.org/\">This is a hyperlink</a></p>", xml);
        // running header and footer are /Artifact content: kept, but set apart
        assertContains("<div class=\"artifact\"><p>This is the header text.", xml);
        assertContains("<p>This is the footer text.", xml);
        // the annotation extraction is untouched
        assertContains("<div class=\"annotation\"><a href=\"http://tika.apache.org/\">", xml);
        assertFalse(xml.contains("<body>") && xml.indexOf("<body>") != xml.lastIndexOf("<body>"));
    }

    @Test
    public void testChildAttachments() throws Exception {
        List<Metadata> metadataList =
                getRecursiveMetadata("testPDF_childAttachments.pdf", MARKUP_CONTEXT);

        //make sure that embedded docs are still getting extracted
        assertEquals(3, metadataList.size());

        String xml = metadataList.get(0).get(TikaCoreProperties.TIKA_CONTENT);
        // the structure tree's Link gives the anchor text; the annotation only has the href
        assertContains("<a href=\"http://www.irs.gov\">IRS.gov</a>", xml);
        assertContains("<a href=\"http://www.irs.gov/pub15\">www.irs.gov/pub15</a>", xml);
        assertEquals(67, metadataList.get(0).getInt(PDF.MARKED_CONTENT_PAGES_TAGGED));
    }
}