AbstractVLMParser.java
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.tika.parser.vlm;
import static org.apache.tika.sax.XHTMLContentHandler.XHTML;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.util.Base64;
import java.util.Collections;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.zip.Deflater;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;
import org.apache.tika.config.Initializable;
import org.apache.tika.config.ParseContextConfig;
import org.apache.tika.config.ParseTimeout;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.exception.TikaException;
import org.apache.tika.extractor.ParentContentHandler;
import org.apache.tika.http.TikaHttpClient;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.Property;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.enricher.OcrWarnings;
import org.apache.tika.parser.enricher.TextRecognizer;
import org.apache.tika.sax.BodyContentHandler;
import org.apache.tika.sax.EmbeddedContentHandler;
import org.apache.tika.sax.TeeContentHandler;
import org.apache.tika.sax.XHTMLContentHandler;
/**
* Abstract base class for parsers that delegate to a remote Vision-Language
* Model (VLM) endpoint for OCR and document understanding.
* <p>
* Subclasses only need to implement the API-specific request/response
* serialization and declare their supported media types. All common logic
* (HTTP transport, timeout handling, inline content, markdown-to-XHTML
* rendering, config resolution) lives here.
*
* @since Apache Tika 4.0
*/
public abstract class AbstractVLMParser implements Parser, Initializable, Closeable,
TextRecognizer {
private static final long serialVersionUID = 1L;
private static final Logger LOG = LoggerFactory.getLogger(AbstractVLMParser.class);
/** Metadata namespace for VLM properties. */
public static final String VLM_META = "vlm:";
public static final Property VLM_MODEL = Property.externalText(VLM_META + "model");
public static final Property VLM_PROMPT_TOKENS =
Property.externalInteger(VLM_META + "prompt-tokens");
public static final Property VLM_COMPLETION_TOKENS =
Property.externalInteger(VLM_META + "completion-tokens");
/** Why the file was not sent to the VLM; absent when it was sent. */
public static final Property VLM_SKIPPED_REASON =
Property.externalText(VLM_META + "skipped-reason");
/** Pixel count of an image that was skipped as too large. */
public static final Property VLM_IMAGE_PIXELS =
Property.externalInteger(VLM_META + "image-pixels");
static final String WARN_TRUNCATED = "truncated";
static final String WARN_REPETITION = "repetition";
static final String WARN_EMPTY = "empty";
/**
* Output of at least this many bytes that deflates below this ratio is a loop. A loop
* deflates to about 1.5 %; an HTML table of numbers to 12 %, dot leaders to 8 %.
*/
private static final int REPETITION_MIN_BYTES = 1000;
private static final double REPETITION_MAX_DEFLATE_RATIO = 0.05;
static final String SKIP_FILE_TOO_SMALL = "file-too-small";
static final String SKIP_FILE_TOO_LARGE = "file-too-large";
static final String SKIP_IMAGE_TOO_LARGE = "image-too-large";
/**
* Encapsulates a fully built HTTP request for a VLM API call.
*
* @param url full request URL (base + path)
* @param json serialized JSON request body
* @param headers additional HTTP headers (e.g. Authorization)
*/
protected record HttpCall(String url, String json, Map<String, String> headers) {}
private VLMOCRConfig defaultConfig;
private transient TikaHttpClient httpClient;
private boolean serverAvailable = false;
protected AbstractVLMParser(VLMOCRConfig config) {
this.defaultConfig = config;
}
/** Built on first use, so the setters that feed it have run. */
synchronized TikaHttpClient httpClient() {
if (httpClient == null) {
httpClient = buildHttpClient();
}
return httpClient;
}
// ---- abstract contract for subclasses ---------------------------------
/**
* Build a fully formed {@link HttpCall} for the target API.
*
* @param config resolved config for this parse
* @param base64Data base64-encoded version of the file bytes
* @param mimeType the MIME type of the input (e.g. {@code image/png})
* @return a ready-to-execute {@link HttpCall}
*/
protected abstract HttpCall buildHttpCall(VLMOCRConfig config,
String base64Data, String mimeType);
/**
* Parse the API response body and extract the model's text output.
* Implementations should also populate {@link #VLM_PROMPT_TOKENS} and
* {@link #VLM_COMPLETION_TOKENS} in metadata when the information is
* available.
*
* @param responseBody raw JSON response body
* @param metadata metadata to enrich with token counts
* @return the extracted text content
*/
protected abstract String extractResponseText(String responseBody, Metadata metadata)
throws TikaException;
/**
* @return the set of media types this parser handles (images, PDFs, etc.)
*/
protected abstract Set<MediaType> getSupportedMediaTypes();
/**
* @return the JSON config key for {@link ParseContextConfig} lookup
* (e.g. {@code "openai-vlm-parser"}, {@code "gemini-vlm-parser"})
*/
protected abstract String configKey();
/**
* @return an optional health-check URL to probe at init time, or
* {@code null} to skip the probe
*/
protected abstract String getHealthCheckUrl(VLMOCRConfig config);
@Override
public boolean parsesDocuments() {
return true;
}
/**
* @return the headers for the health-check probe; Bearer auth by default
*/
protected Map<String, String> getHealthCheckHeaders(VLMOCRConfig config) {
return config.getApiKey() != null
? Map.of("Authorization", "Bearer " + config.getApiKey())
: Map.of();
}
// ---- Parser interface -------------------------------------------------
@Override
public Set<MediaType> getSupportedTypes(ParseContext context) {
if (!serverAvailable) {
return Collections.emptySet();
}
VLMOCRConfig config = context.get(VLMOCRConfig.class);
if (config != null && config.isSkipOcr()) {
return Collections.emptySet();
}
if (defaultConfig.isSkipOcr()) {
return Collections.emptySet();
}
return getSupportedMediaTypes();
}
@Override
public boolean recognizesText(ParseContext context) {
if (!serverAvailable) {
return false;
}
try {
VLMOCRConfig config = getConfig(context);
return !config.isSkipOcr() && config.isTextRecognizer();
} catch (TikaConfigException | IOException e) {
// parse() surfaces the broken config; for the question asked, nothing is recognized
return false;
}
}
@Override
public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata,
ParseContext parseContext) throws IOException, SAXException, TikaException {
if (!serverAvailable) {
return;
}
VLMOCRConfig config = getConfig(parseContext);
if (config.isSkipOcr()) {
return;
}
long size = tis.getLength();
if (size >= 0 && size < config.getMinFileSizeToOcr()) {
skip(parseContext, metadata, SKIP_FILE_TOO_SMALL);
return;
}
if (size >= 0 && size > config.getMaxFileSizeToOcr()) {
skip(parseContext, metadata, SKIP_FILE_TOO_LARGE);
return;
}
String mimeType = detectMimeType(metadata);
long maxPixels = config.getMaxImagePixels();
if (maxPixels > 0 && mimeType.startsWith("image/")) {
tis.mark((int) Math.min(tis.getLength() + 1, 1024 * 1024));
try {
long[] dims = getImageSize(tis);
long pixels = dims == null ? -1 : dims[0] * dims[1];
if (pixels > maxPixels) {
LOG.warn("Image has {} pixels, exceeding maxImagePixels={}. "
+ "Skipping VLM OCR.", pixels, maxPixels);
metadata.set(VLM_IMAGE_PIXELS, (int) Math.min(pixels, Integer.MAX_VALUE));
skip(parseContext, metadata, SKIP_IMAGE_TOO_LARGE);
return;
}
} catch (IOException e) {
LOG.debug("Could not read image dimensions; "
+ "skipping pixel-limit check", e);
} finally {
tis.reset();
}
}
byte[] fileBytes = readFully(tis);
String base64Data = Base64.getEncoder().encodeToString(fileBytes);
HttpCall call = buildHttpCall(config, base64Data, mimeType);
String responseText;
try {
String responseBody = httpClient().postJson(
call.url(), call.json(), call.headers(), config.getTimeoutMillis(), parseContext);
responseText = extractResponseText(responseBody, metadata);
ParseTimeout.checkpoint(parseContext);
} catch (TikaException e) {
throw e;
} catch (IOException e) {
throw new TikaException("VLM request failed: " + e.getMessage(), e);
}
metadata.set(VLM_MODEL, config.getModel());
checkOutput(parseContext, metadata, responseText);
ContentHandler baseHandler = getContentHandler(
config.isInlineContent(), handler, metadata, parseContext);
XHTMLContentHandler xhtml = new XHTMLContentHandler(baseHandler, metadata, parseContext);
xhtml.startDocument();
writeOutput(xhtml, responseText);
xhtml.endDocument();
}
// ---- Initializable ----------------------------------------------------
@Override
public void initialize() throws TikaConfigException {
String healthUrl = getHealthCheckUrl(defaultConfig);
if (healthUrl == null) {
// No health check configured (e.g. Claude) ��� assume available
serverAvailable = true;
return;
}
try {
httpClient().get(healthUrl, getHealthCheckHeaders(defaultConfig),
defaultConfig.getTimeoutMillis());
serverAvailable = true;
LOG.info("VLM server is available at {}", defaultConfig.getBaseUrl());
} catch (TikaException e) {
LOG.warn("VLM server returned error at {}: {}",
defaultConfig.getBaseUrl(), e.getMessage());
serverAvailable = false;
} catch (IOException e) {
LOG.warn("VLM server is not available at {}: {}",
defaultConfig.getBaseUrl(), e.getMessage());
serverAvailable = false;
}
}
@Override
public void close() throws IOException {
if (httpClient != null) {
httpClient.close();
}
}
// ---- shared helpers ---------------------------------------------------
protected VLMOCRConfig getConfig(ParseContext parseContext)
throws TikaConfigException, IOException {
String key = configKey();
if (parseContext.hasJsonConfig(key)) {
VLMOCRConfig config = ParseContextConfig.getConfig(
parseContext, key, VLMOCRConfig.class, defaultConfig);
if (!parseContext.getJsonConfig(key).trusted()) {
checkRuntimeOverrides(config);
}
return config;
}
VLMOCRConfig userConfig = parseContext.get(VLMOCRConfig.class);
if (userConfig != null) {
return userConfig;
}
return defaultConfig;
}
/**
* The per-request rules {@code @OperatorOnly} cannot express: maxTokens may be lowered
* but never raised (the ceiling bounds cost on a paid endpoint), and the prompt, with
* textRecognizer which describes it, changes only when the operator allowed that.
*/
private void checkRuntimeOverrides(VLMOCRConfig config) throws TikaConfigException {
if (config.getMaxTokens() > defaultConfig.getMaxTokens()) {
throw new TikaConfigException("Cannot increase maxTokens beyond the init-time value ("
+ defaultConfig.getMaxTokens() + ") at runtime. Requested: "
+ config.getMaxTokens());
}
if (defaultConfig.isAllowRuntimePrompt()) {
return;
}
if (!Objects.equals(config.getPrompt(), defaultConfig.getPrompt())) {
throw new TikaConfigException("Cannot modify prompt at runtime. Set "
+ "allowRuntimePrompt=true at initialization time to permit per-request "
+ "prompt overrides.");
}
if (config.isTextRecognizer() != defaultConfig.isTextRecognizer()) {
throw new TikaConfigException("Cannot modify textRecognizer at runtime. Set "
+ "allowRuntimePrompt=true at initialization time to permit per-request "
+ "prompt overrides.");
}
}
protected static String stripTrailingSlash(String url) {
return url.replaceAll("/+$", "");
}
String detectMimeType(Metadata metadata) {
String contentType = metadata.get(HttpHeaders.CONTENT_TYPE);
if (contentType != null) {
if (contentType.startsWith("image/") || contentType.equals("application/pdf")) {
return contentType;
}
}
return "image/png";
}
private ContentHandler getContentHandler(boolean isInlineContent, ContentHandler handler,
Metadata metadata, ParseContext parseContext) {
if (!isInlineContent) {
return handler;
}
ParentContentHandler parentContentHandler =
parseContext.get(ParentContentHandler.class);
if (parentContentHandler == null) {
return handler;
}
String embeddedType = metadata.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE);
if (!TikaCoreProperties.EmbeddedResourceType.INLINE.name().equals(embeddedType)) {
return handler;
}
return new TeeContentHandler(
new EmbeddedContentHandler(
new BodyContentHandler(
parentContentHandler.getContentHandler())),
handler);
}
private void writeOutput(XHTMLContentHandler xhtml, String text) throws SAXException {
AttributesImpl attrs = new AttributesImpl();
attrs.addAttribute("", "class", "class", "CDATA", "ocr");
xhtml.startElement(XHTML, "div", "div", attrs);
if (text != null && !text.isEmpty()) {
MarkdownToXHTMLEmitter.emit(text, xhtml);
}
xhtml.endElement(XHTML, "div", "div");
}
private byte[] readFully(InputStream is) throws IOException {
return is.readAllBytes();
}
private static void skip(ParseContext parseContext, Metadata metadata, String reason) {
metadata.set(VLM_SKIPPED_REASON, reason);
warn(metadata, "skipped:" + reason);
OcrWarnings.addToInlineParent(parseContext, metadata, "skipped:" + reason);
}
static void warn(Metadata metadata, String code) {
OcrWarnings.add(metadata, code);
}
private void checkOutput(ParseContext parseContext, Metadata metadata, String text) {
if (text == null || text.isBlank()) {
warn(metadata, WARN_EMPTY);
} else if (looksLikeLoop(text)) {
warn(metadata, WARN_REPETITION);
}
for (String code : metadata.getValues(TikaCoreProperties.OCR_WARNINGS)) {
OcrWarnings.addToInlineParent(parseContext, metadata, code);
}
}
/** Text that deflates this well is a repeated line or sentence, not prose. */
static boolean looksLikeLoop(String text) {
byte[] in = text.getBytes(java.nio.charset.StandardCharsets.UTF_8);
if (in.length < REPETITION_MIN_BYTES) {
return false;
}
Deflater deflater = new Deflater(Deflater.DEFAULT_COMPRESSION);
try {
deflater.setInput(in);
deflater.finish();
byte[] buf = new byte[8192];
long out = 0;
while (!deflater.finished()) {
out += deflater.deflate(buf);
}
return (double) out / in.length < REPETITION_MAX_DEFLATE_RATIO;
} finally {
deflater.end();
}
}
/**
* Reads only the image header to determine width × height
* without decoding the full raster. Returns null if dimensions
* cannot be determined.
*/
private long[] getImageSize(InputStream is) throws IOException {
try (javax.imageio.stream.ImageInputStream iis =
javax.imageio.ImageIO.createImageInputStream(is)) {
if (iis == null) {
return null;
}
java.util.Iterator<javax.imageio.ImageReader> readers =
javax.imageio.ImageIO.getImageReaders(iis);
if (!readers.hasNext()) {
return null;
}
javax.imageio.ImageReader reader = readers.next();
try {
reader.setInput(iis);
return new long[] {reader.getWidth(0), reader.getHeight(0)};
} finally {
reader.dispose();
}
}
}
private TikaHttpClient buildHttpClient() {
return TikaHttpClient.build(30, defaultConfig.getMaxRetries());
}
// ---- delegating config getters/setters --------------------------------
protected VLMOCRConfig getDefaultConfig() {
return defaultConfig;
}
public String getBaseUrl() {
return defaultConfig.getBaseUrl();
}
public void setBaseUrl(String baseUrl) throws TikaConfigException {
defaultConfig.setBaseUrl(baseUrl);
}
public String getModel() {
return defaultConfig.getModel();
}
public void setModel(String model) {
defaultConfig.setModel(model);
}
public String getPrompt() {
return defaultConfig.getPrompt();
}
public void setPrompt(String prompt) {
defaultConfig.setPrompt(prompt);
}
public int getMaxTokens() {
return defaultConfig.getMaxTokens();
}
public void setMaxTokens(int maxTokens) {
defaultConfig.setMaxTokens(maxTokens);
}
public Double getTemperature() {
return defaultConfig.getTemperature();
}
public void setTemperature(Double temperature) {
defaultConfig.setTemperature(temperature);
}
public long getTimeoutMillis() {
return defaultConfig.getTimeoutMillis();
}
public void setTimeoutMillis(long timeoutMillis) {
defaultConfig.setTimeoutMillis(timeoutMillis);
}
/** Retries of a 429/502/503/504 answer or a dropped connection; 0 fails at once. */
public synchronized void setMaxRetries(int maxRetries) {
defaultConfig.setMaxRetries(maxRetries);
if (httpClient != null) {
httpClient.close();
httpClient = null;
}
}
public String getApiKey() {
return defaultConfig.getApiKey();
}
public void setApiKey(String apiKey) throws TikaConfigException {
defaultConfig.setApiKey(apiKey);
}
public boolean isInlineContent() {
return defaultConfig.isInlineContent();
}
public void setInlineContent(boolean inlineContent) {
defaultConfig.setInlineContent(inlineContent);
}
public boolean isSkipOcr() {
return defaultConfig.isSkipOcr();
}
public void setSkipOcr(boolean skipOcr) {
defaultConfig.setSkipOcr(skipOcr);
}
public long getMinFileSizeToOcr() {
return defaultConfig.getMinFileSizeToOcr();
}
public void setMinFileSizeToOcr(long minFileSizeToOcr) {
defaultConfig.setMinFileSizeToOcr(minFileSizeToOcr);
}
public long getMaxFileSizeToOcr() {
return defaultConfig.getMaxFileSizeToOcr();
}
public void setMaxFileSizeToOcr(long maxFileSizeToOcr) {
defaultConfig.setMaxFileSizeToOcr(maxFileSizeToOcr);
}
public boolean isServerAvailable() {
return serverAvailable;
}
}