TikaHttpClientTest.java
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.tika.http;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.net.ConnectException;
import java.net.UnknownHostException;
import java.util.Map;
import javax.net.ssl.SSLHandshakeException;
import org.junit.jupiter.api.Test;
import org.apache.tika.config.ParseTimeout;
import org.apache.tika.config.TimeoutLimits;
import org.apache.tika.exception.TikaException;
import org.apache.tika.exception.TikaTimeoutException;
import org.apache.tika.http.TikaTestHttpServer.MockResponse;
import org.apache.tika.parser.ParseContext;
public class TikaHttpClientTest {
@Test
public void testHeartbeatFiresWhileWaitingForResponse() throws Exception {
try (TikaTestHttpServer server = new TikaTestHttpServer();
TikaHttpClient client = TikaHttpClient.build(30)) {
server.enqueue(new MockResponse(200, "{\"ok\":true}", 6000));
ParseContext context = new ParseContext();
context.set(TimeoutLimits.class, new TimeoutLimits(60_000, 60_000));
ParseTimeout parseTimeout = ParseTimeout.getOrCreate(context);
Thread requester = new Thread(() -> {
try {
client.get(server.url(), Map.of(), 20_000, context);
} catch (Exception e) {
// surfaced via the assertion below if it prevented progress
}
});
requester.start();
try {
// Wait a generous multiple of HEARTBEAT_INTERVAL_MILLIS (~1000ms), well short of
// the server's 6s delay, for slack against jitter under a loaded test run.
Thread.sleep(3000);
// without a mid-wait checkpoint, millisSinceLastProgress() would be >= 3000
assertTrue(parseTimeout.millisSinceLastProgress() < 3000,
"expected a checkpoint to have fired while the request was still in flight");
} finally {
requester.join(10_000);
}
}
}
@Test
public void testNullContextDoesNotThrow() throws Exception {
try (TikaTestHttpServer server = new TikaTestHttpServer();
TikaHttpClient client = TikaHttpClient.build(30)) {
server.enqueue(new MockResponse(200, "{\"ok\":true}"));
String body = client.get(server.url(), Map.of(), 5_000);
assertEquals("{\"ok\":true}", body);
}
}
/**
* A null context must grant the requested timeout unclipped --
* {@code ParseTimeout.getOrCreate(null)} used to clip it against a detached
* default-TimeoutLimits (1 hour) budget. Exercises {@code grantedMillis} directly.
*/
@Test
public void testNullContextGrantsRequestUnclippedEvenAboveDefaultOneHour() throws Exception {
try (TikaHttpClient client = TikaHttpClient.build(30)) {
long requestedTimeoutMillis = TimeoutLimits.DEFAULT_TOTAL_TASK_TIMEOUT_MILLIS + 60_000;
assertEquals(requestedTimeoutMillis, client.grantedMillis(requestedTimeoutMillis, null),
"a null context must not silently clip the request against a default-TimeoutLimits budget");
}
}
@Test
public void testRequestTimeoutStillBoundsTheWait() throws Exception {
try (TikaTestHttpServer server = new TikaTestHttpServer();
TikaHttpClient client = TikaHttpClient.build(30)) {
server.enqueue(new MockResponse(200, "{\"ok\":true}", 5000));
long start = System.currentTimeMillis();
boolean threw = false;
try {
client.get(server.url(), Map.of(), 1_000);
} catch (Exception e) {
threw = true;
}
long elapsed = System.currentTimeMillis() - start;
assertTrue(threw, "a 5s server delay with a 1s request timeout must fail");
assertTrue(elapsed < 4_000,
"the async polling rewrite must still honor the request timeout; took " + elapsed + "ms");
}
}
@Test
public void testSlowBodyAfterHeadersIsBoundedByClientDeadline() throws Exception {
// TIKA-4813 follow-up: headers arrive immediately, but the body then trickles in
// 5s later -- well past the task's 1s budget. HttpRequest.timeout() is not a
// guaranteed bound on the full exchange in every JDK/transport scenario once
// headers have already arrived; without waitWithHeartbeat enforcing its own
// deadline, a document with many such calls (or one truly stuck) could stall far
// past its configured budget instead of failing fast.
try (TikaTestHttpServer server = new TikaTestHttpServer();
TikaHttpClient client = TikaHttpClient.build(30)) {
server.enqueue(new MockResponse(200, "{\"ok\":true}", 0, 5000));
ParseContext context = new ParseContext();
context.set(TimeoutLimits.class, new TimeoutLimits(1000, 1000));
long start = System.currentTimeMillis();
Exception thrown = null;
try {
// Request 30s -- far more than the 1s task budget -- so only the task's
// own remaining-budget deadline (not the request's own timeout) is what
// can be bounding this call.
client.get(server.url(), Map.of(), 30_000, context);
} catch (Exception e) {
thrown = e;
}
long elapsed = System.currentTimeMillis() - start;
assertTrue(thrown != null, "a 5s-delayed body under a 1s task budget must fail");
assertInstanceOf(TikaTimeoutException.class, thrown,
"must be reported as a TikaTimeoutException with requested/granted info, not a bare " +
"IOException/HttpTimeoutException: " + thrown);
assertTrue(elapsed < 4_000,
"client deadline must fire near the ~1s task budget, not wait for the 5s body; took " +
elapsed + "ms");
}
}
@Test
public void testExhaustedBudgetFailsFastWithoutAttemptingRequest() throws Exception {
// TIKA-4813 follow-up: granted==0 must fail immediately (like ProcessUtils does
// for external processes), not be floored up to a 1-second HTTP call -- a
// document with many post-deadline calls would otherwise pay a full extra second
// per call instead of failing fast.
try (TikaTestHttpServer server = new TikaTestHttpServer();
TikaHttpClient client = TikaHttpClient.build(30)) {
server.enqueue(new MockResponse(200, "{\"ok\":true}"));
ParseContext context = new ParseContext();
// total=0 -> remainingMillis() is already 0 by the time anything checks it
context.set(TimeoutLimits.class, new TimeoutLimits(0, 10_000));
long start = System.currentTimeMillis();
TikaTimeoutException thrown = null;
try {
client.get(server.url(), Map.of(), 5_000, context);
} catch (TikaTimeoutException e) {
thrown = e;
}
long elapsed = System.currentTimeMillis() - start;
assertTrue(thrown != null, "an already-exhausted budget must throw TikaTimeoutException");
assertEquals(0, thrown.getGrantedMillis());
assertTrue(elapsed < 500,
"an exhausted budget must fail immediately, not floor up to a 1s HTTP call; took " +
elapsed + "ms");
assertEquals(0, server.getRequestCount(),
"no HTTP request should have been attempted at all with a 0ms granted budget");
}
}
/** A concurrency-capped engine answers 429 at once with no Retry-After: the retry succeeds. */
@Test
public void testRetriesA429ThenSucceeds() throws Exception {
try (TikaTestHttpServer server = new TikaTestHttpServer();
TikaHttpClient client = TikaHttpClient.build(30, 4)) {
server.enqueue(new MockResponse(429, "{\"detail\":\"Concurrency limit exceeded\"}"));
server.enqueue(new MockResponse(429, "{\"detail\":\"Concurrency limit exceeded\"}"));
server.enqueue(new MockResponse(200, "{\"ok\":true}"));
long start = System.currentTimeMillis();
String body = client.postJson(server.url(), "{}", Map.of(), 10_000, new ParseContext());
long elapsed = System.currentTimeMillis() - start;
assertEquals("{\"ok\":true}", body);
assertEquals(3, server.getRequestCount());
assertTrue(elapsed >= 187 + 375, "two backoffs, 250 and 500 ms less jitter: " + elapsed);
}
}
@Test
public void testNonRetryableStatusFailsAtOnce() throws Exception {
try (TikaTestHttpServer server = new TikaTestHttpServer();
TikaHttpClient client = TikaHttpClient.build(30, 4)) {
server.enqueue(new MockResponse(401, "{\"detail\":\"Invalid API key\"}"));
server.enqueue(new MockResponse(200, "{\"ok\":true}"));
TikaException e = assertThrows(TikaException.class,
() -> client.postJson(server.url(), "{}", Map.of(), 10_000, new ParseContext()));
assertTrue(e.getMessage().contains("HTTP 401"), e.getMessage());
assertEquals(1, server.getRequestCount());
}
}
@Test
public void testGivesUpAfterMaxRetriesWithTheLastAnswer() throws Exception {
try (TikaTestHttpServer server = new TikaTestHttpServer();
TikaHttpClient client = TikaHttpClient.build(30, 2)) {
for (int i = 0; i < 4; i++) {
server.enqueue(new MockResponse(503, "{\"detail\":\"busy " + i + "\"}"));
}
TikaException e = assertThrows(TikaException.class,
() -> client.postJson(server.url(), "{}", Map.of(), 10_000, new ParseContext()));
assertTrue(e.getMessage().contains("busy 2"), "the third answer is the one reported: " + e.getMessage());
assertEquals(3, server.getRequestCount(), "one attempt plus two retries");
}
}
@Test
public void testZeroRetriesFailsOnTheFirst429() throws Exception {
try (TikaTestHttpServer server = new TikaTestHttpServer();
TikaHttpClient client = TikaHttpClient.build(30, 0)) {
server.enqueue(new MockResponse(429, "{}"));
server.enqueue(new MockResponse(200, "{}"));
assertThrows(TikaException.class,
() -> client.postJson(server.url(), "{}", Map.of(), 10_000, new ParseContext()));
assertEquals(1, server.getRequestCount());
}
}
@Test
public void testRetryAfterHeaderSetsTheDelay() throws Exception {
try (TikaTestHttpServer server = new TikaTestHttpServer();
TikaHttpClient client = TikaHttpClient.build(30, 4)) {
server.enqueue(new MockResponse(429, "{}", Map.of("Retry-After", "1")));
server.enqueue(new MockResponse(200, "{\"ok\":true}"));
long start = System.currentTimeMillis();
client.postJson(server.url(), "{}", Map.of(), 10_000, new ParseContext());
long elapsed = System.currentTimeMillis() - start;
assertTrue(elapsed >= 1000, "waited the second the server asked for: " + elapsed);
assertEquals(2, server.getRequestCount());
}
}
/** A backoff that would run past the parse budget is not taken: the last answer fails the call. */
@Test
public void testBackoffNeverSleepsPastTheBudget() throws Exception {
try (TikaTestHttpServer server = new TikaTestHttpServer();
TikaHttpClient client = TikaHttpClient.build(30, 4)) {
server.enqueue(new MockResponse(429, "{}", Map.of("Retry-After", "5")));
server.enqueue(new MockResponse(200, "{}"));
long start = System.currentTimeMillis();
TikaException e = assertThrows(TikaException.class,
() -> client.postJson(server.url(), "{}", Map.of(), 800, new ParseContext()));
long elapsed = System.currentTimeMillis() - start;
assertTrue(e.getMessage().contains("HTTP 429"), e.getMessage());
assertTrue(elapsed < 800, "failed without waiting out the budget: " + elapsed);
assertEquals(1, server.getRequestCount());
}
}
/** A service that closes the connection without answering is retried, as a 503 is. */
@Test
public void testRetriesADroppedConnectionThenSucceeds() throws Exception {
try (TikaTestHttpServer server = new TikaTestHttpServer();
TikaHttpClient client = TikaHttpClient.build(30, 4)) {
server.enqueue(MockResponse.dropConnection());
server.enqueue(MockResponse.dropConnection());
server.enqueue(new MockResponse(200, "{\"ok\":true}"));
long start = System.currentTimeMillis();
String body = client.postJson(server.url(), "{}", Map.of(), 10_000, new ParseContext());
long elapsed = System.currentTimeMillis() - start;
assertEquals("{\"ok\":true}", body);
assertEquals(3, server.getRequestCount());
assertTrue(elapsed >= 187 + 375, "two backoffs, 250 and 500 ms less jitter: " + elapsed);
}
}
@Test
public void testGivesUpOnADroppedConnectionAfterMaxRetries() throws Exception {
try (TikaTestHttpServer server = new TikaTestHttpServer();
TikaHttpClient client = TikaHttpClient.build(30, 2)) {
for (int i = 0; i < 4; i++) {
server.enqueue(MockResponse.dropConnection());
}
assertThrows(IOException.class,
() -> client.postJson(server.url(), "{}", Map.of(), 10_000, new ParseContext()));
assertEquals(3, server.getRequestCount(), "one attempt plus two retries");
}
}
@Test
public void testZeroRetriesFailsOnTheFirstDroppedConnection() throws Exception {
try (TikaTestHttpServer server = new TikaTestHttpServer();
TikaHttpClient client = TikaHttpClient.build(30, 0)) {
server.enqueue(MockResponse.dropConnection());
server.enqueue(new MockResponse(200, "{}"));
assertThrows(IOException.class,
() -> client.postJson(server.url(), "{}", Map.of(), 10_000, new ParseContext()));
assertEquals(1, server.getRequestCount());
}
}
/**
* Backoffs of 250 ms doubling to 4 s cannot all fit in a 3 s budget: the call ends with the
* transport failure itself, not a timeout, before the budget is spent.
*/
@Test
public void testDroppedConnectionBackoffNeverSleepsPastTheBudget() throws Exception {
try (TikaTestHttpServer server = new TikaTestHttpServer();
TikaHttpClient client = TikaHttpClient.build(30, 10)) {
for (int i = 0; i < 12; i++) {
server.enqueue(MockResponse.dropConnection());
}
long start = System.currentTimeMillis();
assertThrows(IOException.class,
() -> client.postJson(server.url(), "{}", Map.of(), 3000, new ParseContext()));
long elapsed = System.currentTimeMillis() - start;
assertTrue(elapsed < 3000, "gave up before the budget ran out: " + elapsed);
assertTrue(server.getRequestCount() > 1, "it did retry: " + server.getRequestCount());
assertTrue(server.getRequestCount() < 11, "not every retry was taken: " + server.getRequestCount());
}
}
/** Waiting cannot bring back a host that is not there, a failed handshake, or an interrupted thread. */
@Test
public void testOnlyTransientTransportFailuresAreRetryable() {
assertTrue(TikaHttpClient.isRetryableTransportFailure(
new IOException("HTTP/1.1 header parser received no bytes")));
assertTrue(TikaHttpClient.isRetryableTransportFailure(
new IOException("chunked transfer encoding, state: READING_LENGTH")));
assertFalse(TikaHttpClient.isRetryableTransportFailure(new UnknownHostException("no.such.host")));
assertFalse(TikaHttpClient.isRetryableTransportFailure(new ConnectException("Connection refused")));
assertFalse(TikaHttpClient.isRetryableTransportFailure(new SSLHandshakeException("bad certificate")));
Thread.currentThread().interrupt();
try {
assertFalse(TikaHttpClient.isRetryableTransportFailure(new IOException("HTTP request interrupted")));
} finally {
Thread.interrupted();
}
}
@Test
public void testRetryAfterParsing() {
assertEquals(7000, TikaHttpClient.parseRetryAfterMillis("7"));
assertEquals(-1, TikaHttpClient.parseRetryAfterMillis("soon"));
assertEquals(0, TikaHttpClient.parseRetryAfterMillis("Wed, 21 Oct 2015 07:28:00 GMT"),
"a date in the past is now");
for (int attempt = 0; attempt < 6; attempt++) {
long b = TikaHttpClient.backoffMillis(attempt);
long base = Math.min(4000, 250L << attempt);
assertTrue(b >= base - base / 4 && b <= base + base / 4, attempt + ": " + b);
}
}
}