TestConfiguration.java
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.conf;
import java.io.BufferedInputStream;
import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.io.StringWriter;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.ConcurrentModificationException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Pattern;
import static java.util.concurrent.TimeUnit.*;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.apache.hadoop.conf.StorageUnit.BYTES;
import static org.apache.hadoop.conf.StorageUnit.GB;
import static org.apache.hadoop.conf.StorageUnit.KB;
import static org.apache.hadoop.conf.StorageUnit.MB;
import static org.apache.hadoop.conf.StorageUnit.TB;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration.IntegerRanges;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.LocalJavaKeyStoreProvider;
import org.apache.hadoop.test.GenericTestUtils;
import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
import org.apache.log4j.AppenderSkeleton;
import org.apache.log4j.Logger;
import org.apache.log4j.spi.LoggingEvent;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.Mockito;
public class TestConfiguration {
private static final double DOUBLE_DELTA = 0.000000001f;
private Configuration conf;
final static String CONFIG = new File("./test-config-TestConfiguration.xml").getAbsolutePath();
final static String CONFIG2 = new File("./test-config2-TestConfiguration.xml").getAbsolutePath();
final static String CONFIG_CORE = new File("./core-site.xml")
.getAbsolutePath();
final static String CONFIG_FOR_ENUM = new File("./test-config-enum-TestConfiguration.xml").getAbsolutePath();
final static String CONFIG_FOR_URI = new File(
"./test-config-uri-TestConfiguration.xml").toURI().toString();
private static final String CONFIG_MULTI_BYTE = new File(
"./test-config-multi-byte-TestConfiguration.xml").getAbsolutePath();
private static final String CONFIG_MULTI_BYTE_SAVED = new File(
"./test-config-multi-byte-saved-TestConfiguration.xml").getAbsolutePath();
final static Random RAN = new Random();
final static String XMLHEADER =
IBM_JAVA?"<?xml version=\"1.0\" encoding=\"UTF-8\"?><configuration>":
"<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?><configuration>";
/** Four apostrophes. */
public static final String ESCAPED = "''''";
private static final String SENSITIVE_CONFIG_KEYS =
CommonConfigurationKeysPublic.HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS;
private BufferedWriter out;
@BeforeEach
public void setUp() throws Exception {
conf = new Configuration(false);
}
@AfterEach
public void tearDown() throws Exception {
if(out != null) {
out.close();
}
new File(CONFIG).delete();
new File(CONFIG2).delete();
new File(CONFIG_FOR_ENUM).delete();
new File(new URI(CONFIG_FOR_URI)).delete();
new File(CONFIG_MULTI_BYTE).delete();
new File(CONFIG_MULTI_BYTE_SAVED).delete();
new File(CONFIG_CORE).delete();
}
private void startConfig() throws IOException{
out.write("<?xml version=\"1.0\"?>\n");
out.write("<configuration>\n");
}
private void writeHeader() throws IOException{
out.write("<?xml version=\"1.0\"?>\n");
}
private void writeHeader(String encoding) throws IOException{
out.write("<?xml version=\"1.0\" encoding=\"" + encoding + "\"?>\n");
}
private void writeConfiguration() throws IOException{
out.write("<configuration>\n");
}
private void endConfig() throws IOException{
out.write("</configuration>\n");
out.flush();
out.close();
}
private void startInclude(String filename) throws IOException {
out.write("<xi:include href=\"" + filename + "\" xmlns:xi=\"http://www.w3.org/2001/XInclude\" >\n ");
}
private void endInclude() throws IOException{
out.write("</xi:include>\n ");
}
private void startFallback() throws IOException {
out.write("<xi:fallback>\n ");
}
private void endFallback() throws IOException {
out.write("</xi:fallback>\n ");
}
private void declareEntity(String root, String entity, String value)
throws IOException {
out.write("<!DOCTYPE " + root
+ " [\n<!ENTITY " + entity + " \"" + value + "\">\n]>");
}
private void declareSystemEntity(String root, String entity, String value)
throws IOException {
out.write("<!DOCTYPE " + root
+ " [\n<!ENTITY " + entity + " SYSTEM \"" + value + "\">\n]>");
}
@Test
public void testInputStreamResource() throws Exception {
StringWriter writer = new StringWriter();
out = new BufferedWriter(writer);
startConfig();
declareProperty("prop", "A", "A");
endConfig();
InputStream in1 = Mockito.spy(new ByteArrayInputStream(
writer.toString().getBytes()));
Configuration conf = new Configuration(false);
conf.addResource(in1);
assertEquals("A", conf.get("prop"));
Mockito.verify(in1, Mockito.times(1)).close();
InputStream in2 = new ByteArrayInputStream(writer.toString().getBytes());
conf.addResource(in2);
assertEquals("A", conf.get("prop"));
}
@Test
public void testFinalWarnings() throws Exception {
// Make a configuration file with a final property
StringWriter writer = new StringWriter();
out = new BufferedWriter(writer);
startConfig();
declareProperty("prop", "A", "A", true);
endConfig();
byte[] bytes = writer.toString().getBytes();
InputStream in1 = new ByteArrayInputStream(bytes);
// Make a second config file with a final property with a different value
writer = new StringWriter();
out = new BufferedWriter(writer);
startConfig();
declareProperty("prop", "BB", "BB", true);
endConfig();
byte[] bytes2 = writer.toString().getBytes();
InputStream in2 = new ByteArrayInputStream(bytes2);
// Attach our own log appender so we can verify output
TestAppender appender = new TestAppender();
final Logger logger = Logger.getRootLogger();
logger.addAppender(appender);
try {
// Add the 2 different resources - this should generate a warning
conf.addResource(in1);
conf.addResource(in2);
assertEquals("A", conf.get("prop"), "should see the first value");
List<LoggingEvent> events = appender.getLog();
assertEquals(1, events.size(),
"overriding a final parameter should cause logging");
LoggingEvent loggingEvent = events.get(0);
String renderedMessage = loggingEvent.getRenderedMessage();
assertTrue(renderedMessage.contains(
"an attempt to override final parameter: prop; Ignoring."),
"did not see expected string inside message "+ renderedMessage);
} finally {
// Make sure the appender is removed
logger.removeAppender(appender);
}
}
@Test
public void testNoFinalWarnings() throws Exception {
// Make a configuration file with a final property
StringWriter writer = new StringWriter();
out = new BufferedWriter(writer);
startConfig();
declareProperty("prop", "A", "A", true);
endConfig();
byte[] bytes = writer.toString().getBytes();
// The 2 input streams both have the same config file
InputStream in1 = new ByteArrayInputStream(bytes);
InputStream in2 = new ByteArrayInputStream(bytes);
// Attach our own log appender so we can verify output
TestAppender appender = new TestAppender();
final Logger logger = Logger.getRootLogger();
logger.addAppender(appender);
try {
// Add the resource twice from a stream - should not generate warnings
conf.addResource(in1);
conf.addResource(in2);
assertEquals("A", conf.get("prop"));
List<LoggingEvent> events = appender.getLog();
for (LoggingEvent loggingEvent : events) {
System.out.println("Event = " + loggingEvent.getRenderedMessage());
}
assertTrue(events.isEmpty(), "adding same resource twice should not cause logging");
} finally {
// Make sure the appender is removed
logger.removeAppender(appender);
}
}
@Test
public void testFinalWarningsMultiple() throws Exception {
// Make a configuration file with a repeated final property
StringWriter writer = new StringWriter();
out = new BufferedWriter(writer);
startConfig();
declareProperty("prop", "A", "A", true);
declareProperty("prop", "A", "A", true);
endConfig();
byte[] bytes = writer.toString().getBytes();
InputStream in1 = new ByteArrayInputStream(bytes);
// Attach our own log appender so we can verify output
TestAppender appender = new TestAppender();
final Logger logger = Logger.getRootLogger();
logger.addAppender(appender);
try {
// Add the resource - this should not produce a warning
conf.addResource(in1);
assertEquals("A", conf.get("prop"), "should see the value");
List<LoggingEvent> events = appender.getLog();
for (LoggingEvent loggingEvent : events) {
System.out.println("Event = " + loggingEvent.getRenderedMessage());
}
assertTrue(events.isEmpty(), "adding same resource twice should not cause logging");
} finally {
// Make sure the appender is removed
logger.removeAppender(appender);
}
}
@Test
public void testFinalWarningsMultipleOverride() throws Exception {
// Make a configuration file with 2 final properties with different values
StringWriter writer = new StringWriter();
out = new BufferedWriter(writer);
startConfig();
declareProperty("prop", "A", "A", true);
declareProperty("prop", "BB", "BB", true);
endConfig();
byte[] bytes = writer.toString().getBytes();
InputStream in1 = new ByteArrayInputStream(bytes);
// Attach our own log appender so we can verify output
TestAppender appender = new TestAppender();
final Logger logger = Logger.getRootLogger();
logger.addAppender(appender);
try {
// Add the resource - this should produce a warning
conf.addResource(in1);
assertEquals("A", conf.get("prop"), "should see the value");
List<LoggingEvent> events = appender.getLog();
assertEquals(1, events.size(), "overriding a final parameter should cause logging");
LoggingEvent loggingEvent = events.get(0);
String renderedMessage = loggingEvent.getRenderedMessage();
assertTrue(renderedMessage.contains("an attempt to override final parameter: " +
"prop; Ignoring."), "did not see expected string inside message "+ renderedMessage);
} finally {
// Make sure the appender is removed
logger.removeAppender(appender);
}
}
@Test
public void testDeprecatedPropertyInXMLFileGeneratesLogMessage(@TempDir java.nio.file.Path tmp) throws IOException {
String oldProp = "test.deprecation.old.conf.a";
String newProp = "test.deprecation.new.conf.a";
Configuration.addDeprecation(oldProp, newProp);
java.nio.file.Path confFile = Files.createFile(tmp.resolve("TestConfiguration.xml"));
String confXml = "<configuration><property><name>" + oldProp + "</name><value>a</value></property></configuration>";
Files.write(confFile, confXml.getBytes());
TestAppender appender = new TestAppender();
Logger deprecationLogger = Logger.getLogger("org.apache.hadoop.conf.Configuration.deprecation");
deprecationLogger.addAppender(appender);
try {
conf.addResource(new Path(confFile.toUri()));
// Properties are lazily initialized so access them to trigger the loading of the resource
conf.getProps();
} finally {
deprecationLogger.removeAppender(appender);
}
Pattern deprecationMsgPattern = Pattern.compile(oldProp + " in file:" + confFile + " is deprecated");
boolean hasDeprecationMessage = appender.log.stream().map(LoggingEvent::getRenderedMessage)
.anyMatch(msg -> deprecationMsgPattern.matcher(msg).find());
assertTrue(hasDeprecationMessage);
}
@Test
public void testDeprecatedPropertyLogsWarningOnEveryUse(){
String oldProp = "test.deprecation.old.conf.b";
String newProp = "test.deprecation.new.conf.b";
Configuration.addDeprecation(oldProp, newProp);
TestAppender appender = new TestAppender();
Logger deprecationLogger = Logger.getLogger("org.apache.hadoop.conf.Configuration.deprecation");
deprecationLogger.addAppender(appender);
try {
conf.set(oldProp, "b1");
conf.get(oldProp);
conf.set(oldProp, "b2");
conf.get(oldProp);
// Using the new property should not log a warning
conf.set(newProp, "b3");
conf.get(newProp);
conf.set(newProp, "b4");
conf.get(newProp);
} finally {
deprecationLogger.removeAppender(appender);
}
Pattern deprecationMsgPattern = Pattern.compile(oldProp + " is deprecated");
long count = appender.log.stream().map(LoggingEvent::getRenderedMessage)
.filter(msg -> deprecationMsgPattern.matcher(msg).find()).count();
assertEquals(4, count, "Expected exactly four warnings for deprecated property usage");
}
/**
* A simple appender for white box testing.
*/
private static class TestAppender extends AppenderSkeleton {
private final List<LoggingEvent> log = new ArrayList<>();
@Override public boolean requiresLayout() {
return false;
}
@Override protected void append(final LoggingEvent loggingEvent) {
log.add(loggingEvent);
}
@Override public void close() {
}
public List<LoggingEvent> getLog() {
return new ArrayList<>(log);
}
}
/**
* Tests use of multi-byte characters in property names and values. This test
* round-trips multi-byte string literals through saving and loading of config
* and asserts that the same values were read.
*/
@Test
public void testMultiByteCharacters() throws IOException {
String priorDefaultEncoding = System.getProperty("file.encoding");
try {
System.setProperty("file.encoding", "US-ASCII");
String name = "multi_byte_\u611b_name";
String value = "multi_byte_\u0641_value";
out = new BufferedWriter(new OutputStreamWriter(
new FileOutputStream(CONFIG_MULTI_BYTE), StandardCharsets.UTF_8));
startConfig();
declareProperty(name, value, value);
endConfig();
Configuration conf = new Configuration(false);
conf.addResource(new Path(CONFIG_MULTI_BYTE));
assertEquals(value, conf.get(name));
try (FileOutputStream fos =
new FileOutputStream(CONFIG_MULTI_BYTE_SAVED)) {
conf.writeXml(fos);
}
conf = new Configuration(false);
conf.addResource(new Path(CONFIG_MULTI_BYTE_SAVED));
assertEquals(value, conf.get(name));
} finally {
System.setProperty("file.encoding", priorDefaultEncoding);
}
}
@Test
public void testVariableSubstitution() throws IOException {
// stubbing only environment dependent functions
Configuration mock = Mockito.spy(conf);
Mockito.when(mock.getProperty("user.name")).thenReturn("hadoop_user");
Mockito.when(mock.getenv("FILE_NAME")).thenReturn("hello");
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
declareProperty("my.int", "${intvar}", "42");
declareProperty("intvar", "42", "42");
declareProperty("my.base", "/tmp/${user.name}", "/tmp/hadoop_user");
declareProperty("my.file", "${env.FILE_NAME}", "hello");
declareProperty("my.suffix", ".txt", ".txt");
declareProperty("my.relfile", "${my.file}${my.suffix}", "hello.txt");
declareProperty("my.fullfile", "${my.base}/${my.file}${my.suffix}", "/tmp/hadoop_user/hello.txt");
// check that undefined variables are returned as-is
declareProperty("my.failsexpand", "a${my.undefvar}b", "a${my.undefvar}b");
// check that multiple variable references are resolved
declareProperty("my.user.group", "${user.name} ${user.name}",
"hadoop_user hadoop_user");
endConfig();
Path fileResource = new Path(CONFIG);
mock.addResource(fileResource);
for (Prop p : props) {
System.out.println("p=" + p.name);
String gotVal = mock.get(p.name);
String gotRawVal = mock.getRaw(p.name);
assertEq(p.val, gotRawVal);
assertEq(p.expectEval, gotVal);
}
// check that expansion also occurs for getInt()
assertTrue(mock.getInt("intvar", -1) == 42);
assertTrue(mock.getInt("my.int", -1) == 42);
}
/**
* Checks if variable substitution is accessible via a public API.
*/
@Test
public void testCommonVariableSubstitution() {
conf.set("intvar", String.valueOf(42));
String intVar = conf.substituteCommonVariables("${intvar}");
assertEquals("42", intVar);
}
@Test
public void testEnvDefault() throws IOException {
Configuration mock = Mockito.spy(conf);
Mockito.when(mock.getenv("NULL_VALUE")).thenReturn(null);
Mockito.when(mock.getenv("EMPTY_VALUE")).thenReturn("");
Mockito.when(mock.getenv("SOME_VALUE")).thenReturn("some value");
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
// if var is unbound, literal ${var} is returned
declareProperty("null1", "${env.NULL_VALUE}", "${env.NULL_VALUE}");
declareProperty("null2", "${env.NULL_VALUE-a}", "a");
declareProperty("null3", "${env.NULL_VALUE:-b}", "b");
declareProperty("empty1", "${env.EMPTY_VALUE}", "");
declareProperty("empty2", "${env.EMPTY_VALUE-c}", "");
declareProperty("empty3", "${env.EMPTY_VALUE:-d}", "d");
declareProperty("some1", "${env.SOME_VALUE}", "some value");
declareProperty("some2", "${env.SOME_VALUE-e}", "some value");
declareProperty("some3", "${env.SOME_VALUE:-f}", "some value");
// some edge cases
declareProperty("edge1", "${env.NULL_VALUE-g-h}", "g-h");
declareProperty("edge2", "${env.NULL_VALUE:-i:-j}", "i:-j");
declareProperty("edge3", "${env.NULL_VALUE-}", "");
declareProperty("edge4", "${env.NULL_VALUE:-}", "");
declareProperty("edge5", "${env.NULL_VALUE:}", "${env.NULL_VALUE:}");
endConfig();
Path fileResource = new Path(CONFIG);
mock.addResource(fileResource);
for (Prop p : props) {
System.out.println("p=" + p.name);
String gotVal = mock.get(p.name);
String gotRawVal = mock.getRaw(p.name);
assertEq(p.val, gotRawVal);
assertEq(p.expectEval, gotVal);
}
}
/**
* Verify that when a configuration is restricted, environment
* variables and system properties will be unresolved.
* The fallback patterns for the variables are still parsed.
*/
@Test
public void testRestrictedEnv() throws IOException {
// this test relies on env.PATH being set on all platforms a
// test run will take place on, and the java.version sysprop
// set in all JVMs.
// Restricted configurations will not get access to these values, so
// will either be unresolved or, for env vars with fallbacks: the fallback
// values.
conf.setRestrictSystemProperties(true);
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
// a simple property to reference
declareProperty("d", "D", "D");
// system property evaluation stops working completely
declareProperty("system1", "${java.version}", "${java.version}");
// the env variable does not resolve
declareProperty("secret1", "${env.PATH}", "${env.PATH}");
// but all the fallback options do work
declareProperty("secret2", "${env.PATH-a}", "a");
declareProperty("secret3", "${env.PATH:-b}", "b");
declareProperty("secret4", "${env.PATH:-}", "");
declareProperty("secret5", "${env.PATH-}", "");
// special case
declareProperty("secret6", "${env.PATH:}", "${env.PATH:}");
// safety check
declareProperty("secret7", "${env.PATH:--}", "-");
// recursive eval of the fallback
declareProperty("secret8", "${env.PATH:-${d}}", "D");
// if the fallback doesn't resolve, the result is the whole variable raw.
declareProperty("secret9", "${env.PATH:-$d}}", "${env.PATH:-$d}}");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
for (Prop p : props) {
System.out.println("p=" + p.name);
String gotVal = conf.get(p.name);
String gotRawVal = conf.getRaw(p.name);
assertEq(p.val, gotRawVal);
assertEq(p.expectEval, gotVal);
}
}
@Test
public void testFinalParam() throws IOException {
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
declareProperty("my.var", "", "", true);
endConfig();
Path fileResource = new Path(CONFIG);
Configuration conf1 = new Configuration();
conf1.addResource(fileResource);
assertNull(conf1.get("my.var"), "my var is not null");
out=new BufferedWriter(new FileWriter(CONFIG2));
startConfig();
declareProperty("my.var", "myval", "myval", false);
endConfig();
fileResource = new Path(CONFIG2);
Configuration conf2 = new Configuration(conf1);
conf2.addResource(fileResource);
assertNull(conf2.get("my.var"), "my var is not final");
}
@Test
public void testCompactFormat() throws IOException {
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendCompactFormatProperty("a", "b");
appendCompactFormatProperty("c", "d", true);
appendCompactFormatProperty("e", "f", false, "g");
endConfig();
Path fileResource = new Path(CONFIG);
Configuration conf = new Configuration(false);
conf.addResource(fileResource);
assertEquals("b", conf.get("a"));
assertEquals("d", conf.get("c"));
Set<String> s = conf.getFinalParameters();
assertEquals(1, s.size());
assertTrue(s.contains("c"));
assertEquals("f", conf.get("e"));
String[] sources = conf.getPropertySources("e");
assertEquals(2, sources.length);
assertEquals("g", sources[0]);
assertEquals(fileResource.toString(), sources[1]);
}
public static void assertEq(Object a, Object b) {
System.out.println("assertEq: " + a + ", " + b);
assertEquals(a, b);
}
static class Prop {
String name;
String val;
String expectEval;
}
ArrayList<Prop> props = new ArrayList<Prop>();
void declareProperty(String name, String val, String expectEval)
throws IOException {
declareProperty(name, val, expectEval, false);
}
void declareProperty(String name, String val, String expectEval,
boolean isFinal)
throws IOException {
appendProperty(name, val, isFinal);
Prop p = new Prop();
p.name = name;
p.val = val;
p.expectEval = expectEval;
props.add(p);
}
void appendProperty(String name, String val) throws IOException {
appendProperty(name, val, false);
}
void appendProperty(String name, String val, boolean isFinal,
String ... sources)
throws IOException {
out.write("<property>");
out.write("<name>");
out.write(name);
out.write("</name>");
out.write("<value>");
out.write(val);
out.write("</value>");
if (isFinal) {
out.write("<final>true</final>");
}
for(String s : sources) {
out.write("<source>");
out.write(s);
out.write("</source>");
}
out.write("</property>\n");
}
private void appendPropertyByTag(String name, String val, String tags,
String... sources) throws IOException {
appendPropertyByTag(name, val, false, tags, sources);
}
private void appendPropertyByTag(String name, String val, boolean isFinal,
String tag, String... sources) throws IOException {
out.write("<property>");
out.write("<name>");
out.write(name);
out.write("</name>");
out.write("<value>");
out.write(val);
out.write("</value>");
if (isFinal) {
out.write("<final>true</final>");
}
for (String s : sources) {
out.write("<source>");
out.write(s);
out.write("</source>");
}
out.write("<tag>");
out.write(tag);
out.write("</tag>");
out.write("</property>\n");
}
void appendCompactFormatProperty(String name, String val) throws IOException {
appendCompactFormatProperty(name, val, false);
}
void appendCompactFormatProperty(String name, String val, boolean isFinal)
throws IOException {
appendCompactFormatProperty(name, val, isFinal, null);
}
void appendCompactFormatProperty(String name, String val, boolean isFinal,
String source)
throws IOException {
out.write("<property ");
out.write("name=\"");
out.write(name);
out.write("\" ");
out.write("value=\"");
out.write(val);
out.write("\" ");
if (isFinal) {
out.write("final=\"true\" ");
}
if (source != null) {
out.write("source=\"");
out.write(source);
out.write("\" ");
}
out.write("/>\n");
}
@Test
public void testOverlay() throws IOException{
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("a","b");
appendProperty("b","c");
appendProperty("d","e");
appendProperty("e","f", true);
endConfig();
out=new BufferedWriter(new FileWriter(CONFIG2));
startConfig();
appendProperty("a","b");
appendProperty("b","d");
appendProperty("e","e");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
//set dynamically something
conf.set("c","d");
conf.set("a","d");
Configuration clone=new Configuration(conf);
clone.addResource(new Path(CONFIG2));
assertEquals(clone.get("a"), "d");
assertEquals(clone.get("b"), "d");
assertEquals(clone.get("c"), "d");
assertEquals(clone.get("d"), "e");
assertEquals(clone.get("e"), "f");
}
@Test
public void testCommentsInValue() throws IOException {
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("my.comment", "this <!--comment here--> contains a comment");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
//two spaces one after "this", one before "contains"
assertEquals("this contains a comment", conf.get("my.comment"));
}
@Test
public void testEscapedCharactersInValue() throws IOException {
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("my.comment", ESCAPED);
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
//two spaces one after "this", one before "contains"
assertEquals("''''", conf.get("my.comment"));
}
@Test
public void testTrim() throws IOException {
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
String[] whitespaces = {"", " ", "\n", "\t"};
String[] name = new String[100];
for(int i = 0; i < name.length; i++) {
name[i] = "foo" + i;
StringBuilder prefix = new StringBuilder();
StringBuilder postfix = new StringBuilder();
for(int j = 0; j < 3; j++) {
prefix.append(whitespaces[RAN.nextInt(whitespaces.length)]);
postfix.append(whitespaces[RAN.nextInt(whitespaces.length)]);
}
appendProperty(prefix + name[i] + postfix, name[i] + ".value");
}
endConfig();
conf.addResource(new Path(CONFIG));
for(String n : name) {
assertEquals(n + ".value", conf.get(n));
}
}
@Test
public void testGetLocalPath() throws IOException {
Configuration conf = new Configuration();
String[] dirs = new String[]{"a", "b", "c"};
for (int i = 0; i < dirs.length; i++) {
dirs[i] = new Path(GenericTestUtils.getTempPath(dirs[i])).toString();
}
conf.set("dirs", StringUtils.join(dirs, ","));
for (int i = 0; i < 1000; i++) {
String localPath = conf.getLocalPath("dirs", "dir" + i).toString();
assertTrue(localPath.endsWith("dir" + i),
"Path doesn't end in specified dir: " + localPath);
assertFalse(localPath.contains(" "),
"Path has internal whitespace: " + localPath);
}
}
@Test
public void testGetFile() throws IOException {
Configuration conf = new Configuration();
String[] dirs = new String[]{"a", "b", "c"};
for (int i = 0; i < dirs.length; i++) {
dirs[i] = new Path(GenericTestUtils.getTempPath(dirs[i])).toString();
}
conf.set("dirs", StringUtils.join(dirs, ","));
for (int i = 0; i < 1000; i++) {
String localPath = conf.getFile("dirs", "dir" + i).toString();
assertTrue(localPath.endsWith("dir" + i),
"Path doesn't end in specified dir: " + localPath);
assertFalse(localPath.contains(" "),
"Path has internal whitespace: " + localPath);
}
}
@Test
public void testToString() throws IOException {
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
String expectedOutput =
"Configuration: " + fileResource;
assertEquals(expectedOutput, conf.toString());
}
@Test
public void testWriteXml() throws IOException {
Configuration conf = new Configuration();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
conf.writeXml(baos);
String result = baos.toString();
assertTrue(result.startsWith(XMLHEADER), "Result has proper header");
assertTrue(result.endsWith("</configuration>"), "Result has proper footer");
}
@Test
public void testIncludes() throws Exception {
tearDown();
System.out.println("XXX testIncludes");
out=new BufferedWriter(new FileWriter(CONFIG2));
startConfig();
appendProperty("a","b");
appendProperty("c","d");
endConfig();
File fileUri = new File(new URI(CONFIG_FOR_URI));
out=new BufferedWriter(new FileWriter(fileUri));
startConfig();
appendProperty("e", "f");
appendProperty("g", "h");
endConfig();
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
startInclude(CONFIG2);
endInclude();
startInclude(CONFIG_FOR_URI);
endInclude();
appendProperty("i", "j");
appendProperty("k", "l");
endConfig();
// verify that the includes file contains all properties
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
assertEquals(conf.get("a"), "b");
assertEquals(conf.get("c"), "d");
assertEquals(conf.get("e"), "f");
assertEquals(conf.get("g"), "h");
assertEquals(conf.get("i"), "j");
assertEquals(conf.get("k"), "l");
tearDown();
}
@Test
public void testCharsetInDocumentEncoding() throws Exception {
tearDown();
out=new BufferedWriter(new OutputStreamWriter(new FileOutputStream(CONFIG),
StandardCharsets.ISO_8859_1));
writeHeader(StandardCharsets.ISO_8859_1.displayName());
writeConfiguration();
appendProperty("a", "b");
appendProperty("c", "M��ller");
endConfig();
// verify that the includes file contains all properties
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
assertEquals(conf.get("a"), "b");
assertEquals(conf.get("c"), "M��ller");
tearDown();
}
@Test
public void testEntityReference() throws Exception {
tearDown();
out=new BufferedWriter(new FileWriter(CONFIG));
writeHeader();
declareEntity("configuration", "d", "d");
writeConfiguration();
appendProperty("a", "b");
appendProperty("c", "&d;");
endConfig();
// verify that the includes file contains all properties
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
assertEquals(conf.get("a"), "b");
assertEquals(conf.get("c"), "d");
tearDown();
}
@Test
public void testSystemEntityReference() throws Exception {
tearDown();
out=new BufferedWriter(new FileWriter(CONFIG2));
out.write("d");
out.close();
out=new BufferedWriter(new FileWriter(CONFIG));
writeHeader();
declareSystemEntity("configuration", "d",
new Path(CONFIG2).toUri().toString());
writeConfiguration();
appendProperty("a", "b");
appendProperty("c", "&d;");
endConfig();
// verify that the includes file contains all properties
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
assertEquals(conf.get("a"), "b");
assertEquals(conf.get("c"), "d");
tearDown();
}
@Test
public void testIncludesWithFallback() throws Exception {
tearDown();
out=new BufferedWriter(new FileWriter(CONFIG2));
startConfig();
appendProperty("a","b");
appendProperty("c","d");
endConfig();
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
startInclude(CONFIG2);
startFallback();
appendProperty("a", "b.fallback");
appendProperty("c", "d.fallback", true);
endFallback();
endInclude();
appendProperty("e","f");
appendProperty("g","h");
startInclude("MissingConfig.xml");
startFallback();
appendProperty("i", "j.fallback");
appendProperty("k", "l.fallback", true);
endFallback();
endInclude();
endConfig();
// verify that the includes file contains all properties
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
assertEquals("b", conf.get("a"));
assertEquals("d", conf.get("c"));
assertEquals("f", conf.get("e"));
assertEquals("h", conf.get("g"));
assertEquals("j.fallback", conf.get("i"));
assertEquals("l.fallback", conf.get("k"));
tearDown();
}
// When a resource is parsed as an input stream the first time, included
// properties are saved within the config. However, the included properties
// are not cached in the resource object. So, if an additional resource is
// added after the config is parsed the first time, the config loses the
// prperties that were included from the first resource.
@Test
public void testIncludesFromInputStreamWhenResourceAdded() throws Exception {
tearDown();
// CONFIG includes CONFIG2. CONFIG2 includes CONFIG_FOR_ENUM
out=new BufferedWriter(new FileWriter(CONFIG_FOR_ENUM));
startConfig();
appendProperty("e", "SecondLevelInclude");
appendProperty("f", "SecondLevelInclude");
endConfig();
out=new BufferedWriter(new FileWriter(CONFIG2));
startConfig();
startInclude(CONFIG_FOR_ENUM);
endInclude();
appendProperty("c","FirstLevelInclude");
appendProperty("d","FirstLevelInclude");
endConfig();
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
startInclude(CONFIG2);
endInclude();
appendProperty("a", "1");
appendProperty("b", "2");
endConfig();
// Add CONFIG as an InputStream resource.
File file = new File(CONFIG);
BufferedInputStream bis =
new BufferedInputStream(new FileInputStream(file));
conf.addResource(bis);
// The first time the conf is parsed, verify that all properties were read
// from all levels of includes.
assertEquals("1", conf.get("a"));
assertEquals("2", conf.get("b"));
assertEquals("FirstLevelInclude", conf.get("c"));
assertEquals("FirstLevelInclude", conf.get("d"));
assertEquals("SecondLevelInclude", conf.get("e"));
assertEquals("SecondLevelInclude", conf.get("f"));
// Add another resource to the conf.
out=new BufferedWriter(new FileWriter(CONFIG_MULTI_BYTE));
startConfig();
appendProperty("g", "3");
appendProperty("h", "4");
endConfig();
Path fileResource = new Path(CONFIG_MULTI_BYTE);
conf.addResource(fileResource);
// Verify that all properties were read from all levels of includes the
// second time the conf is parsed.
assertEquals("1", conf.get("a"));
assertEquals("2", conf.get("b"));
assertEquals("FirstLevelInclude", conf.get("c"));
assertEquals("FirstLevelInclude", conf.get("d"));
assertEquals("SecondLevelInclude", conf.get("e"));
assertEquals("SecondLevelInclude", conf.get("f"));
assertEquals("3", conf.get("g"));
assertEquals("4", conf.get("h"));
tearDown();
}
@Test
public void testOrderOfDuplicatePropertiesWithInclude() throws Exception {
tearDown();
// Property "a" is set to different values inside and outside of includes.
out=new BufferedWriter(new FileWriter(CONFIG2));
startConfig();
appendProperty("a", "a-InsideInclude");
appendProperty("b", "b-InsideInclude");
endConfig();
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("a","a-OutsideInclude");
startInclude(CONFIG2);
endInclude();
appendProperty("b","b-OutsideInclude");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
assertEquals("a-InsideInclude", conf.get("a"));
assertEquals("b-OutsideInclude", conf.get("b"));
tearDown();
}
@Test
public void testRelativeIncludes() throws Exception {
tearDown();
String relConfig = new File("./tmp/test-config.xml").getAbsolutePath();
String relConfig2 = new File("./tmp/test-config2.xml").getAbsolutePath();
new File(new File(relConfig).getParent()).mkdirs();
out = new BufferedWriter(new FileWriter(relConfig2));
startConfig();
appendProperty("a", "b");
endConfig();
out = new BufferedWriter(new FileWriter(relConfig));
startConfig();
// Add the relative path instead of the absolute one.
startInclude(new File(relConfig2).getName());
endInclude();
appendProperty("c", "d");
endConfig();
// verify that the includes file contains all properties
Path fileResource = new Path(relConfig);
conf.addResource(fileResource);
assertEquals(conf.get("a"), "b");
assertEquals(conf.get("c"), "d");
// Cleanup
new File(relConfig).delete();
new File(relConfig2).delete();
new File(new File(relConfig).getParent()).delete();
}
@Test
public void testRelativeIncludesWithLoadingViaUri() throws Exception {
tearDown();
File configFile = new File("./tmp/test-config.xml");
File configFile2 = new File("./tmp/test-config2.xml");
new File(configFile.getParent()).mkdirs();
out = new BufferedWriter(new FileWriter(configFile2));
startConfig();
appendProperty("a", "b");
endConfig();
out = new BufferedWriter(new FileWriter(configFile));
startConfig();
// Add the relative path instead of the absolute one.
startInclude(configFile2.getName());
endInclude();
appendProperty("c", "d");
endConfig();
// verify that the includes file contains all properties
Path fileResource = new Path(configFile.toURI());
conf.addResource(fileResource);
assertEquals("b", conf.get("a"));
assertEquals("d", conf.get("c"));
// Cleanup
configFile.delete();
configFile2.delete();
new File(configFile.getParent()).delete();
}
@Test
public void testIntegerRanges() {
Configuration conf = new Configuration();
conf.set("first", "-100");
conf.set("second", "4-6,9-10,27");
conf.set("third", "34-");
Configuration.IntegerRanges range = conf.getRange("first", null);
System.out.println("first = " + range);
assertEquals(true, range.isIncluded(0));
assertEquals(true, range.isIncluded(1));
assertEquals(true, range.isIncluded(100));
assertEquals(false, range.isIncluded(101));
range = conf.getRange("second", null);
System.out.println("second = " + range);
assertEquals(false, range.isIncluded(3));
assertEquals(true, range.isIncluded(4));
assertEquals(true, range.isIncluded(6));
assertEquals(false, range.isIncluded(7));
assertEquals(false, range.isIncluded(8));
assertEquals(true, range.isIncluded(9));
assertEquals(true, range.isIncluded(10));
assertEquals(false, range.isIncluded(11));
assertEquals(false, range.isIncluded(26));
assertEquals(true, range.isIncluded(27));
assertEquals(false, range.isIncluded(28));
range = conf.getRange("third", null);
System.out.println("third = " + range);
assertEquals(false, range.isIncluded(33));
assertEquals(true, range.isIncluded(34));
assertEquals(true, range.isIncluded(100000000));
}
@Test
public void testGetRangeIterator() throws Exception {
Configuration config = new Configuration(false);
IntegerRanges ranges = config.getRange("Test", "");
assertFalse(ranges.iterator().hasNext(), "Empty range has values");
ranges = config.getRange("Test", "5");
Set<Integer> expected = new HashSet<Integer>(Arrays.asList(5));
Set<Integer> found = new HashSet<Integer>();
for(Integer i: ranges) {
found.add(i);
}
assertEquals(expected, found);
ranges = config.getRange("Test", "5-10,13-14");
expected = new HashSet<Integer>(Arrays.asList(5,6,7,8,9,10,13,14));
found = new HashSet<Integer>();
for(Integer i: ranges) {
found.add(i);
}
assertEquals(expected, found);
ranges = config.getRange("Test", "8-12, 5- 7");
expected = new HashSet<Integer>(Arrays.asList(5,6,7,8,9,10,11,12));
found = new HashSet<Integer>();
for(Integer i: ranges) {
found.add(i);
}
assertEquals(expected, found);
}
@Test
public void testHexValues() throws IOException{
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.hex1", "0x10");
appendProperty("test.hex2", "0xF");
appendProperty("test.hex3", "-0x10");
// Invalid?
appendProperty("test.hex4", "-0x10xyz");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
assertEquals(16, conf.getInt("test.hex1", 0));
assertEquals(16, conf.getLong("test.hex1", 0));
assertEquals(15, conf.getInt("test.hex2", 0));
assertEquals(15, conf.getLong("test.hex2", 0));
assertEquals(-16, conf.getInt("test.hex3", 0));
assertEquals(-16, conf.getLong("test.hex3", 0));
try {
conf.getLong("test.hex4", 0);
fail("Property had invalid long value, but was read successfully.");
} catch (NumberFormatException e) {
// pass
}
try {
conf.getInt("test.hex4", 0);
fail("Property had invalid int value, but was read successfully.");
} catch (NumberFormatException e) {
// pass
}
}
@Test
public void testIntegerValues() throws IOException{
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.int1", "20");
appendProperty("test.int2", "020");
appendProperty("test.int3", "-20");
appendProperty("test.int4", " -20 ");
appendProperty("test.int5", " -20xyz ");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
assertEquals(20, conf.getInt("test.int1", 0));
assertEquals(20, conf.getLong("test.int1", 0));
assertEquals(20, conf.getLongBytes("test.int1", 0));
assertEquals(20, conf.getInt("test.int2", 0));
assertEquals(20, conf.getLong("test.int2", 0));
assertEquals(20, conf.getLongBytes("test.int2", 0));
assertEquals(-20, conf.getInt("test.int3", 0));
assertEquals(-20, conf.getLong("test.int3", 0));
assertEquals(-20, conf.getLongBytes("test.int3", 0));
assertEquals(-20, conf.getInt("test.int4", 0));
assertEquals(-20, conf.getLong("test.int4", 0));
assertEquals(-20, conf.getLongBytes("test.int4", 0));
try {
conf.getInt("test.int5", 0);
fail("Property had invalid int value, but was read successfully.");
} catch (NumberFormatException e) {
// pass
}
}
@Test
public void testHumanReadableValues() throws IOException {
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.humanReadableValue1", "1m");
appendProperty("test.humanReadableValue2", "1M");
appendProperty("test.humanReadableValue5", "1MBCDE");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
assertEquals(1048576, conf.getLongBytes("test.humanReadableValue1", 0));
assertEquals(1048576, conf.getLongBytes("test.humanReadableValue2", 0));
try {
conf.getLongBytes("test.humanReadableValue5", 0);
fail("Property had invalid human readable value, but was read successfully.");
} catch (NumberFormatException e) {
// pass
}
}
@Test
public void testBooleanValues() throws IOException {
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.bool1", "true");
appendProperty("test.bool2", "false");
appendProperty("test.bool3", " true ");
appendProperty("test.bool4", " false ");
appendProperty("test.bool5", "foo");
appendProperty("test.bool6", "TRUE");
appendProperty("test.bool7", "FALSE");
appendProperty("test.bool8", "");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
assertEquals(true, conf.getBoolean("test.bool1", false));
assertEquals(false, conf.getBoolean("test.bool2", true));
assertEquals(true, conf.getBoolean("test.bool3", false));
assertEquals(false, conf.getBoolean("test.bool4", true));
assertEquals(true, conf.getBoolean("test.bool5", true));
assertEquals(true, conf.getBoolean("test.bool6", false));
assertEquals(false, conf.getBoolean("test.bool7", true));
assertEquals(false, conf.getBoolean("test.bool8", false));
}
@Test
public void testFloatValues() throws IOException {
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.float1", "3.1415");
appendProperty("test.float2", "003.1415");
appendProperty("test.float3", "-3.1415");
appendProperty("test.float4", " -3.1415 ");
appendProperty("test.float5", "xyz-3.1415xyz");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
assertEquals(3.1415f, conf.getFloat("test.float1", 0.0f), DOUBLE_DELTA);
assertEquals(3.1415f, conf.getFloat("test.float2", 0.0f), DOUBLE_DELTA);
assertEquals(-3.1415f, conf.getFloat("test.float3", 0.0f), DOUBLE_DELTA);
assertEquals(-3.1415f, conf.getFloat("test.float4", 0.0f), DOUBLE_DELTA);
try {
conf.getFloat("test.float5", 0.0f);
fail("Property had invalid float value, but was read successfully.");
} catch (NumberFormatException e) {
// pass
}
}
@Test
public void testDoubleValues() throws IOException {
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.double1", "3.1415");
appendProperty("test.double2", "003.1415");
appendProperty("test.double3", "-3.1415");
appendProperty("test.double4", " -3.1415 ");
appendProperty("test.double5", "xyz-3.1415xyz");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
assertEquals(3.1415, conf.getDouble("test.double1", 0.0), DOUBLE_DELTA);
assertEquals(3.1415, conf.getDouble("test.double2", 0.0), DOUBLE_DELTA);
assertEquals(-3.1415, conf.getDouble("test.double3", 0.0), DOUBLE_DELTA);
assertEquals(-3.1415, conf.getDouble("test.double4", 0.0), DOUBLE_DELTA);
try {
conf.getDouble("test.double5", 0.0);
fail("Property had invalid double value, but was read successfully.");
} catch (NumberFormatException e) {
// pass
}
}
@Test
public void testGetClass() throws IOException {
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.class1", "java.lang.Integer");
appendProperty("test.class2", " java.lang.Integer ");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
assertEquals("java.lang.Integer",
conf.getClass("test.class1", null).getCanonicalName());
assertEquals("java.lang.Integer",
conf.getClass("test.class2", null).getCanonicalName());
}
@Test
public void testGetClasses() throws IOException {
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.classes1", "java.lang.Integer,java.lang.String");
appendProperty("test.classes2", " java.lang.Integer , java.lang.String ");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
String[] expectedNames = {"java.lang.Integer", "java.lang.String"};
Class<?>[] defaultClasses = {};
Class<?>[] classes1 = conf.getClasses("test.classes1", defaultClasses);
Class<?>[] classes2 = conf.getClasses("test.classes2", defaultClasses);
assertArrayEquals(expectedNames, extractClassNames(classes1));
assertArrayEquals(expectedNames, extractClassNames(classes2));
}
@Test
public void testGetStringCollection() {
Configuration c = new Configuration();
c.set("x", " a, b\n,\nc ");
Collection<String> strs = c.getTrimmedStringCollection("x");
assertEquals(3, strs.size());
assertArrayEquals(new String[]{ "a", "b", "c" },
strs.toArray(new String[0]));
// Check that the result is mutable
strs.add("z");
// Make sure same is true for missing config
strs = c.getStringCollection("does-not-exist");
assertEquals(0, strs.size());
strs.add("z");
}
@Test
public void testGetTrimmedStringCollection() {
Configuration c = new Configuration();
c.set("x", "a, b, c");
Collection<String> strs = c.getStringCollection("x");
assertEquals(3, strs.size());
assertArrayEquals(new String[]{ "a", " b", " c" },
strs.toArray(new String[0]));
// Check that the result is mutable
strs.add("z");
// Make sure same is true for missing config
strs = c.getStringCollection("does-not-exist");
assertEquals(0, strs.size());
strs.add("z");
}
private static String[] extractClassNames(Class<?>[] classes) {
String[] classNames = new String[classes.length];
for (int i = 0; i < classNames.length; i++) {
classNames[i] = classes[i].getCanonicalName();
}
return classNames;
}
enum Dingo { FOO, BAR };
enum Yak { RAB, FOO };
@Test
public void testEnum() {
Configuration conf = new Configuration();
conf.setEnum("test.enum", Dingo.FOO);
assertSame(Dingo.FOO, conf.getEnum("test.enum", Dingo.BAR));
assertSame(Yak.FOO, conf.getEnum("test.enum", Yak.RAB));
conf.setEnum("test.enum", Dingo.FOO);
boolean fail = false;
try {
conf.setEnum("test.enum", Dingo.BAR);
conf.getEnum("test.enum", Yak.FOO);
} catch (IllegalArgumentException e) {
fail = true;
}
assertTrue(fail);
}
@Test
public void testEnumFromXml() throws IOException {
out=new BufferedWriter(new FileWriter(CONFIG_FOR_ENUM));
startConfig();
appendProperty("test.enum"," \t \n FOO \t \n");
appendProperty("test.enum2"," \t \n Yak.FOO \t \n");
endConfig();
Configuration conf = new Configuration();
Path fileResource = new Path(CONFIG_FOR_ENUM);
conf.addResource(fileResource);
assertSame(Yak.FOO, conf.getEnum("test.enum", Yak.FOO));
boolean fail = false;
try {
conf.getEnum("test.enum2", Yak.FOO);
} catch (IllegalArgumentException e) {
fail = true;
}
assertTrue(fail);
}
@Test
public void testTimeDuration() {
Configuration conf = new Configuration(false);
assertEquals(7000L,
conf.getTimeDuration("test.time.a", 7L, SECONDS, MILLISECONDS));
conf.setTimeDuration("test.time.a", 7L, SECONDS);
assertEquals("7s", conf.get("test.time.a"));
assertEquals(0L, conf.getTimeDuration("test.time.a", 30, MINUTES));
assertEquals(0L, conf.getTimeDuration("test.time.a", 30, SECONDS, MINUTES));
assertEquals(7L, conf.getTimeDuration("test.time.a", 30, SECONDS));
assertEquals(7L,
conf.getTimeDuration("test.time.a", 30, MILLISECONDS, SECONDS));
assertEquals(7000L, conf.getTimeDuration("test.time.a", 30, MILLISECONDS));
assertEquals(7000000L,
conf.getTimeDuration("test.time.a", 30, MICROSECONDS));
assertEquals(7000000000L,
conf.getTimeDuration("test.time.a", 30, NANOSECONDS));
conf.setTimeDuration("test.time.b", 1, DAYS);
assertEquals("1d", conf.get("test.time.b"));
assertEquals(1, conf.getTimeDuration("test.time.b", 1, DAYS));
assertEquals(24, conf.getTimeDuration("test.time.b", 1, HOURS));
assertEquals(MINUTES.convert(1, DAYS),
conf.getTimeDuration("test.time.b", 1, MINUTES));
// check default
assertEquals(30L, conf.getTimeDuration("test.time.X", 30, SECONDS));
conf.set("test.time.X", "30");
assertEquals(30L, conf.getTimeDuration("test.time.X", 40, SECONDS));
assertEquals(30000L,
conf.getTimeDuration("test.time.X", 40, SECONDS, MILLISECONDS));
assertEquals(10L, conf.getTimeDuration("test.time.c", "10", SECONDS));
assertEquals(30L, conf.getTimeDuration("test.time.c", "30s", SECONDS));
assertEquals(120L, conf.getTimeDuration("test.time.c", "2m", SECONDS));
conf.set("test.time.c", "30");
assertEquals(30L, conf.getTimeDuration("test.time.c", "40s", SECONDS));
// check suffix insensitive
conf.set("test.time.d", "30S");
assertEquals(30L, conf.getTimeDuration("test.time.d", 40, SECONDS));
for (Configuration.ParsedTimeDuration ptd :
Configuration.ParsedTimeDuration.values()) {
conf.setTimeDuration("test.time.unit", 1, ptd.unit());
assertEquals(1 + ptd.suffix(), conf.get("test.time.unit"));
assertEquals(1, conf.getTimeDuration("test.time.unit", 2, ptd.unit()));
}
}
@Test
public void testStorageUnit() {
final String key = "valid.key";
final String nonKey = "not.a.key";
Configuration conf = new Configuration(false);
conf.setStorageSize(key, 10, MB);
// This call returns the value specified in the Key as a double in MBs.
Assertions.assertThat(conf.getStorageSize(key, "1GB", MB))
.isEqualTo(10.0);
// Since this key is missing, This call converts the default value of 1GB
// to MBs are returns that value.
Assertions.assertThat(conf.getStorageSize(nonKey, "1GB", MB))
.isEqualTo(1024.0);
conf.setStorageSize(key, 1024, BYTES);
Assertions.assertThat(conf.getStorageSize(key, 100, KB)).isEqualTo(1.0);
Assertions.assertThat(conf.getStorageSize(nonKey, 100.0, KB))
.isEqualTo(100.0);
// We try out different kind of String formats to see if they work and
// during read, we also try to read using a different Storage Units.
conf.setStrings(key, "1TB");
Assertions.assertThat(conf.getStorageSize(key, "1PB", GB))
.isEqualTo(1024.0);
conf.setStrings(key, "1bytes");
Assertions.assertThat(conf.getStorageSize(key, "1PB", KB))
.isEqualTo(0.001);
conf.setStrings(key, "2048b");
Assertions.assertThat(conf.getStorageSize(key, "1PB", KB)).isEqualTo(2.0);
conf.setStrings(key, "64 GB");
Assertions.assertThat(conf.getStorageSize(key, "1PB", GB)).isEqualTo(64.0);
// Match the parsing patterns of getLongBytes, which takes single char
// suffix.
conf.setStrings(key, "1T");
Assertions.assertThat(conf.getStorageSize(key, "1GB", TB)).isEqualTo(1.0);
conf.setStrings(key, "1k");
Assertions.assertThat(conf.getStorageSize(key, "1GB", KB)).isEqualTo(1.0);
conf.setStrings(key, "10m");
Assertions.assertThat(conf.getStorageSize(key, "1GB", MB)).isEqualTo(10.0);
// Missing format specification, this should throw.
conf.setStrings(key, "100");
assertThrows(IllegalArgumentException.class,
() -> conf.getStorageSize(key, "1PB", GB));
// illegal format specification, this should throw.
conf.setStrings(key, "1HB");
assertThrows(IllegalArgumentException.class,
() -> conf.getStorageSize(key, "1PB", GB));
// Illegal number specification, this should throw.
conf.setStrings(key, "HadoopGB");
assertThrows(IllegalArgumentException.class,
() -> conf.getStorageSize(key, "1PB", GB));
}
@Test
public void testTimeDurationWarning() {
// check warn for possible loss of precision
final String warnFormat = "Possible loss of precision converting %s" +
" to %s for test.time.warn";
final ArrayList<String> warnchk = new ArrayList<>();
Configuration wconf = new Configuration(false) {
@Override
void logDeprecation(String message) {
warnchk.add(message);
}
};
String[] convDAYS = new String[]{"23h", "30m", "40s", "10us", "40000ms"};
for (String s : convDAYS) {
wconf.set("test.time.warn", s);
assertEquals(0, wconf.getTimeDuration("test.time.warn", 1, DAYS));
}
for (int i = 0; i < convDAYS.length; ++i) {
String wchk = String.format(warnFormat, convDAYS[i], "DAYS");
assertEquals(wchk, warnchk.get(i));
}
warnchk.clear();
wconf.setTimeDuration("test.time.warn", 1441, MINUTES);
assertEquals(1, wconf.getTimeDuration("test.time.warn", 0, DAYS));
assertEquals(24, wconf.getTimeDuration("test.time.warn", 0, HOURS));
String dchk = String.format(warnFormat, "1441m", "DAYS");
assertEquals(dchk, warnchk.get(0));
String hchk = String.format(warnFormat, "1441m", "HOURS");
assertEquals(hchk, warnchk.get(1));
assertEquals(1441, wconf.getTimeDuration("test.time.warn", 0, MINUTES));
// no warning
assertEquals(2, warnchk.size());
assertEquals(86460, wconf.getTimeDuration("test.time.warn", 0, SECONDS));
// no warning
assertEquals(2, warnchk.size());
}
@Test
public void testPattern() throws IOException {
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.pattern1", "");
appendProperty("test.pattern2", "(");
appendProperty("test.pattern3", "a+b");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
Pattern defaultPattern = Pattern.compile("x+");
// Return default if missing
assertEquals(defaultPattern.pattern(),
conf.getPattern("xxxxx", defaultPattern).pattern());
// Return null if empty and default is null
assertNull(conf.getPattern("test.pattern1", null));
// Return default for empty
assertEquals(defaultPattern.pattern(),
conf.getPattern("test.pattern1", defaultPattern).pattern());
// Return default for malformed
assertEquals(defaultPattern.pattern(),
conf.getPattern("test.pattern2", defaultPattern).pattern());
// Works for correct patterns
assertEquals("a+b",
conf.getPattern("test.pattern3", defaultPattern).pattern());
}
@Test
public void testPropertySource() throws IOException {
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.foo", "bar");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
conf.set("fs.defaultFS", "value");
String [] sources = conf.getPropertySources("test.foo");
assertEquals(1, sources.length);
assertEquals(fileResource, new Path(sources[0]),
"Resource string returned for a file-loaded property " +
"must be a proper absolute path");
assertArrayEquals(new String[]{"programmatically"},
conf.getPropertySources("fs.defaultFS"),
"Resource string returned for a set() property must be " +
"\"programmatically\"");
assertArrayEquals(null, conf.getPropertySources("fs.defaultFoo"),
"Resource string returned for an unset property must be null");
}
@Test
public void testMultiplePropertySource() throws IOException {
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.foo", "bar", false, "a", "b", "c");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
String [] sources = conf.getPropertySources("test.foo");
assertEquals(4, sources.length);
assertEquals("a", sources[0]);
assertEquals("b", sources[1]);
assertEquals("c", sources[2]);
assertEquals(fileResource, new Path(sources[3]),
"Resource string returned for a file-loaded property" +
" must be a proper absolute path");
}
@Test
public void testSocketAddress() {
Configuration conf = new Configuration();
final String defaultAddr = "host:1";
final int defaultPort = 2;
InetSocketAddress addr = null;
addr = conf.getSocketAddr("myAddress", defaultAddr, defaultPort);
assertEquals(defaultAddr, NetUtils.getHostPortString(addr));
conf.set("myAddress", "host2");
addr = conf.getSocketAddr("myAddress", defaultAddr, defaultPort);
assertEquals("host2:"+defaultPort, NetUtils.getHostPortString(addr));
conf.set("myAddress", "host2:3");
addr = conf.getSocketAddr("myAddress", defaultAddr, defaultPort);
assertEquals("host2:3", NetUtils.getHostPortString(addr));
conf.set("myAddress", " \n \t host4:5 \t \n ");
addr = conf.getSocketAddr("myAddress", defaultAddr, defaultPort);
assertEquals("host4:5", NetUtils.getHostPortString(addr));
boolean threwException = false;
conf.set("myAddress", "bad:-port");
try {
addr = conf.getSocketAddr("myAddress", defaultAddr, defaultPort);
} catch (IllegalArgumentException iae) {
threwException = true;
assertEquals("Does not contain a valid host:port authority: " +
"bad:-port (configuration property 'myAddress')",
iae.getMessage());
} finally {
assertTrue(threwException);
}
}
@Test
public void testSetSocketAddress() {
Configuration conf = new Configuration();
NetUtils.addStaticResolution("host", "127.0.0.1");
final String defaultAddr = "host:1";
InetSocketAddress addr = NetUtils.createSocketAddr(defaultAddr);
conf.setSocketAddr("myAddress", addr);
assertEquals(defaultAddr, NetUtils.getHostPortString(addr));
}
@Test
public void testUpdateSocketAddress() throws IOException {
InetSocketAddress addr = NetUtils.createSocketAddrForHost("host", 1);
InetSocketAddress connectAddr = conf.updateConnectAddr("myAddress", addr);
assertEquals(connectAddr.getHostName(), addr.getHostName());
addr = new InetSocketAddress(1);
connectAddr = conf.updateConnectAddr("myAddress", addr);
assertEquals(connectAddr.getHostName(),
InetAddress.getLocalHost().getHostName());
}
@Test
public void testReload() throws IOException {
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.key1", "final-value1", true);
appendProperty("test.key2", "value2");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
out=new BufferedWriter(new FileWriter(CONFIG2));
startConfig();
appendProperty("test.key1", "value1");
appendProperty("test.key3", "value3");
endConfig();
Path fileResource1 = new Path(CONFIG2);
conf.addResource(fileResource1);
// add a few values via set.
conf.set("test.key3", "value4");
conf.set("test.key4", "value5");
assertEquals("final-value1", conf.get("test.key1"));
assertEquals("value2", conf.get("test.key2"));
assertEquals("value4", conf.get("test.key3"));
assertEquals("value5", conf.get("test.key4"));
// change values in the test file...
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.key1", "final-value1");
appendProperty("test.key3", "final-value3", true);
endConfig();
conf.reloadConfiguration();
assertEquals("value1", conf.get("test.key1"));
// overlayed property overrides.
assertEquals("value4", conf.get("test.key3"));
assertEquals(null, conf.get("test.key2"));
assertEquals("value5", conf.get("test.key4"));
}
@Test
public void testSize() {
Configuration conf = new Configuration(false);
conf.set("a", "A");
conf.set("b", "B");
assertEquals(2, conf.size());
}
@Test
public void testClear() {
Configuration conf = new Configuration(false);
conf.set("a", "A");
conf.set("b", "B");
conf.clear();
assertEquals(0, conf.size());
assertFalse(conf.iterator().hasNext());
}
public static class Fake_ClassLoader extends ClassLoader {
}
@Test
public void testClassLoader() {
Configuration conf = new Configuration(false);
conf.setQuietMode(false);
conf.setClassLoader(new Fake_ClassLoader());
Configuration other = new Configuration(conf);
assertTrue(other.getClassLoader() instanceof Fake_ClassLoader);
}
static class JsonConfiguration {
JsonProperty[] properties;
public JsonProperty[] getProperties() {
return properties;
}
public void setProperties(JsonProperty[] properties) {
this.properties = properties;
}
}
static class SingleJsonConfiguration {
private JsonProperty property;
public JsonProperty getProperty() {
return property;
}
public void setProperty(JsonProperty property) {
this.property = property;
}
}
static class JsonProperty {
String key;
public String getKey() {
return key;
}
public void setKey(String key) {
this.key = key;
}
public String getValue() {
return value;
}
public void setValue(String value) {
this.value = value;
}
public boolean getIsFinal() {
return isFinal;
}
public void setIsFinal(boolean isFinal) {
this.isFinal = isFinal;
}
public String getResource() {
return resource;
}
public void setResource(String resource) {
this.resource = resource;
}
String value;
boolean isFinal;
String resource;
}
private Configuration getActualConf(String xmlStr) {
Configuration ac = new Configuration(false);
InputStream in = new ByteArrayInputStream(xmlStr.getBytes());
ac.addResource(in);
return ac;
}
@Test
public void testGetSetTrimmedNames() throws IOException {
Configuration conf = new Configuration(false);
conf.set(" name", "value");
assertEquals("value", conf.get("name"));
assertEquals("value", conf.get(" name"));
assertEquals("value", conf.getRaw(" name "));
}
@Test
public void testDumpProperty() throws IOException {
StringWriter outWriter = new StringWriter();
ObjectMapper mapper = new ObjectMapper();
String jsonStr = null;
String xmlStr = null;
try {
Configuration testConf = new Configuration(false);
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.key1", "value1");
appendProperty("test.key2", "value2", true);
appendProperty("test.key3", "value3");
endConfig();
Path fileResource = new Path(CONFIG);
testConf.addResource(fileResource);
out.close();
// case 1: dump an existing property
// test json format
outWriter = new StringWriter();
Configuration.dumpConfiguration(testConf, "test.key2", outWriter);
jsonStr = outWriter.toString();
outWriter.close();
mapper = new ObjectMapper();
SingleJsonConfiguration jconf1 =
mapper.readValue(jsonStr, SingleJsonConfiguration.class);
JsonProperty jp1 = jconf1.getProperty();
assertEquals("test.key2", jp1.getKey());
assertEquals("value2", jp1.getValue());
assertEquals(true, jp1.isFinal);
assertEquals(fileResource.toString(), jp1.getResource());
// test xml format
outWriter = new StringWriter();
testConf.writeXml("test.key2", outWriter);
xmlStr = outWriter.toString();
outWriter.close();
Configuration actualConf1 = getActualConf(xmlStr);
assertEquals(1, actualConf1.size());
assertEquals("value2", actualConf1.get("test.key2"));
assertTrue(actualConf1.getFinalParameters().contains("test.key2"));
assertEquals(fileResource.toString(),
actualConf1.getPropertySources("test.key2")[0]);
// case 2: dump an non existing property
// test json format
try {
outWriter = new StringWriter();
Configuration.dumpConfiguration(testConf,
"test.unknown.key", outWriter);
outWriter.close();
} catch (Exception e) {
assertTrue(e instanceof IllegalArgumentException);
assertTrue(e.getMessage().contains("test.unknown.key") &&
e.getMessage().contains("not found"));
}
// test xml format
try {
outWriter = new StringWriter();
testConf.writeXml("test.unknown.key", outWriter);
outWriter.close();
} catch (Exception e) {
assertTrue(e instanceof IllegalArgumentException);
assertTrue(e.getMessage().contains("test.unknown.key") &&
e.getMessage().contains("not found"));
}
// case 3: specify a null property, ensure all configurations are dumped
outWriter = new StringWriter();
Configuration.dumpConfiguration(testConf, null, outWriter);
jsonStr = outWriter.toString();
mapper = new ObjectMapper();
JsonConfiguration jconf3 =
mapper.readValue(jsonStr, JsonConfiguration.class);
assertEquals(3, jconf3.getProperties().length);
outWriter = new StringWriter();
testConf.writeXml(null, outWriter);
xmlStr = outWriter.toString();
outWriter.close();
Configuration actualConf3 = getActualConf(xmlStr);
assertEquals(3, actualConf3.size());
assertTrue(actualConf3.getProps().containsKey("test.key1") &&
actualConf3.getProps().containsKey("test.key2") &&
actualConf3.getProps().containsKey("test.key3"));
// case 4: specify an empty property, ensure all configurations are dumped
outWriter = new StringWriter();
Configuration.dumpConfiguration(testConf, "", outWriter);
jsonStr = outWriter.toString();
mapper = new ObjectMapper();
JsonConfiguration jconf4 =
mapper.readValue(jsonStr, JsonConfiguration.class);
assertEquals(3, jconf4.getProperties().length);
outWriter = new StringWriter();
testConf.writeXml("", outWriter);
xmlStr = outWriter.toString();
outWriter.close();
Configuration actualConf4 = getActualConf(xmlStr);
assertEquals(3, actualConf4.size());
assertTrue(actualConf4.getProps().containsKey("test.key1") &&
actualConf4.getProps().containsKey("test.key2") &&
actualConf4.getProps().containsKey("test.key3"));
} finally {
if(outWriter != null) {
outWriter.close();
}
if(out != null) {
out.close();
}
}
}
@Test
public void testDumpConfiguration() throws IOException {
StringWriter outWriter = new StringWriter();
Configuration.dumpConfiguration(conf, outWriter);
String jsonStr = outWriter.toString();
ObjectMapper mapper = new ObjectMapper();
JsonConfiguration jconf =
mapper.readValue(jsonStr, JsonConfiguration.class);
int defaultLength = jconf.getProperties().length;
// add 3 keys to the existing configuration properties
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.key1", "value1");
appendProperty("test.key2", "value2",true);
appendProperty("test.key3", "value3");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
out.close();
outWriter = new StringWriter();
Configuration.dumpConfiguration(conf, outWriter);
jsonStr = outWriter.toString();
mapper = new ObjectMapper();
jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
int length = jconf.getProperties().length;
// check for consistency in the number of properties parsed in Json format.
assertEquals(length, defaultLength+3);
//change few keys in another resource file
out=new BufferedWriter(new FileWriter(CONFIG2));
startConfig();
appendProperty("test.key1", "newValue1");
appendProperty("test.key2", "newValue2");
endConfig();
Path fileResource1 = new Path(CONFIG2);
conf.addResource(fileResource1);
out.close();
outWriter = new StringWriter();
Configuration.dumpConfiguration(conf, outWriter);
jsonStr = outWriter.toString();
mapper = new ObjectMapper();
jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
// put the keys and their corresponding attributes into a hashmap for their
// efficient retrieval
HashMap<String,JsonProperty> confDump = new HashMap<String,JsonProperty>();
for(JsonProperty prop : jconf.getProperties()) {
confDump.put(prop.getKey(), prop);
}
// check if the value and resource of test.key1 is changed
assertEquals("newValue1", confDump.get("test.key1").getValue());
assertEquals(false, confDump.get("test.key1").getIsFinal());
assertEquals(fileResource1.toString(),
confDump.get("test.key1").getResource());
// check if final parameter test.key2 is not changed, since it is first
// loaded as final parameter
assertEquals("value2", confDump.get("test.key2").getValue());
assertEquals(true, confDump.get("test.key2").getIsFinal());
assertEquals(fileResource.toString(),
confDump.get("test.key2").getResource());
// check for other keys which are not modified later
assertEquals("value3", confDump.get("test.key3").getValue());
assertEquals(false, confDump.get("test.key3").getIsFinal());
assertEquals(fileResource.toString(),
confDump.get("test.key3").getResource());
// check for resource to be "Unknown" for keys which are loaded using 'set'
// and expansion of properties
conf.set("test.key4", "value4");
conf.set("test.key5", "value5");
conf.set("test.key6", "${test.key5}");
outWriter = new StringWriter();
Configuration.dumpConfiguration(conf, outWriter);
jsonStr = outWriter.toString();
mapper = new ObjectMapper();
jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
confDump = new HashMap<String, JsonProperty>();
for(JsonProperty prop : jconf.getProperties()) {
confDump.put(prop.getKey(), prop);
}
assertEquals("value5",confDump.get("test.key6").getValue());
assertEquals("programmatically", confDump.get("test.key4").getResource());
outWriter.close();
}
@Test
public void testDumpConfiguratioWithoutDefaults() throws IOException {
// check for case when default resources are not loaded
Configuration config = new Configuration(false);
StringWriter outWriter = new StringWriter();
Configuration.dumpConfiguration(config, outWriter);
String jsonStr = outWriter.toString();
ObjectMapper mapper = new ObjectMapper();
JsonConfiguration jconf =
mapper.readValue(jsonStr, JsonConfiguration.class);
//ensure that no properties are loaded.
assertEquals(0, jconf.getProperties().length);
// add 2 keys
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.key1", "value1");
appendProperty("test.key2", "value2",true);
endConfig();
Path fileResource = new Path(CONFIG);
config.addResource(fileResource);
out.close();
outWriter = new StringWriter();
Configuration.dumpConfiguration(config, outWriter);
jsonStr = outWriter.toString();
mapper = new ObjectMapper();
jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
HashMap<String, JsonProperty>confDump = new HashMap<String, JsonProperty>();
for (JsonProperty prop : jconf.getProperties()) {
confDump.put(prop.getKey(), prop);
}
//ensure only 2 keys are loaded
assertEquals(2,jconf.getProperties().length);
//ensure the values are consistent
assertEquals(confDump.get("test.key1").getValue(),"value1");
assertEquals(confDump.get("test.key2").getValue(),"value2");
//check the final tag
assertEquals(false, confDump.get("test.key1").getIsFinal());
assertEquals(true, confDump.get("test.key2").getIsFinal());
//check the resource for each property
for (JsonProperty prop : jconf.getProperties()) {
assertEquals(fileResource.toString(),prop.getResource());
}
}
@Test
public void testDumpSensitiveProperty() throws IOException {
final String myPassword = "ThisIsMyPassword";
Configuration testConf = new Configuration(false);
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.password", myPassword);
endConfig();
Path fileResource = new Path(CONFIG);
testConf.addResource(fileResource);
try (StringWriter outWriter = new StringWriter()) {
testConf.set(SENSITIVE_CONFIG_KEYS, "password$");
Configuration.dumpConfiguration(testConf, "test.password", outWriter);
assertFalse(outWriter.toString().contains(myPassword));
}
}
@Test
public void testDumpSensitiveConfiguration() throws IOException {
final String myPassword = "ThisIsMyPassword";
Configuration testConf = new Configuration(false);
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.password", myPassword);
endConfig();
Path fileResource = new Path(CONFIG);
testConf.addResource(fileResource);
try (StringWriter outWriter = new StringWriter()) {
testConf.set(SENSITIVE_CONFIG_KEYS, "password$");
Configuration.dumpConfiguration(testConf, outWriter);
assertFalse(outWriter.toString().contains(myPassword));
}
}
@Test
public void testGetValByRegex() {
Configuration conf = new Configuration();
String key1 = "t.abc.key1";
String key2 = "t.abc.key2";
String key3 = "tt.abc.key3";
String key4 = "t.abc.ey3";
conf.set(key1, "value1");
conf.set(key2, "value2");
conf.set(key3, "value3");
conf.set(key4, "value3");
Map<String,String> res = conf.getValByRegex("^t\\..*\\.key\\d");
assertTrue(res.containsKey(key1), "Conf didn't get key " + key1);
assertTrue(res.containsKey(key2), "Conf didn't get key " + key2);
assertTrue(!res.containsKey(key3), "Picked out wrong key " + key3);
assertTrue(!res.containsKey(key4), "Picked out wrong key " + key4);
}
@Test
public void testGetClassesShouldReturnDefaultValue() throws Exception {
Configuration config = new Configuration();
Class<?>[] classes =
config.getClasses("testClassName", Configuration.class);
assertEquals(1, classes.length,
"Not returning expected number of classes. Number of returned classes =" + classes.length);
assertEquals(Configuration.class, classes[0], "Not returning the default class Name");
}
@Test
public void testGetClassesShouldReturnEmptyArray()
throws Exception {
Configuration config = new Configuration();
config.set("testClassName", "");
Class<?>[] classes = config.getClasses("testClassName", Configuration.class);
assertEquals(0, classes.length,
"Not returning expected number of classes. Number of returned classes =" + classes.length);
}
@Test
public void testSettingValueNull() throws Exception {
Configuration config = new Configuration();
try {
config.set("testClassName", null);
fail("Should throw an IllegalArgumentException exception ");
} catch (Exception e) {
assertTrue(e instanceof IllegalArgumentException);
assertEquals(e.getMessage(),
"The value of property testClassName must not be null");
}
}
@Test
public void testSettingKeyNull() throws Exception {
Configuration config = new Configuration();
try {
config.set(null, "test");
fail("Should throw an IllegalArgumentException exception ");
} catch (Exception e) {
assertTrue(e instanceof IllegalArgumentException);
assertEquals(e.getMessage(), "Property name must not be null");
}
}
@Test
public void testInvalidSubstitution() {
final Configuration configuration = new Configuration(false);
// 2-var loops
//
final String key = "test.random.key";
for (String keyExpression : Arrays.asList(
"${" + key + "}",
"foo${" + key + "}",
"foo${" + key + "}bar",
"${" + key + "}bar")) {
configuration.set(key, keyExpression);
assertEquals(keyExpression, configuration.get(key), "Unexpected value");
}
}
@Test
public void testIncompleteSubbing() {
Configuration configuration = new Configuration(false);
String key = "test.random.key";
for (String keyExpression : Arrays.asList(
"{}",
"${}",
"{" + key,
"${" + key,
"foo${" + key,
"foo${" + key + "bar",
"foo{" + key + "}bar",
"${" + key + "bar")) {
configuration.set(key, keyExpression);
String value = configuration.get(key);
assertTrue(value.equals(keyExpression), "Unexpected value " + value);
}
}
@Test
public void testBoolean() {
boolean value = true;
Configuration configuration = new Configuration();
configuration.setBoolean("value", value);
assertEquals(value, configuration.getBoolean("value", false));
}
@Test
public void testBooleanIfUnset() {
boolean value = true;
Configuration configuration = new Configuration();
configuration.setBooleanIfUnset("value", value);
assertEquals(value, configuration.getBoolean("value", false));
configuration.setBooleanIfUnset("value", false);
assertEquals(value, configuration.getBoolean("value", false));
}
@Test
public void testFloat() {
float value = 1.0F;
Configuration configuration = new Configuration();
configuration.setFloat("value", value);
assertEquals(value, configuration.getFloat("value", 0.0F), DOUBLE_DELTA);
}
@Test
public void testDouble() {
double value = 1.0D;
Configuration configuration = new Configuration();
configuration.setDouble("value", value);
assertEquals(value, configuration.getDouble("value", 0.0D), DOUBLE_DELTA);
}
@Test
public void testInt() {
int value = 1;
Configuration configuration = new Configuration();
configuration.setInt("value", value);
assertEquals(value, configuration.getInt("value", 0));
}
@Test
public void testLong() {
long value = 1L;
Configuration configuration = new Configuration();
configuration.setLong("value", value);
assertEquals(value, configuration.getLong("value", 0L));
}
@Test
public void testStrings() {
String [] strings = {"FOO","BAR"};
Configuration configuration = new Configuration();
configuration.setStrings("strings", strings);
String [] returnStrings = configuration.getStrings("strings");
for(int i=0;i<returnStrings.length;i++) {
assertEquals(strings[i], returnStrings[i]);
}
}
@Test
public void testSetPattern() {
Pattern testPattern = Pattern.compile("a+b");
Configuration configuration = new Configuration();
configuration.setPattern("testPattern", testPattern);
assertEquals(testPattern.pattern(),
configuration.getPattern("testPattern", Pattern.compile("")).pattern());
}
@Test
public void testGetClassByNameOrNull() throws Exception {
Configuration config = new Configuration();
Class<?> clazz = config.getClassByNameOrNull("java.lang.Object");
assertNotNull(clazz);
}
@Test
public void testGetFinalParameters() throws Exception {
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
declareProperty("my.var", "x", "x", true);
endConfig();
Path fileResource = new Path(CONFIG);
Configuration conf = new Configuration();
Set<String> finalParameters = conf.getFinalParameters();
assertFalse(finalParameters.contains("my.var"), "my.var already exists");
conf.addResource(fileResource);
assertEquals("x", conf.get("my.var"), "my.var is undefined");
assertFalse(finalParameters.contains("my.var"), "finalparams not copied");
finalParameters = conf.getFinalParameters();
assertTrue(finalParameters.contains("my.var"), "my.var is not final");
}
/**
* A test to check whether this thread goes into infinite loop because of
* destruction of data structure by resize of Map. This problem was reported
* by SPARK-2546.
* @throws Exception
*/
@Test
public void testConcurrentAccesses() throws Exception {
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
declareProperty("some.config", "xyz", "xyz", false);
endConfig();
Path fileResource = new Path(CONFIG);
Configuration conf = new Configuration();
conf.addResource(fileResource);
class ConfigModifyThread extends Thread {
final private Configuration config;
final private String prefix;
public ConfigModifyThread(Configuration conf, String prefix) {
config = conf;
this.prefix = prefix;
}
@Override
public void run() {
for (int i = 0; i < 10000; i++) {
config.set("some.config.value-" + prefix + i, "value");
}
}
}
ArrayList<ConfigModifyThread> threads = new ArrayList<>();
for (int i = 0; i < 100; i++) {
threads.add(new ConfigModifyThread(conf, String.valueOf(i)));
}
for (Thread t: threads) {
t.start();
}
for (Thread t: threads) {
t.join();
}
// If this test completes without going into infinite loop,
// it's expected behaviour.
}
@Test
public void testNullValueProperties() throws Exception {
Configuration conf = new Configuration();
conf.setAllowNullValueProperties(true);
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("attr", "value", true);
appendProperty("attr", "", true);
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
assertEquals("value", conf.get("attr"));
}
@Test
public void testGetPasswordDeprecatedKeyStored() throws Exception {
final String oldKey = "test.password.old.key";
final String newKey = "test.password.new.key";
final String password = "MyPasswordForDeprecatedKey";
final File tmpDir = GenericTestUtils.getRandomizedTestDir();
tmpDir.mkdirs();
final String ourUrl = new URI(LocalJavaKeyStoreProvider.SCHEME_NAME,
"file", new File(tmpDir, "test.jks").toURI().getPath(),
null).toString();
conf = new Configuration(false);
conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl);
CredentialProvider provider =
CredentialProviderFactory.getProviders(conf).get(0);
provider.createCredentialEntry(oldKey, password.toCharArray());
provider.flush();
Configuration.addDeprecation(oldKey, newKey);
Assertions.assertThat(conf.getPassword(newKey))
.isEqualTo(password.toCharArray());
Assertions.assertThat(conf.getPassword(oldKey))
.isEqualTo(password.toCharArray());
FileUtil.fullyDelete(tmpDir);
}
@Test
public void testGetPasswordByDeprecatedKey() throws Exception {
final String oldKey = "test.password.old.key";
final String newKey = "test.password.new.key";
final String password = "MyPasswordForDeprecatedKey";
final File tmpDir = GenericTestUtils.getRandomizedTestDir();
tmpDir.mkdirs();
final String ourUrl = new URI(LocalJavaKeyStoreProvider.SCHEME_NAME,
"file", new File(tmpDir, "test.jks").toURI().getPath(),
null).toString();
conf = new Configuration(false);
conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl);
CredentialProvider provider =
CredentialProviderFactory.getProviders(conf).get(0);
provider.createCredentialEntry(newKey, password.toCharArray());
provider.flush();
Configuration.addDeprecation(oldKey, newKey);
Assertions.assertThat(conf.getPassword(newKey))
.isEqualTo(password.toCharArray());
Assertions.assertThat(conf.getPassword(oldKey))
.isEqualTo(password.toCharArray());
FileUtil.fullyDelete(tmpDir);
}
@Test
public void testGettingPropertiesWithPrefix() throws Exception {
Configuration conf = new Configuration();
for (int i = 0; i < 10; i++) {
conf.set("prefix." + "name" + i, "value" + i);
}
conf.set("different.prefix" + ".name", "value");
Map<String, String> prefixedProps = conf.getPropsWithPrefix("prefix.");
Assertions.assertThat(prefixedProps).hasSize(10);
for (int i = 0; i < 10; i++) {
assertEquals("value" + i, prefixedProps.get("name" + i));
}
// Repeat test with variable substitution
conf.set("foo", "bar");
for (int i = 0; i < 10; i++) {
conf.set("subprefix." + "subname" + i, "value_${foo}" + i);
}
prefixedProps = conf.getPropsWithPrefix("subprefix.");
Assertions.assertThat(prefixedProps).hasSize(10);
for (int i = 0; i < 10; i++) {
assertEquals("value_bar" + i, prefixedProps.get("subname" + i));
}
// test call with no properties for a given prefix
prefixedProps = conf.getPropsWithPrefix("none");
assertNotNull(prefixedProps.isEmpty());
assertTrue(prefixedProps.isEmpty());
}
public static void main(String[] argv) throws Exception {
junit.textui.TestRunner.main(new String[]{
TestConfiguration.class.getName()
});
}
@Test
public void testGetAllPropertiesByTags() throws Exception {
try{
out = new BufferedWriter(new FileWriter(CONFIG_CORE));
startConfig();
appendProperty("hadoop.tags.system", "YARN,HDFS,NAMENODE");
appendProperty("hadoop.tags.custom", "MYCUSTOMTAG");
appendPropertyByTag("dfs.cblock.trace.io", "false", "YARN");
appendPropertyByTag("dfs.replication", "1", "HDFS");
appendPropertyByTag("dfs.namenode.logging.level", "INFO", "NAMENODE");
appendPropertyByTag("dfs.random.key", "XYZ", "MYCUSTOMTAG");
endConfig();
Path fileResource = new Path(CONFIG_CORE);
conf.addResource(fileResource);
conf.getProps();
} finally {
out.close();
}
System.out.println(Files.readAllLines(Paths.get(CONFIG_CORE)));
List<String> tagList = new ArrayList<>();
tagList.add("YARN");
tagList.add("HDFS");
tagList.add("NAMENODE");
Properties properties = conf.getAllPropertiesByTags(tagList);
String[] sources = conf.getPropertySources("dfs.replication");
assertTrue(sources.length == 1);
assertTrue(Arrays.toString(sources).contains("core-site.xml"));
assertEq(3, properties.size());
assertEq(true, properties.containsKey("dfs.namenode.logging.level"));
assertEq(true, properties.containsKey("dfs.replication"));
assertEq(true, properties.containsKey("dfs.cblock.trace.io"));
assertEq(false, properties.containsKey("namenode.host"));
properties = conf.getAllPropertiesByTag("DEBUG");
assertEq(0, properties.size());
assertEq(false, properties.containsKey("dfs.namenode.logging.level"));
assertEq(true, conf.isPropertyTag("YARN"));
assertEq(true, conf.isPropertyTag("HDFS"));
assertEq(true, conf.isPropertyTag("NAMENODE"));
assertEq(true, conf.isPropertyTag("MYCUSTOMTAG"));
assertEq(false, conf.isPropertyTag("CMYCUSTOMTAG2"));
}
@Test
public void testInvalidTags() throws Exception {
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
conf.getProps();
assertFalse(conf.isPropertyTag("BADTAG"));
assertFalse(conf.isPropertyTag("CUSTOM_TAG"));
assertTrue(conf.isPropertyTag("DEBUG"));
assertTrue(conf.isPropertyTag("HDFS"));
}
/**
* Test race conditions between clone() and getProps().
* Test for race conditions in the way Hadoop handles the Configuration
* class. The scenario is the following. Let's assume that there are two
* threads sharing the same Configuration class. One adds some resources
* to the configuration, while the other one clones it. Resources are
* loaded lazily in a deferred call to loadResources(). If the cloning
* happens after adding the resources but before parsing them, some temporary
* resources like input stream pointers are cloned. Eventually both copies
* will load the same input stream resources.
* One parses the input stream XML and closes it updating it's own copy of
* the resource. The other one has another pointer to the same input stream.
* When it tries to load it, it will crash with a stream closed exception.
*/
@Test
public void testResourceRace() {
InputStream is =
new BufferedInputStream(new ByteArrayInputStream(
"<configuration></configuration>".getBytes()));
Configuration config = new Configuration();
// Thread 1
config.addResource(is);
// Thread 2
Configuration confClone = new Configuration(conf);
// Thread 2
confClone.get("firstParse");
// Thread 1
config.get("secondParse");
}
@Test
public void testCDATA() throws IOException {
String xml = new String(
"<configuration>" +
"<property>" +
"<name>cdata</name>" +
"<value><![CDATA[>cdata]]></value>" +
"</property>\n" +
"<property>" +
"<name>cdata-multiple</name>" +
"<value><![CDATA[>cdata1]]> and <![CDATA[>cdata2]]></value>" +
"</property>\n" +
"<property>" +
"<name>cdata-multiline</name>" +
"<value><![CDATA[>cdata\nmultiline<>]]></value>" +
"</property>\n" +
"<property>" +
"<name>cdata-whitespace</name>" +
"<value> prefix <![CDATA[>cdata]]>\nsuffix </value>" +
"</property>\n" +
"</configuration>");
Configuration conf = checkCDATA(xml.getBytes());
ByteArrayOutputStream os = new ByteArrayOutputStream();
conf.writeXml(os);
checkCDATA(os.toByteArray());
}
private static Configuration checkCDATA(byte[] bytes) {
Configuration conf = new Configuration(false);
conf.addResource(new ByteArrayInputStream(bytes));
assertEquals(">cdata", conf.get("cdata"));
assertEquals(">cdata1 and >cdata2", conf.get("cdata-multiple"));
assertEquals(">cdata\nmultiline<>", conf.get("cdata-multiline"));
assertEquals(" prefix >cdata\nsuffix ", conf.get("cdata-whitespace"));
return conf;
}
@Test
public void testConcurrentModificationDuringIteration() throws InterruptedException {
Configuration configuration = new Configuration();
new Thread(() -> {
while (true) {
configuration.set(String.valueOf(Math.random()), String.valueOf(Math.random()));
}
}).start();
AtomicBoolean exceptionOccurred = new AtomicBoolean(false);
new Thread(() -> {
while (true) {
try {
configuration.iterator();
} catch (final ConcurrentModificationException e) {
exceptionOccurred.set(true);
break;
}
}
}).start();
Thread.sleep(1000); //give enough time for threads to run
assertFalse(exceptionOccurred.get(), "ConcurrentModificationException occurred");
}
}