TestSnapshotDiffReport.java
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode.snapshot;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Random;
import java.util.List;
import java.util.ArrayList;
import java.util.function.Function;
import org.apache.commons.collections4.list.TreeList;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag;
import org.apache.hadoop.hdfs.client.impl.SnapshotDiffReportGenerator;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing;
import org.apache.hadoop.hdfs.protocol.SnapshotException;
import org.apache.hadoop.hdfs.protocol.SnapshotStatus;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.ChunkedArrayList;
import org.apache.hadoop.util.Time;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Tests snapshot deletion.
*/
public class TestSnapshotDiffReport {
private static final Logger LOG =
LoggerFactory.getLogger(TestSnapshotDiffReport.class);
{
SnapshotTestHelper.disableLogs();
}
private static final long SEED = 0;
private static final short REPLICATION = 3;
private static final short REPLICATION_1 = 2;
private static final long BLOCKSIZE = 1024;
private static final long BUFFERLEN = BLOCKSIZE / 2;
private static final long FILELEN = BLOCKSIZE * 2;
private final Path dir = new Path("/TestSnapshot");
private final Path sub1 = new Path(dir, "sub1");
private Configuration conf;
private MiniDFSCluster cluster;
private DistributedFileSystem hdfs;
private final HashMap<Path, Integer> snapshotNumberMap = new HashMap<Path, Integer>();
@Before
public void setUp() throws Exception {
conf = new Configuration();
conf.setBoolean(
DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES, true);
conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, 1);
conf.setBoolean(
DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_SKIP_CAPTURE_ACCESSTIME_ONLY_CHANGE,
true);
conf.setBoolean(
DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DIFF_ALLOW_SNAP_ROOT_DESCENDANT,
true);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DIFF_LISTING_LIMIT, 3);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION)
.format(true).build();
cluster.waitActive();
hdfs = cluster.getFileSystem();
}
@After
public void tearDown() throws Exception {
if (cluster != null) {
cluster.shutdown();
cluster = null;
}
}
private String genSnapshotName(Path snapshotDir) {
int sNum = -1;
if (snapshotNumberMap.containsKey(snapshotDir)) {
sNum = snapshotNumberMap.get(snapshotDir);
}
snapshotNumberMap.put(snapshotDir, ++sNum);
return "s" + sNum;
}
void modifyAndCreateSnapshot(Path modifyDir, Path[] snapshotDirs)
throws Exception {
modifyAndCreateSnapshot(modifyDir, snapshotDirs, hdfs, this::genSnapshotName);
}
/**
* Create/modify/delete files under a given directory, also create snapshots
* of directories.
*/
static void modifyAndCreateSnapshot(Path modifyDir, Path[] snapshotDirs,
DistributedFileSystem hdfs, Function<Path, String> getSnapshotName)
throws Exception {
Path file10 = new Path(modifyDir, "file10");
Path file11 = new Path(modifyDir, "file11");
Path file12 = new Path(modifyDir, "file12");
Path file13 = new Path(modifyDir, "file13");
Path link13 = new Path(modifyDir, "link13");
Path file14 = new Path(modifyDir, "file14");
Path file15 = new Path(modifyDir, "file15");
DFSTestUtil.createFile(hdfs, file10, BLOCKSIZE, REPLICATION_1, SEED);
DFSTestUtil.createFile(hdfs, file11, BLOCKSIZE, REPLICATION_1, SEED);
DFSTestUtil.createFile(hdfs, file12, BLOCKSIZE, REPLICATION_1, SEED);
DFSTestUtil.createFile(hdfs, file13, BLOCKSIZE, REPLICATION_1, SEED);
// create link13
hdfs.createSymlink(file13, link13, false);
// create snapshot
for (Path snapshotDir : snapshotDirs) {
hdfs.allowSnapshot(snapshotDir);
hdfs.createSnapshot(snapshotDir, getSnapshotName.apply(snapshotDir));
}
// delete file11
hdfs.delete(file11, true);
// modify file12
hdfs.setReplication(file12, REPLICATION);
// modify file13
hdfs.setReplication(file13, REPLICATION);
// delete link13
hdfs.delete(link13, false);
// create file14
DFSTestUtil.createFile(hdfs, file14, BLOCKSIZE, REPLICATION, SEED);
// create file15
DFSTestUtil.createFile(hdfs, file15, BLOCKSIZE, REPLICATION, SEED);
// create snapshot
for (Path snapshotDir : snapshotDirs) {
hdfs.createSnapshot(snapshotDir, getSnapshotName.apply(snapshotDir));
}
// create file11 again
DFSTestUtil.createFile(hdfs, file11, BLOCKSIZE, REPLICATION, SEED);
// delete file12
hdfs.delete(file12, true);
// modify file13
hdfs.setReplication(file13, (short) (REPLICATION - 2));
// create link13 again
hdfs.createSymlink(file13, link13, false);
// delete file14
hdfs.delete(file14, true);
// modify file15
hdfs.setReplication(file15, (short) (REPLICATION - 1));
// create snapshot
for (Path snapshotDir : snapshotDirs) {
hdfs.createSnapshot(snapshotDir, getSnapshotName.apply(snapshotDir));
}
// modify file10
hdfs.setReplication(file10, (short) (REPLICATION + 1));
}
/**
* Check the correctness of the diff reports.
*/
private void verifyDiffReport(Path dir, String from, String to,
DiffReportEntry... entries) throws IOException {
DFSTestUtil.verifySnapshotDiffReport(hdfs, dir, from, to, entries);
}
/**
* Test the computation and representation of diff between snapshots.
*/
@Test(timeout = 60000)
public void testDiffReport() throws Exception {
cluster.getNamesystem().getSnapshotManager().setAllowNestedSnapshots(true);
Path subsub1 = new Path(sub1, "subsub1");
Path subsubsub1 = new Path(subsub1, "subsubsub1");
hdfs.mkdirs(subsubsub1);
modifyAndCreateSnapshot(sub1, new Path[]{sub1, subsubsub1});
modifyAndCreateSnapshot(subsubsub1, new Path[]{sub1, subsubsub1});
final String invalidName = "invalid";
try {
hdfs.getSnapshotDiffReport(sub1, invalidName, invalidName);
fail("Expect exception when providing invalid snapshot name " +
"for diff report");
} catch (IOException e) {
GenericTestUtils.assertExceptionContains(
"Cannot find the snapshot of directory " + sub1 + " with name "
+ invalidName, e);
}
// diff between the same snapshot
SnapshotDiffReport report = hdfs.getSnapshotDiffReport(sub1, "s0", "s0");
LOG.info(report.toString());
assertEquals(0, report.getDiffList().size());
report = hdfs.getSnapshotDiffReport(sub1, "", "");
LOG.info(report.toString());
assertEquals(0, report.getDiffList().size());
try {
report = hdfs.getSnapshotDiffReport(subsubsub1, null, "s2");
fail("Expect exception when providing null fromSnapshot ");
} catch (IllegalArgumentException e) {
GenericTestUtils.assertExceptionContains("null fromSnapshot", e);
}
report = hdfs.getSnapshotDiffReport(subsubsub1, "s0", "s2");
LOG.info(report.toString());
assertEquals(0, report.getDiffList().size());
// test path with scheme also works
report = hdfs.getSnapshotDiffReport(hdfs.makeQualified(subsubsub1),
"s0", "s2");
LOG.info(report.toString());
assertEquals(0, report.getDiffList().size());
verifyDiffReport(sub1, "s0", "s2",
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("file15")),
new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("file12")),
new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("file11")),
new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("file11")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("file13")),
new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("link13")),
new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("link13")));
verifyDiffReport(sub1, "s0", "s5",
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("file15")),
new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("file12")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("file10")),
new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("file11")),
new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("file11")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("file13")),
new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("link13")),
new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("link13")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("subsub1/subsubsub1")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file10")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file11")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/link13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file15")));
verifyDiffReport(sub1, "s2", "s5",
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("file10")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("subsub1/subsubsub1")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file10")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file11")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/link13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file15")));
verifyDiffReport(sub1, "s3", "",
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("subsub1/subsubsub1")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file15")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file12")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("subsub1/subsubsub1/file10")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file11")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file11")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("subsub1/subsubsub1/file13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/link13")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("subsub1/subsubsub1/link13")));
}
@Test(timeout = 60000)
public void testSnapRootDescendantDiffReport() throws Exception {
Path subSub = new Path(sub1, "subsub1");
Path subSubSub = new Path(subSub, "subsubsub1");
Path nonSnapDir = new Path(dir, "non_snap");
hdfs.mkdirs(subSubSub);
hdfs.mkdirs(nonSnapDir);
modifyAndCreateSnapshot(sub1, new Path[]{sub1});
modifyAndCreateSnapshot(subSub, new Path[]{sub1});
modifyAndCreateSnapshot(subSubSub, new Path[]{sub1});
try {
hdfs.getSnapshotDiffReport(subSub, "s1", "s2");
hdfs.getSnapshotDiffReport(subSubSub, "s1", "s2");
} catch (IOException e) {
fail("Unexpected exception when getting snapshot diff report " +
subSub + ": " + e);
}
try {
hdfs.getSnapshotDiffReport(nonSnapDir, "s1", "s2");
fail("Snapshot diff report on a non snapshot directory '"
+ nonSnapDir.getName() + "'should fail!");
} catch (SnapshotException e) {
GenericTestUtils.assertExceptionContains(
"The path " + nonSnapDir +
" is neither snapshottable nor under a snapshot root!", e);
}
final String invalidName = "invalid";
try {
hdfs.getSnapshotDiffReport(subSub, invalidName, invalidName);
fail("Expect exception when providing invalid snapshot name " +
"for diff report");
} catch (IOException e) {
GenericTestUtils.assertExceptionContains(
"Cannot find the snapshot of directory " + sub1 + " with name "
+ invalidName, e);
}
// diff between the same snapshot
SnapshotDiffReport report = hdfs.getSnapshotDiffReport(subSub, "s0", "s0");
assertEquals(0, report.getDiffList().size());
report = hdfs.getSnapshotDiffReport(subSub, "", "");
assertEquals(0, report.getDiffList().size());
report = hdfs.getSnapshotDiffReport(subSubSub, "s0", "s2");
assertEquals(0, report.getDiffList().size());
report = hdfs.getSnapshotDiffReport(
hdfs.makeQualified(subSubSub), "s0", "s2");
assertEquals(0, report.getDiffList().size());
verifyDescendantDiffReports(sub1, subSub, subSubSub);
}
private void verifyDescendantDiffReports(final Path snapDir,
final Path snapSubDir, final Path snapSubSubDir) throws
IOException {
verifyDiffReport(snapDir, "s0", "s2",
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("file15")),
new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("file12")),
new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("file11")),
new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("file11")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("file13")),
new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("link13")),
new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("link13")));
verifyDiffReport(snapSubDir, "s0", "s2", new DiffReportEntry[]{});
verifyDiffReport(snapSubSubDir, "s0", "s2", new DiffReportEntry[]{});
verifyDiffReport(snapDir, "s0", "s8",
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("file15")),
new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("file12")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("file10")),
new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("file11")),
new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("file11")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("file13")),
new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("link13")),
new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("link13")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("subsub1")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/file10")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/file11")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/file13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/link13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/file15")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("subsub1/subsubsub1")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file10")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file11")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/link13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file15")));
verifyDiffReport(snapSubDir, "s0", "s8",
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("file10")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("file11")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("file13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("link13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("file15")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("subsubsub1")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsubsub1/file10")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsubsub1/file11")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsubsub1/file13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsubsub1/link13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsubsub1/file15")));
verifyDiffReport(snapSubSubDir, "s0", "s8",
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("file10")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("file11")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("file13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("link13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("file15")));
verifyDiffReport(snapDir, "s2", "s5",
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("file10")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("subsub1")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/file10")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/file11")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/file13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/link13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/file15")));
verifyDiffReport(snapSubDir, "s2", "s5",
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("file10")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("file11")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("file13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("link13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("file15")));
verifyDiffReport(snapSubSubDir, "s2", "s5",
new DiffReportEntry[]{});
verifyDiffReport(snapDir, "s3", "",
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("subsub1")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/file15")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("subsub1/file12")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("subsub1/file10")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("subsub1/file11")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/file11")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("subsub1/file13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/link13")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("subsub1/link13")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("subsub1/subsubsub1")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file10")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file11")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/link13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file15")));
verifyDiffReport(snapSubDir, "s3", "",
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("file15")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("file12")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("file10")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("file11")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("file11")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("file13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("link13")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("link13")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("subsubsub1")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsubsub1/file10")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsubsub1/file11")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsubsub1/file13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsubsub1/link13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsubsub1/file15")));
verifyDiffReport(snapSubSubDir, "s3", "",
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("file10")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("file11")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("file13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("link13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("file15")));
}
@Test
public void testSnapRootDescendantDiffReportWithRename() throws Exception {
Path subSub = new Path(sub1, "subsub1");
Path subSubSub = new Path(subSub, "subsubsub1");
Path nonSnapDir = new Path(dir, "non_snap");
hdfs.mkdirs(subSubSub);
hdfs.mkdirs(nonSnapDir);
hdfs.allowSnapshot(sub1);
hdfs.createSnapshot(sub1, genSnapshotName(sub1));
Path file20 = new Path(subSubSub, "file20");
DFSTestUtil.createFile(hdfs, file20, BLOCKSIZE, REPLICATION_1, SEED);
hdfs.createSnapshot(sub1, genSnapshotName(sub1));
// Case 1: Move a file away from a descendant dir, but within the snap root.
// mv <snaproot>/<subsub>/<subsubsub>/file20 <snaproot>/<subsub>/file20
hdfs.rename(file20, new Path(subSub, file20.getName()));
hdfs.createSnapshot(sub1, genSnapshotName(sub1));
// The snapshot diff for the snap root detects the change as file rename
// as the file move happened within the snap root.
verifyDiffReport(sub1, "s1", "s2",
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("subsub1")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("subsub1/subsubsub1")),
new DiffReportEntry(DiffType.RENAME,
DFSUtil.string2Bytes("subsub1/subsubsub1/file20"),
DFSUtil.string2Bytes("subsub1/file20")));
// The snapshot diff for the descendant dir <subsub> still detects the
// change as file rename as the file move happened under the snap root
// descendant dir.
verifyDiffReport(subSub, "s1", "s2",
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("subsubsub1")),
new DiffReportEntry(DiffType.RENAME,
DFSUtil.string2Bytes("subsubsub1/file20"),
DFSUtil.string2Bytes("file20")));
// The snapshot diff for the descendant dir <subsubsub> detects the
// change as file delete as the file got moved from its scope.
verifyDiffReport(subSubSub, "s1", "s2",
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("file20")));
// Case 2: Move the file from the snap root descendant dir to any
// non snap root dir. mv <snaproot>/<subsub>/file20 <nonsnaproot>/file20.
hdfs.rename(new Path(subSub, file20.getName()),
new Path(dir, file20.getName()));
hdfs.createSnapshot(sub1, genSnapshotName(sub1));
// The snapshot diff for the snap root detects the change as file delete
// as the file got moved away from the snap root dir to some non snap
// root dir.
verifyDiffReport(sub1, "s2", "s3",
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("subsub1")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("subsub1/file20")));
// The snapshot diff for the snap root descendant <subsub> detects the
// change as file delete as the file was previously under its scope and
// got moved away from its scope.
verifyDiffReport(subSub, "s2", "s3",
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("file20")));
// The file was already not under the descendant dir <subsubsub> scope.
// So, the snapshot diff report for the descendant dir doesn't
// show the file rename at all.
verifyDiffReport(subSubSub, "s2", "s3",
new DiffReportEntry[]{});
// Case 3: Move the file from the non-snap root dir to snap root dir
// mv <nonsnaproot>/file20 <snaproot>/file20
hdfs.rename(new Path(dir, file20.getName()),
new Path(sub1, file20.getName()));
hdfs.createSnapshot(sub1, genSnapshotName(sub1));
// Snap root directory should show the file moved in as a new file.
verifyDiffReport(sub1, "s3", "s4",
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("file20")));
// Snap descendant directories don't have visibility to the moved in file.
verifyDiffReport(subSub, "s3", "s4",
new DiffReportEntry[]{});
verifyDiffReport(subSubSub, "s3", "s4",
new DiffReportEntry[]{});
hdfs.rename(new Path(sub1, file20.getName()),
new Path(subSub, file20.getName()));
hdfs.createSnapshot(sub1, genSnapshotName(sub1));
// Snap root directory now shows the rename as both source and
// destination paths are under the snap root.
verifyDiffReport(sub1, "s4", "s5",
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.RENAME,
DFSUtil.string2Bytes("file20"),
DFSUtil.string2Bytes("subsub1/file20")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("subsub1")));
// For the descendant directory under the snap root, the file
// moved in shows up as a new file created.
verifyDiffReport(subSub, "s4", "s5",
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("file20")));
verifyDiffReport(subSubSub, "s4", "s5",
new DiffReportEntry[]{});
// Case 4: Snapshot diff for the newly created descendant directory.
Path subSubSub2 = new Path(subSub, "subsubsub2");
hdfs.mkdirs(subSubSub2);
Path file30 = new Path(subSubSub2, "file30");
DFSTestUtil.createFile(hdfs, file30, BLOCKSIZE, REPLICATION_1, SEED);
hdfs.createFile(file30);
hdfs.createSnapshot(sub1, genSnapshotName(sub1));
verifyDiffReport(sub1, "s5", "s6",
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("subsub1")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub2")));
verifyDiffReport(subSubSub2, "s5", "s6",
new DiffReportEntry[]{});
verifyDiffReport(subSubSub2, "s1", "s2",
new DiffReportEntry[]{});
}
@Test
public void testSnapshotDiffInfo() throws Exception {
Path snapshotRootDirPath = dir;
Path snapshotDirDescendantPath = new Path(snapshotRootDirPath, "desc");
Path snapshotDirNonDescendantPath = new Path("/dummy/non/snap/desc");
hdfs.mkdirs(snapshotDirDescendantPath);
hdfs.mkdirs(snapshotDirNonDescendantPath);
hdfs.allowSnapshot(snapshotRootDirPath);
hdfs.createSnapshot(snapshotRootDirPath, "s0");
hdfs.createSnapshot(snapshotRootDirPath, "s1");
INodeDirectory snapshotRootDir = cluster.getNameNode()
.getNamesystem().getFSDirectory().getINode(
snapshotRootDirPath.toUri().getPath())
.asDirectory();
INodeDirectory snapshotRootDescendantDir = cluster.getNameNode()
.getNamesystem().getFSDirectory().getINode(
snapshotDirDescendantPath.toUri().getPath())
.asDirectory();
INodeDirectory snapshotRootNonDescendantDir = cluster.getNameNode()
.getNamesystem().getFSDirectory().getINode(
snapshotDirNonDescendantPath.toUri().getPath())
.asDirectory();
try {
SnapshotDiffInfo sdi = new SnapshotDiffInfo(
snapshotRootDir,
snapshotRootDescendantDir,
new Snapshot(0, "s0", snapshotRootDescendantDir),
new Snapshot(0, "s1", snapshotRootDescendantDir));
LOG.info("SnapshotDiffInfo: " + sdi.getFrom() + " - " + sdi.getTo());
} catch (IllegalArgumentException iae){
fail("Unexpected exception when constructing SnapshotDiffInfo: " + iae);
}
try {
SnapshotDiffInfo sdi = new SnapshotDiffInfo(
snapshotRootDir,
snapshotRootNonDescendantDir,
new Snapshot(0, "s0", snapshotRootNonDescendantDir),
new Snapshot(0, "s1", snapshotRootNonDescendantDir));
LOG.info("SnapshotDiffInfo: " + sdi.getFrom() + " - " + sdi.getTo());
fail("SnapshotDiffInfo construction should fail for non snapshot root " +
"or non snapshot root descendant directories!");
} catch (IllegalArgumentException iae) {
// expected exception
}
}
/**
* Make changes under a sub-directory, then delete the sub-directory. Make
* sure the diff report computation correctly retrieve the diff from the
* deleted sub-directory.
*/
@Test (timeout=60000)
public void testDiffReport2() throws Exception {
Path subsub1 = new Path(sub1, "subsub1");
Path subsubsub1 = new Path(subsub1, "subsubsub1");
hdfs.mkdirs(subsubsub1);
modifyAndCreateSnapshot(subsubsub1, new Path[]{sub1});
// delete subsub1
hdfs.delete(subsub1, true);
// check diff report between s0 and s2
verifyDiffReport(sub1, "s0", "s2",
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("subsub1/subsubsub1")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file15")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file12")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file11")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/file11")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("subsub1/subsubsub1/file13")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("subsub1/subsubsub1/link13")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("subsub1/subsubsub1/link13")));
// check diff report between s0 and the current status
verifyDiffReport(sub1, "s0", "",
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("subsub1")));
}
@Test
public void testDiffReportWithQuota() throws Exception {
final Path testdir = new Path(sub1, "testdir1");
hdfs.mkdirs(testdir);
hdfs.allowSnapshot(testdir);
// Set quota BEFORE creating the snapshot
hdfs.setQuota(testdir, 10, 10);
hdfs.createSnapshot(testdir, "s0");
final SnapshotDiffReport report =
hdfs.getSnapshotDiffReport(testdir, "s0", "");
// The diff should be null. Snapshot dir inode should keep the quota.
Assert.assertEquals(0, report.getDiffList().size());
// Cleanup
hdfs.deleteSnapshot(testdir, "s0");
hdfs.disallowSnapshot(testdir);
hdfs.delete(testdir, true);
}
/**
* Rename a directory to its prior descendant, and verify the diff report.
*/
@Test
public void testDiffReportWithRename() throws Exception {
final Path root = new Path("/");
final Path sdir1 = new Path(root, "dir1");
final Path sdir2 = new Path(root, "dir2");
final Path foo = new Path(sdir1, "foo");
final Path bar = new Path(foo, "bar");
hdfs.mkdirs(bar);
hdfs.mkdirs(sdir2);
// create snapshot on root
SnapshotTestHelper.createSnapshot(hdfs, root, "s1");
// /dir1/foo/bar -> /dir2/bar
final Path bar2 = new Path(sdir2, "bar");
hdfs.rename(bar, bar2);
// /dir1/foo -> /dir2/bar/foo
final Path foo2 = new Path(bar2, "foo");
hdfs.rename(foo, foo2);
SnapshotTestHelper.createSnapshot(hdfs, root, "s2");
// let's delete /dir2 to make things more complicated
hdfs.delete(sdir2, true);
verifyDiffReport(root, "s1", "s2",
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1")),
new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir1/foo"),
DFSUtil.string2Bytes("dir2/bar/foo")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir2")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes("dir1/foo/bar")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1/foo")),
new DiffReportEntry(DiffType.RENAME, DFSUtil
.string2Bytes("dir1/foo/bar"), DFSUtil.string2Bytes("dir2/bar")));
}
/**
* Rename a file/dir outside of the snapshottable dir should be reported as
* deleted. Rename a file/dir from outside should be reported as created.
*/
@Test
public void testDiffReportWithRenameOutside() throws Exception {
final Path root = new Path("/");
final Path dir1 = new Path(root, "dir1");
final Path dir2 = new Path(root, "dir2");
final Path foo = new Path(dir1, "foo");
final Path fileInFoo = new Path(foo, "file");
final Path bar = new Path(dir2, "bar");
final Path fileInBar = new Path(bar, "file");
DFSTestUtil.createFile(hdfs, fileInFoo, BLOCKSIZE, REPLICATION, SEED);
DFSTestUtil.createFile(hdfs, fileInBar, BLOCKSIZE, REPLICATION, SEED);
// create snapshot on /dir1
SnapshotTestHelper.createSnapshot(hdfs, dir1, "s0");
// move bar into dir1
final Path newBar = new Path(dir1, "newBar");
hdfs.rename(bar, newBar);
// move foo out of dir1 into dir2
final Path newFoo = new Path(dir2, "new");
hdfs.rename(foo, newFoo);
SnapshotTestHelper.createSnapshot(hdfs, dir1, "s1");
verifyDiffReport(dir1, "s0", "s1",
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes(newBar
.getName())),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes(foo.getName())));
}
/**
* Renaming a file/dir then delete the ancestor dir of the rename target
* should be reported as deleted.
*/
@Test
public void testDiffReportWithRenameAndDelete() throws Exception {
final Path root = new Path("/");
final Path dir1 = new Path(root, "dir1");
final Path dir2 = new Path(root, "dir2");
final Path foo = new Path(dir1, "foo");
final Path fileInFoo = new Path(foo, "file");
final Path bar = new Path(dir2, "bar");
final Path fileInBar = new Path(bar, "file");
DFSTestUtil.createFile(hdfs, fileInFoo, BLOCKSIZE, REPLICATION, SEED);
DFSTestUtil.createFile(hdfs, fileInBar, BLOCKSIZE, REPLICATION, SEED);
SnapshotTestHelper.createSnapshot(hdfs, root, "s0");
hdfs.rename(fileInFoo, fileInBar, Rename.OVERWRITE);
SnapshotTestHelper.createSnapshot(hdfs, root, "s1");
verifyDiffReport(root, "s0", "s1",
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1/foo")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir2/bar")),
new DiffReportEntry(DiffType.DELETE, DFSUtil
.string2Bytes("dir2/bar/file")),
new DiffReportEntry(DiffType.RENAME,
DFSUtil.string2Bytes("dir1/foo/file"),
DFSUtil.string2Bytes("dir2/bar/file")));
// delete bar
hdfs.delete(bar, true);
SnapshotTestHelper.createSnapshot(hdfs, root, "s2");
verifyDiffReport(root, "s0", "s2",
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1/foo")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir2")),
new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("dir2/bar")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("dir1/foo/file")));
}
@Test
public void testDiffReportWithRenameToNewDir() throws Exception {
final Path root = new Path("/");
final Path foo = new Path(root, "foo");
final Path fileInFoo = new Path(foo, "file");
DFSTestUtil.createFile(hdfs, fileInFoo, BLOCKSIZE, REPLICATION, SEED);
SnapshotTestHelper.createSnapshot(hdfs, root, "s0");
final Path bar = new Path(root, "bar");
hdfs.mkdirs(bar);
final Path fileInBar = new Path(bar, "file");
hdfs.rename(fileInFoo, fileInBar);
SnapshotTestHelper.createSnapshot(hdfs, root, "s1");
verifyDiffReport(root, "s0", "s1",
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("foo")),
new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("bar")),
new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("foo/file"),
DFSUtil.string2Bytes("bar/file")));
}
/**
* Rename a file and then append some data to it
*/
@Test
public void testDiffReportWithRenameAndAppend() throws Exception {
final Path root = new Path("/");
final Path foo = new Path(root, "foo");
DFSTestUtil.createFile(hdfs, foo, BLOCKSIZE, REPLICATION, SEED);
SnapshotTestHelper.createSnapshot(hdfs, root, "s0");
final Path bar = new Path(root, "bar");
hdfs.rename(foo, bar);
DFSTestUtil.appendFile(hdfs, bar, 10); // append 10 bytes
SnapshotTestHelper.createSnapshot(hdfs, root, "s1");
// we always put modification on the file before rename
verifyDiffReport(root, "s0", "s1",
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("foo")),
new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("foo"),
DFSUtil.string2Bytes("bar")));
}
/**
* Nested renamed dir/file and the withNameList in the WithCount node of the
* parental directory is empty due to snapshot deletion. See HDFS-6996 for
* details.
*/
@Test
public void testDiffReportWithRenameAndSnapshotDeletion() throws Exception {
final Path root = new Path("/");
final Path foo = new Path(root, "foo");
final Path bar = new Path(foo, "bar");
DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPLICATION, SEED);
SnapshotTestHelper.createSnapshot(hdfs, root, "s0");
// rename /foo to /foo2
final Path foo2 = new Path(root, "foo2");
hdfs.rename(foo, foo2);
// now /foo/bar becomes /foo2/bar
final Path bar2 = new Path(foo2, "bar");
// delete snapshot s0 so that the withNameList inside of the WithCount node
// of foo becomes empty
hdfs.deleteSnapshot(root, "s0");
// create snapshot s1 and rename bar again
SnapshotTestHelper.createSnapshot(hdfs, root, "s1");
final Path bar3 = new Path(foo2, "bar-new");
hdfs.rename(bar2, bar3);
// we always put modification on the file before rename
verifyDiffReport(root, "s1", "",
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("foo2")),
new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("foo2/bar"),
DFSUtil.string2Bytes("foo2/bar-new")));
}
private void createFile(final Path filePath) throws IOException {
DFSTestUtil.createFile(hdfs, filePath, (int) BUFFERLEN,
FILELEN, BLOCKSIZE, REPLICATION, SEED);
}
private int writeToStream(final FSDataOutputStream outputStream,
byte[] buf) throws IOException {
outputStream.write(buf);
((HdfsDataOutputStream)outputStream).hsync(
EnumSet.of(SyncFlag.UPDATE_LENGTH));
return buf.length;
}
private void restartNameNode() throws Exception {
cluster.triggerBlockReports();
NameNode nameNode = cluster.getNameNode();
NameNodeAdapter.enterSafeMode(nameNode, false);
NameNodeAdapter.saveNamespace(nameNode);
NameNodeAdapter.leaveSafeMode(nameNode);
cluster.restartNameNode(true);
}
/**
* Test Snapshot diff report for snapshots with open files captures in them.
* Also verify if the diff report remains the same across NameNode restarts.
*/
@Test (timeout = 120000)
public void testDiffReportWithOpenFiles() throws Exception {
// Construct the directory tree
final Path level0A = new Path("/level_0_A");
final Path flumeSnapRootDir = level0A;
final String flumeFileName = "flume.log";
final String flumeSnap1Name = "flume_snap_1";
final String flumeSnap2Name = "flume_snap_2";
// Create files and open a stream
final Path flumeFile = new Path(level0A, flumeFileName);
createFile(flumeFile);
FSDataOutputStream flumeOutputStream = hdfs.append(flumeFile);
// Create Snapshot S1
final Path flumeS1Dir = SnapshotTestHelper.createSnapshot(
hdfs, flumeSnapRootDir, flumeSnap1Name);
final Path flumeS1Path = new Path(flumeS1Dir, flumeFileName);
final long flumeFileLengthAfterS1 = hdfs.getFileStatus(flumeFile).getLen();
// Verify if Snap S1 file length is same as the the live one
Assert.assertEquals(flumeFileLengthAfterS1,
hdfs.getFileStatus(flumeS1Path).getLen());
verifyDiffReport(level0A, flumeSnap1Name, "",
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")));
long flumeFileWrittenDataLength = flumeFileLengthAfterS1;
int newWriteLength = (int) (BLOCKSIZE * 1.5);
byte[] buf = new byte[newWriteLength];
Random random = new Random();
random.nextBytes(buf);
// Write more data to flume file
flumeFileWrittenDataLength += writeToStream(flumeOutputStream, buf);
// Create Snapshot S2
final Path flumeS2Dir = SnapshotTestHelper.createSnapshot(
hdfs, flumeSnapRootDir, flumeSnap2Name);
final Path flumeS2Path = new Path(flumeS2Dir, flumeFileName);
// Verify live files length is same as all data written till now
final long flumeFileLengthAfterS2 = hdfs.getFileStatus(flumeFile).getLen();
Assert.assertEquals(flumeFileWrittenDataLength, flumeFileLengthAfterS2);
// Verify if Snap S2 file length is same as the live one
Assert.assertEquals(flumeFileLengthAfterS2,
hdfs.getFileStatus(flumeS2Path).getLen());
verifyDiffReport(level0A, flumeSnap1Name, "",
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes(flumeFileName)));
verifyDiffReport(level0A, flumeSnap2Name, "");
verifyDiffReport(level0A, flumeSnap1Name, flumeSnap2Name,
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes(flumeFileName)));
// Write more data to flume file
flumeFileWrittenDataLength += writeToStream(flumeOutputStream, buf);
// Verify old flume snapshots have point-in-time / frozen file lengths
// even after the live file have moved forward.
Assert.assertEquals(flumeFileLengthAfterS1,
hdfs.getFileStatus(flumeS1Path).getLen());
Assert.assertEquals(flumeFileLengthAfterS2,
hdfs.getFileStatus(flumeS2Path).getLen());
flumeOutputStream.close();
// Verify if Snap S2 file length is same as the live one
Assert.assertEquals(flumeFileWrittenDataLength,
hdfs.getFileStatus(flumeFile).getLen());
// Verify old flume snapshots have point-in-time / frozen file lengths
// even after the live file have moved forward.
Assert.assertEquals(flumeFileLengthAfterS1,
hdfs.getFileStatus(flumeS1Path).getLen());
Assert.assertEquals(flumeFileLengthAfterS2,
hdfs.getFileStatus(flumeS2Path).getLen());
verifyDiffReport(level0A, flumeSnap1Name, "",
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes(flumeFileName)));
verifyDiffReport(level0A, flumeSnap2Name, "",
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes(flumeFileName)));
verifyDiffReport(level0A, flumeSnap1Name, flumeSnap2Name,
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes(flumeFileName)));
restartNameNode();
verifyDiffReport(level0A, flumeSnap1Name, flumeSnap2Name,
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.MODIFY,
DFSUtil.string2Bytes(flumeFileName)));
}
private long getAccessTime(Path path) throws IOException {
return hdfs.getFileStatus(path).getAccessTime();
}
private String getAccessTimeStr(Path path) throws IOException {
SimpleDateFormat timeFmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
return timeFmt.format(new Date(getAccessTime(path)));
}
private Path getSSpath(Path path, Path ssRoot, String ssName) {
return new Path(ssRoot, ".snapshot/" + ssName + "/" +
path.toString().substring(ssRoot.toString().length()));
}
private void printAtime(Path path, Path ssRoot, String ssName)
throws IOException {
Path ssPath = getSSpath(path, ssRoot, ssName);
LOG.info("Access time "
+ path + ": " + getAccessTimeStr(path)
+ " " + ssPath + ": " + getAccessTimeStr(ssPath));
}
private void assertAtimeEquals(Path path, Path ssRoot,
String ssName1, String ssName2)
throws IOException {
Path ssPath1 = getSSpath(path, ssRoot, ssName1);
Path ssPath2 = getSSpath(path, ssRoot, ssName2);
assertEquals(getAccessTime(ssPath1), getAccessTime(ssPath2));
}
private void assertAtimeNotEquals(Path path, Path ssRoot,
String ssName1, String ssName2)
throws IOException {
Path ssPath1 = getSSpath(path, ssRoot, ssName1);
Path ssPath2 = getSSpath(path, ssRoot, ssName2);
assertNotEquals(getAccessTime(ssPath1), getAccessTime(ssPath2));
}
/**
* Check to see access time is not captured in snapshot when applicable.
* When DFS_NAMENODE_SNAPSHOT_SKIP_CAPTURE_ACCESSTIME_ONLY_CHANGE
* is set to true, and if a file's access time changed between two
* snapshots but has no other modification, then the access time is not
* captured in snapshot.
*/
@Test
public void testDontCaptureAccessTimeOnlyChangeReport() throws Exception {
final Path froot = new Path("/");
final Path root = new Path(froot, "/testSdiffCalc");
// items created pre enabling snapshot
final Path filePreSS = new Path(root, "fParent/filePreSS");
final Path dirPreSS = new Path(root, "dirPreSS");
final Path dirPreSSChild = new Path(dirPreSS, "dirPreSSChild");
// items created after enabling snapshot
final Path filePostSS = new Path(root, "fParent/filePostSS");
final Path dirPostSS = new Path(root, "dirPostSS");
final Path dirPostSSChild = new Path(dirPostSS, "dirPostSSChild");
DFSTestUtil.createFile(hdfs, filePreSS, BLOCKSIZE, REPLICATION, SEED);
DFSTestUtil.createFile(hdfs, dirPreSSChild, BLOCKSIZE, REPLICATION, SEED);
SnapshotTestHelper.createSnapshot(hdfs, root, "s0");
printAtime(filePreSS, root, "s0");
printAtime(dirPreSS, root, "s0");
// items created after creating the first snapshot
DFSTestUtil.createFile(hdfs, filePostSS, BLOCKSIZE, REPLICATION, SEED);
DFSTestUtil.createFile(hdfs, dirPostSSChild, BLOCKSIZE, REPLICATION, SEED);
Thread.sleep(3000);
long now = Time.now();
hdfs.setTimes(filePreSS, -1, now);
hdfs.setTimes(filePostSS, -1, now);
hdfs.setTimes(dirPreSS, -1, now);
hdfs.setTimes(dirPostSS, -1, now);
SnapshotTestHelper.createSnapshot(hdfs, root, "s1");
printAtime(filePreSS, root, "s1");
printAtime(dirPreSS, root, "s1");
printAtime(filePostSS, root, "s1");
printAtime(dirPostSS, root, "s1");
Thread.sleep(3000);
now = Time.now();
hdfs.setTimes(filePreSS, -1, now);
hdfs.setTimes(filePostSS, -1, now);
hdfs.setTimes(dirPreSS, -1, now);
hdfs.setTimes(dirPostSS, -1, now);
SnapshotTestHelper.createSnapshot(hdfs, root, "s2");
printAtime(filePreSS, root, "s2");
printAtime(dirPreSS, root, "s2");
printAtime(filePostSS, root, "s2");
printAtime(dirPostSS, root, "s2");
Thread.sleep(3000);
now = Time.now();
// modify filePostSS, and change access time
hdfs.setReplication(filePostSS, (short) (REPLICATION - 1));
hdfs.setTimes(filePostSS, -1, now);
SnapshotTestHelper.createSnapshot(hdfs, root, "s3");
LOG.info("\nsnapshotDiff s0 -> s1:");
LOG.info(hdfs.getSnapshotDiffReport(root, "s0", "s1").toString());
LOG.info("\nsnapshotDiff s1 -> s2:");
LOG.info(hdfs.getSnapshotDiffReport(root, "s1", "s2").toString());
assertAtimeEquals(filePreSS, root, "s0", "s1");
assertAtimeEquals(dirPreSS, root, "s0", "s1");
assertAtimeEquals(filePreSS, root, "s1", "s2");
assertAtimeEquals(dirPreSS, root, "s1", "s2");
assertAtimeEquals(filePostSS, root, "s1", "s2");
assertAtimeEquals(dirPostSS, root, "s1", "s2");
// access time should be captured in snapshot due to
// other modification
assertAtimeNotEquals(filePostSS, root, "s2", "s3");
// restart NN, and see the access time relationship
// still stands (no change caused by edit logs
// loading)
cluster.restartNameNodes();
cluster.waitActive();
assertAtimeEquals(filePreSS, root, "s0", "s1");
assertAtimeEquals(dirPreSS, root, "s0", "s1");
assertAtimeEquals(filePreSS, root, "s1", "s2");
assertAtimeEquals(dirPreSS, root, "s1", "s2");
assertAtimeEquals(filePostSS, root, "s1", "s2");
assertAtimeEquals(dirPostSS, root, "s1", "s2");
assertAtimeNotEquals(filePostSS, root, "s2", "s3");
}
/**
* Tests to verfy the diff report with maximum SnapsdiffReportEntries limit
* over an rpc being set to 3.
* @throws Exception
*/
@Test
public void testDiffReportWithRpcLimit() throws Exception {
final Path root = new Path("/");
hdfs.mkdirs(root);
for (int i = 1; i < 4; i++) {
final Path path = new Path(root, "dir" + i);
hdfs.mkdirs(path);
}
SnapshotTestHelper.createSnapshot(hdfs, root, "s0");
for (int i = 1; i < 4; i++) {
final Path path = new Path(root, "dir" + i);
for (int j = 1; j < 4; j++) {
final Path file = new Path(path, "file" + j);
DFSTestUtil.createFile(hdfs, file, BLOCKSIZE, REPLICATION, SEED);
}
}
SnapshotTestHelper.createSnapshot(hdfs, root, "s1");
verifyDiffReport(root, "s0", "s1",
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("dir1/file1")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("dir1/file2")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("dir1/file3")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir2")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("dir2/file1")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("dir2/file2")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("dir2/file3")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir3")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("dir3/file1")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("dir3/file2")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("dir3/file3")));
}
@Test
public void testDiffReportWithRpcLimit2() throws Exception {
final Path root = new Path("/");
hdfs.mkdirs(root);
for (int i = 1; i <=3; i++) {
final Path path = new Path(root, "dir" + i);
hdfs.mkdirs(path);
}
for (int i = 1; i <= 3; i++) {
final Path path = new Path(root, "dir" + i);
for (int j = 1; j < 4; j++) {
final Path file = new Path(path, "file" + j);
DFSTestUtil.createFile(hdfs, file, BLOCKSIZE, REPLICATION, SEED);
}
}
SnapshotTestHelper.createSnapshot(hdfs, root, "s0");
Path targetDir = new Path(root, "dir4");
//create directory dir4
hdfs.mkdirs(targetDir);
//moves files from dir1 to dir4
Path path = new Path(root, "dir1");
for (int j = 1; j < 4; j++) {
final Path srcPath = new Path(path, "file" + j);
final Path targetPath = new Path(targetDir, "file" + j);
hdfs.rename(srcPath, targetPath);
}
targetDir = new Path(root, "dir3");
//overwrite existing files in dir3 from files in dir1
path = new Path(root, "dir2");
for (int j = 1; j < 4; j++) {
final Path srcPath = new Path(path, "file" + j);
final Path targetPath = new Path(targetDir, "file" + j);
hdfs.rename(srcPath, targetPath, Rename.OVERWRITE);
}
final Path pathToRename = new Path(root, "dir2");
//move dir2 inside dir3
hdfs.rename(pathToRename, targetDir);
SnapshotTestHelper.createSnapshot(hdfs, root, "s1");
verifyDiffReport(root, "s0", "s1",
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("dir4")),
new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir2"),
DFSUtil.string2Bytes("dir3/dir2")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1")),
new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir1/file1"),
DFSUtil.string2Bytes("dir4/file1")),
new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir1/file2"),
DFSUtil.string2Bytes("dir4/file2")),
new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir1/file3"),
DFSUtil.string2Bytes("dir4/file3")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir2")),
new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir2/file1"),
DFSUtil.string2Bytes("dir3/file1")),
new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir2/file2"),
DFSUtil.string2Bytes("dir3/file2")),
new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir2/file3"),
DFSUtil.string2Bytes("dir3/file3")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir3")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("dir3/file1")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("dir3/file1")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("dir3/file3")));
}
/**
* Tests to verify the diff report with maximum SnapsdiffReportEntries limit
* over an rpc being set to 3.
* @throws Exception
*/
@Test
public void testDiffReportWithRpcLimit3() throws Exception {
final Path root = new Path("/");
hdfs.mkdirs(root);
Path path = new Path(root, "dir1");
hdfs.mkdirs(path);
for (int j = 1; j <= 4; j++) {
final Path file = new Path(path, "file" + j);
DFSTestUtil.createFile(hdfs, file, BLOCKSIZE, REPLICATION, SEED);
}
SnapshotTestHelper.createSnapshot(hdfs, root, "s0");
path = new Path(root, "dir1");
for (int j = 1; j <= 4; j++) {
final Path file = new Path(path, "file" + j);
hdfs.delete(file, false);
}
for (int j = 5; j <= 10; j++) {
final Path file = new Path(path, "file" + j);
DFSTestUtil.createFile(hdfs, file, BLOCKSIZE, REPLICATION, SEED);
}
SnapshotTestHelper.createSnapshot(hdfs, root, "s1");
verifyDiffReport(root, "s0", "s1",
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("dir1/file5")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("dir1/file6")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("dir1/file7")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("dir1/file8")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("dir1/file9")),
new DiffReportEntry(DiffType.CREATE,
DFSUtil.string2Bytes("dir1/file10")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("dir1/file1")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("dir1/file2")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("dir1/file3")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("dir1/file4")));
}
private void verifyDiffReportForGivenReport(Path dirPath, String from,
String to, SnapshotDiffReport report, DiffReportEntry... entries)
throws IOException {
// reverse the order of from and to
SnapshotDiffReport inverseReport =
hdfs.getSnapshotDiffReport(dirPath, to, from);
LOG.info(report.toString());
LOG.info(inverseReport.toString() + "\n");
assertEquals(entries.length, report.getDiffList().size());
assertEquals(entries.length, inverseReport.getDiffList().size());
for (DiffReportEntry entry : entries) {
if (entry.getType() == DiffType.MODIFY) {
assertTrue(report.getDiffList().contains(entry));
assertTrue(inverseReport.getDiffList().contains(entry));
} else if (entry.getType() == DiffType.DELETE) {
assertTrue(report.getDiffList().contains(entry));
assertTrue(inverseReport.getDiffList().contains(
new DiffReportEntry(DiffType.CREATE, entry.getSourcePath())));
} else if (entry.getType() == DiffType.CREATE) {
assertTrue(report.getDiffList().contains(entry));
assertTrue(inverseReport.getDiffList().contains(
new DiffReportEntry(DiffType.DELETE, entry.getSourcePath())));
}
}
}
@Test
public void testSnapshotDiffReportRemoteIterator() throws Exception {
final Path root = new Path("/");
hdfs.mkdirs(root);
for (int i = 1; i <= 3; i++) {
final Path path = new Path(root, "dir" + i);
hdfs.mkdirs(path);
}
for (int i = 1; i <= 3; i++) {
final Path path = new Path(root, "dir" + i);
for (int j = 1; j < 4; j++) {
final Path file = new Path(path, "file" + j);
DFSTestUtil.createFile(hdfs, file, BLOCKSIZE, REPLICATION, SEED);
}
}
SnapshotTestHelper.createSnapshot(hdfs, root, "s0");
Path targetDir = new Path(root, "dir4");
//create directory dir4
hdfs.mkdirs(targetDir);
//moves files from dir1 to dir4
Path path = new Path(root, "dir1");
for (int j = 1; j < 4; j++) {
final Path srcPath = new Path(path, "file" + j);
final Path targetPath = new Path(targetDir, "file" + j);
hdfs.rename(srcPath, targetPath);
}
targetDir = new Path(root, "dir3");
//overwrite existing files in dir3 from files in dir1
path = new Path(root, "dir2");
for (int j = 1; j < 4; j++) {
final Path srcPath = new Path(path, "file" + j);
final Path targetPath = new Path(targetDir, "file" + j);
hdfs.rename(srcPath, targetPath, Rename.OVERWRITE);
}
final Path pathToRename = new Path(root, "dir2");
//move dir2 inside dir3
hdfs.rename(pathToRename, targetDir);
SnapshotTestHelper.createSnapshot(hdfs, root, "s1");
RemoteIterator<SnapshotDiffReportListing> iterator =
hdfs.snapshotDiffReportListingRemoteIterator(root, "s0", "s1");
SnapshotDiffReportGenerator snapshotDiffReport;
List<SnapshotDiffReportListing.DiffReportListingEntry> modifiedList =
new TreeList();
List<SnapshotDiffReportListing.DiffReportListingEntry> createdList =
new ChunkedArrayList<>();
List<SnapshotDiffReportListing.DiffReportListingEntry> deletedList =
new ChunkedArrayList<>();
SnapshotDiffReportListing report = null;
List<SnapshotDiffReportListing> reportList = new ArrayList<>();
while (iterator.hasNext()) {
report = iterator.next();
reportList.add(report);
modifiedList.addAll(report.getModifyList());
createdList.addAll(report.getCreateList());
deletedList.addAll(report.getDeleteList());
}
try {
iterator.next();
} catch (Exception e) {
Assert.assertTrue(
e.getMessage().contains("No more entry in SnapshotDiffReport for /"));
}
Assert.assertNotEquals(0, reportList.size());
// generate the snapshotDiffReport and Verify
snapshotDiffReport = new SnapshotDiffReportGenerator("/", "s0", "s1",
report.getIsFromEarlier(), modifiedList, createdList, deletedList);
verifyDiffReportForGivenReport(root, "s0", "s1",
snapshotDiffReport.generateReport(),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")),
new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("dir4")),
new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir2"),
DFSUtil.string2Bytes("dir3/dir2")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1")),
new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir1/file1"),
DFSUtil.string2Bytes("dir4/file1")),
new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir1/file2"),
DFSUtil.string2Bytes("dir4/file2")),
new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir1/file3"),
DFSUtil.string2Bytes("dir4/file3")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir2")),
new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir2/file1"),
DFSUtil.string2Bytes("dir3/file1")),
new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir2/file2"),
DFSUtil.string2Bytes("dir3/file2")),
new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir2/file3"),
DFSUtil.string2Bytes("dir3/file3")),
new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir3")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("dir3/file1")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("dir3/file1")),
new DiffReportEntry(DiffType.DELETE,
DFSUtil.string2Bytes("dir3/file3")));
}
@Test
public void testSnapshotDiffReportRemoteIterator2() throws Exception {
final Path root = new Path("/");
hdfs.mkdirs(root);
SnapshotTestHelper.createSnapshot(hdfs, root, "s0");
try {
hdfs.snapshotDiffReportListingRemoteIterator(root, "s0", "");
} catch (Exception e) {
Assert.assertTrue(e.getMessage().contains("Remote Iterator is"
+ "supported for snapshotDiffReport between two snapshots"));
}
}
@Test
public void testSubtrees() throws Exception {
final Path root = new Path("/");
final Path foo = new Path(root, "foo");
final Path bar = new Path(foo, "bar");
hdfs.mkdirs(bar);
modifyAndCreateSnapshot(bar, new Path[]{root});
final SnapshottableDirectoryStatus[] snapshottables
= hdfs.getSnapshottableDirListing();
Assert.assertEquals(1, snapshottables.length);
Assert.assertEquals(3, snapshottables[0].getSnapshotNumber());
final SnapshotStatus[] statuses = hdfs.getSnapshotListing(root);
Assert.assertEquals(3, statuses.length);
for (int i = 0; i < statuses.length; i++) {
final SnapshotStatus s = statuses[i];
LOG.info("Snapshot #{}: {}", s.getSnapshotID(), s.getFullPath());
Assert.assertEquals(i, s.getSnapshotID());
}
for (int i = 0; i <= 2; i++) {
for (int j = 0; j <= 2; j++) {
assertDiff(root, foo, bar, "s" + i, "s" + j);
}
}
}
private void assertDiff(Path root, Path foo, Path bar,
String from, String to) throws Exception {
final String barDiff = diff(bar, from, to);
final String fooDiff = diff(foo, from, to);
Assert.assertEquals(barDiff, fooDiff.replace("/bar", ""));
final String rootDiff = diff(root, from, to);
Assert.assertEquals(fooDiff, rootDiff.replace("/foo", ""));
Assert.assertEquals(barDiff, rootDiff.replace("/foo/bar", ""));
}
private String diff(Path path, String from, String to) throws Exception {
final SnapshotDiffReport diff = hdfs.getSnapshotDiffReport(path, from, to);
LOG.info("DIFF {} from {} to {}", path, from, to);
LOG.info("{}", diff);
final String report = diff.toString();
return report.substring(report.indexOf(":") + 1);
}
}