public void testFilesInGetListingOps() throws Exception {
createFile("/tmp1/t1", 3200, (short) 3);
createFile("/tmp1/t2", 3200, (short) 3);
createFile("/tmp2/t1", 3200, (short) 3);
createFile("/tmp2/t2", 3200, (short) 3);
cluster.getNameNode().getListing("/tmp1", HdfsFileStatus.EMPTY_NAME);
assertEquals(2, nnMetrics.numFilesInGetListingOps.getCurrentIntervalValue());
cluster.getNameNode().getListing("/tmp2", HdfsFileStatus.EMPTY_NAME);
assertEquals(4, nnMetrics.numFilesInGetListingOps.getCurrentIntervalValue());
}
@Test
public void testOpenFilesWithRename() throws Exception {
Path path = new Path("/test");
doWriteAndAbort(fs, path);
// check for zero sized blocks
Path fileWithEmptyBlock = new Path("/test/test/test4");
fs.create(fileWithEmptyBlock);
NamenodeProtocols nameNodeRpc = cluster.getNameNodeRpc();
String clientName = fs.getClient().getClientName();
// create one empty block
nameNodeRpc.addBlock(
fileWithEmptyBlock.toString(),
clientName,
null,
null,
HdfsConstants.GRANDFATHER_INODE_ID,
null);
fs.createSnapshot(path, "s2");
fs.rename(new Path("/test/test"), new Path("/test/test-renamed"));
fs.delete(new Path("/test/test-renamed"), true);
NameNode nameNode = cluster.getNameNode();
NameNodeAdapter.enterSafeMode(nameNode, false);
NameNodeAdapter.saveNamespace(nameNode);
NameNodeAdapter.leaveSafeMode(nameNode);
cluster.restartNameNode(true);
}
/**
* test 1. create DFS cluster with 3 storage directories - 2 EDITS_IMAGE, 1 EDITS 2. create a
* cluster and write a file 3. corrupt/disable one storage (or two) by removing 4. run
* doCheckpoint - it will fail on removed dirs (which will invalidate the storages) 5. write
* another file 6. check that edits and fsimage differ 7. run doCheckpoint 8. verify that all the
* image and edits files are the same.
*/
@Test
public void testStorageRestore() throws Exception {
int numDatanodes = 2;
cluster =
new MiniDFSCluster(0, config, numDatanodes, true, false, true, null, null, null, null);
cluster.waitActive();
SecondaryNameNode secondary = new SecondaryNameNode(config);
FileSystem fs = cluster.getFileSystem();
Path path = new Path("/", "test");
writeFile(fs, path, 2);
invalidateStorage(cluster.getNameNode().getFSImage());
path = new Path("/", "test1");
writeFile(fs, path, 2);
checkFiles(false);
secondary.doCheckpoint();
checkFiles(true);
secondary.shutdown();
cluster.shutdown();
}
/**
* Test if fsck can return -1 in case of failure
*
* @throws Exception
*/
public void testFsckError() throws Exception {
MiniDFSCluster cluster = null;
try {
// bring up a one-node cluster
Configuration conf = new Configuration();
cluster = new MiniDFSCluster(conf, 1, true, null);
String fileName = "/test.txt";
Path filePath = new Path(fileName);
FileSystem fs = cluster.getFileSystem();
// create a one-block file
DFSTestUtil.createFile(fs, filePath, 1L, (short) 1, 1L);
DFSTestUtil.waitReplication(fs, filePath, (short) 1);
// intentionally corrupt NN data structure
INodeFile node = (INodeFile) cluster.getNameNode().namesystem.dir.rootDir.getNode(fileName);
assertEquals(node.blocks.length, 1);
node.blocks[0].setNumBytes(-1L); // set the block length to be negative
// run fsck and expect a failure with -1 as the error code
String outStr = runFsck(conf, -1, true, fileName);
System.out.println(outStr);
assertTrue(outStr.contains(NamenodeFsck.FAILURE_STATUS));
// clean up file system
fs.delete(filePath, true);
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
protected void setupCluster(boolean simulated, long minFileSize, String[] racks, String[] hosts)
throws IOException {
conf = new Configuration();
localFileSys = FileSystem.getLocal(conf);
conf.setLong("dfs.blockreport.intervalMsec", 1000L);
conf.set("dfs.replication.pending.timeout.sec", "2");
conf.setLong("dfs.block.size", 1L);
conf.set(
"dfs.block.replicator.classname",
"org.apache.hadoop.hdfs.server.namenode.BlockPlacementPolicyRaid");
conf.setLong("hdfs.raid.min.filesize", minFileSize);
Utils.loadTestCodecs(conf, 5, 5, 1, 3, "/raid", "/raidrs", false, true);
conf.setInt("io.bytes.per.checksum", 1);
excludeFile = new Path(TEST_DIR, "exclude" + System.currentTimeMillis());
cleanFile(excludeFile);
conf.set("dfs.hosts.exclude", excludeFile.toUri().getPath());
writeConfigFile(excludeFile, null);
if (!simulated) {
cluster = new MiniDFSCluster(conf, hosts.length, true, racks, hosts);
} else {
long[] capacities = new long[] {CAPACITY, CAPACITY, CAPACITY};
cluster = new MiniDFSCluster(0, conf, hosts.length, true, true, null, racks, capacities);
}
cluster.waitActive();
namesystem = cluster.getNameNode().getNamesystem();
Assert.assertTrue(
"BlockPlacementPolicy type is not correct.",
namesystem.replicator instanceof BlockPlacementPolicyRaid);
policy = (BlockPlacementPolicyRaid) namesystem.replicator;
fs = cluster.getFileSystem();
dfs = (DistributedFileSystem) fs;
TestDirectoryRaidDfs.setupStripeStore(conf, fs);
}
@Before
public void setupCluster() throws Exception {
conf = new Configuration();
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, SMALL_BLOCK);
// Bump up replication interval so that we only run replication
// checks explicitly.
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 600);
// Increase max streams so that we re-replicate quickly.
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 1000);
// See RandomDeleterPolicy javadoc.
conf.setClass(
DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
RandomDeleterPolicy.class,
BlockPlacementPolicy.class);
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
cluster =
new MiniDFSCluster.Builder(conf)
.nnTopology(MiniDFSNNTopology.simpleHATopology())
.numDataNodes(3)
.build();
nn1 = cluster.getNameNode(0);
nn2 = cluster.getNameNode(1);
cluster.waitActive();
cluster.transitionToActive(0);
// Trigger block reports so that the first NN trusts all
// of the DNs, and will issue deletions
cluster.triggerBlockReports();
fs = HATestUtil.configureFailoverFs(cluster, conf);
}
@Before
public void setUp() throws Exception {
cluster = new MiniDFSCluster.Builder(CONF).build();
cluster.waitActive();
cluster.getNameNode();
fs = (DistributedFileSystem) cluster.getFileSystem();
}
/**
* Verify that the NameNode is able to still use <tt>READ_ONLY_SHARED</tt> replicas even when the
* single NORMAL replica is offline (and the effective replication count is 0).
*/
@Test
public void testNormalReplicaOffline() throws Exception {
// Stop the datanode hosting the NORMAL replica
cluster.stopDataNode(normalDataNode.getXferAddr());
// Force NameNode to detect that the datanode is down
BlockManagerTestUtil.noticeDeadDatanode(cluster.getNameNode(), normalDataNode.getXferAddr());
// The live replica count should now be zero (since the NORMAL replica is offline)
NumberReplicas numberReplicas = blockManager.countNodes(block);
assertThat(numberReplicas.liveReplicas(), is(0));
// The block should be reported as under-replicated
BlockManagerTestUtil.updateState(blockManager);
assertThat(blockManager.getUnderReplicatedBlocksCount(), is(1L));
// The BlockManager should be able to heal the replication count back to 1
// by triggering an inter-datanode replication from one of the READ_ONLY_SHARED replicas
BlockManagerTestUtil.computeAllPendingWork(blockManager);
DFSTestUtil.waitForReplication(cluster, extendedBlock, 1, 1, 0);
// There should now be 2 *locations* for the block, and 1 *replica*
assertThat(getLocatedBlock().getLocations().length, is(2));
validateNumberReplicas(1);
}
@Override
protected void setUp() throws Exception {
cluster = new MiniDFSCluster(CONF, 1, true, null);
cluster.waitActive();
cluster.getNameNode();
nnMetrics = NameNode.getNameNodeMetrics();
fs = (DistributedFileSystem) cluster.getFileSystem();
}
/**
* Setup a {@link MiniDFSCluster}. Create a block with both {@link State#NORMAL} and {@link
* State#READ_ONLY_SHARED} replicas.
*/
@Before
public void setup() throws IOException, InterruptedException {
conf = new HdfsConfiguration();
SimulatedFSDataset.setFactory(conf);
Configuration[] overlays = new Configuration[NUM_DATANODES];
for (int i = 0; i < overlays.length; i++) {
overlays[i] = new Configuration();
if (i == RO_NODE_INDEX) {
overlays[i].setEnum(
SimulatedFSDataset.CONFIG_PROPERTY_STATE,
i == RO_NODE_INDEX ? READ_ONLY_SHARED : NORMAL);
}
}
cluster =
new MiniDFSCluster.Builder(conf)
.numDataNodes(NUM_DATANODES)
.dataNodeConfOverlays(overlays)
.build();
fs = cluster.getFileSystem();
blockManager = cluster.getNameNode().getNamesystem().getBlockManager();
datanodeManager = blockManager.getDatanodeManager();
client =
new DFSClient(
new InetSocketAddress("localhost", cluster.getNameNodePort()),
cluster.getConfiguration(0));
for (int i = 0; i < NUM_DATANODES; i++) {
DataNode dataNode = cluster.getDataNodes().get(i);
validateStorageState(
BlockManagerTestUtil.getStorageReportsForDatanode(
datanodeManager.getDatanode(dataNode.getDatanodeId())),
i == RO_NODE_INDEX ? READ_ONLY_SHARED : NORMAL);
}
// Create a 1 block file
DFSTestUtil.createFile(fs, PATH, BLOCK_SIZE, BLOCK_SIZE, BLOCK_SIZE, (short) 1, seed);
LocatedBlock locatedBlock = getLocatedBlock();
extendedBlock = locatedBlock.getBlock();
block = extendedBlock.getLocalBlock();
assertThat(locatedBlock.getLocations().length, is(1));
normalDataNode = locatedBlock.getLocations()[0];
readOnlyDataNode =
datanodeManager.getDatanode(cluster.getDataNodes().get(RO_NODE_INDEX).getDatanodeId());
assertThat(normalDataNode, is(not(readOnlyDataNode)));
validateNumberReplicas(1);
// Inject the block into the datanode with READ_ONLY_SHARED storage
cluster.injectBlocks(0, RO_NODE_INDEX, Collections.singleton(block));
// There should now be 2 *locations* for the block
// Must wait until the NameNode has processed the block report for the injected blocks
waitForLocations(2);
}
/**
* Restart the cluster, optionally saving a new checkpoint.
*
* @param checkpoint boolean true to save a new checkpoint
* @throws Exception if restart fails
*/
private static void restart(boolean checkpoint) throws Exception {
NameNode nameNode = cluster.getNameNode();
if (checkpoint) {
NameNodeAdapter.enterSafeMode(nameNode, false);
NameNodeAdapter.saveNamespace(nameNode);
}
shutdown();
initCluster(false);
}
/**
* Test to simulate interleaved checkpointing by 2 2NNs after a storage directory has been taken
* offline. The first will cause the directory to come back online, but it won't have any valid
* contents. The second 2NN will then try to perform a checkpoint. The NN should not serve up the
* image or edits from the restored (empty) dir.
*/
@Test
public void testCheckpointWithRestoredDirectory() throws IOException {
SecondaryNameNode secondary = null;
try {
cluster = new MiniDFSCluster(0, config, 1, true, false, true, null, null, null, null);
cluster.waitActive();
secondary = new SecondaryNameNode(config);
FSImage fsImage = cluster.getNameNode().getFSImage();
FileSystem fs = cluster.getFileSystem();
Path path1 = new Path("/", "test");
writeFile(fs, path1, 2);
// Take name3 offline
fsImage.getEditLog().removeEditsAndStorageDir(2);
// Simulate a 2NN beginning a checkpoint, but not finishing. This will
// cause name3 to be restored.
cluster.getNameNode().rollEditLog();
// Now another 2NN comes along to do a full checkpoint.
secondary.doCheckpoint();
// The created file should still exist in the in-memory FS state after the
// checkpoint.
assertTrue("File missing after checkpoint", fs.exists(path1));
secondary.shutdown();
// Restart the NN so it reloads the edits from on-disk.
cluster.restartNameNode();
// The created file should still exist after the restart.
assertTrue("path should still exist after restart", fs.exists(path1));
} finally {
if (cluster != null) {
cluster.shutdown();
}
if (secondary != null) {
secondary.shutdown();
}
}
}
/**
* testing that APPEND operation can handle token expiration when re-establishing pipeline is
* needed
*/
@Test
public void testAppend() throws Exception {
MiniDFSCluster cluster = null;
int numDataNodes = 2;
Configuration conf = getConf(numDataNodes);
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
cluster.waitActive();
assertEquals(numDataNodes, cluster.getDataNodes().size());
final NameNode nn = cluster.getNameNode();
final BlockManager bm = nn.getNamesystem().getBlockManager();
final BlockTokenSecretManager sm = bm.getBlockTokenSecretManager();
// set a short token lifetime (1 second)
SecurityTestUtil.setBlockTokenLifetime(sm, 1000L);
Path fileToAppend = new Path(FILE_TO_APPEND);
FileSystem fs = cluster.getFileSystem();
byte[] expected = generateBytes(FILE_SIZE);
// write a one-byte file
FSDataOutputStream stm = writeFile(fs, fileToAppend, (short) numDataNodes, BLOCK_SIZE);
stm.write(expected, 0, 1);
stm.close();
// open the file again for append
stm = fs.append(fileToAppend);
int mid = expected.length - 1;
stm.write(expected, 1, mid - 1);
stm.hflush();
/*
* wait till token used in stm expires
*/
Token<BlockTokenIdentifier> token = DFSTestUtil.getBlockToken(stm);
while (!SecurityTestUtil.isBlockTokenExpired(token)) {
try {
Thread.sleep(10);
} catch (InterruptedException ignored) {
}
}
// remove a datanode to force re-establishing pipeline
cluster.stopDataNode(0);
// append the rest of the file
stm.write(expected, mid, expected.length - mid);
stm.close();
// check if append is successful
FSDataInputStream in5 = fs.open(fileToAppend);
assertTrue(checkFile1(in5, expected));
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
private void setupCluster() throws IOException {
setupConf();
// start the cluster with one datanode
cluster = new MiniDFSCluster(conf, 6, true, racks, hosts);
cluster.waitActive();
fs = cluster.getFileSystem();
placementMonitor = new PlacementMonitor(conf);
placementMonitor.start();
blockMover = placementMonitor.blockMover;
namenode = cluster.getNameNode();
datanodes = namenode.getDatanodeReport(DatanodeReportType.LIVE);
}
/**
* Test that, when a block is re-opened for append, the related datanode messages are correctly
* queued by the SBN because they have future states and genstamps.
*/
@Test
public void testQueueingWithAppend() throws Exception {
int numQueued = 0;
int numDN = cluster.getDataNodes().size();
FSDataOutputStream out = fs.create(TEST_FILE_PATH);
try {
AppendTestUtil.write(out, 0, 10);
out.hflush();
// Opening the file will report RBW replicas, but will be
// queued on the StandbyNode.
numQueued += numDN; // RBW messages
} finally {
IOUtils.closeStream(out);
numQueued += numDN; // blockReceived messages
}
cluster.triggerBlockReports();
numQueued += numDN;
try {
out = fs.append(TEST_FILE_PATH);
AppendTestUtil.write(out, 10, 10);
// RBW replicas once it's opened for append
numQueued += numDN;
} finally {
IOUtils.closeStream(out);
numQueued += numDN; // blockReceived
}
cluster.triggerBlockReports();
numQueued += numDN;
assertEquals(
numQueued, cluster.getNameNode(1).getNamesystem().getPendingDataNodeMessageCount());
cluster.transitionToStandby(0);
cluster.transitionToActive(1);
// Verify that no replicas are marked corrupt, and that the
// file is readable from the failed-over standby.
BlockManagerTestUtil.updateState(nn1.getNamesystem().getBlockManager());
BlockManagerTestUtil.updateState(nn2.getNamesystem().getBlockManager());
assertEquals(0, nn1.getNamesystem().getCorruptReplicaBlocks());
assertEquals(0, nn2.getNamesystem().getCorruptReplicaBlocks());
AppendTestUtil.check(fs, TEST_FILE_PATH, 20);
}
@Test(timeout = 120000)
public void testFadviseSkippedForSmallReads() throws Exception {
// start a cluster
LOG.info("testFadviseSkippedForSmallReads");
tracker.clear();
Configuration conf = new HdfsConfiguration();
conf.setBoolean(DFSConfigKeys.DFS_DATANODE_DROP_CACHE_BEHIND_READS_KEY, true);
conf.setBoolean(DFSConfigKeys.DFS_DATANODE_DROP_CACHE_BEHIND_WRITES_KEY, true);
MiniDFSCluster cluster = null;
String TEST_PATH = "/test";
int TEST_PATH_LEN = MAX_TEST_FILE_LEN;
FSDataInputStream fis = null;
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
cluster.waitActive();
FileSystem fs = cluster.getFileSystem();
// create new file
createHdfsFile(fs, new Path(TEST_PATH), TEST_PATH_LEN, null);
// Since the DataNode was configured with drop-behind, and we didn't
// specify any policy, we should have done drop-behind.
ExtendedBlock block =
cluster
.getNameNode()
.getRpcServer()
.getBlockLocations(TEST_PATH, 0, Long.MAX_VALUE)
.get(0)
.getBlock();
String fadvisedFileName = cluster.getBlockFile(0, block).getName();
Stats stats = tracker.getStats(fadvisedFileName);
stats.assertDroppedInRange(0, TEST_PATH_LEN - WRITE_PACKET_SIZE);
stats.clear();
stats.assertNotDroppedInRange(0, TEST_PATH_LEN);
// read file
fis = fs.open(new Path(TEST_PATH));
byte buf[] = new byte[17];
fis.readFully(4096, buf, 0, buf.length);
// we should not have dropped anything because of the small read.
stats = tracker.getStats(fadvisedFileName);
stats.assertNotDroppedInRange(0, TEST_PATH_LEN - WRITE_PACKET_SIZE);
} finally {
IOUtils.cleanup(null, fis);
if (cluster != null) {
cluster.shutdown();
}
}
}
private void doTestMultipleSnapshots(boolean saveNamespace) throws IOException {
Path path = new Path("/test");
doWriteAndAbort(fs, path);
fs.createSnapshot(path, "s2");
fs.delete(new Path("/test/test"), true);
fs.deleteSnapshot(path, "s2");
cluster.triggerBlockReports();
if (saveNamespace) {
NameNode nameNode = cluster.getNameNode();
NameNodeAdapter.enterSafeMode(nameNode, false);
NameNodeAdapter.saveNamespace(nameNode);
NameNodeAdapter.leaveSafeMode(nameNode);
}
cluster.restartNameNode(true);
}
@Test
public void testWithCheckpoint() throws Exception {
Path path = new Path("/test");
doWriteAndAbort(fs, path);
fs.delete(new Path("/test/test"), true);
NameNode nameNode = cluster.getNameNode();
NameNodeAdapter.enterSafeMode(nameNode, false);
NameNodeAdapter.saveNamespace(nameNode);
NameNodeAdapter.leaveSafeMode(nameNode);
cluster.restartNameNode(true);
// read snapshot file after restart
String test2snapshotPath = Snapshot.getSnapshotPath(path.toString(), "s1/test/test2");
DFSTestUtil.readFile(fs, new Path(test2snapshotPath));
String test3snapshotPath = Snapshot.getSnapshotPath(path.toString(), "s1/test/test3");
DFSTestUtil.readFile(fs, new Path(test3snapshotPath));
}
/*
* This test start datanodes with simulated mode and keep running
* chooseReplicaToDelete multiple times to get the average processing time
* and number of allocated objects
*/
@Test
public void testDirXORChooseReplicasToDeletePerformance() throws Exception {
try {
setupCluster(true, 1L, racks1, hosts1);
// create test files
int numFiles = 1000;
long blockSize = 1024L;
String parentDir = "/dir/";
for (int i = 0; i < numFiles; i++) {
String file = parentDir + "file" + i;
TestRaidDfs.createTestFile(fs, new Path(file), 3, 1, blockSize);
}
LOG.info("Created " + numFiles + " files");
Codec code = Codec.getCodec("xor");
FSNamesystem fsNameSys = cluster.getNameNode().namesystem;
for (DatanodeDescriptor dd : fsNameSys.datanodeMap.values()) {
LOG.info(dd);
}
// create fake parity file
long numStripes = RaidNode.numStripes(numFiles, code.stripeLength);
TestRaidDfs.createTestFile(
fs,
new Path(code.parityDirectory, "dir"),
3,
(int) numStripes * code.parityLength,
blockSize);
long startTime = System.currentTimeMillis();
long total = 0L;
fsNameSys.readLock();
for (BlocksMap.BlockInfo bi : fsNameSys.blocksMap.getBlocks()) {
fsNameSys.replicator.chooseReplicaToDelete(
bi.getINode(),
bi,
(short) 3,
fsNameSys.datanodeMap.values(),
new ArrayList<DatanodeDescriptor>());
total++;
}
fsNameSys.readUnlock();
LOG.info(
"Average chooseReplicaToDelete time: "
+ ((double) (System.currentTimeMillis() - startTime) / total));
} finally {
closeCluster();
}
}
// Don't move this code to the parent class. There's a binary
// incompatibility between hadoop 1 and 2 wrt MiniDFSCluster and we
// need to have two different shim classes even though they are
// exactly the same.
@Override
public HadoopShims.MiniDFSShim getMiniDfs(
Configuration conf, int numDataNodes, boolean format, String[] racks, boolean isHA)
throws IOException {
configureImpersonation(conf);
MiniDFSCluster miniDFSCluster;
if (isHA) {
MiniDFSNNTopology topo =
new MiniDFSNNTopology()
.addNameservice(
new MiniDFSNNTopology.NSConf("minidfs")
.addNN(new MiniDFSNNTopology.NNConf("nn1"))
.addNN(new MiniDFSNNTopology.NNConf("nn2")));
miniDFSCluster =
new MiniDFSCluster.Builder(conf)
.numDataNodes(numDataNodes)
.format(format)
.racks(racks)
.nnTopology(topo)
.build();
miniDFSCluster.waitActive();
miniDFSCluster.transitionToActive(0);
} else {
miniDFSCluster =
new MiniDFSCluster.Builder(conf)
.numDataNodes(numDataNodes)
.format(format)
.racks(racks)
.build();
}
// Need to set the client's KeyProvider to the NN's for JKS,
// else the updates do not get flushed properly
KeyProviderCryptoExtension keyProvider =
miniDFSCluster.getNameNode(0).getNamesystem().getProvider();
if (keyProvider != null) {
try {
setKeyProvider(miniDFSCluster.getFileSystem(0).getClient(), keyProvider);
} catch (Exception err) {
throw new IOException(err);
}
}
cluster = new MiniDFSShim(miniDFSCluster);
return cluster;
}
@Before
public void startUpCluster() throws IOException {
RaidCodecBuilder.loadDefaultFullBlocksCodecs(conf, numRSParityBlocks, numDataBlocks);
cluster = new MiniDFSCluster(conf, 4, true, null);
assertNotNull("Failed Cluster Creation", cluster);
cluster.waitClusterUp();
dfs = (DistributedFileSystem) cluster.getFileSystem();
assertNotNull("Failed to get FileSystem", dfs);
nn = cluster.getNameNode();
assertNotNull("Failed to get NameNode", nn);
Configuration newConf = new Configuration(conf);
USER1 = new UnixUserGroupInformation("foo", new String[] {"bar"});
UnixUserGroupInformation.saveToConf(newConf, UnixUserGroupInformation.UGI_PROPERTY_NAME, USER1);
userdfs = (DistributedFileSystem) FileSystem.get(newConf); // login as ugi
InjectionHandler h = new FakeBlockGeneratorInjectionHandler();
InjectionHandler.set(h);
rand.nextBytes(bytes);
}
/**
* Test if {@link FSNamesystem#computeInvalidateWork(int)} can schedule invalidate work correctly
*/
public void testCompInvalidate() throws Exception {
final Configuration conf = new Configuration();
final int NUM_OF_DATANODES = 3;
final MiniDFSCluster cluster = new MiniDFSCluster(conf, NUM_OF_DATANODES, true, null);
try {
cluster.waitActive();
final FSNamesystem namesystem = cluster.getNameNode().getNamesystem();
DatanodeDescriptor[] nodes =
namesystem.heartbeats.toArray(new DatanodeDescriptor[NUM_OF_DATANODES]);
assertEquals(nodes.length, NUM_OF_DATANODES);
synchronized (namesystem) {
for (int i = 0; i < nodes.length; i++) {
for (int j = 0; j < 3 * namesystem.blockInvalidateLimit + 1; j++) {
Block block =
new Block(
i * (namesystem.blockInvalidateLimit + 1) + j,
0,
GenerationStamp.FIRST_VALID_STAMP);
namesystem.addToInvalidatesNoLog(block, nodes[i]);
}
}
assertEquals(
namesystem.blockInvalidateLimit * NUM_OF_DATANODES,
namesystem.computeInvalidateWork(NUM_OF_DATANODES + 1));
assertEquals(
namesystem.blockInvalidateLimit * NUM_OF_DATANODES,
namesystem.computeInvalidateWork(NUM_OF_DATANODES));
assertEquals(
namesystem.blockInvalidateLimit * (NUM_OF_DATANODES - 1),
namesystem.computeInvalidateWork(NUM_OF_DATANODES - 1));
int workCount = namesystem.computeInvalidateWork(1);
if (workCount == 1) {
assertEquals(namesystem.blockInvalidateLimit + 1, namesystem.computeInvalidateWork(2));
} else {
assertEquals(workCount, namesystem.blockInvalidateLimit);
assertEquals(2, namesystem.computeInvalidateWork(2));
}
}
} finally {
cluster.shutdown();
}
}
/**
* * Test the scenario where the DataNode defaults to not dropping the cache, but our client
* defaults are set.
*/
@Test(timeout = 120000)
public void testClientDefaults() throws Exception {
// start a cluster
LOG.info("testClientDefaults");
tracker.clear();
Configuration conf = new HdfsConfiguration();
conf.setBoolean(DFSConfigKeys.DFS_DATANODE_DROP_CACHE_BEHIND_READS_KEY, false);
conf.setBoolean(DFSConfigKeys.DFS_DATANODE_DROP_CACHE_BEHIND_WRITES_KEY, false);
conf.setBoolean(DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_READS, true);
conf.setBoolean(DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_WRITES, true);
MiniDFSCluster cluster = null;
String TEST_PATH = "/test";
int TEST_PATH_LEN = MAX_TEST_FILE_LEN;
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
cluster.waitActive();
FileSystem fs = cluster.getFileSystem();
// create new file
createHdfsFile(fs, new Path(TEST_PATH), TEST_PATH_LEN, null);
// verify that we dropped everything from the cache during file creation.
ExtendedBlock block =
cluster
.getNameNode()
.getRpcServer()
.getBlockLocations(TEST_PATH, 0, Long.MAX_VALUE)
.get(0)
.getBlock();
String fadvisedFileName = cluster.getBlockFile(0, block).getName();
Stats stats = tracker.getStats(fadvisedFileName);
stats.assertDroppedInRange(0, TEST_PATH_LEN - WRITE_PACKET_SIZE);
stats.clear();
// read file
readHdfsFile(fs, new Path(TEST_PATH), Long.MAX_VALUE, null);
// verify that we dropped everything from the cache.
Assert.assertNotNull(stats);
stats.assertDroppedInRange(0, TEST_PATH_LEN - WRITE_PACKET_SIZE);
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
@Test(timeout = 120000)
public void testNoFadviseAfterWriteThenRead() throws Exception {
// start a cluster
LOG.info("testNoFadviseAfterWriteThenRead");
tracker.clear();
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = null;
String TEST_PATH = "/test";
int TEST_PATH_LEN = MAX_TEST_FILE_LEN;
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
cluster.waitActive();
FileSystem fs = cluster.getFileSystem();
// create new file
createHdfsFile(fs, new Path(TEST_PATH), TEST_PATH_LEN, false);
// verify that we did not drop everything from the cache during file creation.
ExtendedBlock block =
cluster
.getNameNode()
.getRpcServer()
.getBlockLocations(TEST_PATH, 0, Long.MAX_VALUE)
.get(0)
.getBlock();
String fadvisedFileName = cluster.getBlockFile(0, block).getName();
Stats stats = tracker.getStats(fadvisedFileName);
Assert.assertNull(stats);
// read file
readHdfsFile(fs, new Path(TEST_PATH), Long.MAX_VALUE, false);
// verify that we dropped everything from the cache.
Assert.assertNull(stats);
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
/**
* Test case which restarts the standby node in such a way that, when it exits safemode, it will
* want to invalidate a bunch of over-replicated block replicas. Ensures that if we failover at
* this point it won't lose data.
*/
@Test
public void testNNClearsCommandsOnFailoverAfterStartup() throws Exception {
// Make lots of blocks to increase chances of triggering a bug.
DFSTestUtil.createFile(fs, TEST_FILE_PATH, 30 * SMALL_BLOCK, (short) 3, 1L);
banner("Shutting down NN2");
cluster.shutdownNameNode(1);
banner("Setting replication to 1, rolling edit log.");
nn1.getRpcServer().setReplication(TEST_FILE, (short) 1);
nn1.getRpcServer().rollEditLog();
// Start NN2 again. When it starts up, it will see all of the
// blocks as over-replicated, since it has the metadata for
// replication=1, but the DNs haven't yet processed the deletions.
banner("Starting NN2 again.");
cluster.restartNameNode(1);
nn2 = cluster.getNameNode(1);
banner("triggering BRs");
cluster.triggerBlockReports();
// We expect that both NN1 and NN2 will have some number of
// deletions queued up for the DNs.
banner("computing invalidation on nn1");
BlockManagerTestUtil.computeInvalidationWork(nn1.getNamesystem().getBlockManager());
banner("computing invalidation on nn2");
BlockManagerTestUtil.computeInvalidationWork(nn2.getNamesystem().getBlockManager());
// Dump some info for debugging purposes.
banner("Metadata immediately before failover");
doMetasave(nn2);
// Transition nn2 to active even though nn1 still thinks it's active
banner("Failing to NN2 but let NN1 continue to think it's active");
NameNodeAdapter.abortEditLogs(nn1);
NameNodeAdapter.enterSafeMode(nn1, false);
cluster.transitionToActive(1);
// Check that the standby picked up the replication change.
assertEquals(1, nn2.getRpcServer().getFileInfo(TEST_FILE).getReplication());
// Dump some info for debugging purposes.
banner("Metadata immediately after failover");
doMetasave(nn2);
banner("Triggering heartbeats and block reports so that fencing is completed");
cluster.triggerHeartbeats();
cluster.triggerBlockReports();
banner("Metadata after nodes have all block-reported");
doMetasave(nn2);
// The block should no longer be postponed.
assertEquals(0, nn2.getNamesystem().getPostponedMisreplicatedBlocks());
// Wait for NN2 to enact its deletions (replication monitor has to run, etc)
BlockManagerTestUtil.computeInvalidationWork(nn2.getNamesystem().getBlockManager());
HATestUtil.waitForNNToIssueDeletions(nn2);
cluster.triggerHeartbeats();
HATestUtil.waitForDNDeletions(cluster);
cluster.triggerDeletionReports();
assertEquals(0, nn2.getNamesystem().getUnderReplicatedBlocks());
assertEquals(0, nn2.getNamesystem().getPendingReplicationBlocks());
banner("Making sure the file is still readable");
FileSystem fs2 = cluster.getFileSystem(1);
DFSTestUtil.readFile(fs2, TEST_FILE_PATH);
}
static void checkPath(MiniDFSCluster cluster, FileSystem fileSys) throws IOException {
InetSocketAddress add = cluster.getNameNode().getNameNodeAddress();
// Test upper/lower case
fileSys.checkPath(
new Path("hdfs://" + StringUtils.toUpperCase(add.getHostName()) + ":" + add.getPort()));
}
/*
* This test creates a directory with 3 files and its fake parity file.
* We decommissioned all nodes in the rack2 to make sure all data are stored
* in rack1 machine.
* Then we bring rack2 machines to normal state and create a non-raided file
* which is too small to be raided in the directory with 4 replicas
* (1 in rack1 and 3 in rack2).
* Then we reduce the replication to 3 to trigger chooseReplicatToDelete.
* We verify remaining replicas has 1 in rack1 and 2 in rack2.
*/
@Test
public void testChooseReplicasToDeleteForSmallFile() throws Exception {
try {
setupCluster(false, 512L, racks2, hosts2);
// create test files
int numFiles = 4;
long blockSize = 1024L;
String parentDir = "/dir/";
DFSClient client = getDfsClient(cluster.getNameNode(), conf);
DatanodeInfo[] infos = client.datanodeReport(DatanodeReportType.LIVE);
ArrayList<String> rack2nodes = new ArrayList<String>();
ArrayList<DatanodeInfo> rack2di = new ArrayList<DatanodeInfo>();
for (DatanodeInfo di : infos) {
if (di.getHostName().contains("rack2")) {
rack2nodes.add(di.getName());
rack2di.add(cluster.getNameNode().namesystem.getDatanode(di));
}
}
LOG.info("Decommission rack2 nodes");
writeConfigFile(excludeFile, rack2nodes);
cluster.getNameNode().namesystem.refreshNodes(conf);
waitState(rack2di, AdminStates.DECOMMISSIONED);
for (int i = 0; i < numFiles; i++) {
if (i == 2) {
continue;
}
String file = parentDir + "file" + i;
Path filePath = new Path(file);
TestRaidDfs.createTestFile(fs, filePath, 1, 1, blockSize);
printLocatedBlocks(filePath);
}
LOG.info("Created " + (numFiles - 1) + " files");
// create fake parity file
Codec code = Codec.getCodec("xor");
long numStripes = RaidNode.numStripes(numFiles, code.stripeLength);
Path parityPath = new Path(code.parityDirectory, "dir");
TestRaidDfs.createTestFile(
fs, parityPath, 1, (int) numStripes * code.parityLength, blockSize);
LOG.info("Create parity file: " + parityPath);
printLocatedBlocks(parityPath);
LOG.info("Bring back rack2 nodes out of decommission");
writeConfigFile(excludeFile, null);
cluster.getNameNode().namesystem.refreshNodes(conf);
waitState(rack2di, AdminStates.NORMAL);
Path smallFilePath = new Path(parentDir + "file2");
TestRaidDfs.createTestFile(fs, smallFilePath, 4, 1, 256L);
assertEquals(
"all datanodes should have replicas", hosts2.length, printLocatedBlocks(smallFilePath));
LOG.info("Created small file: " + smallFilePath);
LOG.info("Reduce replication to 3");
dfs.setReplication(smallFilePath, (short) 3);
long startTime = System.currentTimeMillis();
while (System.currentTimeMillis() - startTime < 120000
&& printLocatedBlocks(smallFilePath) == 4) {
Thread.sleep(1000);
}
LocatedBlocks lbs = dfs.getLocatedBlocks(smallFilePath, 0L, Integer.MAX_VALUE);
boolean hasRack1 = false;
for (DatanodeInfo di : lbs.getLocatedBlocks().get(0).getLocations()) {
if (di.getNetworkLocation().contains("rack1")) {
hasRack1 = true;
break;
}
}
assertTrue("We should keep the nodes in rack1", hasRack1);
} finally {
closeCluster();
}
}
/**
* Test that when there is a failure replicating a block the temporary and meta files are cleaned
* up and subsequent replication succeeds.
*/
@Test
public void testReplicationError() throws Exception {
// create a file of replication factor of 1
final Path fileName = new Path("/test.txt");
final int fileLen = 1;
DFSTestUtil.createFile(fs, fileName, 1, (short) 1, 1L);
DFSTestUtil.waitReplication(fs, fileName, (short) 1);
// get the block belonged to the created file
LocatedBlocks blocks =
NameNodeAdapter.getBlockLocations(
cluster.getNameNode(), fileName.toString(), 0, (long) fileLen);
assertEquals("Should only find 1 block", blocks.locatedBlockCount(), 1);
LocatedBlock block = blocks.get(0);
// bring up a second datanode
cluster.startDataNodes(conf, 1, true, null, null);
cluster.waitActive();
final int sndNode = 1;
DataNode datanode = cluster.getDataNodes().get(sndNode);
// replicate the block to the second datanode
InetSocketAddress target = datanode.getXferAddress();
Socket s = new Socket(target.getAddress(), target.getPort());
// write the header.
DataOutputStream out = new DataOutputStream(s.getOutputStream());
DataChecksum checksum = DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, 512);
new Sender(out)
.writeBlock(
block.getBlock(),
StorageType.DEFAULT,
BlockTokenSecretManager.DUMMY_TOKEN,
"",
new DatanodeInfo[0],
new StorageType[0],
null,
BlockConstructionStage.PIPELINE_SETUP_CREATE,
1,
0L,
0L,
0L,
checksum,
CachingStrategy.newDefaultStrategy(),
false);
out.flush();
// close the connection before sending the content of the block
out.close();
// the temporary block & meta files should be deleted
String bpid = cluster.getNamesystem().getBlockPoolId();
File storageDir = cluster.getInstanceStorageDir(sndNode, 0);
File dir1 = MiniDFSCluster.getRbwDir(storageDir, bpid);
storageDir = cluster.getInstanceStorageDir(sndNode, 1);
File dir2 = MiniDFSCluster.getRbwDir(storageDir, bpid);
while (dir1.listFiles().length != 0 || dir2.listFiles().length != 0) {
Thread.sleep(100);
}
// then increase the file's replication factor
fs.setReplication(fileName, (short) 2);
// replication should succeed
DFSTestUtil.waitReplication(fs, fileName, (short) 1);
// clean up the file
fs.delete(fileName, false);
}
@Test
public void testHighAvailability() throws IOException {
Configuration conf = new HdfsConfiguration();
// Create cluster with 3 readers and 1 writer
MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf)
.nnTopology(MiniDFSNNTopology.simpleHOPSTopology(4))
.numDataNodes(2)
.format(true)
.build();
cluster.waitActive();
try {
// Get the filesystem and create a directory
FileSystem fs = cluster.getFileSystem(0);
// Write operation should work since we have one writer
assertTrue(fs.mkdirs(dir));
// Write operation - Create a file and write something to it
Path file1 = new Path(dir, "file1");
createFile(fs, file1);
// Read operation - The file should exist.
assertTrue(fs.exists(file1));
// Read operation - List files in this directory
assertEquals(1, list(fs));
// Read operation - Get file status
FileStatus fileStatus = fs.listStatus(dir)[0];
// Read operation - Get block locations
assertNotSame(0, fs.getFileBlockLocations(file1, 0, 1).length);
// Now we kill all namenodes except the last two
cluster.getNameNode(0).stop();
cluster.getNameNode(1).stop();
// Now lets read again - These operations should be possible
assertTrue(fs.exists(file1));
// Writer operation - concat files
Path file2 = new Path(dir, "file2");
createFile(fs, file2);
assertTrue(fs.exists(file2));
Path file3 = new Path(dir, "file3");
createFile(fs, file3);
assertTrue(fs.exists(file3));
Path file4 = new Path(dir, "file4");
// Read operation - list files (3 files created now under this directory)
assertEquals(3, list(fs));
// Write operation - rename
// [S] commented out because rename is not yet supported
// ((DistributedFileSystem) fs).rename(file1, file4);
// Kill another namenode
cluster.getNameNode(2).stop();
// Read operation - File status
fs.getFileStatus(file2);
// Write operation - Delete
assertTrue(fs.delete(dir, true));
} catch (IOException ex) {
// In case we have any connectivity issues here, there is a problem
// All connectivitiy issues are handled in the above piece of code
LOG.error(ex);
ex.printStackTrace();
assertFalse("Cannot be any connectivity issues", ex instanceof ConnectException);
fail();
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
protected void doTestRead(Configuration conf, MiniDFSCluster cluster, boolean isStriped)
throws Exception {
final int numDataNodes = cluster.getDataNodes().size();
final NameNode nn = cluster.getNameNode();
final NamenodeProtocols nnProto = nn.getRpcServer();
final BlockManager bm = nn.getNamesystem().getBlockManager();
final BlockTokenSecretManager sm = bm.getBlockTokenSecretManager();
// set a short token lifetime (1 second) initially
SecurityTestUtil.setBlockTokenLifetime(sm, 1000L);
Path fileToRead = new Path(FILE_TO_READ);
FileSystem fs = cluster.getFileSystem();
byte[] expected = generateBytes(FILE_SIZE);
createFile(fs, fileToRead, expected);
/*
* setup for testing expiration handling of cached tokens
*/
// read using blockSeekTo(). Acquired tokens are cached in in1
FSDataInputStream in1 = fs.open(fileToRead);
assertTrue(checkFile1(in1, expected));
// read using blockSeekTo(). Acquired tokens are cached in in2
FSDataInputStream in2 = fs.open(fileToRead);
assertTrue(checkFile1(in2, expected));
// read using fetchBlockByteRange(). Acquired tokens are cached in in3
FSDataInputStream in3 = fs.open(fileToRead);
assertTrue(checkFile2(in3, expected));
/*
* testing READ interface on DN using a BlockReader
*/
DFSClient client = null;
try {
client = new DFSClient(new InetSocketAddress("localhost", cluster.getNameNodePort()), conf);
} finally {
if (client != null) client.close();
}
List<LocatedBlock> locatedBlocks =
nnProto.getBlockLocations(FILE_TO_READ, 0, FILE_SIZE).getLocatedBlocks();
LocatedBlock lblock = locatedBlocks.get(0); // first block
// verify token is not expired
assertFalse(isBlockTokenExpired(lblock));
// read with valid token, should succeed
tryRead(conf, lblock, true);
/*
* wait till myToken and all cached tokens in in1, in2 and in3 expire
*/
while (!isBlockTokenExpired(lblock)) {
try {
Thread.sleep(10);
} catch (InterruptedException ignored) {
}
}
/*
* continue testing READ interface on DN using a BlockReader
*/
// verify token is expired
assertTrue(isBlockTokenExpired(lblock));
// read should fail
tryRead(conf, lblock, false);
// use a valid new token
bm.setBlockToken(lblock, BlockTokenIdentifier.AccessMode.READ);
// read should succeed
tryRead(conf, lblock, true);
// use a token with wrong blockID
long rightId = lblock.getBlock().getBlockId();
long wrongId = rightId + 1;
lblock.getBlock().setBlockId(wrongId);
bm.setBlockToken(lblock, BlockTokenIdentifier.AccessMode.READ);
lblock.getBlock().setBlockId(rightId);
// read should fail
tryRead(conf, lblock, false);
// use a token with wrong access modes
bm.setBlockToken(lblock, BlockTokenIdentifier.AccessMode.WRITE);
// read should fail
tryRead(conf, lblock, false);
// set a long token lifetime for future tokens
SecurityTestUtil.setBlockTokenLifetime(sm, 600 * 1000L);
/*
* testing that when cached tokens are expired, DFSClient will re-fetch
* tokens transparently for READ.
*/
// confirm all tokens cached in in1 are expired by now
List<LocatedBlock> lblocks = DFSTestUtil.getAllBlocks(in1);
for (LocatedBlock blk : lblocks) {
assertTrue(isBlockTokenExpired(blk));
}
// verify blockSeekTo() is able to re-fetch token transparently
in1.seek(0);
assertTrue(checkFile1(in1, expected));
// confirm all tokens cached in in2 are expired by now
List<LocatedBlock> lblocks2 = DFSTestUtil.getAllBlocks(in2);
for (LocatedBlock blk : lblocks2) {
assertTrue(isBlockTokenExpired(blk));
}
// verify blockSeekTo() is able to re-fetch token transparently (testing
// via another interface method)
if (isStriped) {
// striped block doesn't support seekToNewSource
in2.seek(0);
} else {
assertTrue(in2.seekToNewSource(0));
}
assertTrue(checkFile1(in2, expected));
// confirm all tokens cached in in3 are expired by now
List<LocatedBlock> lblocks3 = DFSTestUtil.getAllBlocks(in3);
for (LocatedBlock blk : lblocks3) {
assertTrue(isBlockTokenExpired(blk));
}
// verify fetchBlockByteRange() is able to re-fetch token transparently
assertTrue(checkFile2(in3, expected));
/*
* testing that after datanodes are restarted on the same ports, cached
* tokens should still work and there is no need to fetch new tokens from
* namenode. This test should run while namenode is down (to make sure no
* new tokens can be fetched from namenode).
*/
// restart datanodes on the same ports that they currently use
assertTrue(cluster.restartDataNodes(true));
cluster.waitActive();
assertEquals(numDataNodes, cluster.getDataNodes().size());
cluster.shutdownNameNode(0);
// confirm tokens cached in in1 are still valid
lblocks = DFSTestUtil.getAllBlocks(in1);
for (LocatedBlock blk : lblocks) {
assertFalse(isBlockTokenExpired(blk));
}
// verify blockSeekTo() still works (forced to use cached tokens)
in1.seek(0);
assertTrue(checkFile1(in1, expected));
// confirm tokens cached in in2 are still valid
lblocks2 = DFSTestUtil.getAllBlocks(in2);
for (LocatedBlock blk : lblocks2) {
assertFalse(isBlockTokenExpired(blk));
}
// verify blockSeekTo() still works (forced to use cached tokens)
if (isStriped) {
in2.seek(0);
} else {
in2.seekToNewSource(0);
}
assertTrue(checkFile1(in2, expected));
// confirm tokens cached in in3 are still valid
lblocks3 = DFSTestUtil.getAllBlocks(in3);
for (LocatedBlock blk : lblocks3) {
assertFalse(isBlockTokenExpired(blk));
}
// verify fetchBlockByteRange() still works (forced to use cached tokens)
assertTrue(checkFile2(in3, expected));
/*
* testing that when namenode is restarted, cached tokens should still
* work and there is no need to fetch new tokens from namenode. Like the
* previous test, this test should also run while namenode is down. The
* setup for this test depends on the previous test.
*/
// restart the namenode and then shut it down for test
cluster.restartNameNode(0);
cluster.shutdownNameNode(0);
// verify blockSeekTo() still works (forced to use cached tokens)
in1.seek(0);
assertTrue(checkFile1(in1, expected));
// verify again blockSeekTo() still works (forced to use cached tokens)
if (isStriped) {
in2.seek(0);
} else {
in2.seekToNewSource(0);
}
assertTrue(checkFile1(in2, expected));
// verify fetchBlockByteRange() still works (forced to use cached tokens)
assertTrue(checkFile2(in3, expected));
/*
* testing that after both namenode and datanodes got restarted (namenode
* first, followed by datanodes), DFSClient can't access DN without
* re-fetching tokens and is able to re-fetch tokens transparently. The
* setup of this test depends on the previous test.
*/
// restore the cluster and restart the datanodes for test
cluster.restartNameNode(0);
assertTrue(cluster.restartDataNodes(true));
cluster.waitActive();
assertEquals(numDataNodes, cluster.getDataNodes().size());
// shutdown namenode so that DFSClient can't get new tokens from namenode
cluster.shutdownNameNode(0);
// verify blockSeekTo() fails (cached tokens become invalid)
in1.seek(0);
assertFalse(checkFile1(in1, expected));
// verify fetchBlockByteRange() fails (cached tokens become invalid)
assertFalse(checkFile2(in3, expected));
// restart the namenode to allow DFSClient to re-fetch tokens
cluster.restartNameNode(0);
// verify blockSeekTo() works again (by transparently re-fetching
// tokens from namenode)
in1.seek(0);
assertTrue(checkFile1(in1, expected));
if (isStriped) {
in2.seek(0);
} else {
in2.seekToNewSource(0);
}
assertTrue(checkFile1(in2, expected));
// verify fetchBlockByteRange() works again (by transparently
// re-fetching tokens from namenode)
assertTrue(checkFile2(in3, expected));
/*
* testing that when datanodes are restarted on different ports, DFSClient
* is able to re-fetch tokens transparently to connect to them
*/
// restart datanodes on newly assigned ports
assertTrue(cluster.restartDataNodes(false));
cluster.waitActive();
assertEquals(numDataNodes, cluster.getDataNodes().size());
// verify blockSeekTo() is able to re-fetch token transparently
in1.seek(0);
assertTrue(checkFile1(in1, expected));
// verify blockSeekTo() is able to re-fetch token transparently
if (isStriped) {
in2.seek(0);
} else {
in2.seekToNewSource(0);
}
assertTrue(checkFile1(in2, expected));
// verify fetchBlockByteRange() is able to re-fetch token transparently
assertTrue(checkFile2(in3, expected));
}