/**
* Add a namenode to cluster and start it. Configuration of datanodes in the cluster is refreshed
* to register with the new namenode.
*
* @return newly started namenode
*/
public NameNodeInfo addNameNode(Configuration conf) throws IOException {
if (!federation) {
throw new IOException("cannot add namenode to non-federated cluster");
}
int nnIndex = nameNodes.length;
int numNameNodes = nameNodes.length + 1;
NameNodeInfo[] newlist = new NameNodeInfo[numNameNodes];
System.arraycopy(nameNodes, 0, newlist, 0, nameNodes.length);
nameNodes = newlist;
nameNodes[nnIndex] = new NameNodeInfo(nnIndex);
NameNodeInfo nni = nameNodes[nnIndex];
nni.createAvatarDirs();
String nameserviceId = NAMESERVICE_ID_PREFIX + nnIndex;
String nameserviceIds = conf.get(FSConstants.DFS_FEDERATION_NAMESERVICES);
nameserviceIds += "," + nameserviceId;
nni.initGeneralConf(conf, nameserviceId);
conf.set(FSConstants.DFS_FEDERATION_NAMESERVICES, nameserviceIds);
nni.updateAvatarConf(conf);
startAvatarNode(nni);
// Refresh datanodes with the newly started namenode
for (DataNodeProperties dn : dataNodes) {
DataNode datanode = dn.datanode;
datanode.refreshNamenodes(conf);
}
// Wait for new namenode to get registrations from all the datanodes
waitDataNodesActive(nnIndex);
return nni;
}
@Override
public void _processEventIO(InjectionEventI event, Object... args) throws IOException {
if (event == InjectionEvent.DATANODE_PROCESS_RAID_TASK) {
int namespaceId = nn.getNamespaceID();
DataNode dn = (DataNode) args[0];
RaidTaskCommand rtc = (RaidTaskCommand) args[1];
RaidTask[] tasks = rtc.tasks;
for (RaidTask rw : tasks) {
// Generate all parity block locally instead of sending them remotely
try {
for (int idx = 0; idx < rw.toRaidIdxs.length; idx++) {
Block blk = rw.stripeBlocks[rw.toRaidIdxs[idx]];
blk.setNumBytes(blockSize);
BlockDataFile.Writer dataOut =
((BlockInlineChecksumWriter)
dn.getFSDataset()
.writeToBlock(namespaceId, blk, blk, false, false, 1, 512))
.getBlockDataFile()
.getWriter(0);
dataOut.write(bytes);
dataOut.close();
dn.finalizeAndNotifyNamenode(namespaceId, blk);
}
} catch (IOException ioe) {
LOG.warn(ioe);
}
}
}
}
public JspHelper() {
fsn = FSNamesystem.getFSNamesystem();
if (DataNode.getDataNode() != null) {
nameNodeAddr = DataNode.getDataNode().getNameNodeAddr();
} else {
nameNodeAddr = fsn.getDFSNameNodeAddress();
}
}
/**
* Test that an append with no locations fails with an exception showing insufficient locations.
*/
@Test(timeout = 60000)
public void testAppendInsufficientLocations() throws Exception {
Configuration conf = new Configuration();
// lower heartbeat interval for fast recognition of DN
conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000);
conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
conf.setInt(HdfsClientConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, 3000);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
DistributedFileSystem fileSystem = null;
try {
// create a file with replication 3
fileSystem = cluster.getFileSystem();
Path f = new Path("/testAppend");
FSDataOutputStream create = fileSystem.create(f, (short) 2);
create.write("/testAppend".getBytes());
create.close();
// Check for replications
DFSTestUtil.waitReplication(fileSystem, f, (short) 2);
// Shut down all DNs that have the last block location for the file
LocatedBlocks lbs =
fileSystem.dfs.getNamenode().getBlockLocations("/testAppend", 0, Long.MAX_VALUE);
List<DataNode> dnsOfCluster = cluster.getDataNodes();
DatanodeInfo[] dnsWithLocations = lbs.getLastLocatedBlock().getLocations();
for (DataNode dn : dnsOfCluster) {
for (DatanodeInfo loc : dnsWithLocations) {
if (dn.getDatanodeId().equals(loc)) {
dn.shutdown();
DFSTestUtil.waitForDatanodeDeath(dn);
}
}
}
// Wait till 0 replication is recognized
DFSTestUtil.waitReplication(fileSystem, f, (short) 0);
// Append to the file, at this state there are 3 live DNs but none of them
// have the block.
try {
fileSystem.append(f);
fail("Append should fail because insufficient locations");
} catch (IOException e) {
LOG.info("Expected exception: ", e);
}
FSDirectory dir = cluster.getNamesystem().getFSDirectory();
final INodeFile inode = INodeFile.valueOf(dir.getINode("/testAppend"), "/testAppend");
assertTrue("File should remain closed", !inode.isUnderConstruction());
} finally {
if (null != fileSystem) {
fileSystem.close();
}
cluster.shutdown();
}
}
/** Check whether the data-node can be started. */
private boolean canStartDataNode(Configuration conf) throws IOException {
DataNode dn = null;
try {
dn = DataNode.createDataNode(new String[] {}, conf);
} catch (IOException e) {
if (e instanceof java.net.BindException) return false;
throw e;
}
dn.shutdown();
return true;
}
/**
* Another regression test for HDFS-2742. This tests the following sequence: - DN does a block
* report while file is open. This BR contains the block in RBW state. - The block report is
* delayed in reaching the standby. - The file is closed. - The standby processes the OP_ADD and
* OP_CLOSE operations before the RBW block report arrives. - The standby should not mark the
* block as corrupt.
*/
@Test
public void testRBWReportArrivesAfterEdits() throws Exception {
final CountDownLatch brFinished = new CountDownLatch(1);
DelayAnswer delayer =
new GenericTestUtils.DelayAnswer(LOG) {
@Override
protected Object passThrough(InvocationOnMock invocation) throws Throwable {
try {
return super.passThrough(invocation);
} finally {
// inform the test that our block report went through.
brFinished.countDown();
}
}
};
FSDataOutputStream out = fs.create(TEST_FILE_PATH);
try {
AppendTestUtil.write(out, 0, 10);
out.hflush();
DataNode dn = cluster.getDataNodes().get(0);
DatanodeProtocolClientSideTranslatorPB spy = DataNodeTestUtils.spyOnBposToNN(dn, nn2);
Mockito.doAnswer(delayer)
.when(spy)
.blockReport(
Mockito.<DatanodeRegistration>anyObject(),
Mockito.anyString(),
Mockito.<StorageBlockReport[]>anyObject());
dn.scheduleAllBlockReport(0);
delayer.waitForCall();
} finally {
IOUtils.closeStream(out);
}
cluster.transitionToStandby(0);
cluster.transitionToActive(1);
delayer.proceed();
brFinished.await();
// Verify that no replicas are marked corrupt, and that the
// file is readable from the failed-over standby.
BlockManagerTestUtil.updateState(nn1.getNamesystem().getBlockManager());
BlockManagerTestUtil.updateState(nn2.getNamesystem().getBlockManager());
assertEquals(0, nn1.getNamesystem().getCorruptReplicaBlocks());
assertEquals(0, nn2.getNamesystem().getCorruptReplicaBlocks());
DFSTestUtil.readFile(fs, TEST_FILE_PATH);
}
void register() throws IOException {
// get versions from the namenode
nsInfo = nameNode.versionRequest();
dnRegistration.setStorageInfo(new DataStorage(nsInfo, ""));
DataNode.setNewStorageID(dnRegistration);
// register datanode
dnRegistration = nameNode.register(dnRegistration);
}
/** Start the data-node. */
public DataNode startDataNode(int index, Configuration config) throws IOException {
String dataDir = System.getProperty("test.build.data");
File dataNodeDir = new File(dataDir, "data-" + index);
config.set("dfs.data.dir", dataNodeDir.getPath());
String[] args = new String[] {};
// NameNode will modify config with the ports it bound to
return DataNode.createDataNode(args, config);
}
/**
* TC7: Corrupted replicas are present.
*
* @throws IOException an exception might be thrown
*/
public void testTC7() throws Exception {
final short repl = 2;
final Path p = new Path("/TC7/foo");
System.out.println("p=" + p);
// a. Create file with replication factor of 2. Write half block of data. Close file.
final int len1 = (int) (BLOCK_SIZE / 2);
{
FSDataOutputStream out = fs.create(p, false, buffersize, repl, BLOCK_SIZE);
AppendTestUtil.write(out, 0, len1);
out.close();
}
DFSTestUtil.waitReplication(fs, p, repl);
// b. Log into one datanode that has one replica of this block.
// Find the block file on this datanode and truncate it to zero size.
final LocatedBlocks locatedblocks =
fs.dfs.getNamenode().getBlockLocations(p.toString(), 0L, len1);
assertEquals(1, locatedblocks.locatedBlockCount());
final LocatedBlock lb = locatedblocks.get(0);
final Block blk = lb.getBlock();
assertEquals(len1, lb.getBlockSize());
DatanodeInfo[] datanodeinfos = lb.getLocations();
assertEquals(repl, datanodeinfos.length);
final DataNode dn = cluster.getDataNode(datanodeinfos[0].getIpcPort());
final FSDataset data = (FSDataset) dn.getFSDataset();
final RandomAccessFile raf = new RandomAccessFile(data.getBlockFile(blk), "rw");
AppendTestUtil.LOG.info("dn=" + dn + ", blk=" + blk + " (length=" + blk.getNumBytes() + ")");
assertEquals(len1, raf.length());
raf.setLength(0);
raf.close();
// c. Open file in "append mode". Append a new block worth of data. Close file.
final int len2 = (int) BLOCK_SIZE;
{
FSDataOutputStream out = fs.append(p);
AppendTestUtil.write(out, len1, len2);
out.close();
}
// d. Reopen file and read two blocks worth of data.
AppendTestUtil.check(fs, p, len1 + len2);
}
/*
* Wait for the given DN to consider itself dead.
*/
public static void waitForDatanodeDeath(DataNode dn)
throws InterruptedException, TimeoutException {
final int ATTEMPTS = 10;
int count = 0;
do {
Thread.sleep(1000);
count++;
} while (dn.isDatanodeUp() && count < ATTEMPTS);
if (count == ATTEMPTS) {
throw new TimeoutException("Timed out waiting for DN to die");
}
}
/**
* Test to verify that InterDatanode RPC timesout as expected when the server DN does not respond.
*/
@Test(expected = SocketTimeoutException.class)
public void testInterDNProtocolTimeout() throws Throwable {
final Server server = new TestServer(1, true);
server.start();
final InetSocketAddress addr = NetUtils.getConnectAddress(server);
DatanodeID fakeDnId =
new DatanodeID("localhost", "localhost", "fake-storage", addr.getPort(), 0, addr.getPort());
DatanodeInfo dInfo = new DatanodeInfo(fakeDnId);
InterDatanodeProtocol proxy = null;
try {
proxy = DataNode.createInterDataNodeProtocolProxy(dInfo, conf, 500);
proxy.initReplicaRecovery(new RecoveringBlock(new ExtendedBlock("bpid", 1), null, 100));
fail("Expected SocketTimeoutException exception, but did not get.");
} finally {
if (proxy != null) {
RPC.stopProxy(proxy);
}
server.stop();
}
}
/** Test that file data becomes available before file is closed. */
public void testFileCreation() throws IOException {
Configuration conf = new Configuration();
if (simulatedStorage) {
conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true);
}
MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
FileSystem fs = cluster.getFileSystem();
try {
//
// check that / exists
//
Path path = new Path("/");
System.out.println("Path : \"" + path.toString() + "\"");
System.out.println(fs.getFileStatus(path).isDir());
assertTrue("/ should be a directory", fs.getFileStatus(path).isDir() == true);
//
// Create a directory inside /, then try to overwrite it
//
Path dir1 = new Path("/test_dir");
fs.mkdirs(dir1);
System.out.println(
"createFile: Creating " + dir1.getName() + " for overwrite of existing directory.");
try {
fs.create(dir1, true); // Create path, overwrite=true
fs.close();
assertTrue("Did not prevent directory from being overwritten.", false);
} catch (IOException ie) {
if (!ie.getMessage().contains("already exists as a directory.")) throw ie;
}
// create a new file in home directory. Do not close it.
//
Path file1 = new Path("filestatus.dat");
FSDataOutputStream stm = createFile(fs, file1, 1);
// verify that file exists in FS namespace
assertTrue(file1 + " should be a file", fs.getFileStatus(file1).isDir() == false);
System.out.println("Path : \"" + file1 + "\"");
// write to file
writeFile(stm);
// Make sure a client can read it before it is closed.
checkFile(fs, file1, 1);
// verify that file size has changed
long len = fs.getFileStatus(file1).getLen();
assertTrue(
file1
+ " should be of size "
+ (numBlocks * blockSize)
+ " but found to be of size "
+ len,
len == numBlocks * blockSize);
stm.close();
// verify that file size has changed to the full size
len = fs.getFileStatus(file1).getLen();
assertTrue(
file1 + " should be of size " + fileSize + " but found to be of size " + len,
len == fileSize);
// Check storage usage
// can't check capacities for real storage since the OS file system may be changing under us.
if (simulatedStorage) {
DataNode dn = cluster.getDataNodes().get(0);
assertEquals(fileSize, dn.getFSDataset().getDfsUsed());
assertEquals(
SimulatedFSDataset.DEFAULT_CAPACITY - fileSize, dn.getFSDataset().getRemaining());
}
} finally {
cluster.shutdown();
}
}
/** Stop the datanode. */
public void stopDataNode(DataNode dn) {
if (dn != null) {
dn.shutdown();
}
}
/**
* Test the case that a replica is reported corrupt while it is not in blocksMap. Make sure that
* ArrayIndexOutOfBounds does not thrown. See Hadoop-4351.
*
* <p>TODO HOPS This test fails as it tries to remove a non-existing replica. Calling
* findAndMarkBlockAsCorrupt from a DataNode that does not store any replica for this specific
* block will lead to a tuple did not exist exception. The reason for this is that
* BlockManager.removeStoredBlock is called with a node that does not store a replica and hence
* the delete will not be able to succeed during commit.
*/
@Test
public void testArrayOutOfBoundsException() throws Exception {
MiniDFSCluster cluster = null;
try {
Configuration conf = new HdfsConfiguration();
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
cluster.waitActive();
FileSystem fs = cluster.getFileSystem();
final Path FILE_PATH = new Path("/tmp.txt");
final long FILE_LEN = 1L;
DFSTestUtil.createFile(fs, FILE_PATH, FILE_LEN, (short) 2, 1L);
// get the block
final String bpid = cluster.getNamesystem().getBlockPoolId();
File storageDir = cluster.getInstanceStorageDir(0, 0);
File dataDir = MiniDFSCluster.getFinalizedDir(storageDir, bpid);
assertTrue("Data directory does not exist", dataDir.exists());
ExtendedBlock blk = getBlock(bpid, dataDir);
if (blk == null) {
storageDir = cluster.getInstanceStorageDir(0, 1);
dataDir = MiniDFSCluster.getFinalizedDir(storageDir, bpid);
blk = getBlock(bpid, dataDir);
}
assertFalse(
"Data directory does not contain any blocks or there was an " + "IO error", blk == null);
// start a third datanode
cluster.startDataNodes(conf, 1, true, null, null);
ArrayList<DataNode> datanodes = cluster.getDataNodes();
assertEquals(datanodes.size(), 3);
DataNode dataNode = datanodes.get(2);
// report corrupted block by the third datanode
DatanodeRegistration dnR =
DataNodeTestUtils.getDNRegistrationForBP(dataNode, blk.getBlockPoolId());
// Get the storage id of one of the storages on the datanode
String storageId =
cluster
.getNamesystem()
.getBlockManager()
.getDatanodeManager()
.getDatanode(dataNode.getDatanodeId())
.getStorageInfos()[0]
.getStorageID();
cluster
.getNamesystem()
.getBlockManager()
.findAndMarkBlockAsCorrupt(blk, new DatanodeInfo(dnR), storageId, "some test reason");
// open the file
fs.open(FILE_PATH);
// clean up
fs.delete(FILE_PATH, false);
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}