/** * Test that an append with no locations fails with an exception showing insufficient locations. */ @Test(timeout = 60000) public void testAppendInsufficientLocations() throws Exception { Configuration conf = new Configuration(); // lower heartbeat interval for fast recognition of DN conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000); conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); conf.setInt(HdfsClientConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, 3000); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); DistributedFileSystem fileSystem = null; try { // create a file with replication 3 fileSystem = cluster.getFileSystem(); Path f = new Path("/testAppend"); FSDataOutputStream create = fileSystem.create(f, (short) 2); create.write("/testAppend".getBytes()); create.close(); // Check for replications DFSTestUtil.waitReplication(fileSystem, f, (short) 2); // Shut down all DNs that have the last block location for the file LocatedBlocks lbs = fileSystem.dfs.getNamenode().getBlockLocations("/testAppend", 0, Long.MAX_VALUE); List<DataNode> dnsOfCluster = cluster.getDataNodes(); DatanodeInfo[] dnsWithLocations = lbs.getLastLocatedBlock().getLocations(); for (DataNode dn : dnsOfCluster) { for (DatanodeInfo loc : dnsWithLocations) { if (dn.getDatanodeId().equals(loc)) { dn.shutdown(); DFSTestUtil.waitForDatanodeDeath(dn); } } } // Wait till 0 replication is recognized DFSTestUtil.waitReplication(fileSystem, f, (short) 0); // Append to the file, at this state there are 3 live DNs but none of them // have the block. try { fileSystem.append(f); fail("Append should fail because insufficient locations"); } catch (IOException e) { LOG.info("Expected exception: ", e); } FSDirectory dir = cluster.getNamesystem().getFSDirectory(); final INodeFile inode = INodeFile.valueOf(dir.getINode("/testAppend"), "/testAppend"); assertTrue("File should remain closed", !inode.isUnderConstruction()); } finally { if (null != fileSystem) { fileSystem.close(); } cluster.shutdown(); } }
/** * Test the case that a replica is reported corrupt while it is not in blocksMap. Make sure that * ArrayIndexOutOfBounds does not thrown. See Hadoop-4351. * * <p>TODO HOPS This test fails as it tries to remove a non-existing replica. Calling * findAndMarkBlockAsCorrupt from a DataNode that does not store any replica for this specific * block will lead to a tuple did not exist exception. The reason for this is that * BlockManager.removeStoredBlock is called with a node that does not store a replica and hence * the delete will not be able to succeed during commit. */ @Test public void testArrayOutOfBoundsException() throws Exception { MiniDFSCluster cluster = null; try { Configuration conf = new HdfsConfiguration(); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); cluster.waitActive(); FileSystem fs = cluster.getFileSystem(); final Path FILE_PATH = new Path("/tmp.txt"); final long FILE_LEN = 1L; DFSTestUtil.createFile(fs, FILE_PATH, FILE_LEN, (short) 2, 1L); // get the block final String bpid = cluster.getNamesystem().getBlockPoolId(); File storageDir = cluster.getInstanceStorageDir(0, 0); File dataDir = MiniDFSCluster.getFinalizedDir(storageDir, bpid); assertTrue("Data directory does not exist", dataDir.exists()); ExtendedBlock blk = getBlock(bpid, dataDir); if (blk == null) { storageDir = cluster.getInstanceStorageDir(0, 1); dataDir = MiniDFSCluster.getFinalizedDir(storageDir, bpid); blk = getBlock(bpid, dataDir); } assertFalse( "Data directory does not contain any blocks or there was an " + "IO error", blk == null); // start a third datanode cluster.startDataNodes(conf, 1, true, null, null); ArrayList<DataNode> datanodes = cluster.getDataNodes(); assertEquals(datanodes.size(), 3); DataNode dataNode = datanodes.get(2); // report corrupted block by the third datanode DatanodeRegistration dnR = DataNodeTestUtils.getDNRegistrationForBP(dataNode, blk.getBlockPoolId()); // Get the storage id of one of the storages on the datanode String storageId = cluster .getNamesystem() .getBlockManager() .getDatanodeManager() .getDatanode(dataNode.getDatanodeId()) .getStorageInfos()[0] .getStorageID(); cluster .getNamesystem() .getBlockManager() .findAndMarkBlockAsCorrupt(blk, new DatanodeInfo(dnR), storageId, "some test reason"); // open the file fs.open(FILE_PATH); // clean up fs.delete(FILE_PATH, false); } finally { if (cluster != null) { cluster.shutdown(); } } }