/** Test that DataStorage and BlockPoolSliceStorage remove the failed volume after failure. */ @Test(timeout = 150000) public void testFailedVolumeBeingRemovedFromDataNode() throws InterruptedException, IOException, TimeoutException { // The test uses DataNodeTestUtils#injectDataDirFailure() to simulate // volume failures which is currently not supported on Windows. assumeTrue(!Path.WINDOWS); Path file1 = new Path("/test1"); DFSTestUtil.createFile(fs, file1, 1024, (short) 2, 1L); DFSTestUtil.waitReplication(fs, file1, (short) 2); File dn0Vol1 = new File(dataDir, "data" + (2 * 0 + 1)); DataNodeTestUtils.injectDataDirFailure(dn0Vol1); DataNode dn0 = cluster.getDataNodes().get(0); checkDiskErrorSync(dn0); // Verify dn0Vol1 has been completely removed from DN0. // 1. dn0Vol1 is removed from DataStorage. DataStorage storage = dn0.getStorage(); assertEquals(1, storage.getNumStorageDirs()); for (int i = 0; i < storage.getNumStorageDirs(); i++) { Storage.StorageDirectory sd = storage.getStorageDir(i); assertFalse(sd.getRoot().getAbsolutePath().startsWith(dn0Vol1.getAbsolutePath())); } final String bpid = cluster.getNamesystem().getBlockPoolId(); BlockPoolSliceStorage bpsStorage = storage.getBPStorage(bpid); assertEquals(1, bpsStorage.getNumStorageDirs()); for (int i = 0; i < bpsStorage.getNumStorageDirs(); i++) { Storage.StorageDirectory sd = bpsStorage.getStorageDir(i); assertFalse(sd.getRoot().getAbsolutePath().startsWith(dn0Vol1.getAbsolutePath())); } // 2. dn0Vol1 is removed from FsDataset FsDatasetSpi<? extends FsVolumeSpi> data = dn0.getFSDataset(); try (FsDatasetSpi.FsVolumeReferences vols = data.getFsVolumeReferences()) { for (FsVolumeSpi volume : vols) { assertNotEquals( new File(volume.getBasePath()).getAbsoluteFile(), dn0Vol1.getAbsoluteFile()); } } // 3. all blocks on dn0Vol1 have been removed. for (ReplicaInfo replica : FsDatasetTestUtil.getReplicas(data, bpid)) { assertNotNull(replica.getVolume()); assertNotEquals( new File(replica.getVolume().getBasePath()).getAbsoluteFile(), dn0Vol1.getAbsoluteFile()); } // 4. dn0Vol1 is not in DN0's configuration and dataDirs anymore. String[] dataDirStrs = dn0.getConf().get(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY).split(","); assertEquals(1, dataDirStrs.length); assertFalse(dataDirStrs[0].contains(dn0Vol1.getAbsolutePath())); }
/** Test for {@link FsDatasetImpl#updateReplicaUnderRecovery(ExtendedBlock, long, long)} */ @Test public void testUpdateReplicaUnderRecovery() throws IOException { MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); cluster.waitActive(); String bpid = cluster.getNamesystem().getBlockPoolId(); // create a file DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem(); String filestr = "/foo"; Path filepath = new Path(filestr); DFSTestUtil.createFile(dfs, filepath, 1024L, (short) 3, 0L); // get block info final LocatedBlock locatedblock = getLastLocatedBlock(DFSClientAdapter.getDFSClient(dfs).getNamenode(), filestr); final DatanodeInfo[] datanodeinfo = locatedblock.getLocations(); Assert.assertTrue(datanodeinfo.length > 0); // get DataNode and FSDataset objects final DataNode datanode = cluster.getDataNode(datanodeinfo[0].getIpcPort()); Assert.assertTrue(datanode != null); // initReplicaRecovery final ExtendedBlock b = locatedblock.getBlock(); final long recoveryid = b.getGenerationStamp() + 1; final long newlength = b.getNumBytes() - 1; final FsDatasetSpi<?> fsdataset = DataNodeTestUtils.getFSDataset(datanode); final ReplicaRecoveryInfo rri = fsdataset.initReplicaRecovery(new RecoveringBlock(b, null, recoveryid)); // check replica final ReplicaInfo replica = FsDatasetTestUtil.fetchReplicaInfo(fsdataset, bpid, b.getBlockId()); Assert.assertEquals(ReplicaState.RUR, replica.getState()); // check meta data before update FsDatasetImpl.checkReplicaFiles(replica); // case "THIS IS NOT SUPPOSED TO HAPPEN" // with (block length) != (stored replica's on disk length). { // create a block with same id and gs but different length. final ExtendedBlock tmp = new ExtendedBlock( b.getBlockPoolId(), rri.getBlockId(), rri.getNumBytes() - 1, rri.getGenerationStamp()); try { // update should fail fsdataset.updateReplicaUnderRecovery(tmp, recoveryid, newlength); Assert.fail(); } catch (IOException ioe) { System.out.println("GOOD: getting " + ioe); } } // update final String storageID = fsdataset.updateReplicaUnderRecovery( new ExtendedBlock(b.getBlockPoolId(), rri), recoveryid, newlength); assertTrue(storageID != null); } finally { if (cluster != null) cluster.shutdown(); } }