/** Test that DataStorage and BlockPoolSliceStorage remove the failed volume after failure. */ @Test(timeout = 150000) public void testFailedVolumeBeingRemovedFromDataNode() throws InterruptedException, IOException, TimeoutException { // The test uses DataNodeTestUtils#injectDataDirFailure() to simulate // volume failures which is currently not supported on Windows. assumeTrue(!Path.WINDOWS); Path file1 = new Path("/test1"); DFSTestUtil.createFile(fs, file1, 1024, (short) 2, 1L); DFSTestUtil.waitReplication(fs, file1, (short) 2); File dn0Vol1 = new File(dataDir, "data" + (2 * 0 + 1)); DataNodeTestUtils.injectDataDirFailure(dn0Vol1); DataNode dn0 = cluster.getDataNodes().get(0); checkDiskErrorSync(dn0); // Verify dn0Vol1 has been completely removed from DN0. // 1. dn0Vol1 is removed from DataStorage. DataStorage storage = dn0.getStorage(); assertEquals(1, storage.getNumStorageDirs()); for (int i = 0; i < storage.getNumStorageDirs(); i++) { Storage.StorageDirectory sd = storage.getStorageDir(i); assertFalse(sd.getRoot().getAbsolutePath().startsWith(dn0Vol1.getAbsolutePath())); } final String bpid = cluster.getNamesystem().getBlockPoolId(); BlockPoolSliceStorage bpsStorage = storage.getBPStorage(bpid); assertEquals(1, bpsStorage.getNumStorageDirs()); for (int i = 0; i < bpsStorage.getNumStorageDirs(); i++) { Storage.StorageDirectory sd = bpsStorage.getStorageDir(i); assertFalse(sd.getRoot().getAbsolutePath().startsWith(dn0Vol1.getAbsolutePath())); } // 2. dn0Vol1 is removed from FsDataset FsDatasetSpi<? extends FsVolumeSpi> data = dn0.getFSDataset(); try (FsDatasetSpi.FsVolumeReferences vols = data.getFsVolumeReferences()) { for (FsVolumeSpi volume : vols) { assertNotEquals( new File(volume.getBasePath()).getAbsoluteFile(), dn0Vol1.getAbsoluteFile()); } } // 3. all blocks on dn0Vol1 have been removed. for (ReplicaInfo replica : FsDatasetTestUtil.getReplicas(data, bpid)) { assertNotNull(replica.getVolume()); assertNotEquals( new File(replica.getVolume().getBasePath()).getAbsoluteFile(), dn0Vol1.getAbsoluteFile()); } // 4. dn0Vol1 is not in DN0's configuration and dataDirs anymore. String[] dataDirStrs = dn0.getConf().get(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY).split(","); assertEquals(1, dataDirStrs.length); assertFalse(dataDirStrs[0].contains(dn0Vol1.getAbsolutePath())); }