public void testFilesInGetListingOps() throws Exception { createFile("/tmp1/t1", 3200, (short) 3); createFile("/tmp1/t2", 3200, (short) 3); createFile("/tmp2/t1", 3200, (short) 3); createFile("/tmp2/t2", 3200, (short) 3); cluster.getNameNode().getListing("/tmp1", HdfsFileStatus.EMPTY_NAME); assertEquals(2, nnMetrics.numFilesInGetListingOps.getCurrentIntervalValue()); cluster.getNameNode().getListing("/tmp2", HdfsFileStatus.EMPTY_NAME); assertEquals(4, nnMetrics.numFilesInGetListingOps.getCurrentIntervalValue()); }
@Test public void testOpenFilesWithRename() throws Exception { Path path = new Path("/test"); doWriteAndAbort(fs, path); // check for zero sized blocks Path fileWithEmptyBlock = new Path("/test/test/test4"); fs.create(fileWithEmptyBlock); NamenodeProtocols nameNodeRpc = cluster.getNameNodeRpc(); String clientName = fs.getClient().getClientName(); // create one empty block nameNodeRpc.addBlock( fileWithEmptyBlock.toString(), clientName, null, null, HdfsConstants.GRANDFATHER_INODE_ID, null); fs.createSnapshot(path, "s2"); fs.rename(new Path("/test/test"), new Path("/test/test-renamed")); fs.delete(new Path("/test/test-renamed"), true); NameNode nameNode = cluster.getNameNode(); NameNodeAdapter.enterSafeMode(nameNode, false); NameNodeAdapter.saveNamespace(nameNode); NameNodeAdapter.leaveSafeMode(nameNode); cluster.restartNameNode(true); }
/** * test 1. create DFS cluster with 3 storage directories - 2 EDITS_IMAGE, 1 EDITS 2. create a * cluster and write a file 3. corrupt/disable one storage (or two) by removing 4. run * doCheckpoint - it will fail on removed dirs (which will invalidate the storages) 5. write * another file 6. check that edits and fsimage differ 7. run doCheckpoint 8. verify that all the * image and edits files are the same. */ @Test public void testStorageRestore() throws Exception { int numDatanodes = 2; cluster = new MiniDFSCluster(0, config, numDatanodes, true, false, true, null, null, null, null); cluster.waitActive(); SecondaryNameNode secondary = new SecondaryNameNode(config); FileSystem fs = cluster.getFileSystem(); Path path = new Path("/", "test"); writeFile(fs, path, 2); invalidateStorage(cluster.getNameNode().getFSImage()); path = new Path("/", "test1"); writeFile(fs, path, 2); checkFiles(false); secondary.doCheckpoint(); checkFiles(true); secondary.shutdown(); cluster.shutdown(); }
/** * Test if fsck can return -1 in case of failure * * @throws Exception */ public void testFsckError() throws Exception { MiniDFSCluster cluster = null; try { // bring up a one-node cluster Configuration conf = new Configuration(); cluster = new MiniDFSCluster(conf, 1, true, null); String fileName = "/test.txt"; Path filePath = new Path(fileName); FileSystem fs = cluster.getFileSystem(); // create a one-block file DFSTestUtil.createFile(fs, filePath, 1L, (short) 1, 1L); DFSTestUtil.waitReplication(fs, filePath, (short) 1); // intentionally corrupt NN data structure INodeFile node = (INodeFile) cluster.getNameNode().namesystem.dir.rootDir.getNode(fileName); assertEquals(node.blocks.length, 1); node.blocks[0].setNumBytes(-1L); // set the block length to be negative // run fsck and expect a failure with -1 as the error code String outStr = runFsck(conf, -1, true, fileName); System.out.println(outStr); assertTrue(outStr.contains(NamenodeFsck.FAILURE_STATUS)); // clean up file system fs.delete(filePath, true); } finally { if (cluster != null) { cluster.shutdown(); } } }
protected void setupCluster(boolean simulated, long minFileSize, String[] racks, String[] hosts) throws IOException { conf = new Configuration(); localFileSys = FileSystem.getLocal(conf); conf.setLong("dfs.blockreport.intervalMsec", 1000L); conf.set("dfs.replication.pending.timeout.sec", "2"); conf.setLong("dfs.block.size", 1L); conf.set( "dfs.block.replicator.classname", "org.apache.hadoop.hdfs.server.namenode.BlockPlacementPolicyRaid"); conf.setLong("hdfs.raid.min.filesize", minFileSize); Utils.loadTestCodecs(conf, 5, 5, 1, 3, "/raid", "/raidrs", false, true); conf.setInt("io.bytes.per.checksum", 1); excludeFile = new Path(TEST_DIR, "exclude" + System.currentTimeMillis()); cleanFile(excludeFile); conf.set("dfs.hosts.exclude", excludeFile.toUri().getPath()); writeConfigFile(excludeFile, null); if (!simulated) { cluster = new MiniDFSCluster(conf, hosts.length, true, racks, hosts); } else { long[] capacities = new long[] {CAPACITY, CAPACITY, CAPACITY}; cluster = new MiniDFSCluster(0, conf, hosts.length, true, true, null, racks, capacities); } cluster.waitActive(); namesystem = cluster.getNameNode().getNamesystem(); Assert.assertTrue( "BlockPlacementPolicy type is not correct.", namesystem.replicator instanceof BlockPlacementPolicyRaid); policy = (BlockPlacementPolicyRaid) namesystem.replicator; fs = cluster.getFileSystem(); dfs = (DistributedFileSystem) fs; TestDirectoryRaidDfs.setupStripeStore(conf, fs); }
@Before public void setupCluster() throws Exception { conf = new Configuration(); conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, SMALL_BLOCK); // Bump up replication interval so that we only run replication // checks explicitly. conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 600); // Increase max streams so that we re-replicate quickly. conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 1000); // See RandomDeleterPolicy javadoc. conf.setClass( DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY, RandomDeleterPolicy.class, BlockPlacementPolicy.class); conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); cluster = new MiniDFSCluster.Builder(conf) .nnTopology(MiniDFSNNTopology.simpleHATopology()) .numDataNodes(3) .build(); nn1 = cluster.getNameNode(0); nn2 = cluster.getNameNode(1); cluster.waitActive(); cluster.transitionToActive(0); // Trigger block reports so that the first NN trusts all // of the DNs, and will issue deletions cluster.triggerBlockReports(); fs = HATestUtil.configureFailoverFs(cluster, conf); }
@Before public void setUp() throws Exception { cluster = new MiniDFSCluster.Builder(CONF).build(); cluster.waitActive(); cluster.getNameNode(); fs = (DistributedFileSystem) cluster.getFileSystem(); }
/** * Verify that the NameNode is able to still use <tt>READ_ONLY_SHARED</tt> replicas even when the * single NORMAL replica is offline (and the effective replication count is 0). */ @Test public void testNormalReplicaOffline() throws Exception { // Stop the datanode hosting the NORMAL replica cluster.stopDataNode(normalDataNode.getXferAddr()); // Force NameNode to detect that the datanode is down BlockManagerTestUtil.noticeDeadDatanode(cluster.getNameNode(), normalDataNode.getXferAddr()); // The live replica count should now be zero (since the NORMAL replica is offline) NumberReplicas numberReplicas = blockManager.countNodes(block); assertThat(numberReplicas.liveReplicas(), is(0)); // The block should be reported as under-replicated BlockManagerTestUtil.updateState(blockManager); assertThat(blockManager.getUnderReplicatedBlocksCount(), is(1L)); // The BlockManager should be able to heal the replication count back to 1 // by triggering an inter-datanode replication from one of the READ_ONLY_SHARED replicas BlockManagerTestUtil.computeAllPendingWork(blockManager); DFSTestUtil.waitForReplication(cluster, extendedBlock, 1, 1, 0); // There should now be 2 *locations* for the block, and 1 *replica* assertThat(getLocatedBlock().getLocations().length, is(2)); validateNumberReplicas(1); }
@Override protected void setUp() throws Exception { cluster = new MiniDFSCluster(CONF, 1, true, null); cluster.waitActive(); cluster.getNameNode(); nnMetrics = NameNode.getNameNodeMetrics(); fs = (DistributedFileSystem) cluster.getFileSystem(); }
/** * Setup a {@link MiniDFSCluster}. Create a block with both {@link State#NORMAL} and {@link * State#READ_ONLY_SHARED} replicas. */ @Before public void setup() throws IOException, InterruptedException { conf = new HdfsConfiguration(); SimulatedFSDataset.setFactory(conf); Configuration[] overlays = new Configuration[NUM_DATANODES]; for (int i = 0; i < overlays.length; i++) { overlays[i] = new Configuration(); if (i == RO_NODE_INDEX) { overlays[i].setEnum( SimulatedFSDataset.CONFIG_PROPERTY_STATE, i == RO_NODE_INDEX ? READ_ONLY_SHARED : NORMAL); } } cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(NUM_DATANODES) .dataNodeConfOverlays(overlays) .build(); fs = cluster.getFileSystem(); blockManager = cluster.getNameNode().getNamesystem().getBlockManager(); datanodeManager = blockManager.getDatanodeManager(); client = new DFSClient( new InetSocketAddress("localhost", cluster.getNameNodePort()), cluster.getConfiguration(0)); for (int i = 0; i < NUM_DATANODES; i++) { DataNode dataNode = cluster.getDataNodes().get(i); validateStorageState( BlockManagerTestUtil.getStorageReportsForDatanode( datanodeManager.getDatanode(dataNode.getDatanodeId())), i == RO_NODE_INDEX ? READ_ONLY_SHARED : NORMAL); } // Create a 1 block file DFSTestUtil.createFile(fs, PATH, BLOCK_SIZE, BLOCK_SIZE, BLOCK_SIZE, (short) 1, seed); LocatedBlock locatedBlock = getLocatedBlock(); extendedBlock = locatedBlock.getBlock(); block = extendedBlock.getLocalBlock(); assertThat(locatedBlock.getLocations().length, is(1)); normalDataNode = locatedBlock.getLocations()[0]; readOnlyDataNode = datanodeManager.getDatanode(cluster.getDataNodes().get(RO_NODE_INDEX).getDatanodeId()); assertThat(normalDataNode, is(not(readOnlyDataNode))); validateNumberReplicas(1); // Inject the block into the datanode with READ_ONLY_SHARED storage cluster.injectBlocks(0, RO_NODE_INDEX, Collections.singleton(block)); // There should now be 2 *locations* for the block // Must wait until the NameNode has processed the block report for the injected blocks waitForLocations(2); }
/** * Restart the cluster, optionally saving a new checkpoint. * * @param checkpoint boolean true to save a new checkpoint * @throws Exception if restart fails */ private static void restart(boolean checkpoint) throws Exception { NameNode nameNode = cluster.getNameNode(); if (checkpoint) { NameNodeAdapter.enterSafeMode(nameNode, false); NameNodeAdapter.saveNamespace(nameNode); } shutdown(); initCluster(false); }
/** * Test to simulate interleaved checkpointing by 2 2NNs after a storage directory has been taken * offline. The first will cause the directory to come back online, but it won't have any valid * contents. The second 2NN will then try to perform a checkpoint. The NN should not serve up the * image or edits from the restored (empty) dir. */ @Test public void testCheckpointWithRestoredDirectory() throws IOException { SecondaryNameNode secondary = null; try { cluster = new MiniDFSCluster(0, config, 1, true, false, true, null, null, null, null); cluster.waitActive(); secondary = new SecondaryNameNode(config); FSImage fsImage = cluster.getNameNode().getFSImage(); FileSystem fs = cluster.getFileSystem(); Path path1 = new Path("/", "test"); writeFile(fs, path1, 2); // Take name3 offline fsImage.getEditLog().removeEditsAndStorageDir(2); // Simulate a 2NN beginning a checkpoint, but not finishing. This will // cause name3 to be restored. cluster.getNameNode().rollEditLog(); // Now another 2NN comes along to do a full checkpoint. secondary.doCheckpoint(); // The created file should still exist in the in-memory FS state after the // checkpoint. assertTrue("File missing after checkpoint", fs.exists(path1)); secondary.shutdown(); // Restart the NN so it reloads the edits from on-disk. cluster.restartNameNode(); // The created file should still exist after the restart. assertTrue("path should still exist after restart", fs.exists(path1)); } finally { if (cluster != null) { cluster.shutdown(); } if (secondary != null) { secondary.shutdown(); } } }
/** * testing that APPEND operation can handle token expiration when re-establishing pipeline is * needed */ @Test public void testAppend() throws Exception { MiniDFSCluster cluster = null; int numDataNodes = 2; Configuration conf = getConf(numDataNodes); try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build(); cluster.waitActive(); assertEquals(numDataNodes, cluster.getDataNodes().size()); final NameNode nn = cluster.getNameNode(); final BlockManager bm = nn.getNamesystem().getBlockManager(); final BlockTokenSecretManager sm = bm.getBlockTokenSecretManager(); // set a short token lifetime (1 second) SecurityTestUtil.setBlockTokenLifetime(sm, 1000L); Path fileToAppend = new Path(FILE_TO_APPEND); FileSystem fs = cluster.getFileSystem(); byte[] expected = generateBytes(FILE_SIZE); // write a one-byte file FSDataOutputStream stm = writeFile(fs, fileToAppend, (short) numDataNodes, BLOCK_SIZE); stm.write(expected, 0, 1); stm.close(); // open the file again for append stm = fs.append(fileToAppend); int mid = expected.length - 1; stm.write(expected, 1, mid - 1); stm.hflush(); /* * wait till token used in stm expires */ Token<BlockTokenIdentifier> token = DFSTestUtil.getBlockToken(stm); while (!SecurityTestUtil.isBlockTokenExpired(token)) { try { Thread.sleep(10); } catch (InterruptedException ignored) { } } // remove a datanode to force re-establishing pipeline cluster.stopDataNode(0); // append the rest of the file stm.write(expected, mid, expected.length - mid); stm.close(); // check if append is successful FSDataInputStream in5 = fs.open(fileToAppend); assertTrue(checkFile1(in5, expected)); } finally { if (cluster != null) { cluster.shutdown(); } } }
private void setupCluster() throws IOException { setupConf(); // start the cluster with one datanode cluster = new MiniDFSCluster(conf, 6, true, racks, hosts); cluster.waitActive(); fs = cluster.getFileSystem(); placementMonitor = new PlacementMonitor(conf); placementMonitor.start(); blockMover = placementMonitor.blockMover; namenode = cluster.getNameNode(); datanodes = namenode.getDatanodeReport(DatanodeReportType.LIVE); }
/** * Test that, when a block is re-opened for append, the related datanode messages are correctly * queued by the SBN because they have future states and genstamps. */ @Test public void testQueueingWithAppend() throws Exception { int numQueued = 0; int numDN = cluster.getDataNodes().size(); FSDataOutputStream out = fs.create(TEST_FILE_PATH); try { AppendTestUtil.write(out, 0, 10); out.hflush(); // Opening the file will report RBW replicas, but will be // queued on the StandbyNode. numQueued += numDN; // RBW messages } finally { IOUtils.closeStream(out); numQueued += numDN; // blockReceived messages } cluster.triggerBlockReports(); numQueued += numDN; try { out = fs.append(TEST_FILE_PATH); AppendTestUtil.write(out, 10, 10); // RBW replicas once it's opened for append numQueued += numDN; } finally { IOUtils.closeStream(out); numQueued += numDN; // blockReceived } cluster.triggerBlockReports(); numQueued += numDN; assertEquals( numQueued, cluster.getNameNode(1).getNamesystem().getPendingDataNodeMessageCount()); cluster.transitionToStandby(0); cluster.transitionToActive(1); // Verify that no replicas are marked corrupt, and that the // file is readable from the failed-over standby. BlockManagerTestUtil.updateState(nn1.getNamesystem().getBlockManager()); BlockManagerTestUtil.updateState(nn2.getNamesystem().getBlockManager()); assertEquals(0, nn1.getNamesystem().getCorruptReplicaBlocks()); assertEquals(0, nn2.getNamesystem().getCorruptReplicaBlocks()); AppendTestUtil.check(fs, TEST_FILE_PATH, 20); }
@Test(timeout = 120000) public void testFadviseSkippedForSmallReads() throws Exception { // start a cluster LOG.info("testFadviseSkippedForSmallReads"); tracker.clear(); Configuration conf = new HdfsConfiguration(); conf.setBoolean(DFSConfigKeys.DFS_DATANODE_DROP_CACHE_BEHIND_READS_KEY, true); conf.setBoolean(DFSConfigKeys.DFS_DATANODE_DROP_CACHE_BEHIND_WRITES_KEY, true); MiniDFSCluster cluster = null; String TEST_PATH = "/test"; int TEST_PATH_LEN = MAX_TEST_FILE_LEN; FSDataInputStream fis = null; try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); cluster.waitActive(); FileSystem fs = cluster.getFileSystem(); // create new file createHdfsFile(fs, new Path(TEST_PATH), TEST_PATH_LEN, null); // Since the DataNode was configured with drop-behind, and we didn't // specify any policy, we should have done drop-behind. ExtendedBlock block = cluster .getNameNode() .getRpcServer() .getBlockLocations(TEST_PATH, 0, Long.MAX_VALUE) .get(0) .getBlock(); String fadvisedFileName = cluster.getBlockFile(0, block).getName(); Stats stats = tracker.getStats(fadvisedFileName); stats.assertDroppedInRange(0, TEST_PATH_LEN - WRITE_PACKET_SIZE); stats.clear(); stats.assertNotDroppedInRange(0, TEST_PATH_LEN); // read file fis = fs.open(new Path(TEST_PATH)); byte buf[] = new byte[17]; fis.readFully(4096, buf, 0, buf.length); // we should not have dropped anything because of the small read. stats = tracker.getStats(fadvisedFileName); stats.assertNotDroppedInRange(0, TEST_PATH_LEN - WRITE_PACKET_SIZE); } finally { IOUtils.cleanup(null, fis); if (cluster != null) { cluster.shutdown(); } } }
private void doTestMultipleSnapshots(boolean saveNamespace) throws IOException { Path path = new Path("/test"); doWriteAndAbort(fs, path); fs.createSnapshot(path, "s2"); fs.delete(new Path("/test/test"), true); fs.deleteSnapshot(path, "s2"); cluster.triggerBlockReports(); if (saveNamespace) { NameNode nameNode = cluster.getNameNode(); NameNodeAdapter.enterSafeMode(nameNode, false); NameNodeAdapter.saveNamespace(nameNode); NameNodeAdapter.leaveSafeMode(nameNode); } cluster.restartNameNode(true); }
@Test public void testWithCheckpoint() throws Exception { Path path = new Path("/test"); doWriteAndAbort(fs, path); fs.delete(new Path("/test/test"), true); NameNode nameNode = cluster.getNameNode(); NameNodeAdapter.enterSafeMode(nameNode, false); NameNodeAdapter.saveNamespace(nameNode); NameNodeAdapter.leaveSafeMode(nameNode); cluster.restartNameNode(true); // read snapshot file after restart String test2snapshotPath = Snapshot.getSnapshotPath(path.toString(), "s1/test/test2"); DFSTestUtil.readFile(fs, new Path(test2snapshotPath)); String test3snapshotPath = Snapshot.getSnapshotPath(path.toString(), "s1/test/test3"); DFSTestUtil.readFile(fs, new Path(test3snapshotPath)); }
/* * This test start datanodes with simulated mode and keep running * chooseReplicaToDelete multiple times to get the average processing time * and number of allocated objects */ @Test public void testDirXORChooseReplicasToDeletePerformance() throws Exception { try { setupCluster(true, 1L, racks1, hosts1); // create test files int numFiles = 1000; long blockSize = 1024L; String parentDir = "/dir/"; for (int i = 0; i < numFiles; i++) { String file = parentDir + "file" + i; TestRaidDfs.createTestFile(fs, new Path(file), 3, 1, blockSize); } LOG.info("Created " + numFiles + " files"); Codec code = Codec.getCodec("xor"); FSNamesystem fsNameSys = cluster.getNameNode().namesystem; for (DatanodeDescriptor dd : fsNameSys.datanodeMap.values()) { LOG.info(dd); } // create fake parity file long numStripes = RaidNode.numStripes(numFiles, code.stripeLength); TestRaidDfs.createTestFile( fs, new Path(code.parityDirectory, "dir"), 3, (int) numStripes * code.parityLength, blockSize); long startTime = System.currentTimeMillis(); long total = 0L; fsNameSys.readLock(); for (BlocksMap.BlockInfo bi : fsNameSys.blocksMap.getBlocks()) { fsNameSys.replicator.chooseReplicaToDelete( bi.getINode(), bi, (short) 3, fsNameSys.datanodeMap.values(), new ArrayList<DatanodeDescriptor>()); total++; } fsNameSys.readUnlock(); LOG.info( "Average chooseReplicaToDelete time: " + ((double) (System.currentTimeMillis() - startTime) / total)); } finally { closeCluster(); } }
// Don't move this code to the parent class. There's a binary // incompatibility between hadoop 1 and 2 wrt MiniDFSCluster and we // need to have two different shim classes even though they are // exactly the same. @Override public HadoopShims.MiniDFSShim getMiniDfs( Configuration conf, int numDataNodes, boolean format, String[] racks, boolean isHA) throws IOException { configureImpersonation(conf); MiniDFSCluster miniDFSCluster; if (isHA) { MiniDFSNNTopology topo = new MiniDFSNNTopology() .addNameservice( new MiniDFSNNTopology.NSConf("minidfs") .addNN(new MiniDFSNNTopology.NNConf("nn1")) .addNN(new MiniDFSNNTopology.NNConf("nn2"))); miniDFSCluster = new MiniDFSCluster.Builder(conf) .numDataNodes(numDataNodes) .format(format) .racks(racks) .nnTopology(topo) .build(); miniDFSCluster.waitActive(); miniDFSCluster.transitionToActive(0); } else { miniDFSCluster = new MiniDFSCluster.Builder(conf) .numDataNodes(numDataNodes) .format(format) .racks(racks) .build(); } // Need to set the client's KeyProvider to the NN's for JKS, // else the updates do not get flushed properly KeyProviderCryptoExtension keyProvider = miniDFSCluster.getNameNode(0).getNamesystem().getProvider(); if (keyProvider != null) { try { setKeyProvider(miniDFSCluster.getFileSystem(0).getClient(), keyProvider); } catch (Exception err) { throw new IOException(err); } } cluster = new MiniDFSShim(miniDFSCluster); return cluster; }
@Before public void startUpCluster() throws IOException { RaidCodecBuilder.loadDefaultFullBlocksCodecs(conf, numRSParityBlocks, numDataBlocks); cluster = new MiniDFSCluster(conf, 4, true, null); assertNotNull("Failed Cluster Creation", cluster); cluster.waitClusterUp(); dfs = (DistributedFileSystem) cluster.getFileSystem(); assertNotNull("Failed to get FileSystem", dfs); nn = cluster.getNameNode(); assertNotNull("Failed to get NameNode", nn); Configuration newConf = new Configuration(conf); USER1 = new UnixUserGroupInformation("foo", new String[] {"bar"}); UnixUserGroupInformation.saveToConf(newConf, UnixUserGroupInformation.UGI_PROPERTY_NAME, USER1); userdfs = (DistributedFileSystem) FileSystem.get(newConf); // login as ugi InjectionHandler h = new FakeBlockGeneratorInjectionHandler(); InjectionHandler.set(h); rand.nextBytes(bytes); }
/** * Test if {@link FSNamesystem#computeInvalidateWork(int)} can schedule invalidate work correctly */ public void testCompInvalidate() throws Exception { final Configuration conf = new Configuration(); final int NUM_OF_DATANODES = 3; final MiniDFSCluster cluster = new MiniDFSCluster(conf, NUM_OF_DATANODES, true, null); try { cluster.waitActive(); final FSNamesystem namesystem = cluster.getNameNode().getNamesystem(); DatanodeDescriptor[] nodes = namesystem.heartbeats.toArray(new DatanodeDescriptor[NUM_OF_DATANODES]); assertEquals(nodes.length, NUM_OF_DATANODES); synchronized (namesystem) { for (int i = 0; i < nodes.length; i++) { for (int j = 0; j < 3 * namesystem.blockInvalidateLimit + 1; j++) { Block block = new Block( i * (namesystem.blockInvalidateLimit + 1) + j, 0, GenerationStamp.FIRST_VALID_STAMP); namesystem.addToInvalidatesNoLog(block, nodes[i]); } } assertEquals( namesystem.blockInvalidateLimit * NUM_OF_DATANODES, namesystem.computeInvalidateWork(NUM_OF_DATANODES + 1)); assertEquals( namesystem.blockInvalidateLimit * NUM_OF_DATANODES, namesystem.computeInvalidateWork(NUM_OF_DATANODES)); assertEquals( namesystem.blockInvalidateLimit * (NUM_OF_DATANODES - 1), namesystem.computeInvalidateWork(NUM_OF_DATANODES - 1)); int workCount = namesystem.computeInvalidateWork(1); if (workCount == 1) { assertEquals(namesystem.blockInvalidateLimit + 1, namesystem.computeInvalidateWork(2)); } else { assertEquals(workCount, namesystem.blockInvalidateLimit); assertEquals(2, namesystem.computeInvalidateWork(2)); } } } finally { cluster.shutdown(); } }
/** * * Test the scenario where the DataNode defaults to not dropping the cache, but our client * defaults are set. */ @Test(timeout = 120000) public void testClientDefaults() throws Exception { // start a cluster LOG.info("testClientDefaults"); tracker.clear(); Configuration conf = new HdfsConfiguration(); conf.setBoolean(DFSConfigKeys.DFS_DATANODE_DROP_CACHE_BEHIND_READS_KEY, false); conf.setBoolean(DFSConfigKeys.DFS_DATANODE_DROP_CACHE_BEHIND_WRITES_KEY, false); conf.setBoolean(DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_READS, true); conf.setBoolean(DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_WRITES, true); MiniDFSCluster cluster = null; String TEST_PATH = "/test"; int TEST_PATH_LEN = MAX_TEST_FILE_LEN; try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); cluster.waitActive(); FileSystem fs = cluster.getFileSystem(); // create new file createHdfsFile(fs, new Path(TEST_PATH), TEST_PATH_LEN, null); // verify that we dropped everything from the cache during file creation. ExtendedBlock block = cluster .getNameNode() .getRpcServer() .getBlockLocations(TEST_PATH, 0, Long.MAX_VALUE) .get(0) .getBlock(); String fadvisedFileName = cluster.getBlockFile(0, block).getName(); Stats stats = tracker.getStats(fadvisedFileName); stats.assertDroppedInRange(0, TEST_PATH_LEN - WRITE_PACKET_SIZE); stats.clear(); // read file readHdfsFile(fs, new Path(TEST_PATH), Long.MAX_VALUE, null); // verify that we dropped everything from the cache. Assert.assertNotNull(stats); stats.assertDroppedInRange(0, TEST_PATH_LEN - WRITE_PACKET_SIZE); } finally { if (cluster != null) { cluster.shutdown(); } } }
@Test(timeout = 120000) public void testNoFadviseAfterWriteThenRead() throws Exception { // start a cluster LOG.info("testNoFadviseAfterWriteThenRead"); tracker.clear(); Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = null; String TEST_PATH = "/test"; int TEST_PATH_LEN = MAX_TEST_FILE_LEN; try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); cluster.waitActive(); FileSystem fs = cluster.getFileSystem(); // create new file createHdfsFile(fs, new Path(TEST_PATH), TEST_PATH_LEN, false); // verify that we did not drop everything from the cache during file creation. ExtendedBlock block = cluster .getNameNode() .getRpcServer() .getBlockLocations(TEST_PATH, 0, Long.MAX_VALUE) .get(0) .getBlock(); String fadvisedFileName = cluster.getBlockFile(0, block).getName(); Stats stats = tracker.getStats(fadvisedFileName); Assert.assertNull(stats); // read file readHdfsFile(fs, new Path(TEST_PATH), Long.MAX_VALUE, false); // verify that we dropped everything from the cache. Assert.assertNull(stats); } finally { if (cluster != null) { cluster.shutdown(); } } }
/** * Test case which restarts the standby node in such a way that, when it exits safemode, it will * want to invalidate a bunch of over-replicated block replicas. Ensures that if we failover at * this point it won't lose data. */ @Test public void testNNClearsCommandsOnFailoverAfterStartup() throws Exception { // Make lots of blocks to increase chances of triggering a bug. DFSTestUtil.createFile(fs, TEST_FILE_PATH, 30 * SMALL_BLOCK, (short) 3, 1L); banner("Shutting down NN2"); cluster.shutdownNameNode(1); banner("Setting replication to 1, rolling edit log."); nn1.getRpcServer().setReplication(TEST_FILE, (short) 1); nn1.getRpcServer().rollEditLog(); // Start NN2 again. When it starts up, it will see all of the // blocks as over-replicated, since it has the metadata for // replication=1, but the DNs haven't yet processed the deletions. banner("Starting NN2 again."); cluster.restartNameNode(1); nn2 = cluster.getNameNode(1); banner("triggering BRs"); cluster.triggerBlockReports(); // We expect that both NN1 and NN2 will have some number of // deletions queued up for the DNs. banner("computing invalidation on nn1"); BlockManagerTestUtil.computeInvalidationWork(nn1.getNamesystem().getBlockManager()); banner("computing invalidation on nn2"); BlockManagerTestUtil.computeInvalidationWork(nn2.getNamesystem().getBlockManager()); // Dump some info for debugging purposes. banner("Metadata immediately before failover"); doMetasave(nn2); // Transition nn2 to active even though nn1 still thinks it's active banner("Failing to NN2 but let NN1 continue to think it's active"); NameNodeAdapter.abortEditLogs(nn1); NameNodeAdapter.enterSafeMode(nn1, false); cluster.transitionToActive(1); // Check that the standby picked up the replication change. assertEquals(1, nn2.getRpcServer().getFileInfo(TEST_FILE).getReplication()); // Dump some info for debugging purposes. banner("Metadata immediately after failover"); doMetasave(nn2); banner("Triggering heartbeats and block reports so that fencing is completed"); cluster.triggerHeartbeats(); cluster.triggerBlockReports(); banner("Metadata after nodes have all block-reported"); doMetasave(nn2); // The block should no longer be postponed. assertEquals(0, nn2.getNamesystem().getPostponedMisreplicatedBlocks()); // Wait for NN2 to enact its deletions (replication monitor has to run, etc) BlockManagerTestUtil.computeInvalidationWork(nn2.getNamesystem().getBlockManager()); HATestUtil.waitForNNToIssueDeletions(nn2); cluster.triggerHeartbeats(); HATestUtil.waitForDNDeletions(cluster); cluster.triggerDeletionReports(); assertEquals(0, nn2.getNamesystem().getUnderReplicatedBlocks()); assertEquals(0, nn2.getNamesystem().getPendingReplicationBlocks()); banner("Making sure the file is still readable"); FileSystem fs2 = cluster.getFileSystem(1); DFSTestUtil.readFile(fs2, TEST_FILE_PATH); }
static void checkPath(MiniDFSCluster cluster, FileSystem fileSys) throws IOException { InetSocketAddress add = cluster.getNameNode().getNameNodeAddress(); // Test upper/lower case fileSys.checkPath( new Path("hdfs://" + StringUtils.toUpperCase(add.getHostName()) + ":" + add.getPort())); }
/* * This test creates a directory with 3 files and its fake parity file. * We decommissioned all nodes in the rack2 to make sure all data are stored * in rack1 machine. * Then we bring rack2 machines to normal state and create a non-raided file * which is too small to be raided in the directory with 4 replicas * (1 in rack1 and 3 in rack2). * Then we reduce the replication to 3 to trigger chooseReplicatToDelete. * We verify remaining replicas has 1 in rack1 and 2 in rack2. */ @Test public void testChooseReplicasToDeleteForSmallFile() throws Exception { try { setupCluster(false, 512L, racks2, hosts2); // create test files int numFiles = 4; long blockSize = 1024L; String parentDir = "/dir/"; DFSClient client = getDfsClient(cluster.getNameNode(), conf); DatanodeInfo[] infos = client.datanodeReport(DatanodeReportType.LIVE); ArrayList<String> rack2nodes = new ArrayList<String>(); ArrayList<DatanodeInfo> rack2di = new ArrayList<DatanodeInfo>(); for (DatanodeInfo di : infos) { if (di.getHostName().contains("rack2")) { rack2nodes.add(di.getName()); rack2di.add(cluster.getNameNode().namesystem.getDatanode(di)); } } LOG.info("Decommission rack2 nodes"); writeConfigFile(excludeFile, rack2nodes); cluster.getNameNode().namesystem.refreshNodes(conf); waitState(rack2di, AdminStates.DECOMMISSIONED); for (int i = 0; i < numFiles; i++) { if (i == 2) { continue; } String file = parentDir + "file" + i; Path filePath = new Path(file); TestRaidDfs.createTestFile(fs, filePath, 1, 1, blockSize); printLocatedBlocks(filePath); } LOG.info("Created " + (numFiles - 1) + " files"); // create fake parity file Codec code = Codec.getCodec("xor"); long numStripes = RaidNode.numStripes(numFiles, code.stripeLength); Path parityPath = new Path(code.parityDirectory, "dir"); TestRaidDfs.createTestFile( fs, parityPath, 1, (int) numStripes * code.parityLength, blockSize); LOG.info("Create parity file: " + parityPath); printLocatedBlocks(parityPath); LOG.info("Bring back rack2 nodes out of decommission"); writeConfigFile(excludeFile, null); cluster.getNameNode().namesystem.refreshNodes(conf); waitState(rack2di, AdminStates.NORMAL); Path smallFilePath = new Path(parentDir + "file2"); TestRaidDfs.createTestFile(fs, smallFilePath, 4, 1, 256L); assertEquals( "all datanodes should have replicas", hosts2.length, printLocatedBlocks(smallFilePath)); LOG.info("Created small file: " + smallFilePath); LOG.info("Reduce replication to 3"); dfs.setReplication(smallFilePath, (short) 3); long startTime = System.currentTimeMillis(); while (System.currentTimeMillis() - startTime < 120000 && printLocatedBlocks(smallFilePath) == 4) { Thread.sleep(1000); } LocatedBlocks lbs = dfs.getLocatedBlocks(smallFilePath, 0L, Integer.MAX_VALUE); boolean hasRack1 = false; for (DatanodeInfo di : lbs.getLocatedBlocks().get(0).getLocations()) { if (di.getNetworkLocation().contains("rack1")) { hasRack1 = true; break; } } assertTrue("We should keep the nodes in rack1", hasRack1); } finally { closeCluster(); } }
/** * Test that when there is a failure replicating a block the temporary and meta files are cleaned * up and subsequent replication succeeds. */ @Test public void testReplicationError() throws Exception { // create a file of replication factor of 1 final Path fileName = new Path("/test.txt"); final int fileLen = 1; DFSTestUtil.createFile(fs, fileName, 1, (short) 1, 1L); DFSTestUtil.waitReplication(fs, fileName, (short) 1); // get the block belonged to the created file LocatedBlocks blocks = NameNodeAdapter.getBlockLocations( cluster.getNameNode(), fileName.toString(), 0, (long) fileLen); assertEquals("Should only find 1 block", blocks.locatedBlockCount(), 1); LocatedBlock block = blocks.get(0); // bring up a second datanode cluster.startDataNodes(conf, 1, true, null, null); cluster.waitActive(); final int sndNode = 1; DataNode datanode = cluster.getDataNodes().get(sndNode); // replicate the block to the second datanode InetSocketAddress target = datanode.getXferAddress(); Socket s = new Socket(target.getAddress(), target.getPort()); // write the header. DataOutputStream out = new DataOutputStream(s.getOutputStream()); DataChecksum checksum = DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, 512); new Sender(out) .writeBlock( block.getBlock(), StorageType.DEFAULT, BlockTokenSecretManager.DUMMY_TOKEN, "", new DatanodeInfo[0], new StorageType[0], null, BlockConstructionStage.PIPELINE_SETUP_CREATE, 1, 0L, 0L, 0L, checksum, CachingStrategy.newDefaultStrategy(), false); out.flush(); // close the connection before sending the content of the block out.close(); // the temporary block & meta files should be deleted String bpid = cluster.getNamesystem().getBlockPoolId(); File storageDir = cluster.getInstanceStorageDir(sndNode, 0); File dir1 = MiniDFSCluster.getRbwDir(storageDir, bpid); storageDir = cluster.getInstanceStorageDir(sndNode, 1); File dir2 = MiniDFSCluster.getRbwDir(storageDir, bpid); while (dir1.listFiles().length != 0 || dir2.listFiles().length != 0) { Thread.sleep(100); } // then increase the file's replication factor fs.setReplication(fileName, (short) 2); // replication should succeed DFSTestUtil.waitReplication(fs, fileName, (short) 1); // clean up the file fs.delete(fileName, false); }
@Test public void testHighAvailability() throws IOException { Configuration conf = new HdfsConfiguration(); // Create cluster with 3 readers and 1 writer MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .nnTopology(MiniDFSNNTopology.simpleHOPSTopology(4)) .numDataNodes(2) .format(true) .build(); cluster.waitActive(); try { // Get the filesystem and create a directory FileSystem fs = cluster.getFileSystem(0); // Write operation should work since we have one writer assertTrue(fs.mkdirs(dir)); // Write operation - Create a file and write something to it Path file1 = new Path(dir, "file1"); createFile(fs, file1); // Read operation - The file should exist. assertTrue(fs.exists(file1)); // Read operation - List files in this directory assertEquals(1, list(fs)); // Read operation - Get file status FileStatus fileStatus = fs.listStatus(dir)[0]; // Read operation - Get block locations assertNotSame(0, fs.getFileBlockLocations(file1, 0, 1).length); // Now we kill all namenodes except the last two cluster.getNameNode(0).stop(); cluster.getNameNode(1).stop(); // Now lets read again - These operations should be possible assertTrue(fs.exists(file1)); // Writer operation - concat files Path file2 = new Path(dir, "file2"); createFile(fs, file2); assertTrue(fs.exists(file2)); Path file3 = new Path(dir, "file3"); createFile(fs, file3); assertTrue(fs.exists(file3)); Path file4 = new Path(dir, "file4"); // Read operation - list files (3 files created now under this directory) assertEquals(3, list(fs)); // Write operation - rename // [S] commented out because rename is not yet supported // ((DistributedFileSystem) fs).rename(file1, file4); // Kill another namenode cluster.getNameNode(2).stop(); // Read operation - File status fs.getFileStatus(file2); // Write operation - Delete assertTrue(fs.delete(dir, true)); } catch (IOException ex) { // In case we have any connectivity issues here, there is a problem // All connectivitiy issues are handled in the above piece of code LOG.error(ex); ex.printStackTrace(); assertFalse("Cannot be any connectivity issues", ex instanceof ConnectException); fail(); } finally { if (cluster != null) { cluster.shutdown(); } } }
protected void doTestRead(Configuration conf, MiniDFSCluster cluster, boolean isStriped) throws Exception { final int numDataNodes = cluster.getDataNodes().size(); final NameNode nn = cluster.getNameNode(); final NamenodeProtocols nnProto = nn.getRpcServer(); final BlockManager bm = nn.getNamesystem().getBlockManager(); final BlockTokenSecretManager sm = bm.getBlockTokenSecretManager(); // set a short token lifetime (1 second) initially SecurityTestUtil.setBlockTokenLifetime(sm, 1000L); Path fileToRead = new Path(FILE_TO_READ); FileSystem fs = cluster.getFileSystem(); byte[] expected = generateBytes(FILE_SIZE); createFile(fs, fileToRead, expected); /* * setup for testing expiration handling of cached tokens */ // read using blockSeekTo(). Acquired tokens are cached in in1 FSDataInputStream in1 = fs.open(fileToRead); assertTrue(checkFile1(in1, expected)); // read using blockSeekTo(). Acquired tokens are cached in in2 FSDataInputStream in2 = fs.open(fileToRead); assertTrue(checkFile1(in2, expected)); // read using fetchBlockByteRange(). Acquired tokens are cached in in3 FSDataInputStream in3 = fs.open(fileToRead); assertTrue(checkFile2(in3, expected)); /* * testing READ interface on DN using a BlockReader */ DFSClient client = null; try { client = new DFSClient(new InetSocketAddress("localhost", cluster.getNameNodePort()), conf); } finally { if (client != null) client.close(); } List<LocatedBlock> locatedBlocks = nnProto.getBlockLocations(FILE_TO_READ, 0, FILE_SIZE).getLocatedBlocks(); LocatedBlock lblock = locatedBlocks.get(0); // first block // verify token is not expired assertFalse(isBlockTokenExpired(lblock)); // read with valid token, should succeed tryRead(conf, lblock, true); /* * wait till myToken and all cached tokens in in1, in2 and in3 expire */ while (!isBlockTokenExpired(lblock)) { try { Thread.sleep(10); } catch (InterruptedException ignored) { } } /* * continue testing READ interface on DN using a BlockReader */ // verify token is expired assertTrue(isBlockTokenExpired(lblock)); // read should fail tryRead(conf, lblock, false); // use a valid new token bm.setBlockToken(lblock, BlockTokenIdentifier.AccessMode.READ); // read should succeed tryRead(conf, lblock, true); // use a token with wrong blockID long rightId = lblock.getBlock().getBlockId(); long wrongId = rightId + 1; lblock.getBlock().setBlockId(wrongId); bm.setBlockToken(lblock, BlockTokenIdentifier.AccessMode.READ); lblock.getBlock().setBlockId(rightId); // read should fail tryRead(conf, lblock, false); // use a token with wrong access modes bm.setBlockToken(lblock, BlockTokenIdentifier.AccessMode.WRITE); // read should fail tryRead(conf, lblock, false); // set a long token lifetime for future tokens SecurityTestUtil.setBlockTokenLifetime(sm, 600 * 1000L); /* * testing that when cached tokens are expired, DFSClient will re-fetch * tokens transparently for READ. */ // confirm all tokens cached in in1 are expired by now List<LocatedBlock> lblocks = DFSTestUtil.getAllBlocks(in1); for (LocatedBlock blk : lblocks) { assertTrue(isBlockTokenExpired(blk)); } // verify blockSeekTo() is able to re-fetch token transparently in1.seek(0); assertTrue(checkFile1(in1, expected)); // confirm all tokens cached in in2 are expired by now List<LocatedBlock> lblocks2 = DFSTestUtil.getAllBlocks(in2); for (LocatedBlock blk : lblocks2) { assertTrue(isBlockTokenExpired(blk)); } // verify blockSeekTo() is able to re-fetch token transparently (testing // via another interface method) if (isStriped) { // striped block doesn't support seekToNewSource in2.seek(0); } else { assertTrue(in2.seekToNewSource(0)); } assertTrue(checkFile1(in2, expected)); // confirm all tokens cached in in3 are expired by now List<LocatedBlock> lblocks3 = DFSTestUtil.getAllBlocks(in3); for (LocatedBlock blk : lblocks3) { assertTrue(isBlockTokenExpired(blk)); } // verify fetchBlockByteRange() is able to re-fetch token transparently assertTrue(checkFile2(in3, expected)); /* * testing that after datanodes are restarted on the same ports, cached * tokens should still work and there is no need to fetch new tokens from * namenode. This test should run while namenode is down (to make sure no * new tokens can be fetched from namenode). */ // restart datanodes on the same ports that they currently use assertTrue(cluster.restartDataNodes(true)); cluster.waitActive(); assertEquals(numDataNodes, cluster.getDataNodes().size()); cluster.shutdownNameNode(0); // confirm tokens cached in in1 are still valid lblocks = DFSTestUtil.getAllBlocks(in1); for (LocatedBlock blk : lblocks) { assertFalse(isBlockTokenExpired(blk)); } // verify blockSeekTo() still works (forced to use cached tokens) in1.seek(0); assertTrue(checkFile1(in1, expected)); // confirm tokens cached in in2 are still valid lblocks2 = DFSTestUtil.getAllBlocks(in2); for (LocatedBlock blk : lblocks2) { assertFalse(isBlockTokenExpired(blk)); } // verify blockSeekTo() still works (forced to use cached tokens) if (isStriped) { in2.seek(0); } else { in2.seekToNewSource(0); } assertTrue(checkFile1(in2, expected)); // confirm tokens cached in in3 are still valid lblocks3 = DFSTestUtil.getAllBlocks(in3); for (LocatedBlock blk : lblocks3) { assertFalse(isBlockTokenExpired(blk)); } // verify fetchBlockByteRange() still works (forced to use cached tokens) assertTrue(checkFile2(in3, expected)); /* * testing that when namenode is restarted, cached tokens should still * work and there is no need to fetch new tokens from namenode. Like the * previous test, this test should also run while namenode is down. The * setup for this test depends on the previous test. */ // restart the namenode and then shut it down for test cluster.restartNameNode(0); cluster.shutdownNameNode(0); // verify blockSeekTo() still works (forced to use cached tokens) in1.seek(0); assertTrue(checkFile1(in1, expected)); // verify again blockSeekTo() still works (forced to use cached tokens) if (isStriped) { in2.seek(0); } else { in2.seekToNewSource(0); } assertTrue(checkFile1(in2, expected)); // verify fetchBlockByteRange() still works (forced to use cached tokens) assertTrue(checkFile2(in3, expected)); /* * testing that after both namenode and datanodes got restarted (namenode * first, followed by datanodes), DFSClient can't access DN without * re-fetching tokens and is able to re-fetch tokens transparently. The * setup of this test depends on the previous test. */ // restore the cluster and restart the datanodes for test cluster.restartNameNode(0); assertTrue(cluster.restartDataNodes(true)); cluster.waitActive(); assertEquals(numDataNodes, cluster.getDataNodes().size()); // shutdown namenode so that DFSClient can't get new tokens from namenode cluster.shutdownNameNode(0); // verify blockSeekTo() fails (cached tokens become invalid) in1.seek(0); assertFalse(checkFile1(in1, expected)); // verify fetchBlockByteRange() fails (cached tokens become invalid) assertFalse(checkFile2(in3, expected)); // restart the namenode to allow DFSClient to re-fetch tokens cluster.restartNameNode(0); // verify blockSeekTo() works again (by transparently re-fetching // tokens from namenode) in1.seek(0); assertTrue(checkFile1(in1, expected)); if (isStriped) { in2.seek(0); } else { in2.seekToNewSource(0); } assertTrue(checkFile1(in2, expected)); // verify fetchBlockByteRange() works again (by transparently // re-fetching tokens from namenode) assertTrue(checkFile2(in3, expected)); /* * testing that when datanodes are restarted on different ports, DFSClient * is able to re-fetch tokens transparently to connect to them */ // restart datanodes on newly assigned ports assertTrue(cluster.restartDataNodes(false)); cluster.waitActive(); assertEquals(numDataNodes, cluster.getDataNodes().size()); // verify blockSeekTo() is able to re-fetch token transparently in1.seek(0); assertTrue(checkFile1(in1, expected)); // verify blockSeekTo() is able to re-fetch token transparently if (isStriped) { in2.seek(0); } else { in2.seekToNewSource(0); } assertTrue(checkFile1(in2, expected)); // verify fetchBlockByteRange() is able to re-fetch token transparently assertTrue(checkFile2(in3, expected)); }