/** * Verify that the current and previous directories exist. Verify that previous hasn't been * modified by comparing the checksum of all it's containing files with their original checksum. * It is assumed that the server has recovered and upgraded. nsLevelUpgrade specify if the upgrade * is at top level or ns level nsLevelUpgrade=true, we search basedir/current/NS-id/previous * =false, we search basedir/previous */ void checkResult(NodeType nodeType, String[] baseDirs, int nnIndex, boolean simulatedPrevious) throws IOException { switch (nodeType) { case NAME_NODE: for (int i = 0; i < baseDirs.length; i++) { assertTrue(new File(baseDirs[i], "current").isDirectory()); assertTrue(new File(baseDirs[i], "current/VERSION").isFile()); assertTrue(new File(baseDirs[i], "current/edits").isFile()); assertTrue(new File(baseDirs[i], "current/fsimage").isFile()); assertTrue(new File(baseDirs[i], "current/fstime").isFile()); } break; case DATA_NODE: for (int i = 0; i < baseDirs.length; i++) { assertEquals( UpgradeUtilities.checksumContents(nodeType, new File(baseDirs[i], "current")), UpgradeUtilities.checksumMasterContents(nodeType)); File nsBaseDir = NameSpaceSliceStorage.getNsRoot( cluster.getNameNode(nnIndex).getNamespaceID(), new File(baseDirs[i], "current")); assertEquals( UpgradeUtilities.checksumContents( nodeType, new File(nsBaseDir, MiniDFSCluster.FINALIZED_DIR_NAME)), UpgradeUtilities.checksumDatanodeNSStorageContents(nnIndex)); } break; } for (int i = 0; i < baseDirs.length; i++) { switch (nodeType) { case NAME_NODE: assertTrue(new File(baseDirs[i], "previous").isDirectory()); assertEquals( UpgradeUtilities.checksumContents(nodeType, new File(baseDirs[i], "previous")), UpgradeUtilities.checksumMasterContents(nodeType)); break; case DATA_NODE: File nsBaseDir = null; nsBaseDir = NameSpaceSliceStorage.getNsRoot( cluster.getNameNode(nnIndex).getNamespaceID(), new File(baseDirs[i], "current")); // Top level upgrade should not exist. assertFalse(new File(baseDirs[i], "previous").isDirectory() && !simulatedPrevious); assertTrue(new File(nsBaseDir, "previous").isDirectory()); assertEquals( UpgradeUtilities.checksumContents( nodeType, new File(nsBaseDir, "previous/finalized")), UpgradeUtilities.checksumDatanodeNSStorageContents(nnIndex)); } } }
/* * This test attempts to upgrade the datanode from federation * version -35 to upper version * This test is for non-federation cluster with single namenode */ public void testNonFederationClusterUpgradeAfterFederationVersion() throws Exception { File[] baseDirs; UpgradeUtilities.initialize(); for (int numDirs = 1; numDirs <= 2; numDirs++) { conf = new Configuration(); conf.setInt("dfs.datanode.scan.period.hours", -1); conf = UpgradeUtilities.initializeStorageStateConf(numDirs, conf); String[] nameNodeDirs = conf.getStrings("dfs.name.dir"); String[] dataNodeDirs = conf.getStrings("dfs.data.dir"); log("DataNode upgrade with federation layout version in current", numDirs); UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current"); try { cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE); baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current"); UpgradeUtilities.createVersionFile( DATA_NODE, baseDirs, new StorageInfo( FSConstants.FEDERATION_VERSION, UpgradeUtilities.getCurrentNamespaceID(cluster), UpgradeUtilities.getCurrentFsscTime(cluster)), cluster.getNameNode().getNamespaceID()); cluster.startDataNodes(conf, 1, false, StartupOption.REGULAR, null); checkResult(DATA_NODE, dataNodeDirs, 0, false); } finally { if (cluster != null) cluster.shutdown(); UpgradeUtilities.createEmptyDirs(nameNodeDirs); UpgradeUtilities.createEmptyDirs(dataNodeDirs); } } }
/* * This test attempts to upgrade the datanode from federation version -35 to * upper version This test is for federation cluster with 2 namenodes. It * changes the layout version and ctime. */ public void testFederationClusterUpgradeAfterFederationVersionWithCTimeChange() throws Exception { File[] baseDirs; Configuration baseConf = new Configuration(); UpgradeUtilities.initialize(2, baseConf, true); for (int numDirs = 1; numDirs <= 2; numDirs++) { conf = new Configuration(); conf.setInt("dfs.datanode.scan.period.hours", -1); conf = UpgradeUtilities.initializeStorageStateConf(numDirs, conf); String[] nameNodeDirs = conf.getStrings("dfs.name.dir"); String[] dataNodeDirs = conf.getStrings("dfs.data.dir"); log("DataNode upgrade with federation layout version in current and ctime change", numDirs); UpgradeUtilities.createFederatedNameNodeStorageDirs(nameNodeDirs); conf.set( FSConstants.DFS_FEDERATION_NAMESERVICES, baseConf.get(FSConstants.DFS_FEDERATION_NAMESERVICES)); try { cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE, false, 2); baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current"); for (int i = 0; i < 2; i++) { UpgradeUtilities.createVersionFile( DATA_NODE, baseDirs, new StorageInfo( FSConstants.FEDERATION_VERSION, cluster.getNameNode(i).getNamespaceID(), cluster.getNameNode(i).versionRequest().getCTime() - 1), cluster.getNameNode(i).getNamespaceID()); } cluster.startDataNodes(conf, 1, false, StartupOption.REGULAR, null); for (int i = 0; i < 2; i++) { checkResult(DATA_NODE, dataNodeDirs, i, false); } } finally { if (cluster != null) cluster.shutdown(); UpgradeUtilities.createEmptyDirs(nameNodeDirs); UpgradeUtilities.createEmptyDirs(dataNodeDirs); } } }
/** * Initialize the data structures used by this class. IMPORTANT NOTE: This method must be called * once before calling any other public method on this class. * * <p>Creates a singleton master populated storage directory for a Namenode (contains edits, * fsimage, version, and time files) and a Datanode (contains version and block files). This can * be a lengthy operation. */ public static void initialize() throws Exception { createEmptyDirs(new String[] {TEST_ROOT_DIR.toString()}); Configuration config = new Configuration(); config.set("dfs.name.dir", namenodeStorage.toString()); config.set("dfs.data.dir", datanodeStorage.toString()); MiniDFSCluster cluster = null; try { // format data-node createEmptyDirs(new String[] {datanodeStorage.toString()}); // format and start NameNode and start DataNode NameNode.format(config); cluster = new MiniDFSCluster(config, 1, StartupOption.REGULAR); NameNode namenode = cluster.getNameNode(); namenodeStorageNamespaceID = namenode.versionRequest().getNamespaceID(); namenodeStorageFsscTime = namenode.versionRequest().getCTime(); FileSystem fs = FileSystem.get(config); Path baseDir = new Path("/TestUpgrade"); fs.mkdirs(baseDir); // write some files int bufferSize = 4096; byte[] buffer = new byte[bufferSize]; for (int i = 0; i < bufferSize; i++) buffer[i] = (byte) ('0' + i % 50); writeFile(fs, new Path(baseDir, "file1"), buffer, bufferSize); writeFile(fs, new Path(baseDir, "file2"), buffer, bufferSize); // save image namenode.setSafeMode(SafeModeAction.SAFEMODE_ENTER); namenode.saveNamespace(); namenode.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); // write more files writeFile(fs, new Path(baseDir, "file3"), buffer, bufferSize); writeFile(fs, new Path(baseDir, "file4"), buffer, bufferSize); } finally { // shutdown if (cluster != null) cluster.shutdown(); FileUtil.fullyDelete(new File(namenodeStorage, "in_use.lock")); FileUtil.fullyDelete(new File(datanodeStorage, "in_use.lock")); } namenodeStorageChecksum = checksumContents(NAME_NODE, new File(namenodeStorage, "current")); datanodeStorageChecksum = checksumContents(DATA_NODE, new File(datanodeStorage, "current")); }
public void testBlocksScheduledCounter() throws IOException { MiniDFSCluster cluster = new MiniDFSCluster(new Configuration(), 1, true, null); cluster.waitActive(); FileSystem fs = cluster.getFileSystem(); // open a file an write a few bytes: FSDataOutputStream out = fs.create(new Path("/testBlockScheduledCounter")); for (int i = 0; i < 1024; i++) { out.write(i); } // flush to make sure a block is allocated. ((DFSOutputStream) (out.getWrappedStream())).sync(); ArrayList<DatanodeDescriptor> dnList = new ArrayList<DatanodeDescriptor>(); cluster.getNameNode().namesystem.DFSNodesStatus(dnList, dnList); DatanodeDescriptor dn = dnList.get(0); assertEquals(1, dn.getBlocksScheduled()); // close the file and the counter should go to zero. out.close(); assertEquals(0, dn.getBlocksScheduled()); }
/** * Test case that stops a writer after finalizing a block but before calling completeFile, * recovers a file from another writer, starts writing from that writer, and then has the old * lease holder call completeFile */ @Test(timeout = 60000) public void testCompleteOtherLeaseHoldersFile() throws Throwable { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); try { cluster.waitActive(); NameNode preSpyNN = cluster.getNameNode(); NameNode spyNN = spy(preSpyNN); // Delay completeFile DelayAnswer delayer = new DelayAnswer(); doAnswer(delayer).when(spyNN).complete(anyString(), anyString(), (Block) anyObject()); DFSClient client = new DFSClient(null, spyNN, conf, null); file1 = new Path("/testCompleteOtherLease"); final OutputStream stm = client.create("/testCompleteOtherLease", true); // write 1/2 block AppendTestUtil.write(stm, 0, 4096); final AtomicReference<Throwable> err = new AtomicReference<Throwable>(); Thread t = new Thread() { public void run() { try { stm.close(); } catch (Throwable t) { err.set(t); } } }; t.start(); LOG.info("Waiting for close to get to latch..."); delayer.waitForCall(); // At this point, the block is finalized on the DNs, but the file // has not been completed in the NN. // Lose the leases LOG.info("Killing lease checker"); client.leasechecker.interruptAndJoin(); FileSystem fs1 = cluster.getFileSystem(); FileSystem fs2 = AppendTestUtil.createHdfsWithDifferentUsername(fs1.getConf()); LOG.info("Recovering file"); recoverFile(fs2); LOG.info("Opening file for append from new fs"); FSDataOutputStream appenderStream = fs2.append(file1); LOG.info("Writing some data from new appender"); AppendTestUtil.write(appenderStream, 0, 4096); LOG.info("Telling old close to proceed."); delayer.proceed(); LOG.info("Waiting for close to finish."); t.join(); LOG.info("Close finished."); // We expect that close will get a "Lease mismatch" // error. Throwable thrownByClose = err.get(); assertNotNull(thrownByClose); assertTrue(thrownByClose instanceof IOException); if (!thrownByClose.getMessage().contains("Lease mismatch")) throw thrownByClose; // The appender should be able to close properly appenderStream.close(); } finally { cluster.shutdown(); } }
static boolean hasLease(MiniDFSCluster cluster, Path src) { return cluster.getNameNode().namesystem.leaseManager.getLeaseByPath(src.toString()) != null; }
/** * This test attempts to upgrade the NameNode and DataNode under a number of valid and invalid * conditions. */ public void testUpgrade() throws Exception { File[] baseDirs; UpgradeUtilities.initialize(); for (int numDirs = 1; numDirs <= 2; numDirs++) { conf = new Configuration(); conf.setInt("dfs.datanode.scan.period.hours", -1); conf = UpgradeUtilities.initializeStorageStateConf(numDirs, conf); String[] nameNodeDirs = conf.getStrings("dfs.name.dir"); String[] dataNodeDirs = conf.getStrings("dfs.data.dir"); log("Normal NameNode upgrade", numDirs); UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current"); cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE); checkResult(NAME_NODE, nameNodeDirs); cluster.shutdown(); UpgradeUtilities.createEmptyDirs(nameNodeDirs); log("Normal DataNode upgrade", numDirs); UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current"); cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE); UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current"); cluster.startDataNodes(conf, 1, false, StartupOption.REGULAR, null); checkResult(DATA_NODE, dataNodeDirs); cluster.shutdown(); UpgradeUtilities.createEmptyDirs(nameNodeDirs); UpgradeUtilities.createEmptyDirs(dataNodeDirs); log("NameNode upgrade with existing previous dir", numDirs); UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current"); UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "previous"); startNameNodeShouldFail(StartupOption.UPGRADE); UpgradeUtilities.createEmptyDirs(nameNodeDirs); log("DataNode upgrade with existing previous dir", numDirs); UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current"); cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE); UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current"); UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "previous"); startDataNodeShouldFail(StartupOption.REGULAR); cluster.shutdown(); UpgradeUtilities.createEmptyDirs(nameNodeDirs); UpgradeUtilities.createEmptyDirs(dataNodeDirs); log("DataNode upgrade with future stored layout version in current", numDirs); UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current"); cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE); baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current"); UpgradeUtilities.createVersionFile( DATA_NODE, baseDirs, new StorageInfo( Integer.MIN_VALUE, UpgradeUtilities.getCurrentNamespaceID(cluster), UpgradeUtilities.getCurrentFsscTime(cluster)), cluster.getNameNode().getNamespaceID()); startDataNodeShouldFail(StartupOption.REGULAR); cluster.shutdown(); UpgradeUtilities.createEmptyDirs(nameNodeDirs); UpgradeUtilities.createEmptyDirs(dataNodeDirs); log("DataNode upgrade with newer fsscTime in current", numDirs); UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current"); cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE); baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current"); UpgradeUtilities.createVersionFile( DATA_NODE, baseDirs, new StorageInfo( UpgradeUtilities.getCurrentLayoutVersion(), UpgradeUtilities.getCurrentNamespaceID(cluster), Long.MAX_VALUE), cluster.getNameNode().getNamespaceID()); startDataNodeShouldFail(StartupOption.REGULAR); cluster.shutdown(); UpgradeUtilities.createEmptyDirs(nameNodeDirs); UpgradeUtilities.createEmptyDirs(dataNodeDirs); log("NameNode upgrade with no edits file", numDirs); baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current"); for (File f : baseDirs) { FileUtil.fullyDelete(new File(f, "edits")); } startNameNodeShouldFail(StartupOption.UPGRADE); UpgradeUtilities.createEmptyDirs(nameNodeDirs); log("NameNode upgrade with no image file", numDirs); baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current"); for (File f : baseDirs) { FileUtil.fullyDelete(new File(f, "fsimage")); } startNameNodeShouldFail(StartupOption.UPGRADE); UpgradeUtilities.createEmptyDirs(nameNodeDirs); log("NameNode upgrade with corrupt version file", numDirs); baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current"); for (File f : baseDirs) { UpgradeUtilities.corruptFile(new File(f, "VERSION")); } startNameNodeShouldFail(StartupOption.UPGRADE); UpgradeUtilities.createEmptyDirs(nameNodeDirs); log("NameNode upgrade with old layout version in current", numDirs); baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current"); UpgradeUtilities.createVersionFile( NAME_NODE, baseDirs, new StorageInfo( Storage.LAST_UPGRADABLE_LAYOUT_VERSION + 1, UpgradeUtilities.getCurrentNamespaceID(null), UpgradeUtilities.getCurrentFsscTime(null)), 0); startNameNodeShouldFail(StartupOption.UPGRADE); UpgradeUtilities.createEmptyDirs(nameNodeDirs); log("NameNode upgrade with future layout version in current", numDirs); baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current"); UpgradeUtilities.createVersionFile( NAME_NODE, baseDirs, new StorageInfo( Integer.MIN_VALUE, UpgradeUtilities.getCurrentNamespaceID(null), UpgradeUtilities.getCurrentFsscTime(null)), 0); startNameNodeShouldFail(StartupOption.UPGRADE); UpgradeUtilities.createEmptyDirs(nameNodeDirs); log("Normal Datanode upgrade after datanode format", numDirs); UpgradeUtilities.createEmptyDirs(dataNodeDirs); baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current"); UpgradeUtilities.createVersionFile( NAME_NODE, baseDirs, new StorageInfo( FSConstants.LAYOUT_VERSION, UpgradeUtilities.getCurrentNamespaceID(null), 0), 0); cluster = new MiniDFSCluster( 0, conf, 1, false, false, false, StartupOption.UPGRADE, null, null, null, false, false, 1, false); cluster.shutdown(); UpgradeUtilities.createEmptyDirs(nameNodeDirs); } // end numDir loop }
public void testFederationClusterUpgradeAfterFederationVersionWithTopLevelLayout() throws Exception { File[] baseDirs; Configuration baseConf = new Configuration(); UpgradeUtilities.initialize(2, baseConf, true); for (int numDirs = 1; numDirs <= 2; numDirs++) { conf = new Configuration(); conf.setInt("dfs.datanode.scan.period.hours", -1); conf = UpgradeUtilities.initializeStorageStateConf(numDirs, conf); String[] nameNodeDirs = conf.getStrings("dfs.name.dir"); String[] dataNodeDirs = conf.getStrings("dfs.data.dir"); log( "DataNode upgrade with federation layout version in current and no ns level layout version", numDirs); UpgradeUtilities.createFederatedNameNodeStorageDirs(nameNodeDirs); conf.set( FSConstants.DFS_FEDERATION_NAMESERVICES, baseConf.get(FSConstants.DFS_FEDERATION_NAMESERVICES)); try { cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE, false, 2); baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current"); for (int i = 0; i < 2; i++) { UpgradeUtilities.createVersionFile( DATA_NODE, baseDirs, new StorageInfo( FSConstants.FEDERATION_VERSION, cluster.getNameNode(i).getNamespaceID(), cluster.getNameNode(i).versionRequest().getCTime()), cluster.getNameNode(i).getNamespaceID(), false); } cluster.startDataNodes(conf, 1, false, StartupOption.REGULAR, null); for (int i = 0; i < 2; i++) { checkResult(DATA_NODE, dataNodeDirs, i, false); } // Finalize upgrade. for (int i = 0; i < 2; i++) { cluster.getNameNode(i).finalizeUpgrade(); } cluster.restartDataNodes(); // Wait for datanodes to finalize. Thread.sleep(10000); for (int nnIndex = 0; nnIndex < 2; nnIndex++) { for (int i = 0; i < dataNodeDirs.length; i++) { File nsBaseDir = NameSpaceSliceStorage.getNsRoot( cluster.getNameNode(nnIndex).getNamespaceID(), new File(dataNodeDirs[i], "current")); assertFalse(new File(nsBaseDir, "previous").exists()); } } } finally { if (cluster != null) cluster.shutdown(); UpgradeUtilities.createEmptyDirs(nameNodeDirs); UpgradeUtilities.createEmptyDirs(dataNodeDirs); } } }
/** * Return the File System State Creation Timestamp (FSSCTime) inherent in the currently running * Namenode. If no Namenode is running, return the FSSCTime of the master Namenode storage * directory. * * <p>The UpgradeUtilities.initialize() method must be called once before calling this method. */ public static long getCurrentFsscTime(MiniDFSCluster cluster) throws IOException { if (cluster != null) { return cluster.getNameNode().versionRequest().getCTime(); } return namenodeStorageFsscTime; }
/** * Return the namespace ID inherent in the currently running Namenode. If no Namenode is running, * return the namespace ID of the master Namenode storage directory. * * <p>The UpgradeUtilities.initialize() method must be called once before calling this method. */ public static int getCurrentNamespaceID(MiniDFSCluster cluster) throws IOException { if (cluster != null) { return cluster.getNameNode().versionRequest().getNamespaceID(); } return namenodeStorageNamespaceID; }