Exemplo n.º 1
0
 /*
  * This test attempts to upgrade the datanode from federation
  * version -35 to upper version
  * This test is for non-federation cluster with single namenode
  */
 public void testNonFederationClusterUpgradeAfterFederationVersion() throws Exception {
   File[] baseDirs;
   UpgradeUtilities.initialize();
   for (int numDirs = 1; numDirs <= 2; numDirs++) {
     conf = new Configuration();
     conf.setInt("dfs.datanode.scan.period.hours", -1);
     conf = UpgradeUtilities.initializeStorageStateConf(numDirs, conf);
     String[] nameNodeDirs = conf.getStrings("dfs.name.dir");
     String[] dataNodeDirs = conf.getStrings("dfs.data.dir");
     log("DataNode upgrade with federation layout version in current", numDirs);
     UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
     try {
       cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE);
       baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
       UpgradeUtilities.createVersionFile(
           DATA_NODE,
           baseDirs,
           new StorageInfo(
               FSConstants.FEDERATION_VERSION,
               UpgradeUtilities.getCurrentNamespaceID(cluster),
               UpgradeUtilities.getCurrentFsscTime(cluster)),
           cluster.getNameNode().getNamespaceID());
       cluster.startDataNodes(conf, 1, false, StartupOption.REGULAR, null);
       checkResult(DATA_NODE, dataNodeDirs, 0, false);
     } finally {
       if (cluster != null) cluster.shutdown();
       UpgradeUtilities.createEmptyDirs(nameNodeDirs);
       UpgradeUtilities.createEmptyDirs(dataNodeDirs);
     }
   }
 }
Exemplo n.º 2
0
  /**
   * This test attempts to upgrade the NameNode and DataNode under a number of valid and invalid
   * conditions.
   */
  public void testUpgrade() throws Exception {
    File[] baseDirs;
    UpgradeUtilities.initialize();
    for (int numDirs = 1; numDirs <= 2; numDirs++) {
      conf = new Configuration();
      conf.setInt("dfs.datanode.scan.period.hours", -1);
      conf = UpgradeUtilities.initializeStorageStateConf(numDirs, conf);
      String[] nameNodeDirs = conf.getStrings("dfs.name.dir");
      String[] dataNodeDirs = conf.getStrings("dfs.data.dir");

      log("Normal NameNode upgrade", numDirs);
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE);
      checkResult(NAME_NODE, nameNodeDirs);
      cluster.shutdown();
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);

      log("Normal DataNode upgrade", numDirs);
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE);
      UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
      cluster.startDataNodes(conf, 1, false, StartupOption.REGULAR, null);
      checkResult(DATA_NODE, dataNodeDirs);
      cluster.shutdown();
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
      UpgradeUtilities.createEmptyDirs(dataNodeDirs);

      log("NameNode upgrade with existing previous dir", numDirs);
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "previous");
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);

      log("DataNode upgrade with existing previous dir", numDirs);
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE);
      UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
      UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "previous");
      startDataNodeShouldFail(StartupOption.REGULAR);
      cluster.shutdown();
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
      UpgradeUtilities.createEmptyDirs(dataNodeDirs);

      log("DataNode upgrade with future stored layout version in current", numDirs);
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE);
      baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
      UpgradeUtilities.createVersionFile(
          DATA_NODE,
          baseDirs,
          new StorageInfo(
              Integer.MIN_VALUE,
              UpgradeUtilities.getCurrentNamespaceID(cluster),
              UpgradeUtilities.getCurrentFsscTime(cluster)),
          cluster.getNameNode().getNamespaceID());
      startDataNodeShouldFail(StartupOption.REGULAR);
      cluster.shutdown();
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
      UpgradeUtilities.createEmptyDirs(dataNodeDirs);

      log("DataNode upgrade with newer fsscTime in current", numDirs);
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE);
      baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
      UpgradeUtilities.createVersionFile(
          DATA_NODE,
          baseDirs,
          new StorageInfo(
              UpgradeUtilities.getCurrentLayoutVersion(),
              UpgradeUtilities.getCurrentNamespaceID(cluster),
              Long.MAX_VALUE),
          cluster.getNameNode().getNamespaceID());
      startDataNodeShouldFail(StartupOption.REGULAR);
      cluster.shutdown();
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
      UpgradeUtilities.createEmptyDirs(dataNodeDirs);

      log("NameNode upgrade with no edits file", numDirs);
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      for (File f : baseDirs) {
        FileUtil.fullyDelete(new File(f, "edits"));
      }
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);

      log("NameNode upgrade with no image file", numDirs);
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      for (File f : baseDirs) {
        FileUtil.fullyDelete(new File(f, "fsimage"));
      }
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);

      log("NameNode upgrade with corrupt version file", numDirs);
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      for (File f : baseDirs) {
        UpgradeUtilities.corruptFile(new File(f, "VERSION"));
      }
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);

      log("NameNode upgrade with old layout version in current", numDirs);
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      UpgradeUtilities.createVersionFile(
          NAME_NODE,
          baseDirs,
          new StorageInfo(
              Storage.LAST_UPGRADABLE_LAYOUT_VERSION + 1,
              UpgradeUtilities.getCurrentNamespaceID(null),
              UpgradeUtilities.getCurrentFsscTime(null)),
          0);
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);

      log("NameNode upgrade with future layout version in current", numDirs);
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      UpgradeUtilities.createVersionFile(
          NAME_NODE,
          baseDirs,
          new StorageInfo(
              Integer.MIN_VALUE,
              UpgradeUtilities.getCurrentNamespaceID(null),
              UpgradeUtilities.getCurrentFsscTime(null)),
          0);
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);

      log("Normal Datanode upgrade after datanode format", numDirs);
      UpgradeUtilities.createEmptyDirs(dataNodeDirs);
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      UpgradeUtilities.createVersionFile(
          NAME_NODE,
          baseDirs,
          new StorageInfo(
              FSConstants.LAYOUT_VERSION, UpgradeUtilities.getCurrentNamespaceID(null), 0),
          0);
      cluster =
          new MiniDFSCluster(
              0,
              conf,
              1,
              false,
              false,
              false,
              StartupOption.UPGRADE,
              null,
              null,
              null,
              false,
              false,
              1,
              false);
      cluster.shutdown();
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
    } // end numDir loop
  }