Esempio n. 1
0
 /*
  * This test attempts to upgrade the datanode from federation
  * version -35 to upper version
  * This test is for non-federation cluster with single namenode
  */
 public void testNonFederationClusterUpgradeAfterFederationVersion() throws Exception {
   File[] baseDirs;
   UpgradeUtilities.initialize();
   for (int numDirs = 1; numDirs <= 2; numDirs++) {
     conf = new Configuration();
     conf.setInt("dfs.datanode.scan.period.hours", -1);
     conf = UpgradeUtilities.initializeStorageStateConf(numDirs, conf);
     String[] nameNodeDirs = conf.getStrings("dfs.name.dir");
     String[] dataNodeDirs = conf.getStrings("dfs.data.dir");
     log("DataNode upgrade with federation layout version in current", numDirs);
     UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
     try {
       cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE);
       baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
       UpgradeUtilities.createVersionFile(
           DATA_NODE,
           baseDirs,
           new StorageInfo(
               FSConstants.FEDERATION_VERSION,
               UpgradeUtilities.getCurrentNamespaceID(cluster),
               UpgradeUtilities.getCurrentFsscTime(cluster)),
           cluster.getNameNode().getNamespaceID());
       cluster.startDataNodes(conf, 1, false, StartupOption.REGULAR, null);
       checkResult(DATA_NODE, dataNodeDirs, 0, false);
     } finally {
       if (cluster != null) cluster.shutdown();
       UpgradeUtilities.createEmptyDirs(nameNodeDirs);
       UpgradeUtilities.createEmptyDirs(dataNodeDirs);
     }
   }
 }
Esempio n. 2
0
  /*
   * This test attempts to upgrade the datanode from federation version -35 to
   * upper version This test is for federation cluster with 2 namenodes. It
   * changes the layout version and ctime.
   */
  public void testFederationClusterUpgradeAfterFederationVersionWithCTimeChange() throws Exception {
    File[] baseDirs;
    Configuration baseConf = new Configuration();
    UpgradeUtilities.initialize(2, baseConf, true);
    for (int numDirs = 1; numDirs <= 2; numDirs++) {
      conf = new Configuration();
      conf.setInt("dfs.datanode.scan.period.hours", -1);
      conf = UpgradeUtilities.initializeStorageStateConf(numDirs, conf);
      String[] nameNodeDirs = conf.getStrings("dfs.name.dir");
      String[] dataNodeDirs = conf.getStrings("dfs.data.dir");
      log("DataNode upgrade with federation layout version in current and ctime change", numDirs);
      UpgradeUtilities.createFederatedNameNodeStorageDirs(nameNodeDirs);
      conf.set(
          FSConstants.DFS_FEDERATION_NAMESERVICES,
          baseConf.get(FSConstants.DFS_FEDERATION_NAMESERVICES));
      try {
        cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE, false, 2);
        baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
        for (int i = 0; i < 2; i++) {
          UpgradeUtilities.createVersionFile(
              DATA_NODE,
              baseDirs,
              new StorageInfo(
                  FSConstants.FEDERATION_VERSION,
                  cluster.getNameNode(i).getNamespaceID(),
                  cluster.getNameNode(i).versionRequest().getCTime() - 1),
              cluster.getNameNode(i).getNamespaceID());
        }
        cluster.startDataNodes(conf, 1, false, StartupOption.REGULAR, null);

        for (int i = 0; i < 2; i++) {
          checkResult(DATA_NODE, dataNodeDirs, i, false);
        }
      } finally {
        if (cluster != null) cluster.shutdown();
        UpgradeUtilities.createEmptyDirs(nameNodeDirs);
        UpgradeUtilities.createEmptyDirs(dataNodeDirs);
      }
    }
  }
Esempio n. 3
0
  /** This test attempts to finalize the NameNode and DataNode. */
  public void testFinalize() throws Exception {
    UpgradeUtilities.initialize();

    for (int numDirs = 1; numDirs <= 2; numDirs++) {
      /* This test requires that "current" directory not change after
       * the upgrade. Actually it is ok for those contents to change.
       * For now disabling block verification so that the contents are
       * not changed.
       */
      conf = new HdfsConfiguration();
      conf.setInt("dfs.datanode.scan.period.hours", -1);
      conf = UpgradeUtilities.initializeStorageStateConf(numDirs, conf);
      String[] nameNodeDirs = conf.getStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY);
      String[] dataNodeDirs = conf.getStrings(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY);

      log("Finalize with existing previous dir", numDirs);
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "previous");
      UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
      UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "previous");
      cluster =
          new MiniDFSCluster.Builder(conf)
              .format(false)
              .manageDataDfsDirs(false)
              .manageNameDfsDirs(false)
              .startupOption(StartupOption.REGULAR)
              .build();
      cluster.finalizeCluster(conf);
      checkResult(nameNodeDirs, dataNodeDirs);

      log("Finalize without existing previous dir", numDirs);
      cluster.finalizeCluster(conf);
      checkResult(nameNodeDirs, dataNodeDirs);

      cluster.shutdown();
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
      UpgradeUtilities.createEmptyDirs(dataNodeDirs);
    } // end numDir loop
  }
Esempio n. 4
0
  /**
   * This test attempts to upgrade the NameNode and DataNode under a number of valid and invalid
   * conditions.
   */
  public void testUpgrade() throws Exception {
    File[] baseDirs;
    UpgradeUtilities.initialize();
    for (int numDirs = 1; numDirs <= 2; numDirs++) {
      conf = new Configuration();
      conf.setInt("dfs.datanode.scan.period.hours", -1);
      conf = UpgradeUtilities.initializeStorageStateConf(numDirs, conf);
      String[] nameNodeDirs = conf.getStrings("dfs.name.dir");
      String[] dataNodeDirs = conf.getStrings("dfs.data.dir");

      log("Normal NameNode upgrade", numDirs);
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE);
      checkResult(NAME_NODE, nameNodeDirs);
      cluster.shutdown();
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);

      log("Normal DataNode upgrade", numDirs);
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE);
      UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
      cluster.startDataNodes(conf, 1, false, StartupOption.REGULAR, null);
      checkResult(DATA_NODE, dataNodeDirs);
      cluster.shutdown();
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
      UpgradeUtilities.createEmptyDirs(dataNodeDirs);

      log("NameNode upgrade with existing previous dir", numDirs);
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "previous");
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);

      log("DataNode upgrade with existing previous dir", numDirs);
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE);
      UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
      UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "previous");
      startDataNodeShouldFail(StartupOption.REGULAR);
      cluster.shutdown();
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
      UpgradeUtilities.createEmptyDirs(dataNodeDirs);

      log("DataNode upgrade with future stored layout version in current", numDirs);
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE);
      baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
      UpgradeUtilities.createVersionFile(
          DATA_NODE,
          baseDirs,
          new StorageInfo(
              Integer.MIN_VALUE,
              UpgradeUtilities.getCurrentNamespaceID(cluster),
              UpgradeUtilities.getCurrentFsscTime(cluster)),
          cluster.getNameNode().getNamespaceID());
      startDataNodeShouldFail(StartupOption.REGULAR);
      cluster.shutdown();
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
      UpgradeUtilities.createEmptyDirs(dataNodeDirs);

      log("DataNode upgrade with newer fsscTime in current", numDirs);
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE);
      baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
      UpgradeUtilities.createVersionFile(
          DATA_NODE,
          baseDirs,
          new StorageInfo(
              UpgradeUtilities.getCurrentLayoutVersion(),
              UpgradeUtilities.getCurrentNamespaceID(cluster),
              Long.MAX_VALUE),
          cluster.getNameNode().getNamespaceID());
      startDataNodeShouldFail(StartupOption.REGULAR);
      cluster.shutdown();
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
      UpgradeUtilities.createEmptyDirs(dataNodeDirs);

      log("NameNode upgrade with no edits file", numDirs);
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      for (File f : baseDirs) {
        FileUtil.fullyDelete(new File(f, "edits"));
      }
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);

      log("NameNode upgrade with no image file", numDirs);
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      for (File f : baseDirs) {
        FileUtil.fullyDelete(new File(f, "fsimage"));
      }
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);

      log("NameNode upgrade with corrupt version file", numDirs);
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      for (File f : baseDirs) {
        UpgradeUtilities.corruptFile(new File(f, "VERSION"));
      }
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);

      log("NameNode upgrade with old layout version in current", numDirs);
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      UpgradeUtilities.createVersionFile(
          NAME_NODE,
          baseDirs,
          new StorageInfo(
              Storage.LAST_UPGRADABLE_LAYOUT_VERSION + 1,
              UpgradeUtilities.getCurrentNamespaceID(null),
              UpgradeUtilities.getCurrentFsscTime(null)),
          0);
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);

      log("NameNode upgrade with future layout version in current", numDirs);
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      UpgradeUtilities.createVersionFile(
          NAME_NODE,
          baseDirs,
          new StorageInfo(
              Integer.MIN_VALUE,
              UpgradeUtilities.getCurrentNamespaceID(null),
              UpgradeUtilities.getCurrentFsscTime(null)),
          0);
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);

      log("Normal Datanode upgrade after datanode format", numDirs);
      UpgradeUtilities.createEmptyDirs(dataNodeDirs);
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      UpgradeUtilities.createVersionFile(
          NAME_NODE,
          baseDirs,
          new StorageInfo(
              FSConstants.LAYOUT_VERSION, UpgradeUtilities.getCurrentNamespaceID(null), 0),
          0);
      cluster =
          new MiniDFSCluster(
              0,
              conf,
              1,
              false,
              false,
              false,
              StartupOption.UPGRADE,
              null,
              null,
              null,
              false,
              false,
              1,
              false);
      cluster.shutdown();
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
    } // end numDir loop
  }
Esempio n. 5
0
  public void testFederationClusterUpgradeAfterFederationVersionWithTopLevelLayout()
      throws Exception {
    File[] baseDirs;
    Configuration baseConf = new Configuration();
    UpgradeUtilities.initialize(2, baseConf, true);
    for (int numDirs = 1; numDirs <= 2; numDirs++) {
      conf = new Configuration();
      conf.setInt("dfs.datanode.scan.period.hours", -1);
      conf = UpgradeUtilities.initializeStorageStateConf(numDirs, conf);
      String[] nameNodeDirs = conf.getStrings("dfs.name.dir");
      String[] dataNodeDirs = conf.getStrings("dfs.data.dir");
      log(
          "DataNode upgrade with federation layout version in current and no ns level layout version",
          numDirs);
      UpgradeUtilities.createFederatedNameNodeStorageDirs(nameNodeDirs);
      conf.set(
          FSConstants.DFS_FEDERATION_NAMESERVICES,
          baseConf.get(FSConstants.DFS_FEDERATION_NAMESERVICES));
      try {
        cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE, false, 2);
        baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
        for (int i = 0; i < 2; i++) {
          UpgradeUtilities.createVersionFile(
              DATA_NODE,
              baseDirs,
              new StorageInfo(
                  FSConstants.FEDERATION_VERSION,
                  cluster.getNameNode(i).getNamespaceID(),
                  cluster.getNameNode(i).versionRequest().getCTime()),
              cluster.getNameNode(i).getNamespaceID(),
              false);
        }
        cluster.startDataNodes(conf, 1, false, StartupOption.REGULAR, null);

        for (int i = 0; i < 2; i++) {
          checkResult(DATA_NODE, dataNodeDirs, i, false);
        }

        // Finalize upgrade.
        for (int i = 0; i < 2; i++) {
          cluster.getNameNode(i).finalizeUpgrade();
        }
        cluster.restartDataNodes();

        // Wait for datanodes to finalize.
        Thread.sleep(10000);

        for (int nnIndex = 0; nnIndex < 2; nnIndex++) {
          for (int i = 0; i < dataNodeDirs.length; i++) {
            File nsBaseDir =
                NameSpaceSliceStorage.getNsRoot(
                    cluster.getNameNode(nnIndex).getNamespaceID(),
                    new File(dataNodeDirs[i], "current"));
            assertFalse(new File(nsBaseDir, "previous").exists());
          }
        }
      } finally {
        if (cluster != null) cluster.shutdown();
        UpgradeUtilities.createEmptyDirs(nameNodeDirs);
        UpgradeUtilities.createEmptyDirs(dataNodeDirs);
      }
    }
  }