private void myTearDown() throws Exception {
   if (cnode != null) {
     cnode.stop();
     cnode.join();
   }
   if (mr != null) {
     mr.shutdown();
   }
   if (dfsCluster != null) {
     dfsCluster.shutdown();
   }
 }
Beispiel #2
0
 private void estimateSavings() {
   for (RaidPolicyPathPair p : raidPolicyPathPairList) {
     ErasureCodeType code = p.policy.getErasureCode();
     int stripeSize = RaidNode.getStripeLength(jobconf);
     int paritySize = RaidNode.parityLength(code, jobconf);
     int targetRepl = Integer.parseInt(p.policy.getProperty("targetReplication"));
     int parityRepl = Integer.parseInt(p.policy.getProperty("metaReplication"));
     for (FileStatus st : p.srcPaths) {
       long saving =
           RaidNode.savingFromRaidingFile(st, stripeSize, paritySize, targetRepl, parityRepl);
       totalSaving += saving;
     }
   }
 }
Beispiel #3
0
    /** Run a FileOperation */
    public void map(
        Text key,
        PolicyInfo policy,
        OutputCollector<WritableComparable, Text> out,
        Reporter reporter)
        throws IOException {
      this.reporter = reporter;
      try {
        LOG.info("Raiding file=" + key.toString() + " policy=" + policy);
        Path p = new Path(key.toString());
        FileStatus fs = p.getFileSystem(jobconf).getFileStatus(p);
        st.clear();
        RaidNode.doRaid(jobconf, policy, fs, st, reporter);

        ++succeedcount;

        reporter.incrCounter(Counter.PROCESSED_BLOCKS, st.numProcessedBlocks);
        reporter.incrCounter(Counter.PROCESSED_SIZE, st.processedSize);
        reporter.incrCounter(Counter.META_BLOCKS, st.numMetaBlocks);
        reporter.incrCounter(Counter.META_SIZE, st.metaSize);

        reporter.incrCounter(Counter.FILES_SUCCEEDED, 1);
      } catch (IOException e) {
        ++failcount;
        reporter.incrCounter(Counter.FILES_FAILED, 1);

        String s = "FAIL: " + policy + ", " + key + " " + StringUtils.stringifyException(e);
        out.collect(null, new Text(s));
        LOG.info(s);
      } finally {
        reporter.setStatus(getCountString());
      }
    }
 /** {@inheritDocs} */
 @Override
 public void stop() {
   if (stopRequested) {
     return;
   }
   super.stop();
   if (jobMonitor != null) jobMonitor.running = false;
   if (jobMonitorThread != null) jobMonitorThread.interrupt();
 }
 /** {@inheritDocs} */
 @Override
 public void join() {
   super.join();
   try {
     if (jobMonitorThread != null) jobMonitorThread.join();
   } catch (InterruptedException ie) {
     // do nothing
   }
 }
Beispiel #6
0
 public ConfigManager(Configuration conf)
     throws IOException, SAXException, RaidConfigurationException, ClassNotFoundException,
         ParserConfigurationException {
   this.conf = conf;
   this.configFileName = conf.get("raid.config.file");
   this.doReload = conf.getBoolean("raid.config.reload", true);
   this.reloadInterval = conf.getLong("raid.config.reload.interval", RELOAD_INTERVAL);
   this.periodicity = conf.getLong("raid.policy.rescan.interval", RESCAN_INTERVAL);
   this.harPartfileSize = conf.getLong("raid.har.partfile.size", HAR_PARTFILE_SIZE);
   this.maxJobsPerPolicy = conf.getInt("raid.distraid.max.jobs", DISTRAID_MAX_JOBS);
   this.maxFilesPerJob = conf.getInt("raid.distraid.max.files", DISTRAID_MAX_FILES);
   if (configFileName == null) {
     String msg =
         "No raid.config.file given in conf - "
             + "the Hadoop Raid utility cannot run. Aborting....";
     LOG.warn(msg);
     throw new IOException(msg);
   }
   reloadConfigs();
   lastSuccessfulReload = RaidNode.now();
   lastReloadAttempt = RaidNode.now();
   running = true;
 }
Beispiel #7
0
  /**
   * create new job conf based on configuration passed.
   *
   * @param conf
   * @return
   */
  private static JobConf createJobConf(Configuration conf) {
    JobConf jobconf = new JobConf(conf, DistRaid.class);
    jobName = NAME + " " + dateForm.format(new Date(RaidNode.now()));
    jobconf.setUser(RaidNode.JOBUSER);
    jobconf.setJobName(jobName);
    jobconf.setMapSpeculativeExecution(false);
    RaidUtils.parseAndSetOptions(jobconf, SCHEDULER_OPTION_LABEL);

    jobconf.setJarByClass(DistRaid.class);
    jobconf.setInputFormat(DistRaidInputFormat.class);
    jobconf.setOutputKeyClass(Text.class);
    jobconf.setOutputValueClass(Text.class);

    jobconf.setMapperClass(DistRaidMapper.class);
    jobconf.setNumReduceTasks(0);
    return jobconf;
  }
Beispiel #8
0
 /**
  * Reload config file if it hasn't been loaded in a while Returns true if the file was reloaded.
  */
 public synchronized boolean reloadConfigsIfNecessary() {
   long time = RaidNode.now();
   if (time > lastReloadAttempt + reloadInterval) {
     lastReloadAttempt = time;
     try {
       File file = new File(configFileName);
       long lastModified = file.lastModified();
       if (lastModified > lastSuccessfulReload && time > lastModified + RELOAD_WAIT) {
         reloadConfigs();
         lastSuccessfulReload = time;
         lastReloadAttemptFailed = false;
         return true;
       }
     } catch (Exception e) {
       if (!lastReloadAttemptFailed) {
         LOG.error("Failed to reload config file - " + "will use existing configuration.", e);
       }
       lastReloadAttemptFailed = true;
     }
   }
   return false;
 }
  public void testMultiplePriorities() throws Exception {
    long[] crcs = new long[3];
    int[] seeds = new int[3];
    Path dirPath = new Path("/home/test");
    int stripeLength = 3;
    short repl = 1;
    mySetup(stripeLength);
    Codec codec = Codec.getCodec("rs");
    LOG.info("Starting testMultiplePriorities");
    try {
      // Create test file and raid it.
      Path[] files =
          TestRaidDfs.createTestFiles(
              dirPath, fileSizes, blockSizes, crcs, seeds, fileSys, (short) 1);
      FileStatus stat = fileSys.getFileStatus(dirPath);
      RaidNode.doRaid(
          conf,
          stat,
          new Path(codec.parityDirectory),
          codec,
          new RaidNode.Statistics(),
          RaidUtils.NULL_PROGRESSABLE,
          false,
          repl,
          repl);

      Integer[] corruptBlockIdxs = new Integer[] {0, 2};
      LOG.info("Corrupt block " + corruptBlockIdxs + " of directory " + dirPath);
      TestDirectoryRaidDfs.corruptBlocksInDirectory(
          conf, dirPath, crcs, corruptBlockIdxs, fileSys, dfsCluster, false, true);

      // Create Block Fixer and fix.
      FakeDistBlockIntegrityMonitor distBlockFixer = new FakeDistBlockIntegrityMonitor(conf);
      assertEquals(0, distBlockFixer.submittedJobs.size());
      // waiting for one job to submit
      long startTime = System.currentTimeMillis();
      while (System.currentTimeMillis() - startTime < 120000
          && distBlockFixer.submittedJobs.size() == 0) {
        distBlockFixer.getCorruptionMonitor().checkAndReconstructBlocks();
        LOG.info("Waiting for jobs to submit");
        Thread.sleep(10000);
      }
      int submittedJob = distBlockFixer.submittedJobs.size();
      LOG.info("Already Submitted " + submittedJob + " jobs");
      assertTrue("Should submit more than 1 jobs", submittedJob >= 1);

      // Corrupt two more blocks
      corruptBlockIdxs = new Integer[] {4, 5};
      LOG.info("Corrupt block " + corruptBlockIdxs + " of directory " + dirPath);
      TestDirectoryRaidDfs.corruptBlocksInDirectory(
          conf, dirPath, crcs, corruptBlockIdxs, fileSys, dfsCluster, false, true);

      // A new job should be submitted since two blocks are corrupt.
      startTime = System.currentTimeMillis();
      while (System.currentTimeMillis() - startTime < 120000
          && distBlockFixer.submittedJobs.size() == submittedJob) {
        distBlockFixer.getCorruptionMonitor().checkAndReconstructBlocks();
        LOG.info("Waiting for more jobs to submit");
        Thread.sleep(10000);
      }
      LOG.info("Already Submitted " + distBlockFixer.submittedJobs.size() + " jobs");
      assertTrue(
          "should submit more than 1 jobs",
          distBlockFixer.submittedJobs.size() - submittedJob >= 1);
    } finally {
      myTearDown();
    }
  }
  /**
   * tests that the distributed block fixer obeys the limit on how many jobs to submit
   * simultaneously.
   */
  @Test
  public void testMaxPendingJobs() throws Exception {
    LOG.info("Test testMaxPendingJobs started.");
    int stripeLength = 3;
    mySetup(stripeLength);
    long[] crcs1 = new long[3];
    int[] seeds1 = new int[3];
    long[] crcs2 = new long[3];
    int[] seeds2 = new int[3];
    Path dirPath1 = new Path("/user/dhruba/raidtestrs/1");
    Path[] files1 =
        TestRaidDfs.createTestFiles(
            dirPath1, fileSizes, blockSizes, crcs1, seeds1, fileSys, (short) 1);
    Path dirPath2 = new Path("/user/dhruba/raidtestrs/2");
    Path[] files2 =
        TestRaidDfs.createTestFiles(
            dirPath2, fileSizes, blockSizes, crcs2, seeds2, fileSys, (short) 1);
    Path destPath = new Path("/destraidrs/user/dhruba/raidtestrs");
    LOG.info("Test testMaxPendingJobs created test files");
    Configuration localConf = this.getRaidNodeConfig(conf, false);
    localConf.setLong("raid.blockfix.maxpendingjobs", 1L);

    try {
      cnode = RaidNode.createRaidNode(null, localConf);
      TestRaidDfs.waitForDirRaided(LOG, fileSys, dirPath1, destPath);
      TestRaidDfs.waitForDirRaided(LOG, fileSys, dirPath2, destPath);
      cnode.stop();
      cnode.join();

      DistributedFileSystem dfs = (DistributedFileSystem) fileSys;
      String[] corruptFiles = DFSUtil.getCorruptFiles(dfs);
      assertEquals("no corrupt files expected", 0, corruptFiles.length);
      assertEquals(
          "filesFixed() should return 0 before fixing files",
          0,
          cnode.blockIntegrityMonitor.getNumFilesFixed());
      // corrupt directory 1
      this.corruptFiles(
          dirPath1, crcs1, rsCorruptFileIdx1, dfs, files1, rsNumCorruptBlocksInFiles1);

      cnode = RaidNode.createRaidNode(null, localConf);
      DistBlockIntegrityMonitor blockFixer =
          (DistBlockIntegrityMonitor) cnode.blockIntegrityMonitor;
      long start = System.currentTimeMillis();

      while (blockFixer.jobsRunning() < 1 && System.currentTimeMillis() - start < 60000) {
        LOG.info("Test testDirBlockFix waiting for fixing job 1 to start");
        Thread.sleep(1000);
      }
      assertEquals("job not running", 1, blockFixer.jobsRunning());

      // corrupt directory 2
      this.corruptFiles(
          dirPath2, crcs2, rsCorruptFileIdx2, dfs, files2, rsNumCorruptBlocksInFiles2);

      // wait until both files are fixed
      while (blockFixer.getNumFilesFixed() < 6 && System.currentTimeMillis() - start < 240000) {
        // make sure the block fixer does not start a second job while
        // the first one is still running
        assertTrue("too many jobs running", blockFixer.jobsRunning() <= 1);
        Thread.sleep(1000);
      }
      TestBlockFixer.verifyMetrics(
          fileSys,
          cnode,
          false,
          6L,
          getTotal(rsNumCorruptBlocksInFiles1) + getTotal(rsNumCorruptBlocksInFiles2));
      dfs = getDFS(conf, dfs);
      for (int i = 0; i < fileSizes.length; i++) {
        assertTrue(
            "file " + files1[i] + " not fixed",
            TestRaidDfs.validateFile(dfs, files1[i], fileSizes[i], crcs1[i]));
      }
      for (int i = 0; i < fileSizes.length; i++) {
        assertTrue(
            "file " + files2[i] + " not fixed",
            TestRaidDfs.validateFile(dfs, files2[i], fileSizes[i], crcs2[i]));
      }
    } catch (Exception e) {
      LOG.info("Test testMaxPendingJobs exception " + e + StringUtils.stringifyException(e));
      throw e;
    } finally {
      myTearDown();
    }
  }
  /** tests that we can have 2 concurrent jobs fixing files (dist block fixer) */
  @Test
  public void testConcurrentJobs() throws Exception {
    LOG.info("Test testConcurrentJobs started.");
    int stripeLength = 3;
    mySetup(stripeLength);
    long[] crcs1 = new long[3];
    int[] seeds1 = new int[3];
    long[] crcs2 = new long[3];
    int[] seeds2 = new int[3];
    Path dirPath1 = new Path("/user/dhruba/raidtestrs/1");
    Path[] files1 =
        TestRaidDfs.createTestFiles(
            dirPath1, fileSizes, blockSizes, crcs1, seeds1, fileSys, (short) 1);
    Path dirPath2 = new Path("/user/dhruba/raidtestrs/2");
    Path[] files2 =
        TestRaidDfs.createTestFiles(
            dirPath2, fileSizes, blockSizes, crcs2, seeds2, fileSys, (short) 1);
    Path destPath = new Path("/destraidrs/user/dhruba/raidtestrs");

    LOG.info("Test testConcurrentJobs created test files");
    Configuration localConf = this.getRaidNodeConfig(conf, false);
    localConf.setLong(BlockIntegrityMonitor.BLOCKCHECK_INTERVAL, 15000L);
    localConf.setLong(DistBlockIntegrityMonitor.RAIDNODE_BLOCK_FIX_SUBMISSION_INTERVAL_KEY, 15000L);
    localConf.setLong(
        DistBlockIntegrityMonitor.RAIDNODE_BLOCK_FIX_SCAN_SUBMISSION_INTERVAL_KEY, 3600000);

    try {
      cnode = RaidNode.createRaidNode(null, localConf);
      TestRaidDfs.waitForDirRaided(LOG, fileSys, dirPath1, destPath);
      TestRaidDfs.waitForDirRaided(LOG, fileSys, dirPath2, destPath);
      cnode.stop();
      cnode.join();

      DistributedFileSystem dfs = (DistributedFileSystem) fileSys;
      String[] corruptFiles = DFSUtil.getCorruptFiles(dfs);
      assertEquals("no corrupt files expected", 0, corruptFiles.length);
      assertEquals(
          "filesFixed() should return 0 before fixing files",
          0,
          cnode.blockIntegrityMonitor.getNumFilesFixed());
      // corrupt directory 1
      this.corruptFiles(
          dirPath1, crcs1, rsCorruptFileIdx1, dfs, files1, rsNumCorruptBlocksInFiles1);

      cnode = RaidNode.createRaidNode(null, localConf);
      DistBlockIntegrityMonitor blockFixer =
          (DistBlockIntegrityMonitor) cnode.blockIntegrityMonitor;
      long start = System.currentTimeMillis();

      // All files are HIGH-PRI corrupt files
      while (blockFixer.jobsRunning() < 1 && System.currentTimeMillis() - start < 60000) {
        LOG.info("Test testDirBlockFix waiting for fixing job 1 to start");
        Thread.sleep(1000);
      }
      assertEquals("job 1 not running", 1, blockFixer.jobsRunning());

      // Corrupt directory 2
      this.corruptFiles(
          dirPath2, crcs2, rsCorruptFileIdx2, dfs, files2, rsNumCorruptBlocksInFiles2);

      // 1 LOW-PRI file and 2 HIGH-PRI files
      while (blockFixer.jobsRunning() < 3 && System.currentTimeMillis() - start < 60000) {
        LOG.info("Test testDirBlockFix waiting for fixing job 2 and 3 to start");
        Thread.sleep(1000);
      }
      assertTrue("more than 3 jobs are running", blockFixer.jobsRunning() >= 3);

      while (blockFixer.getNumFilesFixed() < 6 && System.currentTimeMillis() - start < 240000) {
        LOG.info("Test testDirBlockFix waiting for files to be fixed.");
        Thread.sleep(1000);
      }
      TestBlockFixer.verifyMetrics(
          fileSys,
          cnode,
          false,
          6L,
          getTotal(rsNumCorruptBlocksInFiles1) + getTotal(rsNumCorruptBlocksInFiles2));
      dfs = getDFS(conf, dfs);
      for (int i = 0; i < fileSizes.length; i++) {
        assertTrue(
            "file " + files1[i] + " not fixed",
            TestRaidDfs.validateFile(dfs, files1[i], fileSizes[i], crcs1[i]));
      }
      for (int i = 0; i < fileSizes.length; i++) {
        assertTrue(
            "file " + files2[i] + " not fixed",
            TestRaidDfs.validateFile(dfs, files2[i], fileSizes[i], crcs2[i]));
      }
    } catch (Exception e) {
      LOG.info("Test testConcurrentJobs exception " + e, e);
      throw e;
    } finally {
      myTearDown();
    }
  }
  /** Corrupt a parity file and wait for it to get fixed. */
  private void implParityBlockFix(String testName, boolean local) throws Exception {
    LOG.info("Test " + testName + " started.");
    int stripeLength = 3;
    mySetup(stripeLength);
    long[] crcs = new long[3];
    int[] seeds = new int[3];
    Path dirPath = new Path("/user/dhruba/raidtest");
    Path[] files =
        TestRaidDfs.createTestFiles(
            dirPath, fileSizes, blockSizes, crcs, seeds, fileSys, (short) 1);
    Path destPath = new Path("/destraid/user/dhruba");
    Path parityFile = new Path("/destraid/user/dhruba/raidtest");
    LOG.info("Test " + testName + " created test files");
    Configuration localConf = this.getRaidNodeConfig(conf, local);

    try {
      cnode = RaidNode.createRaidNode(null, localConf);
      TestRaidDfs.waitForDirRaided(LOG, fileSys, dirPath, destPath);
      cnode.stop();
      cnode.join();

      long parityCRC = RaidDFSUtil.getCRC(fileSys, parityFile);

      FileStatus parityStat = fileSys.getFileStatus(parityFile);
      DistributedFileSystem dfs = (DistributedFileSystem) fileSys;
      LocatedBlocks locs =
          RaidDFSUtil.getBlockLocations(dfs, parityFile.toUri().getPath(), 0, parityStat.getLen());
      String[] corruptFiles = DFSUtil.getCorruptFiles(dfs);

      assertEquals("no corrupt files expected", 0, corruptFiles.length);
      assertEquals(
          "filesFixed() should return 0 before fixing files",
          0,
          cnode.blockIntegrityMonitor.getNumFilesFixed());

      // Corrupt parity blocks for different stripes.
      int[] corruptBlockIdxs = new int[] {0, 1, 2};
      for (int idx : corruptBlockIdxs) corruptBlock(locs.get(idx).getBlock(), dfsCluster);
      RaidDFSUtil.reportCorruptBlocks(dfs, parityFile, corruptBlockIdxs, 2 * blockSize);

      corruptFiles = DFSUtil.getCorruptFiles(dfs);
      assertEquals("file not corrupted", 1, corruptFiles.length);
      assertEquals("wrong file corrupted", corruptFiles[0], parityFile.toUri().getPath());

      cnode = RaidNode.createRaidNode(null, localConf);
      long start = System.currentTimeMillis();
      while (cnode.blockIntegrityMonitor.getNumFilesFixed() < 1
          && System.currentTimeMillis() - start < 120000) {
        LOG.info("Test " + testName + " waiting for files to be fixed.");
        Thread.sleep(3000);
      }
      TestBlockFixer.verifyMetrics(fileSys, cnode, local, 1L, corruptBlockIdxs.length);

      long checkCRC = RaidDFSUtil.getCRC(fileSys, parityFile);

      assertEquals("file not fixed", parityCRC, checkCRC);

    } catch (Exception e) {
      LOG.info("Test " + testName + " Exception " + e + StringUtils.stringifyException(e));
      throw e;
    } finally {
      myTearDown();
    }
    LOG.info("Test " + testName + " completed.");
  }
  /**
   * Tests integrity of generated block. Create a file and delete a block entirely. Wait for the
   * block to be regenerated. Now stop RaidNode and corrupt the generated block. Test that
   * corruption in the generated block can be detected by clients.
   */
  private void generatedBlockTestCommon(String testName, int blockToCorrupt, boolean local)
      throws Exception {
    LOG.info("Test " + testName + " started.");
    int stripeLength = 3;
    mySetup(stripeLength);
    long[] crcs = new long[3];
    int[] seeds = new int[3];
    Path dirPath = new Path("/user/dhruba/raidtest");
    Path[] files =
        TestRaidDfs.createTestFiles(
            dirPath, fileSizes, blockSizes, crcs, seeds, fileSys, (short) 1);
    Path destPath = new Path("/destraid/user/dhruba");
    LOG.info("Test " + testName + " created test files");
    Configuration localConf = this.getRaidNodeConfig(conf, local);

    try {
      cnode = RaidNode.createRaidNode(null, localConf);
      TestRaidDfs.waitForDirRaided(LOG, fileSys, dirPath, destPath);
      cnode.stop();
      cnode.join();

      DistributedFileSystem dfs = (DistributedFileSystem) fileSys;
      String[] corruptFiles = DFSUtil.getCorruptFiles(dfs);
      assertEquals("no corrupt files expected", 0, corruptFiles.length);
      assertEquals(
          "filesFixed() should return 0 before fixing files",
          0,
          cnode.blockIntegrityMonitor.getNumFilesFixed());

      Integer[] corruptBlockIdxs = new Integer[] {blockToCorrupt};
      TestDirectoryRaidDfs.corruptBlocksInDirectory(
          conf, dirPath, crcs, corruptBlockIdxs, fileSys, dfsCluster, false, true);

      corruptFiles = DFSUtil.getCorruptFiles(dfs);
      assertEquals("files not corrupted", corruptBlockIdxs.length, corruptFiles.length);
      int corruptFileIdx = -1;
      for (int i = 0; i < files.length; i++) {
        if (files[i].toUri().getPath().equals(corruptFiles[0])) {
          corruptFileIdx = i;
          break;
        }
      }
      assertNotSame("Wrong corrupt file", -1, corruptFileIdx);
      cnode = RaidNode.createRaidNode(null, localConf);
      long start = System.currentTimeMillis();
      while (cnode.blockIntegrityMonitor.getNumFilesFixed() < 1
          && System.currentTimeMillis() - start < 120000) {
        LOG.info("Test testDirBlockFix waiting for files to be fixed.");
        Thread.sleep(1000);
      }
      TestBlockFixer.verifyMetrics(fileSys, cnode, local, 1L, corruptBlockIdxs.length);
      // Stop RaidNode
      cnode.stop();
      cnode.join();
      cnode = null;

      // The block has successfully been reconstructed.
      dfs = getDFS(conf, dfs);
      assertTrue(
          "file not fixed",
          TestRaidDfs.validateFile(
              dfs, files[corruptFileIdx], fileSizes[corruptFileIdx], crcs[corruptFileIdx]));

      // Now corrupt the generated block.
      TestDirectoryRaidDfs.corruptBlocksInDirectory(
          conf, dirPath, crcs, corruptBlockIdxs, dfs, dfsCluster, false, false);
      try {
        TestRaidDfs.validateFile(
            dfs, files[corruptFileIdx], fileSizes[corruptFileIdx], crcs[corruptFileIdx]);
        fail("Expected exception not thrown");
      } catch (org.apache.hadoop.fs.ChecksumException ce) {
      } catch (org.apache.hadoop.fs.BlockMissingException bme) {
      }
    } catch (Exception e) {
      LOG.info("Test " + testName + " Exception " + e, e);
      throw e;
    } finally {
      myTearDown();
    }
    LOG.info("Test " + testName + " completed.");
  }
  /**
   * Create a file with three stripes, corrupt a block each in two stripes, and wait for the the
   * file to be fixed.
   */
  private void implDirBlockFix(boolean local, boolean hasStripeInfo, boolean corruptStripe)
      throws Exception {
    LOG.info(
        "Test testDirBlockFix started. local:"
            + local
            + " hasStripeInfo:"
            + hasStripeInfo
            + " corruptStripe:"
            + corruptStripe);
    int stripeLength = 3;
    mySetup(stripeLength);
    long[] crcs = new long[3];
    int[] seeds = new int[3];
    Path dirPath = new Path("/user/dhruba/raidtestrs");
    Path[] files =
        TestRaidDfs.createTestFiles(
            dirPath, fileSizes, blockSizes, crcs, seeds, fileSys, (short) 1);
    Path destPath = new Path("/destraidrs/user/dhruba");
    LOG.info("Test testDirBlockFix created test files");
    Configuration localConf = this.getRaidNodeConfig(conf, local);
    // Not allow multiple running jobs
    localConf.setLong("raid.blockfix.maxpendingjobs", 1L);

    try {
      cnode = RaidNode.createRaidNode(null, localConf);
      TestRaidDfs.waitForDirRaided(LOG, fileSys, dirPath, destPath);
      cnode.stop();
      cnode.join();

      DistributedFileSystem dfs = (DistributedFileSystem) fileSys;
      String[] corruptFiles = DFSUtil.getCorruptFiles(dfs);
      assertEquals("no corrupt files expected", 0, corruptFiles.length);
      assertEquals(
          "filesFixed() should return 0 before fixing files",
          0,
          cnode.blockIntegrityMonitor.getNumFilesFixed());
      if (!hasStripeInfo) {
        // clear out all stripes
        LocalStripeStore lss = new LocalStripeStore();
        lss.initialize(localConf, false, dfs);
        lss.clear();
      }
      if (corruptStripe) {
        LocalStripeStore lss = new LocalStripeStore();
        lss.initialize(localConf, false, dfs);
        Set<List<Block>> corruptCandidates = new HashSet<List<Block>>(lss.stripeSet.keySet());
        for (List<Block> lb : corruptCandidates) {
          for (Codec codec : Codec.getCodecs()) {
            StripeInfo si = lss.getStripe(codec, lb.get(0));
            if (si == null) {
              continue;
            }
            String oldSi = si.toString();
            Collections.rotate(si.parityBlocks, 1);
            Collections.rotate(si.srcBlocks, 1);
            lss.putStripe(codec, si.parityBlocks, si.srcBlocks);
            String newSi = lss.getStripe(codec, lb.get(0)).toString();
            LOG.info("Corrupt the stripe info old : " + oldSi + " new : " + newSi);
          }
        }
      }

      this.corruptFiles(dirPath, crcs, rsCorruptFileIdx1, dfs, files, rsNumCorruptBlocksInFiles1);
      cnode = RaidNode.createRaidNode(null, localConf);
      long start = System.currentTimeMillis();
      while (cnode.blockIntegrityMonitor.getNumFilesFixed() < 3
          && cnode.blockIntegrityMonitor.getNumFileFixFailures() < 3
          && System.currentTimeMillis() - start < 120000) {
        LOG.info("Test testDirBlockFix waiting for files to be fixed.");
        Thread.sleep(1000);
      }
      long totalCorruptBlocks = getTotal(rsNumCorruptBlocksInFiles1);
      if (hasStripeInfo) {
        if (!corruptStripe) {
          TestBlockFixer.verifyMetrics(fileSys, cnode, local, 3L, totalCorruptBlocks);

          dfs = getDFS(conf, dfs);
          for (int i = 0; i < fileSizes.length; i++) {
            assertTrue(
                "file " + files[i] + " not fixed",
                TestRaidDfs.validateFile(dfs, files[i], fileSizes[i], crcs[i]));
          }
        } else {
          TestBlockFixer.verifyMetrics(fileSys, cnode, local, 0L, 0L);
          assertTrue(
              "should fail to fix more than 3 files",
              cnode.blockIntegrityMonitor.getNumFileFixFailures() >= 3L);
          TestBlockFixer.verifyMetrics(
              fileSys, cnode, LOGTYPES.OFFLINE_RECONSTRUCTION_FILE, LOGRESULTS.FAILURE, 3L, true);
          // Will throw stripe mismatch exception for the first blocks of 3 files
          TestBlockFixer.verifyMetrics(
              fileSys,
              cnode,
              LOGTYPES.OFFLINE_RECONSTRUCTION_STRIPE_VERIFICATION,
              LOGRESULTS.FAILURE,
              3L,
              true);
        }
      } else {
        TestBlockFixer.verifyMetrics(fileSys, cnode, local, 0L, 0L);
        assertTrue(
            "should fail to fix more than 3 files",
            cnode.blockIntegrityMonitor.getNumFileFixFailures() >= 3L);
        TestBlockFixer.verifyMetrics(
            fileSys,
            cnode,
            LOGTYPES.OFFLINE_RECONSTRUCTION_GET_STRIPE,
            LOGRESULTS.FAILURE,
            totalCorruptBlocks,
            true);
        TestBlockFixer.verifyMetrics(
            fileSys, cnode, LOGTYPES.OFFLINE_RECONSTRUCTION_FILE, LOGRESULTS.FAILURE, 3L, true);
      }
    } catch (Exception e) {
      LOG.info("Test testDirBlockFix Exception " + e, e);
      throw e;
    } finally {
      myTearDown();
    }
    LOG.info("Test testDirBlockFix completed.");
  }