Example #1
0
  /**
   * Check files on DFS, starting from the indicated path.
   *
   * @throws Exception
   */
  public void fsck() throws IOException {
    NameNode.getNameNodeMetrics().numFsckOperations.inc();
    InjectionHandler.processEvent(InjectionEvent.NAMENODE_FSCK_START);

    try {
      FileStatus[] files = nn.namesystem.dir.getListing(path);
      FsckResult res = new FsckResult();
      if (!this.showFiles && !this.showBlocks && !this.showLocations && !this.showRacks) {
        res.totalRacks = nn.getNetworkTopology().getNumOfRacks();
        res.totalDatanodes = nn.namesystem.getNumberOfDatanodes(DatanodeReportType.LIVE);
      }
      res.setReplication((short) conf.getInt("dfs.replication", 3));
      if (files != null) {
        if (showCorruptFileBlocks && showOpenFiles) {
          listCorruptOpenFiles();

          return;
        }

        if (showCorruptFileBlocks) {
          listCorruptFileBlocks();
          return;
        }

        for (int i = 0; i < files.length; i++) {
          check(files[i], res);
        }
        out.println(res);
        // DFSck client scans for the string HEALTHY/CORRUPT to check the status
        // of file system and return appropriate code. Changing the output
        // string might break testcases.
        if (res.isHealthy()) {
          out.print("\n\nThe filesystem under path '" + path + "' " + HEALTHY_STATUS);
        } else {
          out.print("\n\nThe filesystem under path '" + path + "' " + CORRUPT_STATUS);
        }
      } else {
        out.print("\n\nPath '" + path + "' " + NONEXISTENT_STATUS);
      }
    } catch (Throwable e) {
      String errMsg = "Fsck on path '" + path + "' " + FAILURE_STATUS;
      LOG.warn(errMsg, e);
      out.println(e.getMessage());
      out.print("\n\n" + errMsg);
    } finally {
      out.close();
    }
  }
Example #2
0
  private void check(FileStatus file, FsckResult res) throws IOException {
    int minReplication = nn.namesystem.getMinReplication();
    String path = file.getPath().toString();
    boolean isOpen = false;

    if (file.isDir()) {
      FileStatus[] files = nn.namesystem.dir.getListing(path);
      if (files == null) {
        return;
      }
      if (showFiles) {
        out.println(path + " <dir>");
      }
      res.totalDirs++;
      for (int i = 0; i < files.length; i++) {
        check(files[i], res);
      }
      return;
    }
    long fileLen = file.getLen();
    LocatedBlocks blocks = nn.namesystem.getBlockLocations(path, 0, fileLen);
    if (blocks == null) { // the file is deleted
      return;
    }
    isOpen = blocks.isUnderConstruction();
    if (isOpen && !showOpenFiles) {
      // We collect these stats about open files to report with default options
      res.totalOpenFilesSize += fileLen;
      res.totalOpenFilesBlocks += blocks.locatedBlockCount();
      res.totalOpenFiles++;
      return;
    }
    res.totalFiles++;
    res.totalSize += fileLen;
    res.totalBlocks += blocks.locatedBlockCount();
    if (showOpenFiles && isOpen) {
      out.print(
          path
              + " "
              + fileLen
              + " bytes, "
              + blocks.locatedBlockCount()
              + " block(s), OPENFORWRITE: ");
    } else if (showFiles) {
      out.print(path + " " + fileLen + " bytes, " + blocks.locatedBlockCount() + " block(s): ");
    } else {
      out.print('.');
    }
    if (res.totalFiles % 100 == 0) {
      out.println();
      out.flush();
    }
    int missing = 0;
    int corrupt = 0;
    long missize = 0;
    int underReplicatedPerFile = 0;
    int misReplicatedPerFile = 0;
    StringBuffer report = new StringBuffer();
    int i = 0;
    for (LocatedBlock lBlk : blocks.getLocatedBlocks()) {
      Block block = lBlk.getBlock();
      boolean isCorrupt = lBlk.isCorrupt();
      String blkName = block.toString();
      DatanodeInfo[] locs = lBlk.getLocations();
      res.totalReplicas += locs.length;
      short targetFileReplication = file.getReplication();
      if (locs.length > targetFileReplication) {
        res.excessiveReplicas += (locs.length - targetFileReplication);
        res.numOverReplicatedBlocks += 1;
      }
      // Check if block is Corrupt
      if (isCorrupt) {
        corrupt++;
        res.corruptBlocks++;
        out.print("\n" + path + ": CORRUPT block " + block.getBlockName() + "\n");
      }
      if (locs.length >= minReplication) res.numMinReplicatedBlocks++;
      if (locs.length < targetFileReplication && locs.length > 0) {
        res.missingReplicas += (targetFileReplication - locs.length);
        res.numUnderReplicatedBlocks += 1;
        underReplicatedPerFile++;
        if (!showFiles) {
          out.print("\n" + path + ": ");
        }
        out.println(
            " Under replicated "
                + block
                + ". Target Replicas is "
                + targetFileReplication
                + " but found "
                + locs.length
                + " replica(s).");
      }
      // verify block placement policy
      int missingRacks =
          nn.getBlockPlacementPolicy()
              .verifyBlockPlacement(path, lBlk, Math.min(2, targetFileReplication));

      if (missingRacks > 0) {
        res.numMisReplicatedBlocks++;
        misReplicatedPerFile++;
        if (!showFiles) {
          if (underReplicatedPerFile == 0) out.println();
          out.print(path + ": ");
        }
        out.println(
            " Replica placement policy is violated for "
                + block
                + ". Block should be additionally replicated on "
                + missingRacks
                + " more rack(s).");
      }
      report.append(i + ". " + blkName + " len=" + block.getNumBytes());
      if (locs.length == 0) {
        report.append(" MISSING!");
        res.addMissing(block.toString(), block.getNumBytes());
        missing++;
        missize += block.getNumBytes();
      } else {
        report.append(" repl=" + locs.length);
        if (showLocations || showRacks) {
          StringBuffer sb = new StringBuffer("[");
          for (int j = 0; j < locs.length; j++) {
            if (j > 0) {
              sb.append(", ");
            }
            if (showRacks) sb.append(NodeBase.getPath(locs[j]));
            else sb.append(locs[j]);
          }
          sb.append(']');
          report.append(" " + sb.toString());
        }
      }
      report.append('\n');
      i++;
    }
    if ((missing > 0) || (corrupt > 0)) {
      if (!showFiles && (missing > 0)) {
        out.print(
            "\n" + path + ": MISSING " + missing + " blocks of total size " + missize + " B.");
      }
      res.corruptFiles++;
      switch (fixing) {
        case FIXING_NONE:
          break;
        case FIXING_MOVE:
          if (!isOpen) lostFoundMove(file, blocks);
          break;
        case FIXING_DELETE:
          if (!isOpen) nn.namesystem.deleteInternal(path, null, false, false);
      }
    }
    if (showFiles) {
      if (missing > 0) {
        out.print(" MISSING " + missing + " blocks of total size " + missize + " B\n");
      } else if (underReplicatedPerFile == 0 && misReplicatedPerFile == 0) {
        out.print(" OK\n");
      }
      if (showBlocks) {
        out.print(report.toString() + "\n");
      }
    }
  }