コード例 #1
0
ファイル: Hadoop23Shims.java プロジェクト: prongs/apache-hive
 @Override
 public List<HdfsFileStatusWithId> listLocatedHdfsStatus(FileSystem fs, Path p, PathFilter filter)
     throws IOException {
   DistributedFileSystem dfs = ensureDfs(fs);
   DFSClient dfsc = dfs.getClient();
   final String src = p.toUri().getPath();
   DirectoryListing current =
       dfsc.listPaths(src, org.apache.hadoop.hdfs.protocol.HdfsFileStatus.EMPTY_NAME, true);
   if (current == null) { // the directory does not exist
     throw new FileNotFoundException("File " + p + " does not exist.");
   }
   final URI fsUri = fs.getUri();
   List<HdfsFileStatusWithId> result =
       new ArrayList<HdfsFileStatusWithId>(current.getPartialListing().length);
   while (current != null) {
     org.apache.hadoop.hdfs.protocol.HdfsFileStatus[] hfss = current.getPartialListing();
     for (int i = 0; i < hfss.length; ++i) {
       HdfsLocatedFileStatus next = (HdfsLocatedFileStatus) (hfss[i]);
       if (filter != null) {
         Path filterPath = next.getFullPath(p).makeQualified(fsUri, null);
         if (!filter.accept(filterPath)) continue;
       }
       LocatedFileStatus lfs = next.makeQualifiedLocated(fsUri, p);
       result.add(new HdfsFileStatusWithIdImpl(lfs, next.getFileId()));
     }
     current = current.hasMore() ? dfsc.listPaths(src, current.getLastName(), true) : null;
   }
   return result;
 }
コード例 #2
0
  static void generateDirectoryStructure(
      JspWriter out, HttpServletRequest req, HttpServletResponse resp, Configuration conf)
      throws IOException, InterruptedException {
    final String dir =
        JspHelper.validatePath(StringEscapeUtils.unescapeHtml(req.getParameter("dir")));
    if (dir == null) {
      out.print("Invalid input");
      return;
    }
    String tokenString = req.getParameter(JspHelper.DELEGATION_PARAMETER_NAME);
    UserGroupInformation ugi = JspHelper.getUGI(req, conf);
    String namenodeInfoPortStr = req.getParameter("namenodeInfoPort");
    int namenodeInfoPort = -1;
    if (namenodeInfoPortStr != null) namenodeInfoPort = Integer.parseInt(namenodeInfoPortStr);
    final String nnAddr = req.getParameter(JspHelper.NAMENODE_ADDRESS);
    if (nnAddr == null) {
      out.print(JspHelper.NAMENODE_ADDRESS + " url param is null");
      return;
    }

    DFSClient dfs = getDFSClient(ugi, nnAddr, conf);
    String target = dir;
    final HdfsFileStatus targetStatus = dfs.getFileInfo(target);
    if (targetStatus == null) { // not exists
      out.print("<h3>File or directory : " + target + " does not exist</h3>");
      JspHelper.printGotoForm(out, namenodeInfoPort, tokenString, target, nnAddr);
    } else {
      if (!targetStatus.isDir()) { // a file
        List<LocatedBlock> blocks =
            dfs.getNamenode().getBlockLocations(dir, 0, 1).getLocatedBlocks();

        LocatedBlock firstBlock = null;
        DatanodeInfo[] locations = null;
        if (blocks.size() > 0) {
          firstBlock = blocks.get(0);
          locations = firstBlock.getLocations();
        }
        if (locations == null || locations.length == 0) {
          out.print("Empty file");
        } else {
          DatanodeInfo chosenNode = JspHelper.bestNode(firstBlock, conf);
          String fqdn = canonicalize(chosenNode.getIpAddr());
          int datanodePort = chosenNode.getXferPort();
          String redirectLocation =
              "http://"
                  + fqdn
                  + ":"
                  + chosenNode.getInfoPort()
                  + "/browseBlock.jsp?blockId="
                  + firstBlock.getBlock().getBlockId()
                  + "&blockSize="
                  + firstBlock.getBlock().getNumBytes()
                  + "&genstamp="
                  + firstBlock.getBlock().getGenerationStamp()
                  + "&filename="
                  + URLEncoder.encode(dir, "UTF-8")
                  + "&datanodePort="
                  + datanodePort
                  + "&namenodeInfoPort="
                  + namenodeInfoPort
                  + JspHelper.getDelegationTokenUrlParam(tokenString)
                  + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr);
          resp.sendRedirect(redirectLocation);
        }
        return;
      }
      // directory
      // generate a table and dump the info
      String[] headings = {
        "Name",
        "Type",
        "Size",
        "Replication",
        "Block Size",
        "Modification Time",
        "Permission",
        "Owner",
        "Group"
      };
      out.print("<h3>Contents of directory ");
      JspHelper.printPathWithLinks(dir, out, namenodeInfoPort, tokenString, nnAddr);
      out.print("</h3><hr>");
      JspHelper.printGotoForm(out, namenodeInfoPort, tokenString, dir, nnAddr);
      out.print("<hr>");

      File f = new File(dir);
      String parent;
      if ((parent = f.getParent()) != null)
        out.print(
            "<a href=\""
                + req.getRequestURL()
                + "?dir="
                + parent
                + "&namenodeInfoPort="
                + namenodeInfoPort
                + JspHelper.getDelegationTokenUrlParam(tokenString)
                + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr)
                + "\">Go to parent directory</a><br>");

      DirectoryListing thisListing = dfs.listPaths(target, HdfsFileStatus.EMPTY_NAME);
      if (thisListing == null || thisListing.getPartialListing().length == 0) {
        out.print("Empty directory");
      } else {
        JspHelper.addTableHeader(out);
        int row = 0;
        JspHelper.addTableRow(out, headings, row++);
        String cols[] = new String[headings.length];
        do {
          HdfsFileStatus[] files = thisListing.getPartialListing();
          for (int i = 0; i < files.length; i++) {
            String localFileName = files[i].getLocalName();
            // Get the location of the first block of the file
            if (!files[i].isDir()) {
              cols[1] = "file";
              cols[2] = StringUtils.byteDesc(files[i].getLen());
              cols[3] = Short.toString(files[i].getReplication());
              cols[4] = StringUtils.byteDesc(files[i].getBlockSize());
            } else {
              cols[1] = "dir";
              cols[2] = "";
              cols[3] = "";
              cols[4] = "";
            }
            String datanodeUrl =
                req.getRequestURL()
                    + "?dir="
                    + URLEncoder.encode(files[i].getFullName(target), "UTF-8")
                    + "&namenodeInfoPort="
                    + namenodeInfoPort
                    + JspHelper.getDelegationTokenUrlParam(tokenString)
                    + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnAddr);
            cols[0] = "<a href=\"" + datanodeUrl + "\">" + localFileName + "</a>";
            cols[5] = lsDateFormat.format(new Date((files[i].getModificationTime())));
            cols[6] = files[i].getPermission().toString();
            cols[7] = files[i].getOwner();
            cols[8] = files[i].getGroup();
            JspHelper.addTableRow(out, cols, row++);
          }
          if (!thisListing.hasMore()) {
            break;
          }
          thisListing = dfs.listPaths(target, thisListing.getLastName());
        } while (thisListing != null);
        JspHelper.addTableFooter(out);
      }
    }
    out.print(
        "<br><a href=\"http://"
            + canonicalize(nnAddr)
            + ":"
            + namenodeInfoPort
            + "/dfshealth.jsp\">Go back to DFS home</a>");
    dfs.close();
  }