예제 #1
0
파일: ParseUrl.java 프로젝트: ewinli/crawer
  public Map<String, ScoredElement<UrlItem>> getPossiblePageUrls(Node[] allLink, String type)
      throws MalformedURLException {
    Map<String, ScoredElement<UrlItem>> possibleLinks =
        new HashMap<String, ScoredElement<UrlItem>>();

    for (Node link : allLink) {

      String href = link.getAttribute("href");
      if (StringUtil.isEmpty(href)) {
        continue;
      }
      href = href.replaceAll("#.*$", "");
      href = href.replaceAll("/$", "");
      href = Util.getAbsouluteUrl(href, currentUrl);
      if (currentUrl.getUrl().equals(href)) {
        continue;
      }
      String base = "://" + currentUrl.getUri().getHost();
      if (href.indexOf(base) == -1) {
        continue;
      }

      int score = getPageUrlScore(href, link, type);
      UrlItem url = new UrlItem(href);
      ScoredElement<UrlItem> element = new ScoredElement<UrlItem>(score, url);
      possibleLinks.put(url.getKey(), element);
    }
    return possibleLinks;
  }
예제 #2
0
파일: ZipUtil.java 프로젝트: lxmhope/jodd
  /**
   * Adds single entry to ZIP output stream.
   *
   * @param zos zip output stream
   * @param file file or folder to add
   * @param path relative path of file entry; if <code>null</code> files name will be used instead
   * @param comment optional comment
   * @param recursive when set to <code>true</code> content of added folders will be added, too
   */
  public static void addToZip(
      ZipOutputStream zos, File file, String path, String comment, boolean recursive)
      throws IOException {
    if (file.exists() == false) {
      throw new FileNotFoundException(file.toString());
    }

    if (path == null) {
      path = file.getName();
    }

    while (path.length() != 0 && path.charAt(0) == '/') {
      path = path.substring(1);
    }

    boolean isDir = file.isDirectory();

    if (isDir) {
      // add folder record
      if (!StringUtil.endsWithChar(path, '/')) {
        path += '/';
      }
    }

    ZipEntry zipEntry = new ZipEntry(path);
    zipEntry.setTime(file.lastModified());

    if (comment != null) {
      zipEntry.setComment(comment);
    }

    if (isDir) {
      zipEntry.setSize(0);
      zipEntry.setCrc(0);
    }

    zos.putNextEntry(zipEntry);

    if (!isDir) {
      InputStream is = new FileInputStream(file);
      try {
        StreamUtil.copy(is, zos);
      } finally {
        StreamUtil.close(is);
      }
    }

    zos.closeEntry();

    // continue adding

    if (recursive && file.isDirectory()) {
      boolean noRelativePath = StringUtil.isEmpty(path);

      final File[] children = file.listFiles();

      if (children != null && children.length != 0) {
        for (File child : children) {
          String childRelativePath = (noRelativePath ? StringPool.EMPTY : path) + child.getName();
          addToZip(zos, child, childRelativePath, comment, recursive);
        }
      }
    }
  }