Example #1
0
  /*
   ** calculate, given the examine/ignore and save/refuse values, whether
   ** to examine and/or save s.
   */
  private boolean[] EISR(
      String s, String which, String examine[], String ignore[], String save[], String refuse[]) {
    if (s == null) return (null);

    logger.fine(s);
    logger.fine(which);
    logger.fine(java.util.Arrays.toString(examine));
    logger.fine(java.util.Arrays.toString(ignore));
    logger.fine(java.util.Arrays.toString(save));
    logger.fine(java.util.Arrays.toString(refuse));

    boolean E = Utils.blurf(examine, ignore, s, false);
    boolean S = Utils.blurf(save, refuse, s, false);

    if (args.PrintExamine && E) logger.info("Examining " + which + ": " + s);
    if (args.PrintIgnore && !E) logger.info("Ignoring " + which + ": " + s);

    if (args.PrintSave && S) logger.info("Saving " + which + ": " + s);
    if (args.PrintRefuse && !S) logger.info("Refusing " + which + ": " + s);

    boolean ret[] = new boolean[2];
    ret[0] = E;
    ret[1] = S;
    return (ret);
  }
Example #2
0
  private void addToURLs(String baseURL, String strings[]) {
    logger.entering(baseURL);
    logger.entering(strings);

    for (int i = 0; i < strings.length; i++) {
      String next = replaceAll(strings[i], args.URLFixUp);

      String newBase = newBase(next);
      if (newBase != null) {
        logger.fine("Setting base to " + baseURL);
        baseURL = newBase;
      }

      String possible[] = interesting(next);

      for (int j = 0; j < args.Interesting.length; j++) {
        if (possible[j] != null) {
          URL u = makeURL(baseURL, possible[j]);

          if (u == null) continue;

          String total = u.toString();

          String PathAccept[] = args.PathAccept;
          String PathReject[] = args.PathReject;

          boolean accept = Utils.blurf(PathAccept, PathReject, total, true);

          if (args.PrintAccept && accept) logger.info("Accepting path: " + total);
          if (args.PrintReject && !accept) logger.info("Rejecting path: " + total);

          if (accept) {
            if (args.URLRewrite != null) total = REplican.replaceAll(total, args.URLRewrite);

            // if we don't already have it
            if (urls.get(total) == null) {
              if (args.PrintAdd) logger.info("Adding: " + total);
              addOne(total);
            }
          }
        }
      }
    }
  }
Example #3
0
  private void fetch(String url) {
    logger.entering(url);

    boolean Path =
        args.PathExamine != null
            || args.PathIgnore != null
            || args.PathSave != null
            || args.PathRefuse != null;
    boolean MIME =
        args.MIMEExamine != null
            || args.MIMEIgnore != null
            || args.MIMESave != null
            || args.MIMERefuse != null;

    logger.fine("Path = " + Path);
    logger.fine("MIME = " + MIME);

    boolean tb[] =
        EISR(url, "path", args.PathExamine, args.PathIgnore, args.PathSave, args.PathRefuse);

    boolean Pexamine = tb[0];
    boolean Psave = tb[1];

    /*
     * if there is no MIME, and the Path doesn't say to examine or save,
     * we're done.
     */
    if (!MIME && !Pexamine && !Psave) return;

    /*
     * otherwise, we need to Path examine or save, or we need the MIME
     * header.  in either case, we need an InputStream.
     */
    InputStream is = null;
    YouAreEll yrl = null;
    for (int t = 0; t < args.Tries; t++) {
      yrl = new YouAreEll(url, urls, cookies, args);
      is = yrl.getInputStream();
      if (is != null) break;
      if (args.Tries > 1) logger.warning("Trying again");
    }

    if (is == null) return;

    boolean Mexamine = false;
    boolean Msave = false;

    if (MIME && yrl.getContentType() != null) {
      tb =
          EISR(
              yrl.getContentType(),
              "MIME",
              args.MIMEExamine,
              args.MIMEIgnore,
              args.MIMESave,
              args.MIMERefuse);
      Mexamine = tb[0];
      Msave = tb[1];
    }

    // we've looked at both Path and now MIME and there's nothing to do
    if (!Pexamine && !Psave && !Mexamine && !Msave) return;

    fetchOne(Pexamine || Mexamine, Psave || Msave, yrl, is);

    try {
      is.close();
    } catch (IOException IOE) {
      logger.throwing(IOE);
    }
  }