예제 #1
0
  void doit() {
    String username = args.Username;
    String password = args.Password;
    if (username != null || password != null)
      Authenticator.setDefault(new MyAuthenticator(username, password));

    System.setProperty("java.protocol.handler.pkgs", "edu.mscd.cs");

    String[] add = args.additional;

    if (add == null) {
      logger.warning("No URLs specified, exiting");
      System.exit(1);
    }

    /*
     ** add the specified URLs to the list to be fetched.
     */
    String[] t = new String[add.length];
    for (int i = 0; i < add.length; i++) {
      t[i] = "<a href=\"" + add[i] + "\">";
    }

    /*
     ** add to the URLs, with no base
     */
    addToURLs(null, t);
    fetchAll();

    /*
     ** shall we save the cookies to a file?
     */
    String savecookies = args.SaveCookies;
    if (savecookies != null) cookies.saveCookies(savecookies);
  }
예제 #2
0
  REplican(String arguments[]) {
    JCLO jclo = new JCLO(args);

    if (arguments.length == 0) {
      System.out.println("Arguments:\n" + jclo.usage() + "URLs...");
      System.exit(1);
    }

    try {
      jclo.parse(arguments);
    } catch (IllegalArgumentException IAE) {
      System.err.println(IAE);
      System.err.println("Arguments:\n" + jclo.usage() + "URLs...");
      System.exit(0);
    }

    String logLevel = args.LogLevel;
    ConsoleHandler ch = new ConsoleHandler();

    if (logLevel != null) {
      Level level = JavaLN.getLevel(logLevel);
      ch.setLevel(level);
      ch.setFormatter(new LineNumberFormatter());
      logger.setLevel(level);
    } else {
      ch.setFormatter(new NullFormatter());
    }

    logger.addHandler(ch);
    logger.setUseParentHandlers(false);

    if (args.Version) {
      System.out.println(Version.getVersion());
      System.exit(0);
    }

    if (args.Help) {
      System.out.println("Arguments:\n" + jclo.usage() + "URLs...");
      System.exit(0);
    }

    cookies = new Cookies();

    setDefaults();

    if (args.LoadCookies != null) {
      for (int i = 0; i < args.LoadCookies.length; i++) {
        logger.config("Loading cookies from " + args.LoadCookies[i]);
        cookies.loadCookies(args.LoadCookies[i]);
      }
    }

    if (args.PlistCookies != null) {
      for (int i = 0; i < args.PlistCookies.length; i++) {
        logger.config("Loading cookies from " + args.PlistCookies[i]);
        new Plist("file:" + args.PlistCookies[i], cookies);
      }
    }

    if (args.CheckpointEvery != 0) {
      logger.config("Loading urls from " + args.CheckpointFile);

      try {
        ObjectInputStream ois = new ObjectInputStream(new FileInputStream(args.CheckpointFile));
        urls = (Hashtable) ois.readObject();
        ois.close();
      } catch (IOException ioe) {
        logger.throwing(ioe);
      } catch (ClassNotFoundException cnfe) {
        logger.throwing(cnfe);
      }
    }

    if (args.FollowRedirects) HttpURLConnection.setFollowRedirects(false);
  }
예제 #3
0
  private void setDefaults() {
    if (args.Interesting == null) {
      String urlref = "\\s*=\\s*[\"']?([^\"'>]*)";
      String href = "[hH][rR][eE][fF]";
      String src = "[sS][rR][cC]";

      String init[] = {
        href + urlref, src + urlref,
      };

      args.Interesting = init;
    }

    if (args.URLFixUp == null) {
      // so, i don't remember why i collasped multiple spaces and
      // removed \'s. must have been important and i should have
      // documented. 's confuse URLs...
      // args.URLFixUp = new String[]{"\\s+", " ", "\\\\", ""};
      args.URLFixUp = new String[] {"\\s+", " ", "\\\\", "", "\'", "%27"};
    }

    // if they don't specify anything, look at only text.
    if (args.MIMEExamine == null
        && args.MIMEIgnore == null
        && args.PathExamine == null
        && args.PathIgnore == null) {
      args.MIMEExamine = new String[] {"text/.*"};
      if (args.PrintExamine)
        logger.warning("--MIMEExamine=" + java.util.Arrays.toString(args.MIMEExamine));
    }

    // if they don't specify anything, save only what is specified on
    // the command line.
    if (args.MIMESave == null
        && args.MIMERefuse == null
        && args.PathSave == null
        && args.PathRefuse == null) {
      if (args.additional.length == 0) {
        logger.severe("No URLs specified");
        System.exit(1);
      }

      args.PathSave = new String[args.additional.length];

      for (int i = 0; i < args.additional.length; i++)
        args.PathSave[i] = escapeURL(args.additional[i]);

      if (args.PrintSave) logger.warning("--PathSave=" + java.util.Arrays.toString(args.PathSave));
    }

    if (args.PrintAll)
      args.PrintAccept =
          args.PrintReject =
              args.PrintSave = args.PrintRefuse = args.PrintExamine = args.PrintIgnore = true;

    /*
     ** make sure we accept everything we examine, save, and the initial
     ** URLs
     */
    args.PathAccept = Utils.combineArrays(args.PathAccept, args.PathExamine);
    args.PathAccept = Utils.combineArrays(args.PathAccept, args.PathSave);
    args.PathAccept = Utils.combineArrays(args.PathAccept, args.additional);
  }