public void close() {
   _cm.shutdown();
   _eh.close();
 }
  /**
   * @param threads
   * @param proxyHost - the proxy host or <code>null</code> to use
   *     System.getProperties().get("http.proxyHost")
   * @param proxyPort - the proxy port or <code>null</code> to use
   *     System.getProperties().get("http.proxyPort")
   * @param proxyUser - the proxy user or <code>null</code> to use
   *     System.getProperties().get("http.proxyUser")
   * @param proxyPassword - the proxy user password or <code>null</code> to use
   *     System.getProperties().get("http.proxyPassword")
   */
  public Crawler(
      int threads, String proxyHost, String proxyPort, String proxyUser, String proxyPassword) {
    _threads = threads;

    String phost = proxyHost;
    int pport = 0;
    if (proxyPort != null) {
      try {
        pport = Integer.parseInt(proxyPort);
      } catch (NumberFormatException nfe) {
        pport = 0;
      }
    }
    String puser = proxyUser;
    String ppassword = proxyPassword;

    if (phost == null && System.getProperties().get("http.proxyHost") != null) {
      phost = System.getProperties().get("http.proxyHost").toString();
    }
    if (pport == 0 && System.getProperties().get("http.proxyPort") != null) {
      pport = Integer.parseInt(System.getProperties().get("http.proxyPort").toString());
    }

    if (puser == null && System.getProperties().get("http.proxyUser") != null) {
      puser = System.getProperties().get("http.proxyUser").toString();
    }
    if (ppassword == null && System.getProperties().get("http.proxyPassword") != null) {
      ppassword = System.getProperties().get("http.proxyPassword").toString();
    }

    _cm =
        new ConnectionManager(
            phost, pport, puser, ppassword, threads * CrawlerConstants.MAX_CONNECTIONS_PER_THREAD);
    _cm.setRetries(CrawlerConstants.RETRIES);

    // Always use the local TldManager implementation. Changed for the one
    // from NxParser for two reasons:
    // * I fixed a couple of bugs there
    // * I updated the Public suffix list and made a change to it to support
    //   .asia
    // try {
    // _tldm = new TldManager(_cm);
    // } catch (Exception e) {
    // _log.info("cannot get tld file online " + e.getMessage());
    // try {
    // _tldm = new TldManager();
    // } catch (IOException e1) {
    // _log.info("cannot get tld file locally " + e.getMessage());
    // }
    // }

    try {
      _tldm = new TldManager();
    } catch (IOException e1) {
      _log.info("cannot get tld file locally " + e1.getMessage());
    }

    _eh = new ErrorHandlerDummy();

    _robots = new Robots(_cm);
    _robots.setErrorHandler(_eh);

    //	    _sitemaps = new Sitemaps(_cm);
    //	    _sitemaps.setErrorHandler(_eh);

    _contentHandler = new ContentHandlerRdfXml();
    _output = new SinkDummy();
    _ff = new FetchFilterAllow();

    _blacklist = new FetchFilterAllow();
  }