Exemplo n.º 1
0
 void setConfig(Configuration config) {
   log.debug("config: " + config);
   proxyHost = config.get(PARAM_PROXY_HOST);
   proxyPort = config.getInt(PARAM_PROXY_PORT, DEFAULT_PROXY_PORT);
   if (StringUtil.isNullString(proxyHost) || proxyPort <= 0) {
     String http_proxy = System.getenv("http_proxy");
     if (!StringUtil.isNullString(http_proxy)) {
       try {
         HostPortParser hpp = new HostPortParser(http_proxy);
         proxyHost = hpp.getHost();
         proxyPort = hpp.getPort();
       } catch (HostPortParser.InvalidSpec e) {
         log.warning("Can't parse http_proxy environment var, ignoring: " + http_proxy + ": " + e);
       }
     }
   }
   if (StringUtil.isNullString(proxyHost) || proxyPort <= 0) {
     proxyHost = null;
   } else {
     log.info("Proxying through " + proxyHost + ":" + proxyPort);
   }
   userAgent = config.get(PARAM_USER_AGENT);
   if (StringUtil.isNullString(userAgent)) {
     userAgent = null;
   } else {
     log.debug("Setting User-Agent to " + userAgent);
   }
 }
 private void hash(CachedUrlSet set, MessageDigest dig, boolean namesOnly) throws IOException {
   CachedUrlSetHasher hasher = null;
   if (namesOnly) {
     hasher = set.getNameHasher(dig);
   } else {
     hasher = set.getContentHasher(dig);
   }
   int bytesHashed = 0;
   long timeTaken = System.currentTimeMillis();
   while (!hasher.finished()) {
     bytesHashed += hasher.hashStep(256);
   }
   timeTaken = System.currentTimeMillis() - timeTaken;
   if ((timeTaken > 0) && (bytesHashed > 500)) {
     System.out.println("Bytes hashed: " + bytesHashed);
     System.out.println("Time taken: " + timeTaken + "ms");
     System.out.println("Bytes/sec: " + (bytesHashed * 1000 / timeTaken));
   } else {
     System.out.println("No time taken, or insufficient bytes hashed.");
     System.out.println("Bytes hashed: " + bytesHashed);
     System.out.println("Time taken: " + timeTaken + "ms");
   }
 }
 /**
  * Create an array of NodeFilters that combines the atyponBaseFilters with the given array
  *
  * @param nodes The array of NodeFilters to add
  */
 private NodeFilter[] addTo(NodeFilter[] nodes) {
   NodeFilter[] result = Arrays.copyOf(baseAtyponFilters, baseAtyponFilters.length + nodes.length);
   System.arraycopy(nodes, 0, result, baseAtyponFilters.length, nodes.length);
   return result;
 }