void setConfig(Configuration config) {
log.debug("config: " + config);
proxyHost = config.get(PARAM_PROXY_HOST);
proxyPort = config.getInt(PARAM_PROXY_PORT, DEFAULT_PROXY_PORT);
if (StringUtil.isNullString(proxyHost) || proxyPort <= 0) {
String http_proxy = System.getenv("http_proxy");
if (!StringUtil.isNullString(http_proxy)) {
try {
HostPortParser hpp = new HostPortParser(http_proxy);
proxyHost = hpp.getHost();
proxyPort = hpp.getPort();
} catch (HostPortParser.InvalidSpec e) {
log.warning("Can't parse http_proxy environment var, ignoring: " + http_proxy + ": " + e);
}
}
}
if (StringUtil.isNullString(proxyHost) || proxyPort <= 0) {
proxyHost = null;
} else {
log.info("Proxying through " + proxyHost + ":" + proxyPort);
}
userAgent = config.get(PARAM_USER_AGENT);
if (StringUtil.isNullString(userAgent)) {
userAgent = null;
} else {
log.debug("Setting User-Agent to " + userAgent);
}
}
private void hash(CachedUrlSet set, MessageDigest dig, boolean namesOnly) throws IOException {
CachedUrlSetHasher hasher = null;
if (namesOnly) {
hasher = set.getNameHasher(dig);
} else {
hasher = set.getContentHasher(dig);
}
int bytesHashed = 0;
long timeTaken = System.currentTimeMillis();
while (!hasher.finished()) {
bytesHashed += hasher.hashStep(256);
}
timeTaken = System.currentTimeMillis() - timeTaken;
if ((timeTaken > 0) && (bytesHashed > 500)) {
System.out.println("Bytes hashed: " + bytesHashed);
System.out.println("Time taken: " + timeTaken + "ms");
System.out.println("Bytes/sec: " + (bytesHashed * 1000 / timeTaken));
} else {
System.out.println("No time taken, or insufficient bytes hashed.");
System.out.println("Bytes hashed: " + bytesHashed);
System.out.println("Time taken: " + timeTaken + "ms");
}
}
/**
* Create an array of NodeFilters that combines the atyponBaseFilters with the given array
*
* @param nodes The array of NodeFilters to add
*/
private NodeFilter[] addTo(NodeFilter[] nodes) {
NodeFilter[] result = Arrays.copyOf(baseAtyponFilters, baseAtyponFilters.length + nodes.length);
System.arraycopy(nodes, 0, result, baseAtyponFilters.length, nodes.length);
return result;
}