@Override
public void run() {
try {
for (int i = 0; i < otherWorkerIPPort.size(); i++) {
BlockingQueue<URL> queue = getUrlsForOtherCrawlers().get(i);
if (queue.isEmpty()) {
continue;
}
StringBuilder contentBuilder = new StringBuilder();
contentBuilder.append(DispatcherConstants.NEW_URLS_PARAM + "=");
int items = DispatcherConstants.URLS_TO_SEND;
while (!queue.isEmpty() || items < 0) {
URL url = queue.take();
String cleansedURL = URLEncoder.encode(url.toString(), CrawlerConstants.CHARSET);
contentBuilder.append(cleansedURL + ";");
}
String urlString =
"http://" + otherWorkerIPPort.get(i) + "/worker/" + DispatcherConstants.ADD_URLS_URL;
URL url = new URL(urlString);
DispatcherUtils.sendHttpRequest(
url, contentBuilder.toString(), DispatcherUtils.Method.POST, true);
}
} catch (Exception e) {
Utils.logStackTrace(e);
}
}
