@Override public void operate(FlowProcess process, BufferCall<NullContext> buffCall) { QueuedValues values = new QueuedValues(buffCall.getArgumentsIterator()); _collector = buffCall.getOutputCollector(); FetcherPolicy fetcherPolicy = _fetcher.getFetcherPolicy(); // Each value is a PreFetchedDatum that contains a set of URLs to fetch in one request from // a single server, plus other values needed to set state properly. while (!Thread.interrupted() && !fetcherPolicy.isTerminateFetch() && !values.isEmpty()) { FetchSetDatum datum = values.nextOrNull(_fetcherMode); try { if (datum == null) { trace("Nothing ready to fetch, sleeping..."); process.keepAlive(); Thread.sleep(NOTHING_TO_FETCH_SLEEP_TIME); } else { List<ScoredUrlDatum> urls = datum.getUrls(); String ref = datum.getGroupingRef(); trace("Processing %d URLs for %s", urls.size(), ref); Runnable doFetch = new FetchTask(this, _fetcher, urls, ref); if (datum.isLastList()) { makeActive(ref, 0L); trace("Executing fetch of %d URLs from %s (last batch)", urls.size(), ref); } else { Long nextFetchTime = System.currentTimeMillis() + datum.getFetchDelay(); makeActive(ref, nextFetchTime); trace( "Executing fetch of %d URLs from %s (next fetch time %d)", urls.size(), ref, nextFetchTime); } long startTime = System.currentTimeMillis(); try { _executor.execute(doFetch); } catch (RejectedExecutionException e) { // should never happen. LOGGER.error("Fetch pool rejected our fetch list for " + ref); finished(ref); skipUrls( urls, UrlStatus.SKIPPED_DEFERRED, String.format("Execution rejection skipped %d URLs", urls.size())); } // Adjust for how long it took to get the request queued. adjustActive(ref, System.currentTimeMillis() - startTime); } } catch (InterruptedException e) { LOGGER.warn("FetchBuffer interrupted!"); Thread.currentThread().interrupt(); } } // Skip all URLs that we've got left. if (!values.isEmpty()) { trace("Found unprocessed URLs"); UrlStatus status = Thread.interrupted() ? UrlStatus.SKIPPED_INTERRUPTED : UrlStatus.SKIPPED_TIME_LIMIT; while (!values.isEmpty()) { FetchSetDatum datum = values.drain(); List<ScoredUrlDatum> urls = datum.getUrls(); trace( "Skipping %d urls from %s (e.g. %s) ", urls.size(), datum.getGroupingRef(), urls.get(0).getUrl()); skipUrls(urls, status, null); } } }