private void multiThreadedProjectTaskSegmentBuild(
ConcurrencyDependencyGraph analyzer,
ReactorContext reactorContext,
MavenSession rootSession,
CompletionService<ProjectSegment> service,
TaskSegment taskSegment,
Map<MavenProject, ProjectSegment> projectBuildList,
ThreadOutputMuxer muxer) {
// schedule independent projects
for (MavenProject mavenProject : analyzer.getRootSchedulableBuilds()) {
ProjectSegment projectSegment = projectBuildList.get(mavenProject);
logger.debug("Scheduling: " + projectSegment.getProject());
Callable<ProjectSegment> cb =
createBuildCallable(rootSession, projectSegment, reactorContext, taskSegment, muxer);
service.submit(cb);
}
// for each finished project
for (int i = 0; i < analyzer.getNumberOfBuilds(); i++) {
try {
ProjectSegment projectBuild = service.take().get();
if (reactorContext.getReactorBuildStatus().isHalted()) {
break;
}
final List<MavenProject> newItemsThatCanBeBuilt =
analyzer.markAsFinished(projectBuild.getProject());
for (MavenProject mavenProject : newItemsThatCanBeBuilt) {
ProjectSegment scheduledDependent = projectBuildList.get(mavenProject);
logger.debug("Scheduling: " + scheduledDependent);
Callable<ProjectSegment> cb =
createBuildCallable(
rootSession, scheduledDependent, reactorContext, taskSegment, muxer);
service.submit(cb);
}
} catch (InterruptedException e) {
rootSession.getResult().addException(e);
break;
} catch (ExecutionException e) {
// TODO MNG-5766 changes likely made this redundant
rootSession.getResult().addException(e);
break;
}
}
// cancel outstanding builds (if any) - this can happen if an exception is thrown in above
// block
Future<ProjectSegment> unprocessed;
while ((unprocessed = service.poll()) != null) {
try {
unprocessed.get();
} catch (InterruptedException e) {
throw new RuntimeException(e);
} catch (ExecutionException e) {
throw new RuntimeException(e);
}
}
}
public InputStream createIncomingStream(StreamInitiation initiation) throws XMPPException {
PacketCollector collector =
connection.createPacketCollector(
getInitiationPacketFilter(initiation.getFrom(), initiation.getSessionID()));
connection.sendPacket(super.createInitiationAccept(initiation, getNamespaces()));
ExecutorService threadPoolExecutor = Executors.newFixedThreadPool(2);
CompletionService<InputStream> service =
new ExecutorCompletionService<InputStream>(threadPoolExecutor);
List<Future<InputStream>> futures = new ArrayList<Future<InputStream>>();
InputStream stream = null;
XMPPException exception = null;
try {
futures.add(service.submit(new NegotiatorService(collector)));
futures.add(service.submit(new NegotiatorService(collector)));
int i = 0;
while (stream == null && i < futures.size()) {
Future<InputStream> future;
try {
i++;
future = service.poll(10, TimeUnit.SECONDS);
} catch (InterruptedException e) {
continue;
}
if (future == null) {
continue;
}
try {
stream = future.get();
} catch (InterruptedException e) {
/* Do Nothing */
} catch (ExecutionException e) {
exception = new XMPPException(e.getCause());
}
}
} finally {
for (Future<InputStream> future : futures) {
future.cancel(true);
}
collector.cancel();
threadPoolExecutor.shutdownNow();
}
if (stream == null) {
if (exception != null) {
throw exception;
} else {
throw new XMPPException("File transfer negotiation failed.");
}
}
return stream;
}
@Override
public void run() {
while (!end) {
try {
Future<String> result = completionservice.poll(20, TimeUnit.SECONDS);
if (result != null) {
String report = result.get();
System.out.printf("ReportReceiver: Report Received:%s\n", report);
}
} catch (InterruptedException | ExecutionException e) {
e.printStackTrace();
}
}
System.out.printf("ReportSender: End\n");
}
/** 核心方法，如果任务没有完成就一直执行 */
@Override
public void run() {
while (!end) {
try {
// 调用CompletionService接口的poll()方法，获取CompletionService执行的下个已完成任务的Future对象
Future<String> result = service.poll(20, TimeUnit.SECONDS);
if (result != null) {
// 使用Future对象的get()方法获取任务的结果，并且将这些结果写入到控制台
String report = result.get();
System.out.printf("ReportReceiver: Report Received: %s\n", report);
}
} catch (InterruptedException | ExecutionException e) {
e.printStackTrace();
}
}
System.out.printf("ReportSender: End\n");
}
@Override
public void run() {
while (!isEnd) {
try {
// 在结果的Future的列表中查看，如果是空的，则等待20秒。
// 注意，是Future列表。CompletionService内部使用的是BlockingQueue<Future<V>>
// 如果是无参数的poll，则如果是空，立刻返回。
// take()方法则死守，直到非空为止
// 注意：返回的结果在Future队列中。
Future<String> fs = cs.poll(20, TimeUnit.SECONDS);
if (fs != null) {
System.out.println("Result is : " + fs.get());
}
} catch (InterruptedException | ExecutionException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
private void aggregateOnTheFly() throws InterruptedException, ExecutionException {
boolean timedOut = false;
boolean stoppedOnException = false;
final StopWatch watch = new StopWatch();
int aggregated = 0;
boolean done = false;
// not a for loop as on the fly may still run
while (!done) {
// check if we have already aggregate everything
if (allTasksSubmitted.get() && aggregated >= total.get()) {
LOG.debug("Done aggregating {} exchanges on the fly.", aggregated);
break;
}
Future<Exchange> future;
if (timedOut) {
// we are timed out but try to grab if some tasks has been completed
// poll will return null if no tasks is present
future = completion.poll();
LOG.trace(
"Polled completion task #{} after timeout to grab already completed tasks: {}",
aggregated,
future);
} else if (timeout > 0) {
long left = timeout - watch.taken();
if (left < 0) {
left = 0;
}
LOG.trace("Polling completion task #{} using timeout {} millis.", aggregated, left);
future = completion.poll(left, TimeUnit.MILLISECONDS);
} else {
LOG.trace("Polling completion task #{}", aggregated);
// we must not block so poll every second
future = completion.poll(1, TimeUnit.SECONDS);
if (future == null) {
// and continue loop which will recheck if we are done
continue;
}
}
if (future == null && timedOut) {
// we are timed out and no more tasks complete so break out
break;
} else if (future == null) {
// timeout occurred
AggregationStrategy strategy = getAggregationStrategy(null);
if (strategy instanceof TimeoutAwareAggregationStrategy) {
// notify the strategy we timed out
Exchange oldExchange = result.get();
if (oldExchange == null) {
// if they all timed out the result may not have been set yet, so use the original
// exchange
oldExchange = original;
}
((TimeoutAwareAggregationStrategy) strategy)
.timeout(oldExchange, aggregated, total.intValue(), timeout);
} else {
// log a WARN we timed out since it will not be aggregated and the Exchange will be lost
LOG.warn(
"Parallel processing timed out after {} millis for number {}. This task will be cancelled and will not be aggregated.",
timeout,
aggregated);
}
LOG.debug("Timeout occurred after {} millis for number {} task.", timeout, aggregated);
timedOut = true;
// mark that index as timed out, which allows us to try to retrieve
// any already completed tasks in the next loop
if (completion instanceof SubmitOrderedCompletionService) {
((SubmitOrderedCompletionService<?>) completion).timeoutTask();
}
} else {
// there is a result to aggregate
Exchange subExchange = future.get();
// Decide whether to continue with the multicast or not; similar logic to the Pipeline
Integer number = getExchangeIndex(subExchange);
boolean continueProcessing =
PipelineHelper.continueProcessing(
subExchange, "Parallel processing failed for number " + number, LOG);
if (stopOnException && !continueProcessing) {
// we want to stop on exception and an exception or failure occurred
// this is similar to what the pipeline does, so we should do the same to not surprise
// end users
// so we should set the failed exchange as the result and break out
result.set(subExchange);
stoppedOnException = true;
break;
}
// we got a result so aggregate it
AggregationStrategy strategy = getAggregationStrategy(subExchange);
doAggregate(strategy, result, subExchange);
}
aggregated++;
}
if (timedOut || stoppedOnException) {
if (timedOut) {
LOG.debug("Cancelling tasks due timeout after {} millis.", timeout);
}
if (stoppedOnException) {
LOG.debug("Cancelling tasks due stopOnException.");
}
// cancel tasks as we timed out (its safe to cancel done tasks)
running.set(false);
}
}
/** @throws ExecutionException */
protected void monitor() throws ExecutionException {
int displayMillis = DISPLAY_MILLIS;
int futureMillis = FUTURE_MILLIS;
int sleepMillis = SLEEP_MILLIS;
Future<TimedEvent[]> future = null;
/* Initialize lastFutureMillis so that we do not get
* warnings on slow queue startup.
*/
long currentMillis = System.currentTimeMillis();
long lastDisplayMillis = 0;
long lastFutureMillis = currentMillis;
TimedEvent[] lastEvent = null;
logger.finest(
"looping every "
+ sleepMillis
+ ", core="
+ pool.getCorePoolSize()
+ ", active="
+ pool.getActiveCount()
+ ", tasks="
+ taskCount);
timer = new Timer();
// run until all futures have been checked
do {
// try to avoid thread starvation
yield();
// check completed tasks
// sometimes this goes so fast that we never leave the loop,
// so progress is never displayed... so limit the number of loops.
do {
try {
future = completionService.poll(SLEEP_MILLIS, TimeUnit.MILLISECONDS);
if (null != future) {
// record result, or throw exception
lastFutureMillis = System.currentTimeMillis();
try {
lastEvent = future.get();
if (null == lastEvent) {
throw new FatalException("unexpected null event");
}
for (int i = 0; i < lastEvent.length; i++) {
// discard events to reduce memory utilization
if (null != lastEvent[i]) {
timer.add(lastEvent[i], false);
}
}
} catch (ExecutionException e) {
if (fatalErrors) {
throw e;
}
Throwable cause = e.getCause();
if (null != cause && cause instanceof FatalException) {
throw (FatalException) cause;
}
logger.logException("non-fatal", e);
timer.incrementEventCount(false);
}
}
} catch (InterruptedException e) {
// reset interrupt status and continue
Thread.interrupted();
logger.logException("interrupted in poll() or get()", e);
continue;
}
currentMillis = System.currentTimeMillis();
if (currentMillis - lastDisplayMillis > displayMillis) {
lastDisplayMillis = currentMillis;
logger.finer(
"thread count: core="
+ pool.getCorePoolSize()
+ ", active="
+ pool.getActiveCount()
+ ", tasks="
+ taskCount);
if (null != lastEvent) {
logger.info(
""
+ timer.getEventCount()
+ "/"
+ taskCount
+ ", "
+ timer.getProgressMessage(false)
+ ", "
+ lastEvent[0].getDescription());
if (config.doPrintCurrRate()) {
String currMsg = timer.getCurrProgressMessage();
if (currMsg != null) logger.info(currMsg);
}
}
}
} while (null != future);
logger.finer(
"running = "
+ running
+ ", terminated = "
+ pool.isTerminated()
+ ", last future = "
+ lastFutureMillis);
// currentMillis has already been set recently
if (currentMillis - lastFutureMillis > futureMillis) {
logger.warning("no futures received in over " + futureMillis + " ms");
break;
}
} while (running && !pool.isTerminated());
// NB - caller will set running to false to ensure exit
}
