/**
* Create each component in the pipeline and start it.
*
* @param conf Configuration data, no keys specific to this context
* @param traceIn Either a Path to the trace data or "-" for stdin
* @param ioPath <ioPath>/input/ is the dir from which input data is read and
* <ioPath>/distributedCache/ is the gridmix distributed cache directory.
* @param scratchDir Path into which job output is written
* @param startFlag Semaphore for starting job trace pipeline
*/
private void startThreads(
Configuration conf,
String traceIn,
Path ioPath,
Path scratchDir,
CountDownLatch startFlag,
UserResolver userResolver)
throws IOException {
try {
Path inputDir = getGridmixInputDataPath(ioPath);
GridmixJobSubmissionPolicy policy = getJobSubmissionPolicy(conf);
LOG.info(" Submission policy is " + policy.name());
statistics = new Statistics(conf, policy.getPollingInterval(), startFlag);
monitor = createJobMonitor(statistics, conf);
int noOfSubmitterThreads =
(policy == GridmixJobSubmissionPolicy.SERIAL)
? 1
: Runtime.getRuntime().availableProcessors() + 1;
int numThreads = conf.getInt(GRIDMIX_SUB_THR, noOfSubmitterThreads);
int queueDep = conf.getInt(GRIDMIX_QUE_DEP, 5);
submitter =
createJobSubmitter(
monitor,
numThreads,
queueDep,
new FilePool(conf, inputDir),
userResolver,
statistics);
distCacheEmulator = new DistributedCacheEmulator(conf, ioPath);
factory = createJobFactory(submitter, traceIn, scratchDir, conf, startFlag, userResolver);
factory.jobCreator.setDistCacheEmulator(distCacheEmulator);
if (policy == GridmixJobSubmissionPolicy.SERIAL) {
statistics.addJobStatsListeners(factory);
} else {
statistics.addClusterStatsObservers(factory);
}
// add the gridmix run summarizer to the statistics
statistics.addJobStatsListeners(summarizer.getExecutionSummarizer());
statistics.addClusterStatsObservers(summarizer.getClusterSummarizer());
monitor.start();
submitter.start();
} catch (Exception e) {
LOG.error(" Exception at start ", e);
throw new IOException(e);
}
}
