int start(
Configuration conf, String traceIn, Path ioPath, long genbytes, UserResolver userResolver)
throws IOException, InterruptedException {
InputStream trace = null;
try {
Path scratchDir = new Path(ioPath, conf.get(GRIDMIX_OUT_DIR, "gridmix"));
final FileSystem scratchFs = scratchDir.getFileSystem(conf);
scratchFs.mkdirs(scratchDir, new FsPermission((short) 0777));
scratchFs.setPermission(scratchDir, new FsPermission((short) 0777));
// add shutdown hook for SIGINT, etc.
Runtime.getRuntime().addShutdownHook(sdh);
CountDownLatch startFlag = new CountDownLatch(1);
try {
// Create, start job submission threads
startThreads(conf, traceIn, ioPath, scratchDir, startFlag, userResolver);
// Write input data if specified
if (genbytes > 0) {
writeInputData(genbytes, ioPath);
}
// scan input dir contents
submitter.refreshFilePool();
factory.start();
statistics.start();
} catch (Throwable e) {
LOG.error("Startup failed", e);
if (factory != null) factory.abort(); // abort pipeline
} finally {
// signal for factory to start; sets start time
startFlag.countDown();
}
if (factory != null) {
// wait for input exhaustion
factory.join(Long.MAX_VALUE);
final Throwable badTraceException = factory.error();
if (null != badTraceException) {
LOG.error("Error in trace", badTraceException);
throw new IOException("Error in trace", badTraceException);
}
// wait for pending tasks to be submitted
submitter.shutdown();
submitter.join(Long.MAX_VALUE);
// wait for running tasks to complete
monitor.shutdown();
monitor.join(Long.MAX_VALUE);
statistics.shutdown();
statistics.join(Long.MAX_VALUE);
}
} finally {
IOUtils.cleanup(LOG, trace);
}
return 0;
}
@BeforeClass
public static void setUpBeforeClass() throws Exception {
savedList = new JobArchive<JobSeeker>();
seeker1 = new JobSeeker(new Name("David"));
seeker2 = new JobSeeker(new Name("Adam"));
Job job1 = JobFactory.createATSJob(new JobTitle("Developer"));
Job job2 = JobFactory.createATSJob(new JobTitle("Architect"));
Job job3 = JobFactory.createATSJob(new JobTitle("Programmer"));
recruiter1 = new Recruiter(new Name("John"));
recruiter2 = new Recruiter(new Name("Henry"));
postedJob1 = new PostedJob(recruiter1, job1);
postedJob2 = new PostedJob(recruiter2, job2);
postedJob3 = new PostedJob(recruiter2, job3);
}
/**
* @param conf gridmix configuration
* @param traceIn trace file path(if it is '-', then trace comes from the stream stdin)
* @param ioPath Working directory for gridmix. GenerateData job will generate data in the
* directory <ioPath>/input/ and distributed cache data is generated in the directory
* <ioPath>/distributedCache/, if -generate option is specified.
* @param genbytes size of input data to be generated under the directory <ioPath>/input/
* @param userResolver gridmix user resolver
* @param generate true if -generate option was specified
* @return exit code
* @throws IOException
* @throws InterruptedException
*/
int start(
Configuration conf,
String traceIn,
Path ioPath,
long genbytes,
UserResolver userResolver,
boolean generate)
throws IOException, InterruptedException {
DataStatistics stats = null;
InputStream trace = null;
ioPath = ioPath.makeQualified(ioPath.getFileSystem(conf));
try {
Path scratchDir = new Path(ioPath, conf.get(GRIDMIX_OUT_DIR, "gridmix"));
// add shutdown hook for SIGINT, etc.
Runtime.getRuntime().addShutdownHook(sdh);
CountDownLatch startFlag = new CountDownLatch(1);
try {
// Create, start job submission threads
startThreads(conf, traceIn, ioPath, scratchDir, startFlag, userResolver);
Path inputDir = getGridmixInputDataPath(ioPath);
// Write input data if specified
if (genbytes > 0) {
writeInputData(genbytes, inputDir);
}
// publish the data statistics
stats = GenerateData.publishDataStatistics(inputDir, genbytes, conf);
// scan input dir contents
submitter.refreshFilePool();
// set up the needed things for emulation of various loads
int exitCode = setupEmulation(conf, traceIn, scratchDir, ioPath, generate);
if (exitCode != 0) {
return exitCode;
}
// start the summarizer
summarizer.start(conf);
factory.start();
statistics.start();
} catch (Throwable e) {
LOG.error("Startup failed", e);
if (factory != null) factory.abort(); // abort pipeline
} finally {
// signal for factory to start; sets start time
startFlag.countDown();
}
if (factory != null) {
// wait for input exhaustion
factory.join(Long.MAX_VALUE);
final Throwable badTraceException = factory.error();
if (null != badTraceException) {
LOG.error("Error in trace", badTraceException);
throw new IOException("Error in trace", badTraceException);
}
// wait for pending tasks to be submitted
submitter.shutdown();
submitter.join(Long.MAX_VALUE);
// wait for running tasks to complete
monitor.shutdown();
monitor.join(Long.MAX_VALUE);
statistics.shutdown();
statistics.join(Long.MAX_VALUE);
}
} finally {
if (factory != null) {
summarizer.finalize(factory, traceIn, genbytes, userResolver, stats, conf);
}
IOUtils.cleanup(LOG, trace);
}
return 0;
}
