public static boolean runJob(DMLConfig conf) throws Exception {
boolean ret = false;
try {
JobConf job;
job = new JobConf(CleanupMR.class);
job.setJobName("Cleanup-MR");
// set up SystemML local tmp dir
String dir = conf.getTextValue(DMLConfig.LOCAL_TMP_DIR);
MRJobConfiguration.setSystemMLLocalTmpDir(job, dir);
// set mappers, reducers
int numNodes = InfrastructureAnalyzer.getRemoteParallelNodes();
job.setMapperClass(CleanupMapper.class); // map-only
job.setNumMapTasks(numNodes); // numMappers
job.setNumReduceTasks(0);
// set input/output format, input path
String inFileName = conf.getTextValue(DMLConfig.SCRATCH_SPACE) + "/cleanup_tasks";
job.setInputFormat(NLineInputFormat.class);
job.setOutputFormat(NullOutputFormat.class);
Path path = new Path(inFileName);
FileInputFormat.setInputPaths(job, path);
writeCleanupTasksToFile(path, numNodes);
// disable automatic tasks timeouts and speculative task exec
job.setInt(MRConfigurationNames.MR_TASK_TIMEOUT, 0);
job.setMapSpeculativeExecution(false);
/////
// execute the MR job
RunningJob runjob = JobClient.runJob(job);
ret = runjob.isSuccessful();
} catch (Exception ex) {
// don't raise an exception, just gracefully an error message.
LOG.error("Failed to run cleanup MR job. ", ex);
}
return ret;
}
