private boolean runSecondPhaseEnrichmentJob( AminoEnrichmentJob aej, Configuration conf, int jobType) throws Exception { System.out.println("Running Amino Job"); final Job job = new Job(conf, aej.getJobName() + " phase 2"); job.setJarByClass(aej.getClass()); AminoDriverUtils.setAminoJob(job.getConfiguration(), aej.getClass()); if (jobType == JOB_TYPE_ENRICHMENT) { job.getConfiguration().set(AminoDriverUtils.ENRICHMENT_OUTPUT, this.enrichmentOutput); } else if (jobType == JOB_TYPE_REUSE_ENRICHMENT) { String root = conf.get(AminoDriverUtils.ENRICHMENT_ROOT_OUTPUT); String front = ""; if (!root.endsWith("/")) { front = "/"; } root += front; final Iterable<String> inputs = ((AminoReuseEnrichmentJob) aej) .getSecondPhaseEnrichmentInputDirectories(job.getConfiguration()); String inputStr = ""; System.out.println("Using enrichment input paths:"); for (String input : inputs) { if (inputStr.length() > 0) { inputStr += "," + PathUtils.getJobDataPath(root + input); } else { inputStr += PathUtils.getJobDataPath(root + input); } System.out.println(PathUtils.getJobDataPath(root + input)); } job.getConfiguration().set(AminoDriverUtils.ENRICHMENT_OUTPUT, inputStr); // Need to do this because the first phase data loader is sitting in this slot currently AminoInputFormat.setDataLoader( job.getConfiguration(), aej.getDataLoaderClass().newInstance()); } int numReducers = job.getConfiguration() .getInt( AMINO_NUM_REDUCERS_ENRICH_PHASE2, job.getConfiguration().getInt(AMINO_NUM_REDUCERS, DEFAULT_NUM_REDUCERS)); job.setNumReduceTasks(numReducers); job.setMapperClass(FrameworkMapper.class); job.setReducerClass(FrameworkReducer.class); job.setMapOutputKeyClass(BucketStripped.class); job.setMapOutputValueClass(MapWritable.class); job.setOutputKeyClass(BucketStripped.class); job.setOutputValueClass(AminoWritable.class); job.setInputFormatClass(AminoMultiInputFormat.class); AminoMultiInputFormat.setDataLoader( job.getConfiguration(), aej.getDataLoaderClass().newInstance()); // Call job configuration for special properties jobConfiguration(job); @SuppressWarnings("serial") ArrayList<Class<? extends DataLoader>> joinSource = new ArrayList<Class<? extends DataLoader>>() { { add(EnrichmentDataLoader.class); } }; AminoMultiInputFormat.setJoinDataLoaders(job.getConfiguration(), joinSource); job.setOutputFormatClass(AminoOutputFormat.class); AminoOutputFormat.setAminoConfigPath( job, job.getConfiguration().get(AminoConfiguration.DEFAULT_CONFIGURATION_PATH_KEY)); String output = job.getConfiguration().get("amino.output"); System.out.println("Output will be written to: " + PathUtils.getJobDataPath(output)); AminoOutputFormat.setOutputPath(job, new Path(PathUtils.getJobDataPath(output))); JobUtilities.deleteDirectory(job.getConfiguration(), output); CacheBuilder.buildCaches( AminoDataUtils.getDataLoader(job.getConfiguration()), aej, output, job.getConfiguration()); return job.waitForCompletion(true); }