protected JobOutput executeJellyfishMerger(
Args args, String ecqName, Set<File> fileSet, File outputDir)
throws InterruptedException, ProcessExecutionException, ConanParameterException, IOException {
String suffix = "jellyfish_" + ecqName + "_all.jf31_0";
String jobName = args.getJobPrefix() + "-merge-" + suffix;
List<File> files = new ArrayList<>();
files.addAll(fileSet);
File outputFile = new File(outputDir, suffix);
JellyfishMergeV11 jellyfishMerge =
this.makeJellyfishMerge(files, outputFile, args.getOrganism());
ExecutionResult id =
this.conanExecutorService.executeProcess(
jellyfishMerge,
args.getOutputDir(),
jobName,
args.getThreadsPerProcess(),
args.getMemoryPerProcess(),
args.isRunParallel());
id.setName("merge-" + suffix);
return new JobOutput(id, outputFile);
}
protected JobOutput executeJellyfishCount(Args args, String ecqName, File outputDir, Library lib)
throws ProcessExecutionException, InterruptedException, ConanParameterException, IOException {
String suffix = "jellyfish_" + ecqName + "_" + lib.getName() + ".jf31";
// Create the process
JellyfishCountV11 jellyfishProcess =
this.makeJellyfishCount(
this.makeInputStringFromLib(lib),
new File(new File(outputDir, ecqName), suffix).getAbsolutePath(),
args.getOrganism(),
args.getThreadsPerProcess());
File outputFile = jellyfishProcess.getArgs().getOutputFile();
// Create a job name
String jobName = args.getJobPrefix() + "-count-" + suffix;
// Start jellyfish
final ExecutionResult id =
this.conanExecutorService.executeProcess(
jellyfishProcess,
new File(outputDir, ecqName),
jobName,
args.getThreadsPerProcess(),
args.getMemoryPerProcess(),
args.isRunParallel());
id.setName("count-" + suffix);
return new JobOutput(id, outputFile);
}
@Test
public void exampleQuery() throws Exception {
// START SNIPPET: JavaQuery
ExecutionEngine engine = new ExecutionEngine(db);
ExecutionResult result = engine.execute("start n=node(0) where 1=1 return n");
assertThat(result.columns(), hasItem("n"));
Iterator<Node> n_column = result.columnAs("n");
assertThat(asIterable(n_column), hasItem(db.getNodeById(0)));
assertThat(result.toString(), containsString("Node[0]"));
// END SNIPPET: JavaQuery
}
@Test
public void exampleWithParameterForNodeObject() throws Exception {
// START SNIPPET: exampleWithParameterForNodeObject
Map<String, Object> params = new HashMap<String, Object>();
params.put("node", andreasNode);
ExecutionResult result = engine.execute("start n=node({node}) return n.name", params);
// END SNIPPET: exampleWithParameterForNodeObject
assertThat(result.columns(), hasItem("n.name"));
Iterator<Object> n_column = result.columnAs("n.name");
assertEquals("Andreas", n_column.next());
}
@Test
public void testColumnAreInTheRightOrder() throws Exception {
createTenNodes();
String q =
"start one=node(1), two=node(2), three=node(3), four=node(4), five=node(5), six=node(6), "
+ "seven=node(7), eight=node(8), nine=node(9), ten=node(10) "
+ "return one, two, three, four, five, six, seven, eight, nine, ten";
ExecutionResult result = engine.execute(q);
assertThat(
result.toString(),
matchesPattern("one.*two.*three.*four.*five.*six.*seven.*eight.*nine.*ten"));
}
@Test
public void shouldBeAbleToEmitJavaIterables() throws Exception {
makeFriends(michaelaNode, andreasNode);
makeFriends(michaelaNode, johanNode);
ExecutionEngine engine = new ExecutionEngine(db);
ExecutionResult result =
engine.execute("start n=node(0) match n-->friend return collect(friend)");
Iterable<Node> friends = (Iterable<Node>) result.columnAs("collect(friend)").next();
assertThat(friends, hasItems(andreasNode, johanNode));
assertThat(friends, instanceOf(Iterable.class));
}
@Test
public void exampleWithParameterRegularExpression() throws Exception {
// START SNIPPET: exampleWithParameterRegularExpression
Map<String, Object> params = new HashMap<String, Object>();
params.put("regex", ".*h.*");
ExecutionResult result =
engine.execute("start n=node(0,1,2) where n.name =~ {regex} return n.name", params);
// END SNIPPET: exampleWithParameterRegularExpression
assertThat(result.columns(), hasItem("n.name"));
Iterator<Object> n_column = result.columnAs("n.name");
assertEquals("Michaela", n_column.next());
assertEquals("Johan", n_column.next());
}
@Test
public void exampleWithParameterForSkipAndLimit() throws Exception {
// START SNIPPET: exampleWithParameterForSkipLimit
Map<String, Object> params = new HashMap<String, Object>();
params.put("s", 1);
params.put("l", 1);
ExecutionResult result =
engine.execute("start n=node(0,1,2) return n.name skip {s} limit {l}", params);
// END SNIPPET: exampleWithParameterForSkipLimit
assertThat(result.columns(), hasItem("n.name"));
Iterator<Object> n_column = result.columnAs("n.name");
assertEquals("Andreas", n_column.next());
}
public void assertErrorHasNoStackTraces(ExecutionResult result) {
String error = result.getError();
if (result instanceof ExecutionFailure) {
// Axe everything after the expected exception
int pos = error.indexOf("* Exception is:" + TextUtil.getPlatformLineSeparator());
if (pos >= 0) {
error = error.substring(0, pos);
}
}
assertNoStackTraces(error, "Standard error");
}
private <T> List<T> toList(ExecutionResult result, String column) {
List<T> results = new ArrayList<T>();
IteratorUtil.addToCollection(result.<T>columnAs(column), results);
return results;
}
@Override
public ExecutionResult execute(ExecutionContext executionContext)
throws ProcessExecutionException, InterruptedException {
try {
StopWatch stopWatch = new StopWatch();
stopWatch.start();
log.info("Starting Kmer Counting on all Reads");
// Create shortcut to args for convienience
Args args = this.getArgs();
// Force run parallel to false if not using a scheduler
if (!executionContext.usingScheduler() && args.isRunParallel()) {
log.warn("Forcing linear execution due to lack of job scheduler");
args.setRunParallel(false);
}
// Create the output directory
args.getOutputDir().mkdirs();
JobOutputMap jfCountOutputs = new JobOutputMap();
List<ExecutionResult> jobResults = new ArrayList<>();
List<ExecutionResult> allJobResults = new ArrayList<>();
// Create the output directory for the RAW datasets
File rawOutputDir = new File(args.getOutputDir(), "raw");
if (!rawOutputDir.exists()) {
rawOutputDir.mkdirs();
}
// Start jellyfish on all RAW datasets
for (Library lib : args.getAllLibraries()) {
// Execute jellyfish and add id to list of job ids
JobOutput jfOut = this.executeJellyfishCount(args, "raw", args.getOutputDir(), lib);
jobResults.add(jfOut.getResult());
allJobResults.add(jfOut.getResult());
jfCountOutputs.updateTracker("raw", jfOut.getOutputFile());
}
// Also start jellyfish on all the prep-processed libraries from MECQ
if (args.getAllMecqs() != null) {
for (Mecq.EcqArgs ecqArgs : args.getAllMecqs()) {
// Create the output directory for the RAW datasets
File ecqOutputDir = new File(args.getOutputDir(), ecqArgs.getName());
if (!ecqOutputDir.exists()) {
ecqOutputDir.mkdirs();
}
for (Library lib : ecqArgs.getOutputLibraries()) {
// Add jellyfish id to list of job ids
JobOutput jfOut =
this.executeJellyfishCount(args, ecqArgs.getName(), args.getOutputDir(), lib);
jobResults.add(jfOut.getResult());
allJobResults.add(jfOut.getResult());
jfCountOutputs.updateTracker(ecqArgs.getName(), jfOut.getOutputFile());
}
}
}
// If we're using a scheduler and we have been asked to run each job
// in parallel, then we should wait for all those to complete before continueing.
if (executionContext.usingScheduler() && args.isRunParallel()) {
log.info("Kmer counting all ECQ groups in parallel, waiting for completion");
this.conanExecutorService.executeScheduledWait(
jobResults,
args.getJobPrefix() + "-count-*",
ExitStatus.Type.COMPLETED_ANY,
args.getJobPrefix() + "-kmer-count-wait",
args.getOutputDir());
}
// Waiting point... clear job ids.
jobResults.clear();
JobOutputMap mergedOutputs = new JobOutputMap();
// Now execute merge jobs if required
for (Map.Entry<String, Set<File>> entry : jfCountOutputs.entrySet()) {
String ecqName = entry.getKey();
Set<File> fileSet = entry.getValue();
// Only merge if there's more than one library
if (fileSet.size() > 1) {
JobOutput jfOut =
this.executeJellyfishMerger(
args, ecqName, fileSet, new File(args.getOutputDir(), ecqName));
jobResults.add(jfOut.getResult());
allJobResults.add(jfOut.getResult());
mergedOutputs.updateTracker(ecqName, jfOut.getOutputFile());
}
}
// If we're using a scheduler and we have been asked to run each job
// in parallel, then we should wait for all those to complete before continueing.
if (executionContext.usingScheduler() && args.isRunParallel()) {
log.info(
"Creating merged kmer counts for all ECQ groups in parallel, waiting for completion");
this.conanExecutorService.executeScheduledWait(
jobResults,
args.getJobPrefix() + "-merge-*",
ExitStatus.Type.COMPLETED_ANY,
args.getJobPrefix() + "-kmer-merge-wait",
args.getOutputDir());
}
// Waiting point... clear job ids.
jobResults.clear();
// Combine all jellyfish out maps
jfCountOutputs.combine(mergedOutputs);
String katGcpJobPrefix = args.getJobPrefix() + "-kat-gcp";
// Run KAT GCP on everything
List<ExecutionResult> katGcpResults =
this.executeKatGcp(
jfCountOutputs,
katGcpJobPrefix,
args.getThreadsPerProcess(),
args.getMemoryPerProcess(),
args.isRunParallel());
for (ExecutionResult result : katGcpResults) {
result.setName(result.getName().substring(args.getJobPrefix().length() + 1));
jobResults.add(result);
allJobResults.add(result);
}
// If we're using a scheduler and we have been asked to run each job
// in parallel, then we should wait for all those to complete before continueing.
if (executionContext.usingScheduler() && args.isRunParallel()) {
log.info("Running \"kat gcp\" for all ECQ groups in parallel, waiting for completion");
this.conanExecutorService.executeScheduledWait(
jobResults,
katGcpJobPrefix + "*",
ExitStatus.Type.COMPLETED_ANY,
args.getJobPrefix() + "-kat-gcp-wait",
args.getOutputDir());
}
// Waiting point... clear job ids.
jobResults.clear();
log.info("Kmer counting of all reads finished.");
stopWatch.stop();
TaskResult taskResult =
new DefaultTaskResult(
"rampart-read_analysis-kmer", true, allJobResults, stopWatch.getTime() / 1000L);
// Output the resource usage to file
FileUtils.writeLines(
new File(args.getOutputDir(), args.getJobPrefix() + ".summary"), taskResult.getOutput());
return new DefaultExecutionResult(
taskResult.getTaskName(),
0,
new String[] {},
null,
-1,
new ResourceUsage(
taskResult.getMaxMemUsage(),
taskResult.getActualTotalRuntime(),
taskResult.getTotalExternalCputime()));
} catch (ConanParameterException | IOException e) {
throw new ProcessExecutionException(-1, e);
}
}
public void assertOutputHasNoDeprecationWarnings(ExecutionResult result) {
assertNoDeprecationWarnings(result.getOutput(), "Standard output");
assertNoDeprecationWarnings(result.getError(), "Standard error");
}
private void assertOutputHasNoStackTraces(ExecutionResult result) {
assertNoStackTraces(result.getOutput(), "Standard output");
}
protected ResultSet toResultSet(ExecutionResult result) throws SQLException {
return new IteratorResultSet(this, result.columns(), result.getResult());
}