Пример #1
0
  protected JobOutput executeJellyfishMerger(
      Args args, String ecqName, Set<File> fileSet, File outputDir)
      throws InterruptedException, ProcessExecutionException, ConanParameterException, IOException {

    String suffix = "jellyfish_" + ecqName + "_all.jf31_0";

    String jobName = args.getJobPrefix() + "-merge-" + suffix;

    List<File> files = new ArrayList<>();
    files.addAll(fileSet);

    File outputFile = new File(outputDir, suffix);

    JellyfishMergeV11 jellyfishMerge =
        this.makeJellyfishMerge(files, outputFile, args.getOrganism());

    ExecutionResult id =
        this.conanExecutorService.executeProcess(
            jellyfishMerge,
            args.getOutputDir(),
            jobName,
            args.getThreadsPerProcess(),
            args.getMemoryPerProcess(),
            args.isRunParallel());

    id.setName("merge-" + suffix);

    return new JobOutput(id, outputFile);
  }
Пример #2
0
  protected JobOutput executeJellyfishCount(Args args, String ecqName, File outputDir, Library lib)
      throws ProcessExecutionException, InterruptedException, ConanParameterException, IOException {
    String suffix = "jellyfish_" + ecqName + "_" + lib.getName() + ".jf31";

    // Create the process
    JellyfishCountV11 jellyfishProcess =
        this.makeJellyfishCount(
            this.makeInputStringFromLib(lib),
            new File(new File(outputDir, ecqName), suffix).getAbsolutePath(),
            args.getOrganism(),
            args.getThreadsPerProcess());

    File outputFile = jellyfishProcess.getArgs().getOutputFile();

    // Create a job name
    String jobName = args.getJobPrefix() + "-count-" + suffix;

    // Start jellyfish
    final ExecutionResult id =
        this.conanExecutorService.executeProcess(
            jellyfishProcess,
            new File(outputDir, ecqName),
            jobName,
            args.getThreadsPerProcess(),
            args.getMemoryPerProcess(),
            args.isRunParallel());

    id.setName("count-" + suffix);

    return new JobOutput(id, outputFile);
  }
  @Test
  public void exampleQuery() throws Exception {
    // START SNIPPET: JavaQuery
    ExecutionEngine engine = new ExecutionEngine(db);
    ExecutionResult result = engine.execute("start n=node(0) where 1=1 return n");

    assertThat(result.columns(), hasItem("n"));
    Iterator<Node> n_column = result.columnAs("n");
    assertThat(asIterable(n_column), hasItem(db.getNodeById(0)));
    assertThat(result.toString(), containsString("Node[0]"));
    // END SNIPPET: JavaQuery
  }
  @Test
  public void exampleWithParameterForNodeObject() throws Exception {
    // START SNIPPET: exampleWithParameterForNodeObject
    Map<String, Object> params = new HashMap<String, Object>();
    params.put("node", andreasNode);
    ExecutionResult result = engine.execute("start n=node({node}) return n.name", params);
    // END SNIPPET: exampleWithParameterForNodeObject

    assertThat(result.columns(), hasItem("n.name"));
    Iterator<Object> n_column = result.columnAs("n.name");
    assertEquals("Andreas", n_column.next());
  }
 @Test
 public void testColumnAreInTheRightOrder() throws Exception {
   createTenNodes();
   String q =
       "start one=node(1), two=node(2), three=node(3), four=node(4), five=node(5), six=node(6), "
           + "seven=node(7), eight=node(8), nine=node(9), ten=node(10) "
           + "return one, two, three, four, five, six, seven, eight, nine, ten";
   ExecutionResult result = engine.execute(q);
   assertThat(
       result.toString(),
       matchesPattern("one.*two.*three.*four.*five.*six.*seven.*eight.*nine.*ten"));
 }
  @Test
  public void shouldBeAbleToEmitJavaIterables() throws Exception {
    makeFriends(michaelaNode, andreasNode);
    makeFriends(michaelaNode, johanNode);

    ExecutionEngine engine = new ExecutionEngine(db);

    ExecutionResult result =
        engine.execute("start n=node(0) match n-->friend return collect(friend)");

    Iterable<Node> friends = (Iterable<Node>) result.columnAs("collect(friend)").next();
    assertThat(friends, hasItems(andreasNode, johanNode));
    assertThat(friends, instanceOf(Iterable.class));
  }
  @Test
  public void exampleWithParameterRegularExpression() throws Exception {
    // START SNIPPET: exampleWithParameterRegularExpression
    Map<String, Object> params = new HashMap<String, Object>();
    params.put("regex", ".*h.*");
    ExecutionResult result =
        engine.execute("start n=node(0,1,2) where n.name =~ {regex} return n.name", params);
    // END SNIPPET: exampleWithParameterRegularExpression

    assertThat(result.columns(), hasItem("n.name"));
    Iterator<Object> n_column = result.columnAs("n.name");
    assertEquals("Michaela", n_column.next());
    assertEquals("Johan", n_column.next());
  }
  @Test
  public void exampleWithParameterForSkipAndLimit() throws Exception {
    // START SNIPPET: exampleWithParameterForSkipLimit
    Map<String, Object> params = new HashMap<String, Object>();
    params.put("s", 1);
    params.put("l", 1);
    ExecutionResult result =
        engine.execute("start n=node(0,1,2) return n.name skip {s} limit {l}", params);
    // END SNIPPET: exampleWithParameterForSkipLimit

    assertThat(result.columns(), hasItem("n.name"));
    Iterator<Object> n_column = result.columnAs("n.name");
    assertEquals("Andreas", n_column.next());
  }
 public void assertErrorHasNoStackTraces(ExecutionResult result) {
   String error = result.getError();
   if (result instanceof ExecutionFailure) {
     // Axe everything after the expected exception
     int pos = error.indexOf("* Exception is:" + TextUtil.getPlatformLineSeparator());
     if (pos >= 0) {
       error = error.substring(0, pos);
     }
   }
   assertNoStackTraces(error, "Standard error");
 }
 private <T> List<T> toList(ExecutionResult result, String column) {
   List<T> results = new ArrayList<T>();
   IteratorUtil.addToCollection(result.<T>columnAs(column), results);
   return results;
 }
Пример #11
0
  @Override
  public ExecutionResult execute(ExecutionContext executionContext)
      throws ProcessExecutionException, InterruptedException {

    try {

      StopWatch stopWatch = new StopWatch();
      stopWatch.start();

      log.info("Starting Kmer Counting on all Reads");

      // Create shortcut to args for convienience
      Args args = this.getArgs();

      // Force run parallel to false if not using a scheduler
      if (!executionContext.usingScheduler() && args.isRunParallel()) {
        log.warn("Forcing linear execution due to lack of job scheduler");
        args.setRunParallel(false);
      }

      // Create the output directory
      args.getOutputDir().mkdirs();

      JobOutputMap jfCountOutputs = new JobOutputMap();
      List<ExecutionResult> jobResults = new ArrayList<>();
      List<ExecutionResult> allJobResults = new ArrayList<>();

      // Create the output directory for the RAW datasets
      File rawOutputDir = new File(args.getOutputDir(), "raw");

      if (!rawOutputDir.exists()) {
        rawOutputDir.mkdirs();
      }

      // Start jellyfish on all RAW datasets
      for (Library lib : args.getAllLibraries()) {

        // Execute jellyfish and add id to list of job ids
        JobOutput jfOut = this.executeJellyfishCount(args, "raw", args.getOutputDir(), lib);
        jobResults.add(jfOut.getResult());
        allJobResults.add(jfOut.getResult());
        jfCountOutputs.updateTracker("raw", jfOut.getOutputFile());
      }

      // Also start jellyfish on all the prep-processed libraries from MECQ
      if (args.getAllMecqs() != null) {
        for (Mecq.EcqArgs ecqArgs : args.getAllMecqs()) {

          // Create the output directory for the RAW datasets
          File ecqOutputDir = new File(args.getOutputDir(), ecqArgs.getName());

          if (!ecqOutputDir.exists()) {
            ecqOutputDir.mkdirs();
          }

          for (Library lib : ecqArgs.getOutputLibraries()) {

            // Add jellyfish id to list of job ids
            JobOutput jfOut =
                this.executeJellyfishCount(args, ecqArgs.getName(), args.getOutputDir(), lib);

            jobResults.add(jfOut.getResult());
            allJobResults.add(jfOut.getResult());
            jfCountOutputs.updateTracker(ecqArgs.getName(), jfOut.getOutputFile());
          }
        }
      }

      // If we're using a scheduler and we have been asked to run each job
      // in parallel, then we should wait for all those to complete before continueing.
      if (executionContext.usingScheduler() && args.isRunParallel()) {
        log.info("Kmer counting all ECQ groups in parallel, waiting for completion");
        this.conanExecutorService.executeScheduledWait(
            jobResults,
            args.getJobPrefix() + "-count-*",
            ExitStatus.Type.COMPLETED_ANY,
            args.getJobPrefix() + "-kmer-count-wait",
            args.getOutputDir());
      }

      // Waiting point... clear job ids.
      jobResults.clear();

      JobOutputMap mergedOutputs = new JobOutputMap();

      // Now execute merge jobs if required
      for (Map.Entry<String, Set<File>> entry : jfCountOutputs.entrySet()) {

        String ecqName = entry.getKey();
        Set<File> fileSet = entry.getValue();

        // Only merge if there's more than one library
        if (fileSet.size() > 1) {
          JobOutput jfOut =
              this.executeJellyfishMerger(
                  args, ecqName, fileSet, new File(args.getOutputDir(), ecqName));

          jobResults.add(jfOut.getResult());
          allJobResults.add(jfOut.getResult());
          mergedOutputs.updateTracker(ecqName, jfOut.getOutputFile());
        }
      }

      // If we're using a scheduler and we have been asked to run each job
      // in parallel, then we should wait for all those to complete before continueing.
      if (executionContext.usingScheduler() && args.isRunParallel()) {
        log.info(
            "Creating merged kmer counts for all ECQ groups in parallel, waiting for completion");
        this.conanExecutorService.executeScheduledWait(
            jobResults,
            args.getJobPrefix() + "-merge-*",
            ExitStatus.Type.COMPLETED_ANY,
            args.getJobPrefix() + "-kmer-merge-wait",
            args.getOutputDir());
      }

      // Waiting point... clear job ids.
      jobResults.clear();

      // Combine all jellyfish out maps
      jfCountOutputs.combine(mergedOutputs);

      String katGcpJobPrefix = args.getJobPrefix() + "-kat-gcp";

      // Run KAT GCP on everything
      List<ExecutionResult> katGcpResults =
          this.executeKatGcp(
              jfCountOutputs,
              katGcpJobPrefix,
              args.getThreadsPerProcess(),
              args.getMemoryPerProcess(),
              args.isRunParallel());

      for (ExecutionResult result : katGcpResults) {
        result.setName(result.getName().substring(args.getJobPrefix().length() + 1));
        jobResults.add(result);
        allJobResults.add(result);
      }

      // If we're using a scheduler and we have been asked to run each job
      // in parallel, then we should wait for all those to complete before continueing.
      if (executionContext.usingScheduler() && args.isRunParallel()) {
        log.info("Running \"kat gcp\" for all ECQ groups in parallel, waiting for completion");
        this.conanExecutorService.executeScheduledWait(
            jobResults,
            katGcpJobPrefix + "*",
            ExitStatus.Type.COMPLETED_ANY,
            args.getJobPrefix() + "-kat-gcp-wait",
            args.getOutputDir());
      }

      // Waiting point... clear job ids.
      jobResults.clear();

      log.info("Kmer counting of all reads finished.");

      stopWatch.stop();

      TaskResult taskResult =
          new DefaultTaskResult(
              "rampart-read_analysis-kmer", true, allJobResults, stopWatch.getTime() / 1000L);

      // Output the resource usage to file
      FileUtils.writeLines(
          new File(args.getOutputDir(), args.getJobPrefix() + ".summary"), taskResult.getOutput());

      return new DefaultExecutionResult(
          taskResult.getTaskName(),
          0,
          new String[] {},
          null,
          -1,
          new ResourceUsage(
              taskResult.getMaxMemUsage(),
              taskResult.getActualTotalRuntime(),
              taskResult.getTotalExternalCputime()));
    } catch (ConanParameterException | IOException e) {
      throw new ProcessExecutionException(-1, e);
    }
  }
 public void assertOutputHasNoDeprecationWarnings(ExecutionResult result) {
   assertNoDeprecationWarnings(result.getOutput(), "Standard output");
   assertNoDeprecationWarnings(result.getError(), "Standard error");
 }
 private void assertOutputHasNoStackTraces(ExecutionResult result) {
   assertNoStackTraces(result.getOutput(), "Standard output");
 }
Пример #14
0
 protected ResultSet toResultSet(ExecutionResult result) throws SQLException {
   return new IteratorResultSet(this, result.columns(), result.getResult());
 }