/**
   * GML (Graph Modeling Language) is a text file format supporting network data with a very easy
   * syntax. It is used by Graphlet, Pajek, yEd, LEDA and NetworkX.
   */
  public void exportAsGML(File gmlFile, List<SemanticPath> paths) {
    gmlFile.getParentFile().mkdirs();

    try (PrintWriter writer = new PrintWriter(Files.newWriter(gmlFile, Charsets.UTF_8)); ) {

      writer.println("graph\n[");
      // write vertex
      for (SemanticPath path : paths) {
        for (int i = 0; i < path.length(); i++) {
          writer.println("\tnode\n\t[");
          writer.println("\t\tid " + path.id(i));
          writer.println("\t\tlabel \"" + path.name(i) + "\"");
          writer.println("\t]");
        }
      }

      // write edge
      for (SemanticPath path : paths) {
        for (int i = 0; i < path.length() - 1; i++) {
          writer.println("\tedge\n\t[");
          writer.println("\t\tsource " + path.id(i + 1));
          writer.println("\t\ttarget " + path.id(i));
          writer.println("\t]");
        }
      }
      writer.println(']');
      writer.flush();
    } catch (IOException e) {
      LOG.error("Export to GML file error!", e);
    }
  }
  @Override
  public void setUp() throws Exception {
    super.setUp();

    File inputDir = getTestTempDir("transactions");
    File input = new File(inputDir, "test.txt");

    paramsImpl1.set(PFPGrowth.MIN_SUPPORT, "100");
    paramsImpl1.set(PFPGrowth.MAX_HEAPSIZE, "10000");
    paramsImpl1.set(PFPGrowth.NUM_GROUPS, "50");
    paramsImpl1.set(PFPGrowth.ENCODING, "UTF-8");
    paramsImpl1.set(PFPGrowth.INPUT, input.getAbsolutePath());

    paramsImpl2.set(PFPGrowth.MIN_SUPPORT, "100");
    paramsImpl2.set(PFPGrowth.MAX_HEAPSIZE, "10000");
    paramsImpl2.set(PFPGrowth.NUM_GROUPS, "50");
    paramsImpl2.set(PFPGrowth.ENCODING, "UTF-8");
    paramsImpl2.set(PFPGrowth.INPUT, input.getAbsolutePath());
    paramsImpl2.set(PFPGrowth.USE_FPG2, "true");

    File outputDir1 = getTestTempDir("frequentpatterns1");
    paramsImpl1.set(PFPGrowth.OUTPUT, outputDir1.getAbsolutePath());

    File outputDir2 = getTestTempDir("frequentpatterns2");
    paramsImpl2.set(PFPGrowth.OUTPUT, outputDir2.getAbsolutePath());

    Writer writer = Files.newWriter(input, Charsets.UTF_8);
    try {
      StringRecordIterator it =
          new StringRecordIterator(
              new FileLineIterable(Resources.getResource("retail.dat").openStream()), "\\s+");
      Collection<List<String>> transactions = Lists.newArrayList();

      while (it.hasNext()) {
        Pair<List<String>, Long> next = it.next();
        transactions.add(next.getFirst());
      }

      for (List<String> transaction : transactions) {
        String sep = "";
        for (String item : transaction) {
          writer.write(sep + item);
          sep = ",";
        }
        writer.write("\n");
      }

    } finally {
      Closeables.closeQuietly(writer);
    }
  }
Ejemplo n.º 3
0
  @BeforeClass
  public static void buildUserSchema() throws Exception {
    sample = "target/users.csv";
    BufferedWriter writer = Files.newWriter(new File(sample), CSVSchemaCommand.SCHEMA_CHARSET);
    writer.append("id, username, email\n");
    writer.append("1, test, [email protected]\n");
    writer.close();

    schema =
        SchemaBuilder.record("User")
            .fields()
            .optionalLong("id")
            .optionalString("username")
            .optionalString("email")
            .endRecord();
  }
Ejemplo n.º 4
0
 public static void main(String[] args) throws IOException {
   WikidumpXmlAnalyzer analyzer =
       new WikidumpXmlAnalyzer("../wikihowcom-20141208-current.xml", null);
   List<WikidumpArticle> articles = analyzer.getArticles();
   TaskExtractor extractor = new TaskExtractor();
   extractor.addWikidumpArticles(articles);
   extractor.normalizeTasks();
   extractor.buildHierarchy();
   extractor.fillTaskIds();
   List<Task> tasks = extractor.getTasks();
   BufferedWriter bw = Files.newWriter(new File("data/wikihow-id-summary.tsv"), Charsets.UTF_8);
   for (Task task : tasks) {
     bw.write(task.getId() + "\t" + task.getSummary() + "\n");
   }
   bw.close();
 }
  /**
   * Generates Incremental DOM JS source files given a Soy parse tree, an options object, an
   * optional bundle of translated messages, and information on where to put the output files.
   *
   * @param soyTree The Soy parse tree to generate JS source code for.
   * @param jsSrcOptions The compilation options relevant to this backend.
   * @param outputPathFormat The format string defining how to build the output file path
   *     corresponding to an input file path.
   * @throws SoySyntaxException If a syntax error is found.
   * @throws IOException If there is an error in opening/writing an output JS file.
   */
  public void genJsFiles(
      SoyFileSetNode soyTree, SoyJsSrcOptions jsSrcOptions, String outputPathFormat)
      throws SoySyntaxException, IOException {

    List<String> jsFileContents = genJsSrc(soyTree, jsSrcOptions);

    ImmutableList<SoyFileNode> srcsToCompile =
        ImmutableList.copyOf(
            Iterables.filter(soyTree.getChildren(), SoyFileNode.MATCH_SRC_FILENODE));

    if (srcsToCompile.size() != jsFileContents.size()) {
      throw new AssertionError(
          String.format(
              "Expected to generate %d code chunk(s), got %d",
              srcsToCompile.size(), jsFileContents.size()));
    }

    Multimap<String, Integer> outputs =
        MainEntryPointUtils.mapOutputsToSrcs(
            null /* locale */, outputPathFormat, "" /* inputPathsPrefix */, srcsToCompile);

    for (String outputFilePath : outputs.keySet()) {
      Writer out = Files.newWriter(new File(outputFilePath), UTF_8);
      try {
        boolean isFirst = true;
        for (int inputFileIndex : outputs.get(outputFilePath)) {
          if (isFirst) {
            isFirst = false;
          } else {
            // Concatenating JS files is not safe unless we know that the last statement from one
            // couldn't combine with the isFirst statement of the next.  Inserting a semicolon will
            // prevent this from happening.
            out.write("\n;\n");
          }
          out.write(jsFileContents.get(inputFileIndex));
        }
      } finally {
        out.close();
      }
    }
  }
Ejemplo n.º 6
0
  @Override
  public void runInternal(GeogigCLI cli) throws IOException {
    checkParameter(patchFiles.size() < 2, "Only one single patch file accepted");
    checkParameter(!patchFiles.isEmpty(), "No patch file specified");

    ConsoleReader console = cli.getConsole();
    GeoGIG geogig = cli.getGeogig();

    File patchFile = new File(patchFiles.get(0));
    checkParameter(patchFile.exists(), "Patch file cannot be found");
    FileInputStream stream;
    try {
      stream = new FileInputStream(patchFile);
    } catch (FileNotFoundException e1) {
      throw new CommandFailedException("Can't open patch file " + patchFile, e1);
    }
    BufferedReader reader = null;
    try {
      reader = new BufferedReader(new InputStreamReader(stream, "UTF-8"));
    } catch (UnsupportedEncodingException e) {
      Closeables.closeQuietly(reader);
      Closeables.closeQuietly(stream);
      throw new CommandFailedException("Error reading patch file " + patchFile, e);
    }
    Patch patch = PatchSerializer.read(reader);
    Closeables.closeQuietly(reader);
    Closeables.closeQuietly(stream);

    if (reverse) {
      patch = patch.reversed();
    }

    if (summary) {
      console.println(patch.toString());
    } else if (check) {
      VerifyPatchResults verify =
          cli.getGeogig().command(VerifyPatchOp.class).setPatch(patch).call();
      Patch toReject = verify.getToReject();
      Patch toApply = verify.getToApply();
      if (toReject.isEmpty()) {
        console.println("Patch can be applied.");
      } else {
        console.println("Error: Patch cannot be applied\n");
        console.println("Applicable entries:\n");
        console.println(toApply.toString());
        console.println("\nConflicting entries:\n");
        console.println(toReject.toString());
      }
    } else {
      try {
        Patch rejected =
            geogig.command(ApplyPatchOp.class).setPatch(patch).setApplyPartial(reject).call();
        if (reject) {
          if (rejected.isEmpty()) {
            console.println("Patch applied succesfully");
          } else {
            int accepted = patch.count() - rejected.count();
            StringBuilder sb = new StringBuilder();
            File file = new File(patchFile.getAbsolutePath() + ".rej");
            sb.append("Patch applied only partially.\n");
            sb.append(Integer.toString(accepted) + " changes were applied.\n");
            sb.append(Integer.toString(rejected.count()) + " changes were rejected.\n");
            BufferedWriter writer = Files.newWriter(file, Charsets.UTF_8);
            PatchSerializer.write(writer, patch);
            writer.flush();
            writer.close();
            sb.append(
                "Patch file with rejected changes created at " + file.getAbsolutePath() + "\n");
            throw new CommandFailedException(sb.toString());
          }
        } else {
          console.println("Patch applied succesfully");
        }
      } catch (CannotApplyPatchException e) {
        throw new CommandFailedException(e);
      }
    }
  }
Ejemplo n.º 7
0
  @SuppressWarnings("PMD.EmptyCatchBlock")
  public static int runTests(
      final CommandRunnerParams params,
      Iterable<TestRule> tests,
      BuildContext buildContext,
      ExecutionContext executionContext,
      final TestRunningOptions options,
      ListeningExecutorService service,
      BuildEngine buildEngine,
      final StepRunner stepRunner)
      throws IOException, ExecutionException, InterruptedException {

    if (options.isUsingOneTimeOutputDirectories()) {
      BuckConstant.setOneTimeTestSubdirectory(UUID.randomUUID().toString());
    }

    ImmutableSet<JavaLibrary> rulesUnderTest;
    // If needed, we first run instrumentation on the class files.
    if (options.isCodeCoverageEnabled()) {
      rulesUnderTest = getRulesUnderTest(tests);
      if (!rulesUnderTest.isEmpty()) {
        try {
          stepRunner.runStepForBuildTarget(
              new MakeCleanDirectoryStep(JUnitStep.JACOCO_OUTPUT_DIR),
              Optional.<BuildTarget>absent());
        } catch (StepFailedException e) {
          params.getConsole().printBuildFailureWithoutStacktrace(e);
          return 1;
        }
      }
    } else {
      rulesUnderTest = ImmutableSet.of();
    }

    final ImmutableSet<String> testTargets =
        FluentIterable.from(tests)
            .transform(HasBuildTarget.TO_TARGET)
            .transform(Functions.toStringFunction())
            .toSet();

    final int totalNumberOfTests = Iterables.size(tests);

    params
        .getBuckEventBus()
        .post(
            TestRunEvent.started(
                options.isRunAllTests(),
                options.getTestSelectorList(),
                options.shouldExplainTestSelectorList(),
                testTargets));

    // Start running all of the tests. The result of each java_test() rule is represented as a
    // ListenableFuture.
    List<ListenableFuture<TestResults>> results = Lists.newArrayList();

    // Unless `--verbose 0` is specified, print out test results as they become available.
    // Failures with the ListenableFuture should always be printed, as they indicate an error with
    // Buck, not the test being run.
    Verbosity verbosity = params.getConsole().getVerbosity();
    final boolean printTestResults = (verbosity != Verbosity.SILENT);

    // For grouping results!
    final TestResultsGrouper grouper;
    if (options.isIgnoreFailingDependencies()) {
      grouper = new TestResultsGrouper(tests);
    } else {
      grouper = null;
    }

    TestRuleKeyFileHelper testRuleKeyFileHelper =
        new TestRuleKeyFileHelper(executionContext.getProjectFilesystem(), buildEngine);
    final AtomicInteger lastReportedTestSequenceNumber = new AtomicInteger();
    final List<TestRun> separateTestRuns = Lists.newArrayList();
    List<TestRun> parallelTestRuns = Lists.newArrayList();
    for (final TestRule test : tests) {
      // Determine whether the test needs to be executed.
      boolean isTestRunRequired;
      isTestRunRequired =
          isTestRunRequiredForTest(
              test,
              buildEngine,
              executionContext,
              testRuleKeyFileHelper,
              options.isResultsCacheEnabled(),
              !options.getTestSelectorList().isEmpty());

      List<Step> steps;
      if (isTestRunRequired) {
        params.getBuckEventBus().post(IndividualTestEvent.started(testTargets));
        ImmutableList.Builder<Step> stepsBuilder = ImmutableList.builder();
        Preconditions.checkState(buildEngine.isRuleBuilt(test.getBuildTarget()));
        final Map<String, UUID> testUUIDMap = new HashMap<>();
        List<Step> testSteps =
            test.runTests(
                buildContext,
                executionContext,
                options.isDryRun(),
                options.isShufflingTests(),
                options.getTestSelectorList(),
                new TestRule.TestReportingCallback() {
                  @Override
                  public void testsDidBegin() {
                    LOG.debug("Tests for rule %s began", test.getBuildTarget());
                  }

                  @Override
                  public void testDidBegin(String testCaseName, String testName) {
                    LOG.debug(
                        "Test rule %s test case %s test name %s began",
                        test.getBuildTarget(), testCaseName, testName);
                    UUID testUUID = UUID.randomUUID();
                    // UUID is immutable and thread-safe as of Java 7, so it's
                    // safe to stash in a map and use later:
                    //
                    // http://bugs.java.com/view_bug.do?bug_id=6611830
                    testUUIDMap.put(testCaseName + ":" + testName, testUUID);
                    params
                        .getBuckEventBus()
                        .post(TestSummaryEvent.started(testUUID, testCaseName, testName));
                  }

                  @Override
                  public void testDidEnd(TestResultSummary testResultSummary) {
                    LOG.debug(
                        "Test rule %s test did end: %s", test.getBuildTarget(), testResultSummary);
                    UUID testUUID =
                        testUUIDMap.get(
                            testResultSummary.getTestCaseName()
                                + ":"
                                + testResultSummary.getTestName());
                    Preconditions.checkNotNull(testUUID);
                    params
                        .getBuckEventBus()
                        .post(TestSummaryEvent.finished(testUUID, testResultSummary));
                  }

                  @Override
                  public void testsDidEnd(List<TestCaseSummary> testCaseSummaries) {
                    LOG.debug(
                        "Test rule %s tests did end: %s", test.getBuildTarget(), testCaseSummaries);
                  }
                });
        if (!testSteps.isEmpty()) {
          stepsBuilder.addAll(testSteps);
          stepsBuilder.add(testRuleKeyFileHelper.createRuleKeyInDirStep(test));
        }
        steps = stepsBuilder.build();
      } else {
        steps = ImmutableList.of();
      }

      TestRun testRun =
          TestRun.of(
              test,
              steps,
              getCachingStatusTransformingCallable(
                  isTestRunRequired,
                  test.interpretTestResults(
                      executionContext,
                      /*isUsingTestSelectors*/ !options.getTestSelectorList().isEmpty(),
                      /*isDryRun*/ options.isDryRun())));

      // Always run the commands, even if the list of commands as empty. There may be zero
      // commands because the rule is cached, but its results must still be processed.
      if (test.runTestSeparately()) {
        LOG.debug("Running test %s in serial", test);
        separateTestRuns.add(testRun);
      } else {
        LOG.debug("Running test %s in parallel", test);
        parallelTestRuns.add(testRun);
      }
    }

    final StepRunner.StepRunningCallback testStepRunningCallback =
        new StepRunner.StepRunningCallback() {
          @Override
          public void stepsWillRun(Optional<BuildTarget> buildTarget) {
            Preconditions.checkState(buildTarget.isPresent());
            LOG.debug("Test steps will run for %s", buildTarget);
            params.getBuckEventBus().post(TestRuleEvent.started(buildTarget.get()));
          }

          @Override
          public void stepsDidRun(Optional<BuildTarget> buildTarget) {
            Preconditions.checkState(buildTarget.isPresent());
            LOG.debug("Test steps did run for %s", buildTarget);
            params.getBuckEventBus().post(TestRuleEvent.finished(buildTarget.get()));
          }
        };

    for (TestRun testRun : parallelTestRuns) {
      ListenableFuture<TestResults> testResults =
          stepRunner.runStepsAndYieldResult(
              testRun.getSteps(),
              testRun.getTestResultsCallable(),
              Optional.of(testRun.getTest().getBuildTarget()),
              service,
              testStepRunningCallback);
      results.add(
          transformTestResults(
              params,
              testResults,
              grouper,
              testRun.getTest(),
              testTargets,
              printTestResults,
              lastReportedTestSequenceNumber,
              totalNumberOfTests));
    }

    ListenableFuture<List<TestResults>> parallelTestStepsFuture = Futures.allAsList(results);

    final List<TestResults> completedResults = Lists.newArrayList();

    final ListeningExecutorService directExecutorService = MoreExecutors.newDirectExecutorService();
    ListenableFuture<Void> uberFuture =
        stepRunner.addCallback(
            parallelTestStepsFuture,
            new FutureCallback<List<TestResults>>() {
              @Override
              public void onSuccess(List<TestResults> parallelTestResults) {
                LOG.debug("Parallel tests completed, running separate tests...");
                completedResults.addAll(parallelTestResults);
                List<ListenableFuture<TestResults>> separateResultsList = Lists.newArrayList();
                for (TestRun testRun : separateTestRuns) {
                  separateResultsList.add(
                      transformTestResults(
                          params,
                          stepRunner.runStepsAndYieldResult(
                              testRun.getSteps(),
                              testRun.getTestResultsCallable(),
                              Optional.of(testRun.getTest().getBuildTarget()),
                              directExecutorService,
                              testStepRunningCallback),
                          grouper,
                          testRun.getTest(),
                          testTargets,
                          printTestResults,
                          lastReportedTestSequenceNumber,
                          totalNumberOfTests));
                }
                ListenableFuture<List<TestResults>> serialResults =
                    Futures.allAsList(separateResultsList);
                try {
                  completedResults.addAll(serialResults.get());
                } catch (ExecutionException e) {
                  LOG.error(e, "Error fetching serial test results");
                  throw new HumanReadableException(e, "Error fetching serial test results");
                } catch (InterruptedException e) {
                  LOG.error(e, "Interrupted fetching serial test results");
                  try {
                    serialResults.cancel(true);
                  } catch (CancellationException ignored) {
                    // Rethrow original InterruptedException instead.
                  }
                  Thread.currentThread().interrupt();
                  throw new HumanReadableException(e, "Test cancelled");
                }
                LOG.debug("Done running serial tests.");
              }

              @Override
              public void onFailure(Throwable e) {
                LOG.error(e, "Parallel tests failed, not running serial tests");
                throw new HumanReadableException(e, "Parallel tests failed");
              }
            },
            directExecutorService);

    try {
      // Block until all the tests have finished running.
      uberFuture.get();
    } catch (ExecutionException e) {
      e.printStackTrace(params.getConsole().getStdErr());
      return 1;
    } catch (InterruptedException e) {
      try {
        uberFuture.cancel(true);
      } catch (CancellationException ignored) {
        // Rethrow original InterruptedException instead.
      }
      Thread.currentThread().interrupt();
      throw e;
    }

    params.getBuckEventBus().post(TestRunEvent.finished(testTargets, completedResults));

    // Write out the results as XML, if requested.
    Optional<String> path = options.getPathToXmlTestOutput();
    if (path.isPresent()) {
      try (Writer writer = Files.newWriter(new File(path.get()), Charsets.UTF_8)) {
        writeXmlOutput(completedResults, writer);
      }
    }

    // Generate the code coverage report.
    if (options.isCodeCoverageEnabled() && !rulesUnderTest.isEmpty()) {
      try {
        Optional<DefaultJavaPackageFinder> defaultJavaPackageFinderOptional =
            Optional.fromNullable(params.getBuckConfig().createDefaultJavaPackageFinder());
        stepRunner.runStepForBuildTarget(
            getReportCommand(
                rulesUnderTest,
                defaultJavaPackageFinderOptional,
                params.getRepository().getFilesystem(),
                JUnitStep.JACOCO_OUTPUT_DIR,
                options.getCoverageReportFormat()),
            Optional.<BuildTarget>absent());
      } catch (StepFailedException e) {
        params.getConsole().printBuildFailureWithoutStacktrace(e);
        return 1;
      }
    }

    boolean failures =
        Iterables.any(
            completedResults,
            new Predicate<TestResults>() {
              @Override
              public boolean apply(TestResults results) {
                LOG.debug("Checking result %s for failure", results);
                return !results.isSuccess();
              }
            });

    return failures ? TEST_FAILURES_EXIT_CODE : 0;
  }