/**
* GML (Graph Modeling Language) is a text file format supporting network data with a very easy
* syntax. It is used by Graphlet, Pajek, yEd, LEDA and NetworkX.
*/
public void exportAsGML(File gmlFile, List<SemanticPath> paths) {
gmlFile.getParentFile().mkdirs();
try (PrintWriter writer = new PrintWriter(Files.newWriter(gmlFile, Charsets.UTF_8)); ) {
writer.println("graph\n[");
// write vertex
for (SemanticPath path : paths) {
for (int i = 0; i < path.length(); i++) {
writer.println("\tnode\n\t[");
writer.println("\t\tid " + path.id(i));
writer.println("\t\tlabel \"" + path.name(i) + "\"");
writer.println("\t]");
}
}
// write edge
for (SemanticPath path : paths) {
for (int i = 0; i < path.length() - 1; i++) {
writer.println("\tedge\n\t[");
writer.println("\t\tsource " + path.id(i + 1));
writer.println("\t\ttarget " + path.id(i));
writer.println("\t]");
}
}
writer.println(']');
writer.flush();
} catch (IOException e) {
LOG.error("Export to GML file error!", e);
}
}
@Override
public void setUp() throws Exception {
super.setUp();
File inputDir = getTestTempDir("transactions");
File input = new File(inputDir, "test.txt");
paramsImpl1.set(PFPGrowth.MIN_SUPPORT, "100");
paramsImpl1.set(PFPGrowth.MAX_HEAPSIZE, "10000");
paramsImpl1.set(PFPGrowth.NUM_GROUPS, "50");
paramsImpl1.set(PFPGrowth.ENCODING, "UTF-8");
paramsImpl1.set(PFPGrowth.INPUT, input.getAbsolutePath());
paramsImpl2.set(PFPGrowth.MIN_SUPPORT, "100");
paramsImpl2.set(PFPGrowth.MAX_HEAPSIZE, "10000");
paramsImpl2.set(PFPGrowth.NUM_GROUPS, "50");
paramsImpl2.set(PFPGrowth.ENCODING, "UTF-8");
paramsImpl2.set(PFPGrowth.INPUT, input.getAbsolutePath());
paramsImpl2.set(PFPGrowth.USE_FPG2, "true");
File outputDir1 = getTestTempDir("frequentpatterns1");
paramsImpl1.set(PFPGrowth.OUTPUT, outputDir1.getAbsolutePath());
File outputDir2 = getTestTempDir("frequentpatterns2");
paramsImpl2.set(PFPGrowth.OUTPUT, outputDir2.getAbsolutePath());
Writer writer = Files.newWriter(input, Charsets.UTF_8);
try {
StringRecordIterator it =
new StringRecordIterator(
new FileLineIterable(Resources.getResource("retail.dat").openStream()), "\\s+");
Collection<List<String>> transactions = Lists.newArrayList();
while (it.hasNext()) {
Pair<List<String>, Long> next = it.next();
transactions.add(next.getFirst());
}
for (List<String> transaction : transactions) {
String sep = "";
for (String item : transaction) {
writer.write(sep + item);
sep = ",";
}
writer.write("\n");
}
} finally {
Closeables.closeQuietly(writer);
}
}
@BeforeClass
public static void buildUserSchema() throws Exception {
sample = "target/users.csv";
BufferedWriter writer = Files.newWriter(new File(sample), CSVSchemaCommand.SCHEMA_CHARSET);
writer.append("id, username, email\n");
writer.append("1, test, [email protected]\n");
writer.close();
schema =
SchemaBuilder.record("User")
.fields()
.optionalLong("id")
.optionalString("username")
.optionalString("email")
.endRecord();
}
public static void main(String[] args) throws IOException {
WikidumpXmlAnalyzer analyzer =
new WikidumpXmlAnalyzer("../wikihowcom-20141208-current.xml", null);
List<WikidumpArticle> articles = analyzer.getArticles();
TaskExtractor extractor = new TaskExtractor();
extractor.addWikidumpArticles(articles);
extractor.normalizeTasks();
extractor.buildHierarchy();
extractor.fillTaskIds();
List<Task> tasks = extractor.getTasks();
BufferedWriter bw = Files.newWriter(new File("data/wikihow-id-summary.tsv"), Charsets.UTF_8);
for (Task task : tasks) {
bw.write(task.getId() + "\t" + task.getSummary() + "\n");
}
bw.close();
}
/**
* Generates Incremental DOM JS source files given a Soy parse tree, an options object, an
* optional bundle of translated messages, and information on where to put the output files.
*
* @param soyTree The Soy parse tree to generate JS source code for.
* @param jsSrcOptions The compilation options relevant to this backend.
* @param outputPathFormat The format string defining how to build the output file path
* corresponding to an input file path.
* @throws SoySyntaxException If a syntax error is found.
* @throws IOException If there is an error in opening/writing an output JS file.
*/
public void genJsFiles(
SoyFileSetNode soyTree, SoyJsSrcOptions jsSrcOptions, String outputPathFormat)
throws SoySyntaxException, IOException {
List<String> jsFileContents = genJsSrc(soyTree, jsSrcOptions);
ImmutableList<SoyFileNode> srcsToCompile =
ImmutableList.copyOf(
Iterables.filter(soyTree.getChildren(), SoyFileNode.MATCH_SRC_FILENODE));
if (srcsToCompile.size() != jsFileContents.size()) {
throw new AssertionError(
String.format(
"Expected to generate %d code chunk(s), got %d",
srcsToCompile.size(), jsFileContents.size()));
}
Multimap<String, Integer> outputs =
MainEntryPointUtils.mapOutputsToSrcs(
null /* locale */, outputPathFormat, "" /* inputPathsPrefix */, srcsToCompile);
for (String outputFilePath : outputs.keySet()) {
Writer out = Files.newWriter(new File(outputFilePath), UTF_8);
try {
boolean isFirst = true;
for (int inputFileIndex : outputs.get(outputFilePath)) {
if (isFirst) {
isFirst = false;
} else {
// Concatenating JS files is not safe unless we know that the last statement from one
// couldn't combine with the isFirst statement of the next. Inserting a semicolon will
// prevent this from happening.
out.write("\n;\n");
}
out.write(jsFileContents.get(inputFileIndex));
}
} finally {
out.close();
}
}
}
@Override
public void runInternal(GeogigCLI cli) throws IOException {
checkParameter(patchFiles.size() < 2, "Only one single patch file accepted");
checkParameter(!patchFiles.isEmpty(), "No patch file specified");
ConsoleReader console = cli.getConsole();
GeoGIG geogig = cli.getGeogig();
File patchFile = new File(patchFiles.get(0));
checkParameter(patchFile.exists(), "Patch file cannot be found");
FileInputStream stream;
try {
stream = new FileInputStream(patchFile);
} catch (FileNotFoundException e1) {
throw new CommandFailedException("Can't open patch file " + patchFile, e1);
}
BufferedReader reader = null;
try {
reader = new BufferedReader(new InputStreamReader(stream, "UTF-8"));
} catch (UnsupportedEncodingException e) {
Closeables.closeQuietly(reader);
Closeables.closeQuietly(stream);
throw new CommandFailedException("Error reading patch file " + patchFile, e);
}
Patch patch = PatchSerializer.read(reader);
Closeables.closeQuietly(reader);
Closeables.closeQuietly(stream);
if (reverse) {
patch = patch.reversed();
}
if (summary) {
console.println(patch.toString());
} else if (check) {
VerifyPatchResults verify =
cli.getGeogig().command(VerifyPatchOp.class).setPatch(patch).call();
Patch toReject = verify.getToReject();
Patch toApply = verify.getToApply();
if (toReject.isEmpty()) {
console.println("Patch can be applied.");
} else {
console.println("Error: Patch cannot be applied\n");
console.println("Applicable entries:\n");
console.println(toApply.toString());
console.println("\nConflicting entries:\n");
console.println(toReject.toString());
}
} else {
try {
Patch rejected =
geogig.command(ApplyPatchOp.class).setPatch(patch).setApplyPartial(reject).call();
if (reject) {
if (rejected.isEmpty()) {
console.println("Patch applied succesfully");
} else {
int accepted = patch.count() - rejected.count();
StringBuilder sb = new StringBuilder();
File file = new File(patchFile.getAbsolutePath() + ".rej");
sb.append("Patch applied only partially.\n");
sb.append(Integer.toString(accepted) + " changes were applied.\n");
sb.append(Integer.toString(rejected.count()) + " changes were rejected.\n");
BufferedWriter writer = Files.newWriter(file, Charsets.UTF_8);
PatchSerializer.write(writer, patch);
writer.flush();
writer.close();
sb.append(
"Patch file with rejected changes created at " + file.getAbsolutePath() + "\n");
throw new CommandFailedException(sb.toString());
}
} else {
console.println("Patch applied succesfully");
}
} catch (CannotApplyPatchException e) {
throw new CommandFailedException(e);
}
}
}
@SuppressWarnings("PMD.EmptyCatchBlock")
public static int runTests(
final CommandRunnerParams params,
Iterable<TestRule> tests,
BuildContext buildContext,
ExecutionContext executionContext,
final TestRunningOptions options,
ListeningExecutorService service,
BuildEngine buildEngine,
final StepRunner stepRunner)
throws IOException, ExecutionException, InterruptedException {
if (options.isUsingOneTimeOutputDirectories()) {
BuckConstant.setOneTimeTestSubdirectory(UUID.randomUUID().toString());
}
ImmutableSet<JavaLibrary> rulesUnderTest;
// If needed, we first run instrumentation on the class files.
if (options.isCodeCoverageEnabled()) {
rulesUnderTest = getRulesUnderTest(tests);
if (!rulesUnderTest.isEmpty()) {
try {
stepRunner.runStepForBuildTarget(
new MakeCleanDirectoryStep(JUnitStep.JACOCO_OUTPUT_DIR),
Optional.<BuildTarget>absent());
} catch (StepFailedException e) {
params.getConsole().printBuildFailureWithoutStacktrace(e);
return 1;
}
}
} else {
rulesUnderTest = ImmutableSet.of();
}
final ImmutableSet<String> testTargets =
FluentIterable.from(tests)
.transform(HasBuildTarget.TO_TARGET)
.transform(Functions.toStringFunction())
.toSet();
final int totalNumberOfTests = Iterables.size(tests);
params
.getBuckEventBus()
.post(
TestRunEvent.started(
options.isRunAllTests(),
options.getTestSelectorList(),
options.shouldExplainTestSelectorList(),
testTargets));
// Start running all of the tests. The result of each java_test() rule is represented as a
// ListenableFuture.
List<ListenableFuture<TestResults>> results = Lists.newArrayList();
// Unless `--verbose 0` is specified, print out test results as they become available.
// Failures with the ListenableFuture should always be printed, as they indicate an error with
// Buck, not the test being run.
Verbosity verbosity = params.getConsole().getVerbosity();
final boolean printTestResults = (verbosity != Verbosity.SILENT);
// For grouping results!
final TestResultsGrouper grouper;
if (options.isIgnoreFailingDependencies()) {
grouper = new TestResultsGrouper(tests);
} else {
grouper = null;
}
TestRuleKeyFileHelper testRuleKeyFileHelper =
new TestRuleKeyFileHelper(executionContext.getProjectFilesystem(), buildEngine);
final AtomicInteger lastReportedTestSequenceNumber = new AtomicInteger();
final List<TestRun> separateTestRuns = Lists.newArrayList();
List<TestRun> parallelTestRuns = Lists.newArrayList();
for (final TestRule test : tests) {
// Determine whether the test needs to be executed.
boolean isTestRunRequired;
isTestRunRequired =
isTestRunRequiredForTest(
test,
buildEngine,
executionContext,
testRuleKeyFileHelper,
options.isResultsCacheEnabled(),
!options.getTestSelectorList().isEmpty());
List<Step> steps;
if (isTestRunRequired) {
params.getBuckEventBus().post(IndividualTestEvent.started(testTargets));
ImmutableList.Builder<Step> stepsBuilder = ImmutableList.builder();
Preconditions.checkState(buildEngine.isRuleBuilt(test.getBuildTarget()));
final Map<String, UUID> testUUIDMap = new HashMap<>();
List<Step> testSteps =
test.runTests(
buildContext,
executionContext,
options.isDryRun(),
options.isShufflingTests(),
options.getTestSelectorList(),
new TestRule.TestReportingCallback() {
@Override
public void testsDidBegin() {
LOG.debug("Tests for rule %s began", test.getBuildTarget());
}
@Override
public void testDidBegin(String testCaseName, String testName) {
LOG.debug(
"Test rule %s test case %s test name %s began",
test.getBuildTarget(), testCaseName, testName);
UUID testUUID = UUID.randomUUID();
// UUID is immutable and thread-safe as of Java 7, so it's
// safe to stash in a map and use later:
//
// http://bugs.java.com/view_bug.do?bug_id=6611830
testUUIDMap.put(testCaseName + ":" + testName, testUUID);
params
.getBuckEventBus()
.post(TestSummaryEvent.started(testUUID, testCaseName, testName));
}
@Override
public void testDidEnd(TestResultSummary testResultSummary) {
LOG.debug(
"Test rule %s test did end: %s", test.getBuildTarget(), testResultSummary);
UUID testUUID =
testUUIDMap.get(
testResultSummary.getTestCaseName()
+ ":"
+ testResultSummary.getTestName());
Preconditions.checkNotNull(testUUID);
params
.getBuckEventBus()
.post(TestSummaryEvent.finished(testUUID, testResultSummary));
}
@Override
public void testsDidEnd(List<TestCaseSummary> testCaseSummaries) {
LOG.debug(
"Test rule %s tests did end: %s", test.getBuildTarget(), testCaseSummaries);
}
});
if (!testSteps.isEmpty()) {
stepsBuilder.addAll(testSteps);
stepsBuilder.add(testRuleKeyFileHelper.createRuleKeyInDirStep(test));
}
steps = stepsBuilder.build();
} else {
steps = ImmutableList.of();
}
TestRun testRun =
TestRun.of(
test,
steps,
getCachingStatusTransformingCallable(
isTestRunRequired,
test.interpretTestResults(
executionContext,
/*isUsingTestSelectors*/ !options.getTestSelectorList().isEmpty(),
/*isDryRun*/ options.isDryRun())));
// Always run the commands, even if the list of commands as empty. There may be zero
// commands because the rule is cached, but its results must still be processed.
if (test.runTestSeparately()) {
LOG.debug("Running test %s in serial", test);
separateTestRuns.add(testRun);
} else {
LOG.debug("Running test %s in parallel", test);
parallelTestRuns.add(testRun);
}
}
final StepRunner.StepRunningCallback testStepRunningCallback =
new StepRunner.StepRunningCallback() {
@Override
public void stepsWillRun(Optional<BuildTarget> buildTarget) {
Preconditions.checkState(buildTarget.isPresent());
LOG.debug("Test steps will run for %s", buildTarget);
params.getBuckEventBus().post(TestRuleEvent.started(buildTarget.get()));
}
@Override
public void stepsDidRun(Optional<BuildTarget> buildTarget) {
Preconditions.checkState(buildTarget.isPresent());
LOG.debug("Test steps did run for %s", buildTarget);
params.getBuckEventBus().post(TestRuleEvent.finished(buildTarget.get()));
}
};
for (TestRun testRun : parallelTestRuns) {
ListenableFuture<TestResults> testResults =
stepRunner.runStepsAndYieldResult(
testRun.getSteps(),
testRun.getTestResultsCallable(),
Optional.of(testRun.getTest().getBuildTarget()),
service,
testStepRunningCallback);
results.add(
transformTestResults(
params,
testResults,
grouper,
testRun.getTest(),
testTargets,
printTestResults,
lastReportedTestSequenceNumber,
totalNumberOfTests));
}
ListenableFuture<List<TestResults>> parallelTestStepsFuture = Futures.allAsList(results);
final List<TestResults> completedResults = Lists.newArrayList();
final ListeningExecutorService directExecutorService = MoreExecutors.newDirectExecutorService();
ListenableFuture<Void> uberFuture =
stepRunner.addCallback(
parallelTestStepsFuture,
new FutureCallback<List<TestResults>>() {
@Override
public void onSuccess(List<TestResults> parallelTestResults) {
LOG.debug("Parallel tests completed, running separate tests...");
completedResults.addAll(parallelTestResults);
List<ListenableFuture<TestResults>> separateResultsList = Lists.newArrayList();
for (TestRun testRun : separateTestRuns) {
separateResultsList.add(
transformTestResults(
params,
stepRunner.runStepsAndYieldResult(
testRun.getSteps(),
testRun.getTestResultsCallable(),
Optional.of(testRun.getTest().getBuildTarget()),
directExecutorService,
testStepRunningCallback),
grouper,
testRun.getTest(),
testTargets,
printTestResults,
lastReportedTestSequenceNumber,
totalNumberOfTests));
}
ListenableFuture<List<TestResults>> serialResults =
Futures.allAsList(separateResultsList);
try {
completedResults.addAll(serialResults.get());
} catch (ExecutionException e) {
LOG.error(e, "Error fetching serial test results");
throw new HumanReadableException(e, "Error fetching serial test results");
} catch (InterruptedException e) {
LOG.error(e, "Interrupted fetching serial test results");
try {
serialResults.cancel(true);
} catch (CancellationException ignored) {
// Rethrow original InterruptedException instead.
}
Thread.currentThread().interrupt();
throw new HumanReadableException(e, "Test cancelled");
}
LOG.debug("Done running serial tests.");
}
@Override
public void onFailure(Throwable e) {
LOG.error(e, "Parallel tests failed, not running serial tests");
throw new HumanReadableException(e, "Parallel tests failed");
}
},
directExecutorService);
try {
// Block until all the tests have finished running.
uberFuture.get();
} catch (ExecutionException e) {
e.printStackTrace(params.getConsole().getStdErr());
return 1;
} catch (InterruptedException e) {
try {
uberFuture.cancel(true);
} catch (CancellationException ignored) {
// Rethrow original InterruptedException instead.
}
Thread.currentThread().interrupt();
throw e;
}
params.getBuckEventBus().post(TestRunEvent.finished(testTargets, completedResults));
// Write out the results as XML, if requested.
Optional<String> path = options.getPathToXmlTestOutput();
if (path.isPresent()) {
try (Writer writer = Files.newWriter(new File(path.get()), Charsets.UTF_8)) {
writeXmlOutput(completedResults, writer);
}
}
// Generate the code coverage report.
if (options.isCodeCoverageEnabled() && !rulesUnderTest.isEmpty()) {
try {
Optional<DefaultJavaPackageFinder> defaultJavaPackageFinderOptional =
Optional.fromNullable(params.getBuckConfig().createDefaultJavaPackageFinder());
stepRunner.runStepForBuildTarget(
getReportCommand(
rulesUnderTest,
defaultJavaPackageFinderOptional,
params.getRepository().getFilesystem(),
JUnitStep.JACOCO_OUTPUT_DIR,
options.getCoverageReportFormat()),
Optional.<BuildTarget>absent());
} catch (StepFailedException e) {
params.getConsole().printBuildFailureWithoutStacktrace(e);
return 1;
}
}
boolean failures =
Iterables.any(
completedResults,
new Predicate<TestResults>() {
@Override
public boolean apply(TestResults results) {
LOG.debug("Checking result %s for failure", results);
return !results.isSuccess();
}
});
return failures ? TEST_FAILURES_EXIT_CODE : 0;
}