@VisibleForTesting
int printJsonInputs(final CommandRunnerParams params, TargetGraph graph) throws IOException {
final SortedMap<String, ImmutableSortedSet<Path>> targetToInputs = new TreeMap<>();
new AbstractBottomUpTraversal<TargetNode<?>, Void>(graph) {
@Override
public void visit(TargetNode<?> node) {
LOG.debug("Looking at inputs for %s", node.getBuildTarget().getFullyQualifiedName());
SortedSet<Path> targetInputs = new TreeSet<>();
for (Path input : node.getInputs()) {
LOG.debug("Walking input %s", input);
try {
if (!params.getCell().getFilesystem().exists(input)) {
throw new HumanReadableException(
"Target %s refers to non-existent input file: %s", node, input);
}
targetInputs.addAll(params.getCell().getFilesystem().getFilesUnderPath(input));
} catch (IOException e) {
throw new RuntimeException(e);
}
}
targetToInputs.put(
node.getBuildTarget().getFullyQualifiedName(), ImmutableSortedSet.copyOf(targetInputs));
}
}.traverse();
params.getObjectMapper().writeValue(params.getConsole().getStdOut(), targetToInputs);
return 0;
}
private int runTestsExternal(
final CommandRunnerParams params,
Build build,
Iterable<String> command,
Iterable<TestRule> testRules)
throws InterruptedException, IOException {
TestRunningOptions options = getTestRunningOptions(params);
// Walk the test rules, collecting all the specs.
List<ExternalTestRunnerTestSpec> specs = Lists.newArrayList();
for (TestRule testRule : testRules) {
if (!(testRule instanceof ExternalTestRunnerRule)) {
params
.getBuckEventBus()
.post(
ConsoleEvent.severe(
String.format(
"Test %s does not support external test running",
testRule.getBuildTarget())));
return 1;
}
ExternalTestRunnerRule rule = (ExternalTestRunnerRule) testRule;
specs.add(rule.getExternalTestRunnerSpec(build.getExecutionContext(), options));
}
// Serialize the specs to a file to pass into the test runner.
Path infoFile =
params
.getCell()
.getFilesystem()
.resolve(BuckConstant.SCRATCH_PATH.resolve("external_runner_specs.json"));
Files.createDirectories(infoFile.getParent());
Files.deleteIfExists(infoFile);
params.getObjectMapper().writerWithDefaultPrettyPrinter().writeValue(infoFile.toFile(), specs);
// Launch and run the external test runner, forwarding it's stdout/stderr to the console.
// We wait for it to complete then returns its error code.
ListeningProcessExecutor processExecutor = new ListeningProcessExecutor();
ProcessExecutorParams processExecutorParams =
ProcessExecutorParams.builder()
.addAllCommand(command)
.addAllCommand(withDashArguments)
.addCommand("--buck-test-info", infoFile.toString())
.setDirectory(params.getCell().getFilesystem().getRootPath().toFile())
.build();
ForwardingProcessListener processListener =
new ForwardingProcessListener(
Channels.newChannel(params.getConsole().getStdOut()),
Channels.newChannel(params.getConsole().getStdErr()));
ListeningProcessExecutor.LaunchedProcess process =
processExecutor.launchProcess(processExecutorParams, processListener);
try {
return processExecutor.waitForProcess(process, Long.MAX_VALUE, TimeUnit.DAYS);
} finally {
processExecutor.destroyProcess(process, /* force */ false);
processExecutor.waitForProcess(process, Long.MAX_VALUE, TimeUnit.DAYS);
}
}
public BuckQueryEnvironment(CommandRunnerParams params, boolean enableProfiling) {
this.params = params;
this.enableProfiling = enableProfiling;
this.parserConfig = new ParserConfig(params.getBuckConfig());
this.buildFileTree =
new FilesystemBackedBuildFileTree(
params.getCell().getFilesystem(), parserConfig.getBuildFileName());
this.targetPatternEvaluator = new TargetPatternEvaluator(params, enableProfiling);
}
TargetNode<?> getNode(QueryTarget target) throws QueryException, InterruptedException {
Preconditions.checkState(target instanceof QueryBuildTarget);
try {
return params
.getParser()
.getTargetNode(
((QueryBuildTarget) target).getBuildTarget(),
params.getBuckEventBus(),
params.getConsole(),
enableProfiling);
} catch (BuildTargetException | BuildFileParseException | IOException e) {
throw new QueryException("Error getting target node for %s\n%s", target, e.getMessage());
}
}
/** Print only targets which were identified as owners in JSON. */
@VisibleForTesting
void printOwnersOnlyJsonReport(CommandRunnerParams params, OwnersReport report)
throws IOException {
final Multimap<String, String> output = TreeMultimap.create();
Set<TargetNode<?>> sortedTargetNodes = report.owners.keySet();
for (TargetNode<?> targetNode : sortedTargetNodes) {
Set<Path> files = report.owners.get(targetNode);
for (Path input : files) {
output.put(input.toString(), targetNode.getBuildTarget().getFullyQualifiedName());
}
}
params.getObjectMapper().writeValue(params.getConsole().getStdOut(), output.asMap());
}
@Override
public int runWithoutHelp(CommandRunnerParams params) throws IOException, InterruptedException {
ParserConfig parserConfig = new ParserConfig(params.getBuckConfig());
BuildFileTree buildFileTree =
new FilesystemBackedBuildFileTree(
params.getCell().getFilesystem(), parserConfig.getBuildFileName());
try {
OwnersReport report =
buildOwnersReport(
params, parserConfig, buildFileTree, getArguments(), isGuessForDeletedEnabled());
printReport(params, report);
} catch (BuildFileParseException | BuildTargetException e) {
return BUILD_TARGET_ERROR;
}
return 0;
}
private void buildGraphForBuildTargets(Set<BuildTarget> targets)
throws QueryException, InterruptedException {
try {
graph =
params
.getParser()
.buildTargetGraphForBuildTargets(
targets,
parserConfig,
params.getBuckEventBus(),
params.getConsole(),
params.getEnvironment(),
enableProfiling);
} catch (BuildTargetException | BuildFileParseException | IOException e) {
throw new QueryException("Error in building depencency graph");
}
}
@Override
public int runWithoutHelp(final CommandRunnerParams params)
throws IOException, InterruptedException {
// Create a TargetGraph that is composed of the transitive closure of all of the dependent
// TargetNodes for the specified BuildTargets.
final ImmutableSet<String> fullyQualifiedBuildTargets =
ImmutableSet.copyOf(getArgumentsFormattedAsBuildTargets(params.getBuckConfig()));
if (fullyQualifiedBuildTargets.isEmpty()) {
params.getConsole().printBuildFailure("Please specify at least one build target.");
return 1;
}
ImmutableSet<BuildTarget> targets =
FluentIterable.from(getArgumentsFormattedAsBuildTargets(params.getBuckConfig()))
.transform(
new Function<String, BuildTarget>() {
@Override
public BuildTarget apply(String input) {
return BuildTargetParser.INSTANCE.parse(
input, BuildTargetPatternParser.fullyQualified());
}
})
.toSet();
LOG.debug("Getting input for targets: %s", targets);
TargetGraph graph;
try {
graph =
params
.getParser()
.buildTargetGraphForBuildTargets(
targets,
new ParserConfig(params.getBuckConfig()),
params.getBuckEventBus(),
params.getConsole(),
params.getEnvironment(),
getEnableProfiling());
} catch (BuildTargetException | BuildFileParseException e) {
params.getConsole().printBuildFailureWithoutStacktrace(e);
return 1;
}
if (shouldGenerateJsonOutput()) {
return printJsonInputs(params, graph);
}
return printInputs(params, graph);
}
private static void printArtifactsInformation(
CommandRunnerParams params, DeployResult deployResult) {
params
.getConsole()
.getStdOut()
.println(
"\nPublished artifacts:\n"
+ Joiner.on('\n')
.join(
FluentIterable.from(deployResult.getArtifacts())
.transform(
new Function<Artifact, String>() {
@Override
public String apply(Artifact input) {
return atrifactToString(input);
}
})));
params.getConsole().getStdOut().println("\nDeployRequest:\n" + deployResult.getRequest());
}
/** Print detailed report on all owners. */
private void printFullReport(CommandRunnerParams params, OwnersReport report) {
PrintStream out = params.getConsole().getStdOut();
Ansi ansi = params.getConsole().getAnsi();
if (report.owners.isEmpty()) {
out.println(ansi.asErrorText("No owners found"));
} else {
out.println(ansi.asSuccessText("Owners:"));
for (TargetNode<?> targetNode : report.owners.keySet()) {
out.println(targetNode.getBuildTarget().getFullyQualifiedName());
Set<Path> files = report.owners.get(targetNode);
for (Path input : files) {
out.println(FILE_INDENT + input);
}
}
}
if (!report.inputsWithNoOwners.isEmpty()) {
out.println();
out.println(ansi.asErrorText("Files without owners:"));
for (Path input : report.inputsWithNoOwners) {
out.println(FILE_INDENT + input);
}
}
if (!report.nonExistentInputs.isEmpty()) {
out.println();
out.println(ansi.asErrorText("Non existent files:"));
for (String input : report.nonExistentInputs) {
out.println(FILE_INDENT + input);
}
}
if (!report.nonFileInputs.isEmpty()) {
out.println();
out.println(ansi.asErrorText("Non-file inputs:"));
for (String input : report.nonFileInputs) {
out.println(FILE_INDENT + input);
}
}
}
@VisibleForTesting
static OwnersReport generateOwnersReport(
CommandRunnerParams params,
TargetNode<?> targetNode,
Iterable<String> filePaths,
boolean guessForDeletedEnabled) {
// Process arguments assuming they are all relative file paths.
Set<Path> inputs = Sets.newHashSet();
Set<String> nonExistentInputs = Sets.newHashSet();
Set<String> nonFileInputs = Sets.newHashSet();
for (String filePath : filePaths) {
File file = params.getCell().getFilesystem().getFileForRelativePath(filePath);
if (!file.exists()) {
nonExistentInputs.add(filePath);
} else if (!file.isFile()) {
nonFileInputs.add(filePath);
} else {
inputs.add(Paths.get(filePath));
}
}
// Try to find owners for each valid and existing file.
Set<Path> inputsWithNoOwners = Sets.newHashSet(inputs);
SetMultimap<TargetNode<?>, Path> owners = TreeMultimap.create();
for (final Path commandInput : inputs) {
Predicate<Path> startsWith =
new Predicate<Path>() {
@Override
public boolean apply(Path input) {
return !commandInput.equals(input) && commandInput.startsWith(input);
}
};
Set<Path> ruleInputs = targetNode.getInputs();
if (ruleInputs.contains(commandInput)
|| FluentIterable.from(ruleInputs).anyMatch(startsWith)) {
inputsWithNoOwners.remove(commandInput);
owners.put(targetNode, commandInput);
}
}
// Try to guess owners for nonexistent files.
if (guessForDeletedEnabled) {
for (String nonExistentInput : nonExistentInputs) {
owners.put(targetNode, new File(nonExistentInput).toPath());
}
}
return new OwnersReport(owners, inputsWithNoOwners, nonExistentInputs, nonFileInputs);
}
private int runTestsInternal(
CommandRunnerParams params,
BuildEngine buildEngine,
Build build,
Iterable<TestRule> testRules)
throws InterruptedException, IOException {
if (!withDashArguments.isEmpty()) {
params
.getBuckEventBus()
.post(
ConsoleEvent.severe("Unexpected arguments after \"--\" when using internal runner"));
return 1;
}
ConcurrencyLimit concurrencyLimit =
new ConcurrencyLimit(
getNumTestThreads(params.getBuckConfig()), params.getBuckConfig().getLoadLimit());
try (CommandThreadManager testPool =
new CommandThreadManager(
"Test-Run", params.getBuckConfig().getWorkQueueExecutionOrder(), concurrencyLimit)) {
return TestRunning.runTests(
params,
testRules,
Preconditions.checkNotNull(build.getBuildContext()),
build.getExecutionContext(),
getTestRunningOptions(params),
testPool.getExecutor(),
buildEngine,
new DefaultStepRunner(build.getExecutionContext()));
} catch (ExecutionException e) {
params
.getBuckEventBus()
.post(ConsoleEvent.severe(MoreExceptions.getHumanReadableOrLocalizedMessage(e)));
return 1;
}
}
/** @return whether successful */
private boolean publishTarget(BuildTarget buildTarget, CommandRunnerParams params) {
BuildRule buildRule =
Preconditions.checkNotNull(getBuild().getActionGraph().findBuildRuleByTarget(buildTarget));
if (!(buildRule instanceof MavenPublishable)) {
printError(
params,
"Cannot retrieve maven coordinates for rule of type " + buildRule.getClass().getName());
return false;
}
Publisher publisher =
new Publisher(
params.getRepository().getFilesystem(), Optional.fromNullable(remoteRepo), dryRun);
try {
DeployResult deployResult = publisher.publish((MavenPublishable) buildRule);
printArtifactsInformation(params, deployResult);
} catch (DeploymentException e) {
params.getConsole().printBuildFailureWithoutStacktraceDontUnwrap(e);
return false;
}
return true;
}
private TestRunningOptions getTestRunningOptions(CommandRunnerParams params) {
return TestRunningOptions.builder()
.setUsingOneTimeOutputDirectories(isUsingOneTimeOutput)
.setCodeCoverageEnabled(isCodeCoverageEnabled)
.setRunAllTests(isRunAllTests())
.setTestSelectorList(testSelectorOptions.getTestSelectorList())
.setShouldExplainTestSelectorList(testSelectorOptions.shouldExplain())
.setResultsCacheEnabled(isResultsCacheEnabled(params.getBuckConfig()))
.setDryRun(isDryRun)
.setShufflingTests(isShufflingTests)
.setPathToXmlTestOutput(Optional.fromNullable(pathToXmlTestOutput))
.setPathToJavaAgent(Optional.fromNullable(pathToJavaAgent))
.setCoverageReportFormat(coverageReportFormat)
.setCoverageReportTitle(coverageReportTitle)
.build();
}
@SuppressWarnings("PMD.EmptyCatchBlock")
public static int runTests(
final CommandRunnerParams params,
Iterable<TestRule> tests,
BuildContext buildContext,
ExecutionContext executionContext,
final TestRunningOptions options,
ListeningExecutorService service,
BuildEngine buildEngine,
final StepRunner stepRunner)
throws IOException, ExecutionException, InterruptedException {
if (options.isUsingOneTimeOutputDirectories()) {
BuckConstant.setOneTimeTestSubdirectory(UUID.randomUUID().toString());
}
ImmutableSet<JavaLibrary> rulesUnderTest;
// If needed, we first run instrumentation on the class files.
if (options.isCodeCoverageEnabled()) {
rulesUnderTest = getRulesUnderTest(tests);
if (!rulesUnderTest.isEmpty()) {
try {
stepRunner.runStepForBuildTarget(
new MakeCleanDirectoryStep(JUnitStep.JACOCO_OUTPUT_DIR),
Optional.<BuildTarget>absent());
} catch (StepFailedException e) {
params.getConsole().printBuildFailureWithoutStacktrace(e);
return 1;
}
}
} else {
rulesUnderTest = ImmutableSet.of();
}
final ImmutableSet<String> testTargets =
FluentIterable.from(tests)
.transform(HasBuildTarget.TO_TARGET)
.transform(Functions.toStringFunction())
.toSet();
final int totalNumberOfTests = Iterables.size(tests);
params
.getBuckEventBus()
.post(
TestRunEvent.started(
options.isRunAllTests(),
options.getTestSelectorList(),
options.shouldExplainTestSelectorList(),
testTargets));
// Start running all of the tests. The result of each java_test() rule is represented as a
// ListenableFuture.
List<ListenableFuture<TestResults>> results = Lists.newArrayList();
// Unless `--verbose 0` is specified, print out test results as they become available.
// Failures with the ListenableFuture should always be printed, as they indicate an error with
// Buck, not the test being run.
Verbosity verbosity = params.getConsole().getVerbosity();
final boolean printTestResults = (verbosity != Verbosity.SILENT);
// For grouping results!
final TestResultsGrouper grouper;
if (options.isIgnoreFailingDependencies()) {
grouper = new TestResultsGrouper(tests);
} else {
grouper = null;
}
TestRuleKeyFileHelper testRuleKeyFileHelper =
new TestRuleKeyFileHelper(executionContext.getProjectFilesystem(), buildEngine);
final AtomicInteger lastReportedTestSequenceNumber = new AtomicInteger();
final List<TestRun> separateTestRuns = Lists.newArrayList();
List<TestRun> parallelTestRuns = Lists.newArrayList();
for (final TestRule test : tests) {
// Determine whether the test needs to be executed.
boolean isTestRunRequired;
isTestRunRequired =
isTestRunRequiredForTest(
test,
buildEngine,
executionContext,
testRuleKeyFileHelper,
options.isResultsCacheEnabled(),
!options.getTestSelectorList().isEmpty());
List<Step> steps;
if (isTestRunRequired) {
params.getBuckEventBus().post(IndividualTestEvent.started(testTargets));
ImmutableList.Builder<Step> stepsBuilder = ImmutableList.builder();
Preconditions.checkState(buildEngine.isRuleBuilt(test.getBuildTarget()));
final Map<String, UUID> testUUIDMap = new HashMap<>();
List<Step> testSteps =
test.runTests(
buildContext,
executionContext,
options.isDryRun(),
options.isShufflingTests(),
options.getTestSelectorList(),
new TestRule.TestReportingCallback() {
@Override
public void testsDidBegin() {
LOG.debug("Tests for rule %s began", test.getBuildTarget());
}
@Override
public void testDidBegin(String testCaseName, String testName) {
LOG.debug(
"Test rule %s test case %s test name %s began",
test.getBuildTarget(), testCaseName, testName);
UUID testUUID = UUID.randomUUID();
// UUID is immutable and thread-safe as of Java 7, so it's
// safe to stash in a map and use later:
//
// http://bugs.java.com/view_bug.do?bug_id=6611830
testUUIDMap.put(testCaseName + ":" + testName, testUUID);
params
.getBuckEventBus()
.post(TestSummaryEvent.started(testUUID, testCaseName, testName));
}
@Override
public void testDidEnd(TestResultSummary testResultSummary) {
LOG.debug(
"Test rule %s test did end: %s", test.getBuildTarget(), testResultSummary);
UUID testUUID =
testUUIDMap.get(
testResultSummary.getTestCaseName()
+ ":"
+ testResultSummary.getTestName());
Preconditions.checkNotNull(testUUID);
params
.getBuckEventBus()
.post(TestSummaryEvent.finished(testUUID, testResultSummary));
}
@Override
public void testsDidEnd(List<TestCaseSummary> testCaseSummaries) {
LOG.debug(
"Test rule %s tests did end: %s", test.getBuildTarget(), testCaseSummaries);
}
});
if (!testSteps.isEmpty()) {
stepsBuilder.addAll(testSteps);
stepsBuilder.add(testRuleKeyFileHelper.createRuleKeyInDirStep(test));
}
steps = stepsBuilder.build();
} else {
steps = ImmutableList.of();
}
TestRun testRun =
TestRun.of(
test,
steps,
getCachingStatusTransformingCallable(
isTestRunRequired,
test.interpretTestResults(
executionContext,
/*isUsingTestSelectors*/ !options.getTestSelectorList().isEmpty(),
/*isDryRun*/ options.isDryRun())));
// Always run the commands, even if the list of commands as empty. There may be zero
// commands because the rule is cached, but its results must still be processed.
if (test.runTestSeparately()) {
LOG.debug("Running test %s in serial", test);
separateTestRuns.add(testRun);
} else {
LOG.debug("Running test %s in parallel", test);
parallelTestRuns.add(testRun);
}
}
final StepRunner.StepRunningCallback testStepRunningCallback =
new StepRunner.StepRunningCallback() {
@Override
public void stepsWillRun(Optional<BuildTarget> buildTarget) {
Preconditions.checkState(buildTarget.isPresent());
LOG.debug("Test steps will run for %s", buildTarget);
params.getBuckEventBus().post(TestRuleEvent.started(buildTarget.get()));
}
@Override
public void stepsDidRun(Optional<BuildTarget> buildTarget) {
Preconditions.checkState(buildTarget.isPresent());
LOG.debug("Test steps did run for %s", buildTarget);
params.getBuckEventBus().post(TestRuleEvent.finished(buildTarget.get()));
}
};
for (TestRun testRun : parallelTestRuns) {
ListenableFuture<TestResults> testResults =
stepRunner.runStepsAndYieldResult(
testRun.getSteps(),
testRun.getTestResultsCallable(),
Optional.of(testRun.getTest().getBuildTarget()),
service,
testStepRunningCallback);
results.add(
transformTestResults(
params,
testResults,
grouper,
testRun.getTest(),
testTargets,
printTestResults,
lastReportedTestSequenceNumber,
totalNumberOfTests));
}
ListenableFuture<List<TestResults>> parallelTestStepsFuture = Futures.allAsList(results);
final List<TestResults> completedResults = Lists.newArrayList();
final ListeningExecutorService directExecutorService = MoreExecutors.newDirectExecutorService();
ListenableFuture<Void> uberFuture =
stepRunner.addCallback(
parallelTestStepsFuture,
new FutureCallback<List<TestResults>>() {
@Override
public void onSuccess(List<TestResults> parallelTestResults) {
LOG.debug("Parallel tests completed, running separate tests...");
completedResults.addAll(parallelTestResults);
List<ListenableFuture<TestResults>> separateResultsList = Lists.newArrayList();
for (TestRun testRun : separateTestRuns) {
separateResultsList.add(
transformTestResults(
params,
stepRunner.runStepsAndYieldResult(
testRun.getSteps(),
testRun.getTestResultsCallable(),
Optional.of(testRun.getTest().getBuildTarget()),
directExecutorService,
testStepRunningCallback),
grouper,
testRun.getTest(),
testTargets,
printTestResults,
lastReportedTestSequenceNumber,
totalNumberOfTests));
}
ListenableFuture<List<TestResults>> serialResults =
Futures.allAsList(separateResultsList);
try {
completedResults.addAll(serialResults.get());
} catch (ExecutionException e) {
LOG.error(e, "Error fetching serial test results");
throw new HumanReadableException(e, "Error fetching serial test results");
} catch (InterruptedException e) {
LOG.error(e, "Interrupted fetching serial test results");
try {
serialResults.cancel(true);
} catch (CancellationException ignored) {
// Rethrow original InterruptedException instead.
}
Thread.currentThread().interrupt();
throw new HumanReadableException(e, "Test cancelled");
}
LOG.debug("Done running serial tests.");
}
@Override
public void onFailure(Throwable e) {
LOG.error(e, "Parallel tests failed, not running serial tests");
throw new HumanReadableException(e, "Parallel tests failed");
}
},
directExecutorService);
try {
// Block until all the tests have finished running.
uberFuture.get();
} catch (ExecutionException e) {
e.printStackTrace(params.getConsole().getStdErr());
return 1;
} catch (InterruptedException e) {
try {
uberFuture.cancel(true);
} catch (CancellationException ignored) {
// Rethrow original InterruptedException instead.
}
Thread.currentThread().interrupt();
throw e;
}
params.getBuckEventBus().post(TestRunEvent.finished(testTargets, completedResults));
// Write out the results as XML, if requested.
Optional<String> path = options.getPathToXmlTestOutput();
if (path.isPresent()) {
try (Writer writer = Files.newWriter(new File(path.get()), Charsets.UTF_8)) {
writeXmlOutput(completedResults, writer);
}
}
// Generate the code coverage report.
if (options.isCodeCoverageEnabled() && !rulesUnderTest.isEmpty()) {
try {
Optional<DefaultJavaPackageFinder> defaultJavaPackageFinderOptional =
Optional.fromNullable(params.getBuckConfig().createDefaultJavaPackageFinder());
stepRunner.runStepForBuildTarget(
getReportCommand(
rulesUnderTest,
defaultJavaPackageFinderOptional,
params.getRepository().getFilesystem(),
JUnitStep.JACOCO_OUTPUT_DIR,
options.getCoverageReportFormat()),
Optional.<BuildTarget>absent());
} catch (StepFailedException e) {
params.getConsole().printBuildFailureWithoutStacktrace(e);
return 1;
}
}
boolean failures =
Iterables.any(
completedResults,
new Predicate<TestResults>() {
@Override
public boolean apply(TestResults results) {
LOG.debug("Checking result %s for failure", results);
return !results.isSuccess();
}
});
return failures ? TEST_FAILURES_EXIT_CODE : 0;
}
private static void printError(CommandRunnerParams params, String errorMessage) {
params.getConsole().printErrorText(errorMessage);
}
@Override
public int runWithoutHelp(CommandRunnerParams params) throws IOException, InterruptedException {
LOG.debug("Running with arguments %s", getArguments());
try (CommandThreadManager pool =
new CommandThreadManager(
"Test",
params.getBuckConfig().getWorkQueueExecutionOrder(),
getConcurrencyLimit(params.getBuckConfig()))) {
// Post the build started event, setting it to the Parser recorded start time if appropriate.
BuildEvent.Started started = BuildEvent.started(getArguments());
if (params.getParser().getParseStartTime().isPresent()) {
params.getBuckEventBus().post(started, params.getParser().getParseStartTime().get());
} else {
params.getBuckEventBus().post(started);
}
// The first step is to parse all of the build files. This will populate the parser and find
// all of the test rules.
TargetGraph targetGraph;
ImmutableSet<BuildTarget> explicitBuildTargets;
try {
// If the user asked to run all of the tests, parse all of the build files looking for any
// test rules.
if (isRunAllTests()) {
targetGraph =
params
.getParser()
.buildTargetGraphForTargetNodeSpecs(
params.getBuckEventBus(),
params.getCell(),
getEnableProfiling(),
pool.getExecutor(),
ImmutableList.of(
TargetNodePredicateSpec.of(
new Predicate<TargetNode<?>>() {
@Override
public boolean apply(TargetNode<?> input) {
return input.getType().isTestRule();
}
},
BuildFileSpec.fromRecursivePath(Paths.get("")))))
.getSecond();
explicitBuildTargets = ImmutableSet.of();
// Otherwise, the user specified specific test targets to build and run, so build a graph
// around these.
} else {
LOG.debug("Parsing graph for arguments %s", getArguments());
Pair<ImmutableSet<BuildTarget>, TargetGraph> result =
params
.getParser()
.buildTargetGraphForTargetNodeSpecs(
params.getBuckEventBus(),
params.getCell(),
getEnableProfiling(),
pool.getExecutor(),
parseArgumentsAsTargetNodeSpecs(params.getBuckConfig(), getArguments()));
targetGraph = result.getSecond();
explicitBuildTargets = result.getFirst();
LOG.debug("Got explicit build targets %s", explicitBuildTargets);
ImmutableSet.Builder<BuildTarget> testTargetsBuilder = ImmutableSet.builder();
for (TargetNode<?> node : targetGraph.getAll(explicitBuildTargets)) {
ImmutableSortedSet<BuildTarget> nodeTests = TargetNodes.getTestTargetsForNode(node);
if (!nodeTests.isEmpty()) {
LOG.debug("Got tests for target %s: %s", node.getBuildTarget(), nodeTests);
testTargetsBuilder.addAll(nodeTests);
}
}
ImmutableSet<BuildTarget> testTargets = testTargetsBuilder.build();
if (!testTargets.isEmpty()) {
LOG.debug("Got related test targets %s, building new target graph...", testTargets);
targetGraph =
params
.getParser()
.buildTargetGraph(
params.getBuckEventBus(),
params.getCell(),
getEnableProfiling(),
pool.getExecutor(),
Iterables.concat(explicitBuildTargets, testTargets));
LOG.debug("Finished building new target graph with tests.");
}
}
} catch (BuildTargetException | BuildFileParseException e) {
params
.getBuckEventBus()
.post(ConsoleEvent.severe(MoreExceptions.getHumanReadableOrLocalizedMessage(e)));
return 1;
}
TargetGraphToActionGraph targetGraphToActionGraph =
new TargetGraphToActionGraph(
params.getBuckEventBus(), new BuildTargetNodeToBuildRuleTransformer());
Pair<ActionGraph, BuildRuleResolver> actionGraphAndResolver =
Preconditions.checkNotNull(targetGraphToActionGraph.apply(targetGraph));
// Look up all of the test rules in the action graph.
Iterable<TestRule> testRules =
Iterables.filter(actionGraphAndResolver.getFirst().getNodes(), TestRule.class);
// Unless the user requests that we build filtered tests, filter them out here, before
// the build.
if (!isBuildFiltered(params.getBuckConfig())) {
testRules = filterTestRules(params.getBuckConfig(), explicitBuildTargets, testRules);
}
if (isDryRun()) {
printMatchingTestRules(params.getConsole(), testRules);
}
CachingBuildEngine cachingBuildEngine =
new CachingBuildEngine(
pool.getExecutor(),
params.getFileHashCache(),
getBuildEngineMode().or(params.getBuckConfig().getBuildEngineMode()),
params.getBuckConfig().getDependencySchedulingOrder(),
params.getBuckConfig().getBuildDepFiles(),
params.getBuckConfig().getBuildMaxDepFileCacheEntries(),
actionGraphAndResolver.getSecond());
try (Build build =
createBuild(
params.getBuckConfig(),
actionGraphAndResolver.getFirst(),
actionGraphAndResolver.getSecond(),
params.getAndroidPlatformTargetSupplier(),
cachingBuildEngine,
params.getArtifactCache(),
params.getConsole(),
params.getBuckEventBus(),
getTargetDeviceOptional(),
params.getPlatform(),
params.getEnvironment(),
params.getObjectMapper(),
params.getClock(),
Optional.of(getAdbOptions(params.getBuckConfig())),
Optional.of(getTargetDeviceOptions()))) {
// Build all of the test rules.
int exitCode =
build.executeAndPrintFailuresToEventBus(
testRules,
isKeepGoing(),
params.getBuckEventBus(),
params.getConsole(),
getPathToBuildReport(params.getBuckConfig()));
params.getBuckEventBus().post(BuildEvent.finished(started, exitCode));
if (exitCode != 0) {
return exitCode;
}
// If the user requests that we build tests that we filter out, then we perform
// the filtering here, after we've done the build but before we run the tests.
if (isBuildFiltered(params.getBuckConfig())) {
testRules = filterTestRules(params.getBuckConfig(), explicitBuildTargets, testRules);
}
// Once all of the rules are built, then run the tests.
Optional<ImmutableList<String>> externalTestRunner =
params.getBuckConfig().getExternalTestRunner();
if (externalTestRunner.isPresent()) {
return runTestsExternal(params, build, externalTestRunner.get(), testRules);
}
return runTestsInternal(params, cachingBuildEngine, build, testRules);
}
}
}
@Override
public int runWithoutHelp(CommandRunnerParams params) throws IOException, InterruptedException {
if (getArguments().isEmpty()) {
params.getConsole().printBuildFailure("Must specify at least one build target to fetch.");
return 1;
}
// Post the build started event, setting it to the Parser recorded start time if appropriate.
BuildEvent.Started started = BuildEvent.started(getArguments());
if (params.getParser().getParseStartTime().isPresent()) {
params.getBuckEventBus().post(started, params.getParser().getParseStartTime().get());
} else {
params.getBuckEventBus().post(started);
}
FetchTargetNodeToBuildRuleTransformer ruleGenerator = createFetchTransformer(params);
TargetGraphToActionGraph transformer =
new TargetGraphToActionGraph(
params.getBuckEventBus(), ruleGenerator, params.getFileHashCache());
ActionGraph actionGraph;
ImmutableSet<BuildTarget> buildTargets;
try {
Pair<ImmutableSet<BuildTarget>, TargetGraph> result =
params
.getParser()
.buildTargetGraphForTargetNodeSpecs(
parseArgumentsAsTargetNodeSpecs(
params.getBuckConfig(),
params.getRepository().getFilesystem().getIgnorePaths(),
getArguments()),
new ParserConfig(params.getBuckConfig()),
params.getBuckEventBus(),
params.getConsole(),
params.getEnvironment(),
getEnableProfiling());
actionGraph = transformer.apply(result.getSecond());
buildTargets = ruleGenerator.getDownloadableTargets();
} catch (BuildTargetException | BuildFileParseException e) {
params.getConsole().printBuildFailureWithoutStacktrace(e);
return 1;
}
int exitCode;
SourcePathResolver pathResolver = new SourcePathResolver(transformer.getRuleResolver());
try (CommandThreadManager pool =
new CommandThreadManager("Fetch", getConcurrencyLimit(params.getBuckConfig()));
Build build =
createBuild(
params.getBuckConfig(),
actionGraph,
params.getAndroidPlatformTargetSupplier(),
new CachingBuildEngine(
pool.getExecutor(),
params.getFileHashCache(),
getBuildEngineMode().or(params.getBuckConfig().getBuildEngineMode()),
params.getBuckConfig().getBuildDepFiles(),
new InputBasedRuleKeyBuilderFactory(params.getFileHashCache(), pathResolver),
new AbiRuleKeyBuilderFactory(params.getFileHashCache(), pathResolver),
new DependencyFileRuleKeyBuilderFactory(
params.getFileHashCache(), pathResolver)),
getArtifactCache(params),
params.getConsole(),
params.getBuckEventBus(),
Optional.<TargetDevice>absent(),
params.getPlatform(),
params.getEnvironment(),
params.getObjectMapper(),
params.getClock(),
Optional.<AdbOptions>absent(),
Optional.<TargetDeviceOptions>absent())) {
exitCode =
build.executeAndPrintFailuresToEventBus(
buildTargets,
isKeepGoing(),
params.getBuckEventBus(),
params.getConsole().getAnsi(),
getPathToBuildReport(params.getBuckConfig()));
}
params.getBuckEventBus().post(BuildEvent.finished(started, exitCode));
return exitCode;
}
/** Print only targets which were identified as owners. */
private void printOwnersOnlyReport(CommandRunnerParams params, OwnersReport report) {
Set<TargetNode<?>> sortedTargetNodes = report.owners.keySet();
for (TargetNode<?> targetNode : sortedTargetNodes) {
params.getConsole().getStdOut().println(targetNode.getBuildTarget().getFullyQualifiedName());
}
}
private FetchTargetNodeToBuildRuleTransformer createFetchTransformer(CommandRunnerParams params) {
Optional<String> defaultMavenRepo = params.getBuckConfig().getValue("download", "maven_repo");
Downloader downloader = new HttpDownloader(Optional.<Proxy>absent(), defaultMavenRepo);
Description<?> description = new RemoteFileDescription(downloader);
return new FetchTargetNodeToBuildRuleTransformer(ImmutableSet.<Description<?>>of(description));
}
static OwnersReport buildOwnersReport(
CommandRunnerParams params,
ParserConfig parserConfig,
BuildFileTree buildFileTree,
Iterable<String> arguments,
boolean guessForDeletedEnabled)
throws IOException, InterruptedException, BuildFileParseException, BuildTargetException {
final Path rootPath = params.getCell().getFilesystem().getRootPath();
Preconditions.checkState(rootPath.isAbsolute());
Map<Path, List<TargetNode<?>>> targetNodes = Maps.newHashMap();
OwnersReport report = OwnersReport.emptyReport();
for (Path filePath : getArgumentsAsPaths(rootPath, arguments)) {
Optional<Path> basePath = buildFileTree.getBasePathOfAncestorTarget(filePath);
if (!basePath.isPresent()) {
report =
report.updatedWith(
new OwnersReport(
ImmutableSetMultimap.<TargetNode<?>, Path>of(),
/* inputWithNoOwners */ ImmutableSet.of(filePath),
Sets.<String>newHashSet(),
Sets.<String>newHashSet()));
continue;
}
Path buckFile = basePath.get().resolve(parserConfig.getBuildFileName());
Preconditions.checkState(params.getCell().getFilesystem().exists(buckFile));
// Parse buck files and load target nodes.
if (!targetNodes.containsKey(buckFile)) {
try {
targetNodes.put(
buckFile,
params
.getParser()
.getOrLoadTargetNodes(
buckFile,
parserConfig,
params.getBuckEventBus(),
params.getConsole(),
params.getEnvironment()));
} catch (BuildFileParseException | BuildTargetException e) {
Path targetBasePath = MorePaths.relativize(rootPath, rootPath.resolve(basePath.get()));
String targetBaseName = "//" + MorePaths.pathWithUnixSeparators(targetBasePath);
params
.getConsole()
.getStdErr()
.format("Could not parse build targets for %s", targetBaseName);
throw e;
}
}
for (TargetNode<?> targetNode : targetNodes.get(buckFile)) {
report =
report.updatedWith(
generateOwnersReport(
params,
targetNode,
ImmutableList.of(filePath.toString()),
guessForDeletedEnabled));
}
}
return report;
}
