@VisibleForTesting int printJsonInputs(final CommandRunnerParams params, TargetGraph graph) throws IOException { final SortedMap<String, ImmutableSortedSet<Path>> targetToInputs = new TreeMap<>(); new AbstractBottomUpTraversal<TargetNode<?>, Void>(graph) { @Override public void visit(TargetNode<?> node) { LOG.debug("Looking at inputs for %s", node.getBuildTarget().getFullyQualifiedName()); SortedSet<Path> targetInputs = new TreeSet<>(); for (Path input : node.getInputs()) { LOG.debug("Walking input %s", input); try { if (!params.getCell().getFilesystem().exists(input)) { throw new HumanReadableException( "Target %s refers to non-existent input file: %s", node, input); } targetInputs.addAll(params.getCell().getFilesystem().getFilesUnderPath(input)); } catch (IOException e) { throw new RuntimeException(e); } } targetToInputs.put( node.getBuildTarget().getFullyQualifiedName(), ImmutableSortedSet.copyOf(targetInputs)); } }.traverse(); params.getObjectMapper().writeValue(params.getConsole().getStdOut(), targetToInputs); return 0; }
private int runTestsExternal( final CommandRunnerParams params, Build build, Iterable<String> command, Iterable<TestRule> testRules) throws InterruptedException, IOException { TestRunningOptions options = getTestRunningOptions(params); // Walk the test rules, collecting all the specs. List<ExternalTestRunnerTestSpec> specs = Lists.newArrayList(); for (TestRule testRule : testRules) { if (!(testRule instanceof ExternalTestRunnerRule)) { params .getBuckEventBus() .post( ConsoleEvent.severe( String.format( "Test %s does not support external test running", testRule.getBuildTarget()))); return 1; } ExternalTestRunnerRule rule = (ExternalTestRunnerRule) testRule; specs.add(rule.getExternalTestRunnerSpec(build.getExecutionContext(), options)); } // Serialize the specs to a file to pass into the test runner. Path infoFile = params .getCell() .getFilesystem() .resolve(BuckConstant.SCRATCH_PATH.resolve("external_runner_specs.json")); Files.createDirectories(infoFile.getParent()); Files.deleteIfExists(infoFile); params.getObjectMapper().writerWithDefaultPrettyPrinter().writeValue(infoFile.toFile(), specs); // Launch and run the external test runner, forwarding it's stdout/stderr to the console. // We wait for it to complete then returns its error code. ListeningProcessExecutor processExecutor = new ListeningProcessExecutor(); ProcessExecutorParams processExecutorParams = ProcessExecutorParams.builder() .addAllCommand(command) .addAllCommand(withDashArguments) .addCommand("--buck-test-info", infoFile.toString()) .setDirectory(params.getCell().getFilesystem().getRootPath().toFile()) .build(); ForwardingProcessListener processListener = new ForwardingProcessListener( Channels.newChannel(params.getConsole().getStdOut()), Channels.newChannel(params.getConsole().getStdErr())); ListeningProcessExecutor.LaunchedProcess process = processExecutor.launchProcess(processExecutorParams, processListener); try { return processExecutor.waitForProcess(process, Long.MAX_VALUE, TimeUnit.DAYS); } finally { processExecutor.destroyProcess(process, /* force */ false); processExecutor.waitForProcess(process, Long.MAX_VALUE, TimeUnit.DAYS); } }
public BuckQueryEnvironment(CommandRunnerParams params, boolean enableProfiling) { this.params = params; this.enableProfiling = enableProfiling; this.parserConfig = new ParserConfig(params.getBuckConfig()); this.buildFileTree = new FilesystemBackedBuildFileTree( params.getCell().getFilesystem(), parserConfig.getBuildFileName()); this.targetPatternEvaluator = new TargetPatternEvaluator(params, enableProfiling); }
TargetNode<?> getNode(QueryTarget target) throws QueryException, InterruptedException { Preconditions.checkState(target instanceof QueryBuildTarget); try { return params .getParser() .getTargetNode( ((QueryBuildTarget) target).getBuildTarget(), params.getBuckEventBus(), params.getConsole(), enableProfiling); } catch (BuildTargetException | BuildFileParseException | IOException e) { throw new QueryException("Error getting target node for %s\n%s", target, e.getMessage()); } }
/** Print only targets which were identified as owners in JSON. */ @VisibleForTesting void printOwnersOnlyJsonReport(CommandRunnerParams params, OwnersReport report) throws IOException { final Multimap<String, String> output = TreeMultimap.create(); Set<TargetNode<?>> sortedTargetNodes = report.owners.keySet(); for (TargetNode<?> targetNode : sortedTargetNodes) { Set<Path> files = report.owners.get(targetNode); for (Path input : files) { output.put(input.toString(), targetNode.getBuildTarget().getFullyQualifiedName()); } } params.getObjectMapper().writeValue(params.getConsole().getStdOut(), output.asMap()); }
@Override public int runWithoutHelp(CommandRunnerParams params) throws IOException, InterruptedException { ParserConfig parserConfig = new ParserConfig(params.getBuckConfig()); BuildFileTree buildFileTree = new FilesystemBackedBuildFileTree( params.getCell().getFilesystem(), parserConfig.getBuildFileName()); try { OwnersReport report = buildOwnersReport( params, parserConfig, buildFileTree, getArguments(), isGuessForDeletedEnabled()); printReport(params, report); } catch (BuildFileParseException | BuildTargetException e) { return BUILD_TARGET_ERROR; } return 0; }
private void buildGraphForBuildTargets(Set<BuildTarget> targets) throws QueryException, InterruptedException { try { graph = params .getParser() .buildTargetGraphForBuildTargets( targets, parserConfig, params.getBuckEventBus(), params.getConsole(), params.getEnvironment(), enableProfiling); } catch (BuildTargetException | BuildFileParseException | IOException e) { throw new QueryException("Error in building depencency graph"); } }
@Override public int runWithoutHelp(final CommandRunnerParams params) throws IOException, InterruptedException { // Create a TargetGraph that is composed of the transitive closure of all of the dependent // TargetNodes for the specified BuildTargets. final ImmutableSet<String> fullyQualifiedBuildTargets = ImmutableSet.copyOf(getArgumentsFormattedAsBuildTargets(params.getBuckConfig())); if (fullyQualifiedBuildTargets.isEmpty()) { params.getConsole().printBuildFailure("Please specify at least one build target."); return 1; } ImmutableSet<BuildTarget> targets = FluentIterable.from(getArgumentsFormattedAsBuildTargets(params.getBuckConfig())) .transform( new Function<String, BuildTarget>() { @Override public BuildTarget apply(String input) { return BuildTargetParser.INSTANCE.parse( input, BuildTargetPatternParser.fullyQualified()); } }) .toSet(); LOG.debug("Getting input for targets: %s", targets); TargetGraph graph; try { graph = params .getParser() .buildTargetGraphForBuildTargets( targets, new ParserConfig(params.getBuckConfig()), params.getBuckEventBus(), params.getConsole(), params.getEnvironment(), getEnableProfiling()); } catch (BuildTargetException | BuildFileParseException e) { params.getConsole().printBuildFailureWithoutStacktrace(e); return 1; } if (shouldGenerateJsonOutput()) { return printJsonInputs(params, graph); } return printInputs(params, graph); }
private static void printArtifactsInformation( CommandRunnerParams params, DeployResult deployResult) { params .getConsole() .getStdOut() .println( "\nPublished artifacts:\n" + Joiner.on('\n') .join( FluentIterable.from(deployResult.getArtifacts()) .transform( new Function<Artifact, String>() { @Override public String apply(Artifact input) { return atrifactToString(input); } }))); params.getConsole().getStdOut().println("\nDeployRequest:\n" + deployResult.getRequest()); }
/** Print detailed report on all owners. */ private void printFullReport(CommandRunnerParams params, OwnersReport report) { PrintStream out = params.getConsole().getStdOut(); Ansi ansi = params.getConsole().getAnsi(); if (report.owners.isEmpty()) { out.println(ansi.asErrorText("No owners found")); } else { out.println(ansi.asSuccessText("Owners:")); for (TargetNode<?> targetNode : report.owners.keySet()) { out.println(targetNode.getBuildTarget().getFullyQualifiedName()); Set<Path> files = report.owners.get(targetNode); for (Path input : files) { out.println(FILE_INDENT + input); } } } if (!report.inputsWithNoOwners.isEmpty()) { out.println(); out.println(ansi.asErrorText("Files without owners:")); for (Path input : report.inputsWithNoOwners) { out.println(FILE_INDENT + input); } } if (!report.nonExistentInputs.isEmpty()) { out.println(); out.println(ansi.asErrorText("Non existent files:")); for (String input : report.nonExistentInputs) { out.println(FILE_INDENT + input); } } if (!report.nonFileInputs.isEmpty()) { out.println(); out.println(ansi.asErrorText("Non-file inputs:")); for (String input : report.nonFileInputs) { out.println(FILE_INDENT + input); } } }
@VisibleForTesting static OwnersReport generateOwnersReport( CommandRunnerParams params, TargetNode<?> targetNode, Iterable<String> filePaths, boolean guessForDeletedEnabled) { // Process arguments assuming they are all relative file paths. Set<Path> inputs = Sets.newHashSet(); Set<String> nonExistentInputs = Sets.newHashSet(); Set<String> nonFileInputs = Sets.newHashSet(); for (String filePath : filePaths) { File file = params.getCell().getFilesystem().getFileForRelativePath(filePath); if (!file.exists()) { nonExistentInputs.add(filePath); } else if (!file.isFile()) { nonFileInputs.add(filePath); } else { inputs.add(Paths.get(filePath)); } } // Try to find owners for each valid and existing file. Set<Path> inputsWithNoOwners = Sets.newHashSet(inputs); SetMultimap<TargetNode<?>, Path> owners = TreeMultimap.create(); for (final Path commandInput : inputs) { Predicate<Path> startsWith = new Predicate<Path>() { @Override public boolean apply(Path input) { return !commandInput.equals(input) && commandInput.startsWith(input); } }; Set<Path> ruleInputs = targetNode.getInputs(); if (ruleInputs.contains(commandInput) || FluentIterable.from(ruleInputs).anyMatch(startsWith)) { inputsWithNoOwners.remove(commandInput); owners.put(targetNode, commandInput); } } // Try to guess owners for nonexistent files. if (guessForDeletedEnabled) { for (String nonExistentInput : nonExistentInputs) { owners.put(targetNode, new File(nonExistentInput).toPath()); } } return new OwnersReport(owners, inputsWithNoOwners, nonExistentInputs, nonFileInputs); }
private int runTestsInternal( CommandRunnerParams params, BuildEngine buildEngine, Build build, Iterable<TestRule> testRules) throws InterruptedException, IOException { if (!withDashArguments.isEmpty()) { params .getBuckEventBus() .post( ConsoleEvent.severe("Unexpected arguments after \"--\" when using internal runner")); return 1; } ConcurrencyLimit concurrencyLimit = new ConcurrencyLimit( getNumTestThreads(params.getBuckConfig()), params.getBuckConfig().getLoadLimit()); try (CommandThreadManager testPool = new CommandThreadManager( "Test-Run", params.getBuckConfig().getWorkQueueExecutionOrder(), concurrencyLimit)) { return TestRunning.runTests( params, testRules, Preconditions.checkNotNull(build.getBuildContext()), build.getExecutionContext(), getTestRunningOptions(params), testPool.getExecutor(), buildEngine, new DefaultStepRunner(build.getExecutionContext())); } catch (ExecutionException e) { params .getBuckEventBus() .post(ConsoleEvent.severe(MoreExceptions.getHumanReadableOrLocalizedMessage(e))); return 1; } }
/** @return whether successful */ private boolean publishTarget(BuildTarget buildTarget, CommandRunnerParams params) { BuildRule buildRule = Preconditions.checkNotNull(getBuild().getActionGraph().findBuildRuleByTarget(buildTarget)); if (!(buildRule instanceof MavenPublishable)) { printError( params, "Cannot retrieve maven coordinates for rule of type " + buildRule.getClass().getName()); return false; } Publisher publisher = new Publisher( params.getRepository().getFilesystem(), Optional.fromNullable(remoteRepo), dryRun); try { DeployResult deployResult = publisher.publish((MavenPublishable) buildRule); printArtifactsInformation(params, deployResult); } catch (DeploymentException e) { params.getConsole().printBuildFailureWithoutStacktraceDontUnwrap(e); return false; } return true; }
private TestRunningOptions getTestRunningOptions(CommandRunnerParams params) { return TestRunningOptions.builder() .setUsingOneTimeOutputDirectories(isUsingOneTimeOutput) .setCodeCoverageEnabled(isCodeCoverageEnabled) .setRunAllTests(isRunAllTests()) .setTestSelectorList(testSelectorOptions.getTestSelectorList()) .setShouldExplainTestSelectorList(testSelectorOptions.shouldExplain()) .setResultsCacheEnabled(isResultsCacheEnabled(params.getBuckConfig())) .setDryRun(isDryRun) .setShufflingTests(isShufflingTests) .setPathToXmlTestOutput(Optional.fromNullable(pathToXmlTestOutput)) .setPathToJavaAgent(Optional.fromNullable(pathToJavaAgent)) .setCoverageReportFormat(coverageReportFormat) .setCoverageReportTitle(coverageReportTitle) .build(); }
@SuppressWarnings("PMD.EmptyCatchBlock") public static int runTests( final CommandRunnerParams params, Iterable<TestRule> tests, BuildContext buildContext, ExecutionContext executionContext, final TestRunningOptions options, ListeningExecutorService service, BuildEngine buildEngine, final StepRunner stepRunner) throws IOException, ExecutionException, InterruptedException { if (options.isUsingOneTimeOutputDirectories()) { BuckConstant.setOneTimeTestSubdirectory(UUID.randomUUID().toString()); } ImmutableSet<JavaLibrary> rulesUnderTest; // If needed, we first run instrumentation on the class files. if (options.isCodeCoverageEnabled()) { rulesUnderTest = getRulesUnderTest(tests); if (!rulesUnderTest.isEmpty()) { try { stepRunner.runStepForBuildTarget( new MakeCleanDirectoryStep(JUnitStep.JACOCO_OUTPUT_DIR), Optional.<BuildTarget>absent()); } catch (StepFailedException e) { params.getConsole().printBuildFailureWithoutStacktrace(e); return 1; } } } else { rulesUnderTest = ImmutableSet.of(); } final ImmutableSet<String> testTargets = FluentIterable.from(tests) .transform(HasBuildTarget.TO_TARGET) .transform(Functions.toStringFunction()) .toSet(); final int totalNumberOfTests = Iterables.size(tests); params .getBuckEventBus() .post( TestRunEvent.started( options.isRunAllTests(), options.getTestSelectorList(), options.shouldExplainTestSelectorList(), testTargets)); // Start running all of the tests. The result of each java_test() rule is represented as a // ListenableFuture. List<ListenableFuture<TestResults>> results = Lists.newArrayList(); // Unless `--verbose 0` is specified, print out test results as they become available. // Failures with the ListenableFuture should always be printed, as they indicate an error with // Buck, not the test being run. Verbosity verbosity = params.getConsole().getVerbosity(); final boolean printTestResults = (verbosity != Verbosity.SILENT); // For grouping results! final TestResultsGrouper grouper; if (options.isIgnoreFailingDependencies()) { grouper = new TestResultsGrouper(tests); } else { grouper = null; } TestRuleKeyFileHelper testRuleKeyFileHelper = new TestRuleKeyFileHelper(executionContext.getProjectFilesystem(), buildEngine); final AtomicInteger lastReportedTestSequenceNumber = new AtomicInteger(); final List<TestRun> separateTestRuns = Lists.newArrayList(); List<TestRun> parallelTestRuns = Lists.newArrayList(); for (final TestRule test : tests) { // Determine whether the test needs to be executed. boolean isTestRunRequired; isTestRunRequired = isTestRunRequiredForTest( test, buildEngine, executionContext, testRuleKeyFileHelper, options.isResultsCacheEnabled(), !options.getTestSelectorList().isEmpty()); List<Step> steps; if (isTestRunRequired) { params.getBuckEventBus().post(IndividualTestEvent.started(testTargets)); ImmutableList.Builder<Step> stepsBuilder = ImmutableList.builder(); Preconditions.checkState(buildEngine.isRuleBuilt(test.getBuildTarget())); final Map<String, UUID> testUUIDMap = new HashMap<>(); List<Step> testSteps = test.runTests( buildContext, executionContext, options.isDryRun(), options.isShufflingTests(), options.getTestSelectorList(), new TestRule.TestReportingCallback() { @Override public void testsDidBegin() { LOG.debug("Tests for rule %s began", test.getBuildTarget()); } @Override public void testDidBegin(String testCaseName, String testName) { LOG.debug( "Test rule %s test case %s test name %s began", test.getBuildTarget(), testCaseName, testName); UUID testUUID = UUID.randomUUID(); // UUID is immutable and thread-safe as of Java 7, so it's // safe to stash in a map and use later: // // http://bugs.java.com/view_bug.do?bug_id=6611830 testUUIDMap.put(testCaseName + ":" + testName, testUUID); params .getBuckEventBus() .post(TestSummaryEvent.started(testUUID, testCaseName, testName)); } @Override public void testDidEnd(TestResultSummary testResultSummary) { LOG.debug( "Test rule %s test did end: %s", test.getBuildTarget(), testResultSummary); UUID testUUID = testUUIDMap.get( testResultSummary.getTestCaseName() + ":" + testResultSummary.getTestName()); Preconditions.checkNotNull(testUUID); params .getBuckEventBus() .post(TestSummaryEvent.finished(testUUID, testResultSummary)); } @Override public void testsDidEnd(List<TestCaseSummary> testCaseSummaries) { LOG.debug( "Test rule %s tests did end: %s", test.getBuildTarget(), testCaseSummaries); } }); if (!testSteps.isEmpty()) { stepsBuilder.addAll(testSteps); stepsBuilder.add(testRuleKeyFileHelper.createRuleKeyInDirStep(test)); } steps = stepsBuilder.build(); } else { steps = ImmutableList.of(); } TestRun testRun = TestRun.of( test, steps, getCachingStatusTransformingCallable( isTestRunRequired, test.interpretTestResults( executionContext, /*isUsingTestSelectors*/ !options.getTestSelectorList().isEmpty(), /*isDryRun*/ options.isDryRun()))); // Always run the commands, even if the list of commands as empty. There may be zero // commands because the rule is cached, but its results must still be processed. if (test.runTestSeparately()) { LOG.debug("Running test %s in serial", test); separateTestRuns.add(testRun); } else { LOG.debug("Running test %s in parallel", test); parallelTestRuns.add(testRun); } } final StepRunner.StepRunningCallback testStepRunningCallback = new StepRunner.StepRunningCallback() { @Override public void stepsWillRun(Optional<BuildTarget> buildTarget) { Preconditions.checkState(buildTarget.isPresent()); LOG.debug("Test steps will run for %s", buildTarget); params.getBuckEventBus().post(TestRuleEvent.started(buildTarget.get())); } @Override public void stepsDidRun(Optional<BuildTarget> buildTarget) { Preconditions.checkState(buildTarget.isPresent()); LOG.debug("Test steps did run for %s", buildTarget); params.getBuckEventBus().post(TestRuleEvent.finished(buildTarget.get())); } }; for (TestRun testRun : parallelTestRuns) { ListenableFuture<TestResults> testResults = stepRunner.runStepsAndYieldResult( testRun.getSteps(), testRun.getTestResultsCallable(), Optional.of(testRun.getTest().getBuildTarget()), service, testStepRunningCallback); results.add( transformTestResults( params, testResults, grouper, testRun.getTest(), testTargets, printTestResults, lastReportedTestSequenceNumber, totalNumberOfTests)); } ListenableFuture<List<TestResults>> parallelTestStepsFuture = Futures.allAsList(results); final List<TestResults> completedResults = Lists.newArrayList(); final ListeningExecutorService directExecutorService = MoreExecutors.newDirectExecutorService(); ListenableFuture<Void> uberFuture = stepRunner.addCallback( parallelTestStepsFuture, new FutureCallback<List<TestResults>>() { @Override public void onSuccess(List<TestResults> parallelTestResults) { LOG.debug("Parallel tests completed, running separate tests..."); completedResults.addAll(parallelTestResults); List<ListenableFuture<TestResults>> separateResultsList = Lists.newArrayList(); for (TestRun testRun : separateTestRuns) { separateResultsList.add( transformTestResults( params, stepRunner.runStepsAndYieldResult( testRun.getSteps(), testRun.getTestResultsCallable(), Optional.of(testRun.getTest().getBuildTarget()), directExecutorService, testStepRunningCallback), grouper, testRun.getTest(), testTargets, printTestResults, lastReportedTestSequenceNumber, totalNumberOfTests)); } ListenableFuture<List<TestResults>> serialResults = Futures.allAsList(separateResultsList); try { completedResults.addAll(serialResults.get()); } catch (ExecutionException e) { LOG.error(e, "Error fetching serial test results"); throw new HumanReadableException(e, "Error fetching serial test results"); } catch (InterruptedException e) { LOG.error(e, "Interrupted fetching serial test results"); try { serialResults.cancel(true); } catch (CancellationException ignored) { // Rethrow original InterruptedException instead. } Thread.currentThread().interrupt(); throw new HumanReadableException(e, "Test cancelled"); } LOG.debug("Done running serial tests."); } @Override public void onFailure(Throwable e) { LOG.error(e, "Parallel tests failed, not running serial tests"); throw new HumanReadableException(e, "Parallel tests failed"); } }, directExecutorService); try { // Block until all the tests have finished running. uberFuture.get(); } catch (ExecutionException e) { e.printStackTrace(params.getConsole().getStdErr()); return 1; } catch (InterruptedException e) { try { uberFuture.cancel(true); } catch (CancellationException ignored) { // Rethrow original InterruptedException instead. } Thread.currentThread().interrupt(); throw e; } params.getBuckEventBus().post(TestRunEvent.finished(testTargets, completedResults)); // Write out the results as XML, if requested. Optional<String> path = options.getPathToXmlTestOutput(); if (path.isPresent()) { try (Writer writer = Files.newWriter(new File(path.get()), Charsets.UTF_8)) { writeXmlOutput(completedResults, writer); } } // Generate the code coverage report. if (options.isCodeCoverageEnabled() && !rulesUnderTest.isEmpty()) { try { Optional<DefaultJavaPackageFinder> defaultJavaPackageFinderOptional = Optional.fromNullable(params.getBuckConfig().createDefaultJavaPackageFinder()); stepRunner.runStepForBuildTarget( getReportCommand( rulesUnderTest, defaultJavaPackageFinderOptional, params.getRepository().getFilesystem(), JUnitStep.JACOCO_OUTPUT_DIR, options.getCoverageReportFormat()), Optional.<BuildTarget>absent()); } catch (StepFailedException e) { params.getConsole().printBuildFailureWithoutStacktrace(e); return 1; } } boolean failures = Iterables.any( completedResults, new Predicate<TestResults>() { @Override public boolean apply(TestResults results) { LOG.debug("Checking result %s for failure", results); return !results.isSuccess(); } }); return failures ? TEST_FAILURES_EXIT_CODE : 0; }
private static void printError(CommandRunnerParams params, String errorMessage) { params.getConsole().printErrorText(errorMessage); }
@Override public int runWithoutHelp(CommandRunnerParams params) throws IOException, InterruptedException { LOG.debug("Running with arguments %s", getArguments()); try (CommandThreadManager pool = new CommandThreadManager( "Test", params.getBuckConfig().getWorkQueueExecutionOrder(), getConcurrencyLimit(params.getBuckConfig()))) { // Post the build started event, setting it to the Parser recorded start time if appropriate. BuildEvent.Started started = BuildEvent.started(getArguments()); if (params.getParser().getParseStartTime().isPresent()) { params.getBuckEventBus().post(started, params.getParser().getParseStartTime().get()); } else { params.getBuckEventBus().post(started); } // The first step is to parse all of the build files. This will populate the parser and find // all of the test rules. TargetGraph targetGraph; ImmutableSet<BuildTarget> explicitBuildTargets; try { // If the user asked to run all of the tests, parse all of the build files looking for any // test rules. if (isRunAllTests()) { targetGraph = params .getParser() .buildTargetGraphForTargetNodeSpecs( params.getBuckEventBus(), params.getCell(), getEnableProfiling(), pool.getExecutor(), ImmutableList.of( TargetNodePredicateSpec.of( new Predicate<TargetNode<?>>() { @Override public boolean apply(TargetNode<?> input) { return input.getType().isTestRule(); } }, BuildFileSpec.fromRecursivePath(Paths.get(""))))) .getSecond(); explicitBuildTargets = ImmutableSet.of(); // Otherwise, the user specified specific test targets to build and run, so build a graph // around these. } else { LOG.debug("Parsing graph for arguments %s", getArguments()); Pair<ImmutableSet<BuildTarget>, TargetGraph> result = params .getParser() .buildTargetGraphForTargetNodeSpecs( params.getBuckEventBus(), params.getCell(), getEnableProfiling(), pool.getExecutor(), parseArgumentsAsTargetNodeSpecs(params.getBuckConfig(), getArguments())); targetGraph = result.getSecond(); explicitBuildTargets = result.getFirst(); LOG.debug("Got explicit build targets %s", explicitBuildTargets); ImmutableSet.Builder<BuildTarget> testTargetsBuilder = ImmutableSet.builder(); for (TargetNode<?> node : targetGraph.getAll(explicitBuildTargets)) { ImmutableSortedSet<BuildTarget> nodeTests = TargetNodes.getTestTargetsForNode(node); if (!nodeTests.isEmpty()) { LOG.debug("Got tests for target %s: %s", node.getBuildTarget(), nodeTests); testTargetsBuilder.addAll(nodeTests); } } ImmutableSet<BuildTarget> testTargets = testTargetsBuilder.build(); if (!testTargets.isEmpty()) { LOG.debug("Got related test targets %s, building new target graph...", testTargets); targetGraph = params .getParser() .buildTargetGraph( params.getBuckEventBus(), params.getCell(), getEnableProfiling(), pool.getExecutor(), Iterables.concat(explicitBuildTargets, testTargets)); LOG.debug("Finished building new target graph with tests."); } } } catch (BuildTargetException | BuildFileParseException e) { params .getBuckEventBus() .post(ConsoleEvent.severe(MoreExceptions.getHumanReadableOrLocalizedMessage(e))); return 1; } TargetGraphToActionGraph targetGraphToActionGraph = new TargetGraphToActionGraph( params.getBuckEventBus(), new BuildTargetNodeToBuildRuleTransformer()); Pair<ActionGraph, BuildRuleResolver> actionGraphAndResolver = Preconditions.checkNotNull(targetGraphToActionGraph.apply(targetGraph)); // Look up all of the test rules in the action graph. Iterable<TestRule> testRules = Iterables.filter(actionGraphAndResolver.getFirst().getNodes(), TestRule.class); // Unless the user requests that we build filtered tests, filter them out here, before // the build. if (!isBuildFiltered(params.getBuckConfig())) { testRules = filterTestRules(params.getBuckConfig(), explicitBuildTargets, testRules); } if (isDryRun()) { printMatchingTestRules(params.getConsole(), testRules); } CachingBuildEngine cachingBuildEngine = new CachingBuildEngine( pool.getExecutor(), params.getFileHashCache(), getBuildEngineMode().or(params.getBuckConfig().getBuildEngineMode()), params.getBuckConfig().getDependencySchedulingOrder(), params.getBuckConfig().getBuildDepFiles(), params.getBuckConfig().getBuildMaxDepFileCacheEntries(), actionGraphAndResolver.getSecond()); try (Build build = createBuild( params.getBuckConfig(), actionGraphAndResolver.getFirst(), actionGraphAndResolver.getSecond(), params.getAndroidPlatformTargetSupplier(), cachingBuildEngine, params.getArtifactCache(), params.getConsole(), params.getBuckEventBus(), getTargetDeviceOptional(), params.getPlatform(), params.getEnvironment(), params.getObjectMapper(), params.getClock(), Optional.of(getAdbOptions(params.getBuckConfig())), Optional.of(getTargetDeviceOptions()))) { // Build all of the test rules. int exitCode = build.executeAndPrintFailuresToEventBus( testRules, isKeepGoing(), params.getBuckEventBus(), params.getConsole(), getPathToBuildReport(params.getBuckConfig())); params.getBuckEventBus().post(BuildEvent.finished(started, exitCode)); if (exitCode != 0) { return exitCode; } // If the user requests that we build tests that we filter out, then we perform // the filtering here, after we've done the build but before we run the tests. if (isBuildFiltered(params.getBuckConfig())) { testRules = filterTestRules(params.getBuckConfig(), explicitBuildTargets, testRules); } // Once all of the rules are built, then run the tests. Optional<ImmutableList<String>> externalTestRunner = params.getBuckConfig().getExternalTestRunner(); if (externalTestRunner.isPresent()) { return runTestsExternal(params, build, externalTestRunner.get(), testRules); } return runTestsInternal(params, cachingBuildEngine, build, testRules); } } }
@Override public int runWithoutHelp(CommandRunnerParams params) throws IOException, InterruptedException { if (getArguments().isEmpty()) { params.getConsole().printBuildFailure("Must specify at least one build target to fetch."); return 1; } // Post the build started event, setting it to the Parser recorded start time if appropriate. BuildEvent.Started started = BuildEvent.started(getArguments()); if (params.getParser().getParseStartTime().isPresent()) { params.getBuckEventBus().post(started, params.getParser().getParseStartTime().get()); } else { params.getBuckEventBus().post(started); } FetchTargetNodeToBuildRuleTransformer ruleGenerator = createFetchTransformer(params); TargetGraphToActionGraph transformer = new TargetGraphToActionGraph( params.getBuckEventBus(), ruleGenerator, params.getFileHashCache()); ActionGraph actionGraph; ImmutableSet<BuildTarget> buildTargets; try { Pair<ImmutableSet<BuildTarget>, TargetGraph> result = params .getParser() .buildTargetGraphForTargetNodeSpecs( parseArgumentsAsTargetNodeSpecs( params.getBuckConfig(), params.getRepository().getFilesystem().getIgnorePaths(), getArguments()), new ParserConfig(params.getBuckConfig()), params.getBuckEventBus(), params.getConsole(), params.getEnvironment(), getEnableProfiling()); actionGraph = transformer.apply(result.getSecond()); buildTargets = ruleGenerator.getDownloadableTargets(); } catch (BuildTargetException | BuildFileParseException e) { params.getConsole().printBuildFailureWithoutStacktrace(e); return 1; } int exitCode; SourcePathResolver pathResolver = new SourcePathResolver(transformer.getRuleResolver()); try (CommandThreadManager pool = new CommandThreadManager("Fetch", getConcurrencyLimit(params.getBuckConfig())); Build build = createBuild( params.getBuckConfig(), actionGraph, params.getAndroidPlatformTargetSupplier(), new CachingBuildEngine( pool.getExecutor(), params.getFileHashCache(), getBuildEngineMode().or(params.getBuckConfig().getBuildEngineMode()), params.getBuckConfig().getBuildDepFiles(), new InputBasedRuleKeyBuilderFactory(params.getFileHashCache(), pathResolver), new AbiRuleKeyBuilderFactory(params.getFileHashCache(), pathResolver), new DependencyFileRuleKeyBuilderFactory( params.getFileHashCache(), pathResolver)), getArtifactCache(params), params.getConsole(), params.getBuckEventBus(), Optional.<TargetDevice>absent(), params.getPlatform(), params.getEnvironment(), params.getObjectMapper(), params.getClock(), Optional.<AdbOptions>absent(), Optional.<TargetDeviceOptions>absent())) { exitCode = build.executeAndPrintFailuresToEventBus( buildTargets, isKeepGoing(), params.getBuckEventBus(), params.getConsole().getAnsi(), getPathToBuildReport(params.getBuckConfig())); } params.getBuckEventBus().post(BuildEvent.finished(started, exitCode)); return exitCode; }
/** Print only targets which were identified as owners. */ private void printOwnersOnlyReport(CommandRunnerParams params, OwnersReport report) { Set<TargetNode<?>> sortedTargetNodes = report.owners.keySet(); for (TargetNode<?> targetNode : sortedTargetNodes) { params.getConsole().getStdOut().println(targetNode.getBuildTarget().getFullyQualifiedName()); } }
private FetchTargetNodeToBuildRuleTransformer createFetchTransformer(CommandRunnerParams params) { Optional<String> defaultMavenRepo = params.getBuckConfig().getValue("download", "maven_repo"); Downloader downloader = new HttpDownloader(Optional.<Proxy>absent(), defaultMavenRepo); Description<?> description = new RemoteFileDescription(downloader); return new FetchTargetNodeToBuildRuleTransformer(ImmutableSet.<Description<?>>of(description)); }
static OwnersReport buildOwnersReport( CommandRunnerParams params, ParserConfig parserConfig, BuildFileTree buildFileTree, Iterable<String> arguments, boolean guessForDeletedEnabled) throws IOException, InterruptedException, BuildFileParseException, BuildTargetException { final Path rootPath = params.getCell().getFilesystem().getRootPath(); Preconditions.checkState(rootPath.isAbsolute()); Map<Path, List<TargetNode<?>>> targetNodes = Maps.newHashMap(); OwnersReport report = OwnersReport.emptyReport(); for (Path filePath : getArgumentsAsPaths(rootPath, arguments)) { Optional<Path> basePath = buildFileTree.getBasePathOfAncestorTarget(filePath); if (!basePath.isPresent()) { report = report.updatedWith( new OwnersReport( ImmutableSetMultimap.<TargetNode<?>, Path>of(), /* inputWithNoOwners */ ImmutableSet.of(filePath), Sets.<String>newHashSet(), Sets.<String>newHashSet())); continue; } Path buckFile = basePath.get().resolve(parserConfig.getBuildFileName()); Preconditions.checkState(params.getCell().getFilesystem().exists(buckFile)); // Parse buck files and load target nodes. if (!targetNodes.containsKey(buckFile)) { try { targetNodes.put( buckFile, params .getParser() .getOrLoadTargetNodes( buckFile, parserConfig, params.getBuckEventBus(), params.getConsole(), params.getEnvironment())); } catch (BuildFileParseException | BuildTargetException e) { Path targetBasePath = MorePaths.relativize(rootPath, rootPath.resolve(basePath.get())); String targetBaseName = "//" + MorePaths.pathWithUnixSeparators(targetBasePath); params .getConsole() .getStdErr() .format("Could not parse build targets for %s", targetBaseName); throw e; } } for (TargetNode<?> targetNode : targetNodes.get(buckFile)) { report = report.updatedWith( generateOwnersReport( params, targetNode, ImmutableList.of(filePath.toString()), guessForDeletedEnabled)); } } return report; }