@VisibleForTesting int printJsonInputs(final CommandRunnerParams params, TargetGraph graph) throws IOException { final SortedMap<String, ImmutableSortedSet<Path>> targetToInputs = new TreeMap<>(); new AbstractBottomUpTraversal<TargetNode<?>, Void>(graph) { @Override public void visit(TargetNode<?> node) { LOG.debug("Looking at inputs for %s", node.getBuildTarget().getFullyQualifiedName()); SortedSet<Path> targetInputs = new TreeSet<>(); for (Path input : node.getInputs()) { LOG.debug("Walking input %s", input); try { if (!params.getCell().getFilesystem().exists(input)) { throw new HumanReadableException( "Target %s refers to non-existent input file: %s", node, input); } targetInputs.addAll(params.getCell().getFilesystem().getFilesUnderPath(input)); } catch (IOException e) { throw new RuntimeException(e); } } targetToInputs.put( node.getBuildTarget().getFullyQualifiedName(), ImmutableSortedSet.copyOf(targetInputs)); } }.traverse(); params.getObjectMapper().writeValue(params.getConsole().getStdOut(), targetToInputs); return 0; }
private int runTestsExternal( final CommandRunnerParams params, Build build, Iterable<String> command, Iterable<TestRule> testRules) throws InterruptedException, IOException { TestRunningOptions options = getTestRunningOptions(params); // Walk the test rules, collecting all the specs. List<ExternalTestRunnerTestSpec> specs = Lists.newArrayList(); for (TestRule testRule : testRules) { if (!(testRule instanceof ExternalTestRunnerRule)) { params .getBuckEventBus() .post( ConsoleEvent.severe( String.format( "Test %s does not support external test running", testRule.getBuildTarget()))); return 1; } ExternalTestRunnerRule rule = (ExternalTestRunnerRule) testRule; specs.add(rule.getExternalTestRunnerSpec(build.getExecutionContext(), options)); } // Serialize the specs to a file to pass into the test runner. Path infoFile = params .getCell() .getFilesystem() .resolve(BuckConstant.SCRATCH_PATH.resolve("external_runner_specs.json")); Files.createDirectories(infoFile.getParent()); Files.deleteIfExists(infoFile); params.getObjectMapper().writerWithDefaultPrettyPrinter().writeValue(infoFile.toFile(), specs); // Launch and run the external test runner, forwarding it's stdout/stderr to the console. // We wait for it to complete then returns its error code. ListeningProcessExecutor processExecutor = new ListeningProcessExecutor(); ProcessExecutorParams processExecutorParams = ProcessExecutorParams.builder() .addAllCommand(command) .addAllCommand(withDashArguments) .addCommand("--buck-test-info", infoFile.toString()) .setDirectory(params.getCell().getFilesystem().getRootPath().toFile()) .build(); ForwardingProcessListener processListener = new ForwardingProcessListener( Channels.newChannel(params.getConsole().getStdOut()), Channels.newChannel(params.getConsole().getStdErr())); ListeningProcessExecutor.LaunchedProcess process = processExecutor.launchProcess(processExecutorParams, processListener); try { return processExecutor.waitForProcess(process, Long.MAX_VALUE, TimeUnit.DAYS); } finally { processExecutor.destroyProcess(process, /* force */ false); processExecutor.waitForProcess(process, Long.MAX_VALUE, TimeUnit.DAYS); } }
/** Print only targets which were identified as owners in JSON. */ @VisibleForTesting void printOwnersOnlyJsonReport(CommandRunnerParams params, OwnersReport report) throws IOException { final Multimap<String, String> output = TreeMultimap.create(); Set<TargetNode<?>> sortedTargetNodes = report.owners.keySet(); for (TargetNode<?> targetNode : sortedTargetNodes) { Set<Path> files = report.owners.get(targetNode); for (Path input : files) { output.put(input.toString(), targetNode.getBuildTarget().getFullyQualifiedName()); } } params.getObjectMapper().writeValue(params.getConsole().getStdOut(), output.asMap()); }
@Override public int runWithoutHelp(CommandRunnerParams params) throws IOException, InterruptedException { LOG.debug("Running with arguments %s", getArguments()); try (CommandThreadManager pool = new CommandThreadManager( "Test", params.getBuckConfig().getWorkQueueExecutionOrder(), getConcurrencyLimit(params.getBuckConfig()))) { // Post the build started event, setting it to the Parser recorded start time if appropriate. BuildEvent.Started started = BuildEvent.started(getArguments()); if (params.getParser().getParseStartTime().isPresent()) { params.getBuckEventBus().post(started, params.getParser().getParseStartTime().get()); } else { params.getBuckEventBus().post(started); } // The first step is to parse all of the build files. This will populate the parser and find // all of the test rules. TargetGraph targetGraph; ImmutableSet<BuildTarget> explicitBuildTargets; try { // If the user asked to run all of the tests, parse all of the build files looking for any // test rules. if (isRunAllTests()) { targetGraph = params .getParser() .buildTargetGraphForTargetNodeSpecs( params.getBuckEventBus(), params.getCell(), getEnableProfiling(), pool.getExecutor(), ImmutableList.of( TargetNodePredicateSpec.of( new Predicate<TargetNode<?>>() { @Override public boolean apply(TargetNode<?> input) { return input.getType().isTestRule(); } }, BuildFileSpec.fromRecursivePath(Paths.get(""))))) .getSecond(); explicitBuildTargets = ImmutableSet.of(); // Otherwise, the user specified specific test targets to build and run, so build a graph // around these. } else { LOG.debug("Parsing graph for arguments %s", getArguments()); Pair<ImmutableSet<BuildTarget>, TargetGraph> result = params .getParser() .buildTargetGraphForTargetNodeSpecs( params.getBuckEventBus(), params.getCell(), getEnableProfiling(), pool.getExecutor(), parseArgumentsAsTargetNodeSpecs(params.getBuckConfig(), getArguments())); targetGraph = result.getSecond(); explicitBuildTargets = result.getFirst(); LOG.debug("Got explicit build targets %s", explicitBuildTargets); ImmutableSet.Builder<BuildTarget> testTargetsBuilder = ImmutableSet.builder(); for (TargetNode<?> node : targetGraph.getAll(explicitBuildTargets)) { ImmutableSortedSet<BuildTarget> nodeTests = TargetNodes.getTestTargetsForNode(node); if (!nodeTests.isEmpty()) { LOG.debug("Got tests for target %s: %s", node.getBuildTarget(), nodeTests); testTargetsBuilder.addAll(nodeTests); } } ImmutableSet<BuildTarget> testTargets = testTargetsBuilder.build(); if (!testTargets.isEmpty()) { LOG.debug("Got related test targets %s, building new target graph...", testTargets); targetGraph = params .getParser() .buildTargetGraph( params.getBuckEventBus(), params.getCell(), getEnableProfiling(), pool.getExecutor(), Iterables.concat(explicitBuildTargets, testTargets)); LOG.debug("Finished building new target graph with tests."); } } } catch (BuildTargetException | BuildFileParseException e) { params .getBuckEventBus() .post(ConsoleEvent.severe(MoreExceptions.getHumanReadableOrLocalizedMessage(e))); return 1; } TargetGraphToActionGraph targetGraphToActionGraph = new TargetGraphToActionGraph( params.getBuckEventBus(), new BuildTargetNodeToBuildRuleTransformer()); Pair<ActionGraph, BuildRuleResolver> actionGraphAndResolver = Preconditions.checkNotNull(targetGraphToActionGraph.apply(targetGraph)); // Look up all of the test rules in the action graph. Iterable<TestRule> testRules = Iterables.filter(actionGraphAndResolver.getFirst().getNodes(), TestRule.class); // Unless the user requests that we build filtered tests, filter them out here, before // the build. if (!isBuildFiltered(params.getBuckConfig())) { testRules = filterTestRules(params.getBuckConfig(), explicitBuildTargets, testRules); } if (isDryRun()) { printMatchingTestRules(params.getConsole(), testRules); } CachingBuildEngine cachingBuildEngine = new CachingBuildEngine( pool.getExecutor(), params.getFileHashCache(), getBuildEngineMode().or(params.getBuckConfig().getBuildEngineMode()), params.getBuckConfig().getDependencySchedulingOrder(), params.getBuckConfig().getBuildDepFiles(), params.getBuckConfig().getBuildMaxDepFileCacheEntries(), actionGraphAndResolver.getSecond()); try (Build build = createBuild( params.getBuckConfig(), actionGraphAndResolver.getFirst(), actionGraphAndResolver.getSecond(), params.getAndroidPlatformTargetSupplier(), cachingBuildEngine, params.getArtifactCache(), params.getConsole(), params.getBuckEventBus(), getTargetDeviceOptional(), params.getPlatform(), params.getEnvironment(), params.getObjectMapper(), params.getClock(), Optional.of(getAdbOptions(params.getBuckConfig())), Optional.of(getTargetDeviceOptions()))) { // Build all of the test rules. int exitCode = build.executeAndPrintFailuresToEventBus( testRules, isKeepGoing(), params.getBuckEventBus(), params.getConsole(), getPathToBuildReport(params.getBuckConfig())); params.getBuckEventBus().post(BuildEvent.finished(started, exitCode)); if (exitCode != 0) { return exitCode; } // If the user requests that we build tests that we filter out, then we perform // the filtering here, after we've done the build but before we run the tests. if (isBuildFiltered(params.getBuckConfig())) { testRules = filterTestRules(params.getBuckConfig(), explicitBuildTargets, testRules); } // Once all of the rules are built, then run the tests. Optional<ImmutableList<String>> externalTestRunner = params.getBuckConfig().getExternalTestRunner(); if (externalTestRunner.isPresent()) { return runTestsExternal(params, build, externalTestRunner.get(), testRules); } return runTestsInternal(params, cachingBuildEngine, build, testRules); } } }
@Override public int runWithoutHelp(CommandRunnerParams params) throws IOException, InterruptedException { if (getArguments().isEmpty()) { params.getConsole().printBuildFailure("Must specify at least one build target to fetch."); return 1; } // Post the build started event, setting it to the Parser recorded start time if appropriate. BuildEvent.Started started = BuildEvent.started(getArguments()); if (params.getParser().getParseStartTime().isPresent()) { params.getBuckEventBus().post(started, params.getParser().getParseStartTime().get()); } else { params.getBuckEventBus().post(started); } FetchTargetNodeToBuildRuleTransformer ruleGenerator = createFetchTransformer(params); TargetGraphToActionGraph transformer = new TargetGraphToActionGraph( params.getBuckEventBus(), ruleGenerator, params.getFileHashCache()); ActionGraph actionGraph; ImmutableSet<BuildTarget> buildTargets; try { Pair<ImmutableSet<BuildTarget>, TargetGraph> result = params .getParser() .buildTargetGraphForTargetNodeSpecs( parseArgumentsAsTargetNodeSpecs( params.getBuckConfig(), params.getRepository().getFilesystem().getIgnorePaths(), getArguments()), new ParserConfig(params.getBuckConfig()), params.getBuckEventBus(), params.getConsole(), params.getEnvironment(), getEnableProfiling()); actionGraph = transformer.apply(result.getSecond()); buildTargets = ruleGenerator.getDownloadableTargets(); } catch (BuildTargetException | BuildFileParseException e) { params.getConsole().printBuildFailureWithoutStacktrace(e); return 1; } int exitCode; SourcePathResolver pathResolver = new SourcePathResolver(transformer.getRuleResolver()); try (CommandThreadManager pool = new CommandThreadManager("Fetch", getConcurrencyLimit(params.getBuckConfig())); Build build = createBuild( params.getBuckConfig(), actionGraph, params.getAndroidPlatformTargetSupplier(), new CachingBuildEngine( pool.getExecutor(), params.getFileHashCache(), getBuildEngineMode().or(params.getBuckConfig().getBuildEngineMode()), params.getBuckConfig().getBuildDepFiles(), new InputBasedRuleKeyBuilderFactory(params.getFileHashCache(), pathResolver), new AbiRuleKeyBuilderFactory(params.getFileHashCache(), pathResolver), new DependencyFileRuleKeyBuilderFactory( params.getFileHashCache(), pathResolver)), getArtifactCache(params), params.getConsole(), params.getBuckEventBus(), Optional.<TargetDevice>absent(), params.getPlatform(), params.getEnvironment(), params.getObjectMapper(), params.getClock(), Optional.<AdbOptions>absent(), Optional.<TargetDeviceOptions>absent())) { exitCode = build.executeAndPrintFailuresToEventBus( buildTargets, isKeepGoing(), params.getBuckEventBus(), params.getConsole().getAnsi(), getPathToBuildReport(params.getBuckConfig())); } params.getBuckEventBus().post(BuildEvent.finished(started, exitCode)); return exitCode; }