private int runTestsExternal( final CommandRunnerParams params, Build build, Iterable<String> command, Iterable<TestRule> testRules) throws InterruptedException, IOException { TestRunningOptions options = getTestRunningOptions(params); // Walk the test rules, collecting all the specs. List<ExternalTestRunnerTestSpec> specs = Lists.newArrayList(); for (TestRule testRule : testRules) { if (!(testRule instanceof ExternalTestRunnerRule)) { params .getBuckEventBus() .post( ConsoleEvent.severe( String.format( "Test %s does not support external test running", testRule.getBuildTarget()))); return 1; } ExternalTestRunnerRule rule = (ExternalTestRunnerRule) testRule; specs.add(rule.getExternalTestRunnerSpec(build.getExecutionContext(), options)); } // Serialize the specs to a file to pass into the test runner. Path infoFile = params .getCell() .getFilesystem() .resolve(BuckConstant.SCRATCH_PATH.resolve("external_runner_specs.json")); Files.createDirectories(infoFile.getParent()); Files.deleteIfExists(infoFile); params.getObjectMapper().writerWithDefaultPrettyPrinter().writeValue(infoFile.toFile(), specs); // Launch and run the external test runner, forwarding it's stdout/stderr to the console. // We wait for it to complete then returns its error code. ListeningProcessExecutor processExecutor = new ListeningProcessExecutor(); ProcessExecutorParams processExecutorParams = ProcessExecutorParams.builder() .addAllCommand(command) .addAllCommand(withDashArguments) .addCommand("--buck-test-info", infoFile.toString()) .setDirectory(params.getCell().getFilesystem().getRootPath().toFile()) .build(); ForwardingProcessListener processListener = new ForwardingProcessListener( Channels.newChannel(params.getConsole().getStdOut()), Channels.newChannel(params.getConsole().getStdErr())); ListeningProcessExecutor.LaunchedProcess process = processExecutor.launchProcess(processExecutorParams, processListener); try { return processExecutor.waitForProcess(process, Long.MAX_VALUE, TimeUnit.DAYS); } finally { processExecutor.destroyProcess(process, /* force */ false); processExecutor.waitForProcess(process, Long.MAX_VALUE, TimeUnit.DAYS); } }
public BuckQueryEnvironment(CommandRunnerParams params, boolean enableProfiling) { this.params = params; this.enableProfiling = enableProfiling; this.parserConfig = new ParserConfig(params.getBuckConfig()); this.buildFileTree = new FilesystemBackedBuildFileTree( params.getCell().getFilesystem(), parserConfig.getBuildFileName()); this.targetPatternEvaluator = new TargetPatternEvaluator(params, enableProfiling); }
@VisibleForTesting static OwnersReport generateOwnersReport( CommandRunnerParams params, TargetNode<?> targetNode, Iterable<String> filePaths, boolean guessForDeletedEnabled) { // Process arguments assuming they are all relative file paths. Set<Path> inputs = Sets.newHashSet(); Set<String> nonExistentInputs = Sets.newHashSet(); Set<String> nonFileInputs = Sets.newHashSet(); for (String filePath : filePaths) { File file = params.getCell().getFilesystem().getFileForRelativePath(filePath); if (!file.exists()) { nonExistentInputs.add(filePath); } else if (!file.isFile()) { nonFileInputs.add(filePath); } else { inputs.add(Paths.get(filePath)); } } // Try to find owners for each valid and existing file. Set<Path> inputsWithNoOwners = Sets.newHashSet(inputs); SetMultimap<TargetNode<?>, Path> owners = TreeMultimap.create(); for (final Path commandInput : inputs) { Predicate<Path> startsWith = new Predicate<Path>() { @Override public boolean apply(Path input) { return !commandInput.equals(input) && commandInput.startsWith(input); } }; Set<Path> ruleInputs = targetNode.getInputs(); if (ruleInputs.contains(commandInput) || FluentIterable.from(ruleInputs).anyMatch(startsWith)) { inputsWithNoOwners.remove(commandInput); owners.put(targetNode, commandInput); } } // Try to guess owners for nonexistent files. if (guessForDeletedEnabled) { for (String nonExistentInput : nonExistentInputs) { owners.put(targetNode, new File(nonExistentInput).toPath()); } } return new OwnersReport(owners, inputsWithNoOwners, nonExistentInputs, nonFileInputs); }
@Override public int runWithoutHelp(CommandRunnerParams params) throws IOException, InterruptedException { ParserConfig parserConfig = new ParserConfig(params.getBuckConfig()); BuildFileTree buildFileTree = new FilesystemBackedBuildFileTree( params.getCell().getFilesystem(), parserConfig.getBuildFileName()); try { OwnersReport report = buildOwnersReport( params, parserConfig, buildFileTree, getArguments(), isGuessForDeletedEnabled()); printReport(params, report); } catch (BuildFileParseException | BuildTargetException e) { return BUILD_TARGET_ERROR; } return 0; }
@Override public int runWithoutHelp(CommandRunnerParams params) throws IOException, InterruptedException { LOG.debug("Running with arguments %s", getArguments()); try (CommandThreadManager pool = new CommandThreadManager( "Test", params.getBuckConfig().getWorkQueueExecutionOrder(), getConcurrencyLimit(params.getBuckConfig()))) { // Post the build started event, setting it to the Parser recorded start time if appropriate. BuildEvent.Started started = BuildEvent.started(getArguments()); if (params.getParser().getParseStartTime().isPresent()) { params.getBuckEventBus().post(started, params.getParser().getParseStartTime().get()); } else { params.getBuckEventBus().post(started); } // The first step is to parse all of the build files. This will populate the parser and find // all of the test rules. TargetGraph targetGraph; ImmutableSet<BuildTarget> explicitBuildTargets; try { // If the user asked to run all of the tests, parse all of the build files looking for any // test rules. if (isRunAllTests()) { targetGraph = params .getParser() .buildTargetGraphForTargetNodeSpecs( params.getBuckEventBus(), params.getCell(), getEnableProfiling(), pool.getExecutor(), ImmutableList.of( TargetNodePredicateSpec.of( new Predicate<TargetNode<?>>() { @Override public boolean apply(TargetNode<?> input) { return input.getType().isTestRule(); } }, BuildFileSpec.fromRecursivePath(Paths.get(""))))) .getSecond(); explicitBuildTargets = ImmutableSet.of(); // Otherwise, the user specified specific test targets to build and run, so build a graph // around these. } else { LOG.debug("Parsing graph for arguments %s", getArguments()); Pair<ImmutableSet<BuildTarget>, TargetGraph> result = params .getParser() .buildTargetGraphForTargetNodeSpecs( params.getBuckEventBus(), params.getCell(), getEnableProfiling(), pool.getExecutor(), parseArgumentsAsTargetNodeSpecs(params.getBuckConfig(), getArguments())); targetGraph = result.getSecond(); explicitBuildTargets = result.getFirst(); LOG.debug("Got explicit build targets %s", explicitBuildTargets); ImmutableSet.Builder<BuildTarget> testTargetsBuilder = ImmutableSet.builder(); for (TargetNode<?> node : targetGraph.getAll(explicitBuildTargets)) { ImmutableSortedSet<BuildTarget> nodeTests = TargetNodes.getTestTargetsForNode(node); if (!nodeTests.isEmpty()) { LOG.debug("Got tests for target %s: %s", node.getBuildTarget(), nodeTests); testTargetsBuilder.addAll(nodeTests); } } ImmutableSet<BuildTarget> testTargets = testTargetsBuilder.build(); if (!testTargets.isEmpty()) { LOG.debug("Got related test targets %s, building new target graph...", testTargets); targetGraph = params .getParser() .buildTargetGraph( params.getBuckEventBus(), params.getCell(), getEnableProfiling(), pool.getExecutor(), Iterables.concat(explicitBuildTargets, testTargets)); LOG.debug("Finished building new target graph with tests."); } } } catch (BuildTargetException | BuildFileParseException e) { params .getBuckEventBus() .post(ConsoleEvent.severe(MoreExceptions.getHumanReadableOrLocalizedMessage(e))); return 1; } TargetGraphToActionGraph targetGraphToActionGraph = new TargetGraphToActionGraph( params.getBuckEventBus(), new BuildTargetNodeToBuildRuleTransformer()); Pair<ActionGraph, BuildRuleResolver> actionGraphAndResolver = Preconditions.checkNotNull(targetGraphToActionGraph.apply(targetGraph)); // Look up all of the test rules in the action graph. Iterable<TestRule> testRules = Iterables.filter(actionGraphAndResolver.getFirst().getNodes(), TestRule.class); // Unless the user requests that we build filtered tests, filter them out here, before // the build. if (!isBuildFiltered(params.getBuckConfig())) { testRules = filterTestRules(params.getBuckConfig(), explicitBuildTargets, testRules); } if (isDryRun()) { printMatchingTestRules(params.getConsole(), testRules); } CachingBuildEngine cachingBuildEngine = new CachingBuildEngine( pool.getExecutor(), params.getFileHashCache(), getBuildEngineMode().or(params.getBuckConfig().getBuildEngineMode()), params.getBuckConfig().getDependencySchedulingOrder(), params.getBuckConfig().getBuildDepFiles(), params.getBuckConfig().getBuildMaxDepFileCacheEntries(), actionGraphAndResolver.getSecond()); try (Build build = createBuild( params.getBuckConfig(), actionGraphAndResolver.getFirst(), actionGraphAndResolver.getSecond(), params.getAndroidPlatformTargetSupplier(), cachingBuildEngine, params.getArtifactCache(), params.getConsole(), params.getBuckEventBus(), getTargetDeviceOptional(), params.getPlatform(), params.getEnvironment(), params.getObjectMapper(), params.getClock(), Optional.of(getAdbOptions(params.getBuckConfig())), Optional.of(getTargetDeviceOptions()))) { // Build all of the test rules. int exitCode = build.executeAndPrintFailuresToEventBus( testRules, isKeepGoing(), params.getBuckEventBus(), params.getConsole(), getPathToBuildReport(params.getBuckConfig())); params.getBuckEventBus().post(BuildEvent.finished(started, exitCode)); if (exitCode != 0) { return exitCode; } // If the user requests that we build tests that we filter out, then we perform // the filtering here, after we've done the build but before we run the tests. if (isBuildFiltered(params.getBuckConfig())) { testRules = filterTestRules(params.getBuckConfig(), explicitBuildTargets, testRules); } // Once all of the rules are built, then run the tests. Optional<ImmutableList<String>> externalTestRunner = params.getBuckConfig().getExternalTestRunner(); if (externalTestRunner.isPresent()) { return runTestsExternal(params, build, externalTestRunner.get(), testRules); } return runTestsInternal(params, cachingBuildEngine, build, testRules); } } }
static OwnersReport buildOwnersReport( CommandRunnerParams params, ParserConfig parserConfig, BuildFileTree buildFileTree, Iterable<String> arguments, boolean guessForDeletedEnabled) throws IOException, InterruptedException, BuildFileParseException, BuildTargetException { final Path rootPath = params.getCell().getFilesystem().getRootPath(); Preconditions.checkState(rootPath.isAbsolute()); Map<Path, List<TargetNode<?>>> targetNodes = Maps.newHashMap(); OwnersReport report = OwnersReport.emptyReport(); for (Path filePath : getArgumentsAsPaths(rootPath, arguments)) { Optional<Path> basePath = buildFileTree.getBasePathOfAncestorTarget(filePath); if (!basePath.isPresent()) { report = report.updatedWith( new OwnersReport( ImmutableSetMultimap.<TargetNode<?>, Path>of(), /* inputWithNoOwners */ ImmutableSet.of(filePath), Sets.<String>newHashSet(), Sets.<String>newHashSet())); continue; } Path buckFile = basePath.get().resolve(parserConfig.getBuildFileName()); Preconditions.checkState(params.getCell().getFilesystem().exists(buckFile)); // Parse buck files and load target nodes. if (!targetNodes.containsKey(buckFile)) { try { targetNodes.put( buckFile, params .getParser() .getOrLoadTargetNodes( buckFile, parserConfig, params.getBuckEventBus(), params.getConsole(), params.getEnvironment())); } catch (BuildFileParseException | BuildTargetException e) { Path targetBasePath = MorePaths.relativize(rootPath, rootPath.resolve(basePath.get())); String targetBaseName = "//" + MorePaths.pathWithUnixSeparators(targetBasePath); params .getConsole() .getStdErr() .format("Could not parse build targets for %s", targetBaseName); throw e; } } for (TargetNode<?> targetNode : targetNodes.get(buckFile)) { report = report.updatedWith( generateOwnersReport( params, targetNode, ImmutableList.of(filePath.toString()), guessForDeletedEnabled)); } } return report; }