/**
* Create all the modules we are capable of representing in IntelliJ from the supplied graph.
*
* @param targetGraph graph whose nodes will be converted to {@link IjModule}s.
* @return map which for every BuildTarget points to the corresponding IjModule. Multiple
* BuildTarget can point to one IjModule (many:one mapping), the BuildTargets which can't be
* prepresented in IntelliJ are missing from this mapping.
*/
public static ImmutableMap<BuildTarget, IjModule> createModules(
TargetGraph targetGraph, IjModuleFactory moduleFactory) {
ImmutableSet<TargetNode<?>> supportedTargets =
FluentIterable.from(targetGraph.getNodes())
.filter(IjModuleFactory.SUPPORTED_MODULE_TYPES_PREDICATE)
.toSet();
ImmutableListMultimap<Path, TargetNode<?>> baseTargetPathMultimap =
FluentIterable.from(supportedTargets)
.index(
new Function<TargetNode<?>, Path>() {
@Override
public Path apply(TargetNode<?> input) {
return input.getBuildTarget().getBasePath();
}
});
ImmutableMap.Builder<BuildTarget, IjModule> moduleMapBuilder = new ImmutableMap.Builder<>();
for (Path baseTargetPath : baseTargetPathMultimap.keySet()) {
ImmutableSet<TargetNode<?>> targets =
FluentIterable.from(baseTargetPathMultimap.get(baseTargetPath)).toSet();
IjModule module = moduleFactory.createModule(baseTargetPath, targets);
for (TargetNode<?> target : targets) {
moduleMapBuilder.put(target.getBuildTarget(), module);
}
}
return moduleMapBuilder.build();
}
private void registerInputsUnderSymlinks(Path buildFile, TargetNode<?> node) throws IOException {
Map<Path, Path> newSymlinksEncountered = Maps.newHashMap();
if (inputFilesUnderSymlink(
node.getInputs(),
node.getRuleFactoryParams().getProjectFilesystem(),
symlinkExistenceCache,
newSymlinksEncountered)) {
ParserConfig.AllowSymlinks allowSymlinks =
Preconditions.checkNotNull(
cellSymlinkAllowability.get(node.getBuildTarget().getCellPath()));
if (allowSymlinks == ParserConfig.AllowSymlinks.FORBID) {
throw new HumanReadableException(
"Target %s contains input files under a path which contains a symbolic link "
+ "(%s). To resolve this, use separate rules and declare dependencies instead of "
+ "using symbolic links.",
node.getBuildTarget(), newSymlinksEncountered);
}
LOG.warn(
"Disabling caching for target %s, because one or more input files are under a "
+ "symbolic link (%s). This will severely impact performance! To resolve this, use "
+ "separate rules and declare dependencies instead of using symbolic links.",
node.getBuildTarget(), newSymlinksEncountered);
buildInputPathsUnderSymlink.add(buildFile);
}
}
/** Verify that owners are correctly detected: - one owner, multiple inputs */
@Test
public void verifyInputsWithOneOwnerAreCorrectlyReported()
throws CmdLineException, IOException, InterruptedException {
FakeProjectFilesystem filesystem =
new FakeProjectFilesystem() {
@Override
public File getFileForRelativePath(String pathRelativeToProjectRoot) {
return new ExistingFile(getRootPath(), pathRelativeToProjectRoot);
}
};
ImmutableSet<String> inputs =
ImmutableSet.of(
"java/somefolder/badfolder/somefile.java",
"java/somefolder/perfect.java",
"com/test/subtest/random.java");
ImmutableSet<Path> inputPaths = MorePaths.asPaths(inputs);
BuildTarget target = BuildTargetFactory.newInstance("//base:name");
TargetNode<?> targetNode = createTargetNode(target, inputPaths);
CommandRunnerParams params = createAuditOwnerCommandRunnerParams(filesystem);
AuditOwnerCommand.OwnersReport report =
AuditOwnerCommand.generateOwnersReport(params, targetNode, inputs, false);
assertTrue(report.nonFileInputs.isEmpty());
assertTrue(report.nonExistentInputs.isEmpty());
assertTrue(report.inputsWithNoOwners.isEmpty());
assertEquals(inputs.size(), report.owners.size());
assertTrue(report.owners.containsKey(targetNode));
assertEquals(targetNode.getInputs(), report.owners.get(targetNode));
}
/** Given a set of target nodes, returns the build targets. */
private static Set<BuildTarget> getTargetsFromNodes(Iterable<TargetNode<?>> input) {
Set<BuildTarget> result = new LinkedHashSet<>();
for (TargetNode<?> node : input) {
result.add(node.getBuildTarget());
}
return result;
}
/**
* @param buildTarget target to process.
* @return the set of {@link BuildTarget}s that must be appended to the dependencies of a node Y
* if node Y depends on X.
*/
public ImmutableSet<BuildTarget> getExportedDepsClosure(BuildTarget buildTarget) {
if (index.containsKey(buildTarget)) {
return index.get(buildTarget);
}
ImmutableSet<BuildTarget> exportedDeps = ImmutableSet.of();
TargetNode<?> targetNode = Preconditions.checkNotNull(targetGraph.get(buildTarget));
if (targetNode.getType().equals(JavaLibraryDescription.TYPE)) {
JavaLibraryDescription.Arg arg = (JavaLibraryDescription.Arg) targetNode.getConstructorArg();
exportedDeps = arg.exportedDeps.get();
} else if (targetNode.getType().equals(AndroidLibraryDescription.TYPE)) {
AndroidLibraryDescription.Arg arg =
(AndroidLibraryDescription.Arg) targetNode.getConstructorArg();
exportedDeps = arg.exportedDeps.get();
}
ImmutableSet<BuildTarget> exportedDepsClosure =
FluentIterable.from(exportedDeps)
.transformAndConcat(
new Function<BuildTarget, Iterable<BuildTarget>>() {
@Override
public Iterable<BuildTarget> apply(BuildTarget input) {
return Iterables.concat(ImmutableSet.of(input), getExportedDepsClosure(input));
}
})
.toSet();
index.put(buildTarget, exportedDepsClosure);
return exportedDepsClosure;
}
private static <T> BuildRule createBuildRule(
TargetGraph targetGraph,
BuildRuleParams params,
BuildRuleResolver ruleResolver,
TargetNode<T> node,
Flavor... flavors) {
BuildTarget target = BuildTarget.builder(params.getBuildTarget()).addFlavors(flavors).build();
Description<T> description = node.getDescription();
T args = node.getConstructorArg();
return description.createBuildRule(
targetGraph,
params.copyWithChanges(target, params.getDeclaredDeps(), params.getExtraDeps()),
ruleResolver,
args);
}
/** Print only targets which were identified as owners in JSON. */
@VisibleForTesting
void printOwnersOnlyJsonReport(CommandRunnerParams params, OwnersReport report)
throws IOException {
final Multimap<String, String> output = TreeMultimap.create();
Set<TargetNode<?>> sortedTargetNodes = report.owners.keySet();
for (TargetNode<?> targetNode : sortedTargetNodes) {
Set<Path> files = report.owners.get(targetNode);
for (Path input : files) {
output.put(input.toString(), targetNode.getBuildTarget().getFullyQualifiedName());
}
}
params.getObjectMapper().writeValue(params.getConsole().getStdOut(), output.asMap());
}
@Test
@SuppressWarnings("unchecked")
public void testWriteModule() throws Exception {
TargetNode<?> guavaTargetNode =
JavaLibraryBuilder.createBuilder(
BuildTargetFactory.newInstance("//third_party/guava:guava"))
.addSrc(Paths.get("third_party/guava/src/Collections.java"))
.build();
TargetNode<?> baseTargetNode =
JavaLibraryBuilder.createBuilder(
BuildTargetFactory.newInstance("//java/com/example/base:base"))
.addDep(guavaTargetNode.getBuildTarget())
.addSrc(Paths.get("java/com/example/base/Base.java"))
.build();
IjModuleGraph moduleGraph =
IjModuleGraphTest.createModuleGraph(ImmutableSet.of(guavaTargetNode, baseTargetNode));
IjModule baseModule = IjModuleGraphTest.getModuleForTarget(moduleGraph, baseTargetNode);
IjProjectTemplateDataPreparer dataPreparer =
new IjProjectTemplateDataPreparer(javaPackageFinder, moduleGraph, filesystem);
ContentRoot contentRoot = dataPreparer.getContentRoot(baseModule);
assertEquals("file://$MODULE_DIR$/../../java/com/example/base", contentRoot.getUrl());
IjSourceFolder baseSourceFolder = contentRoot.getFolders().first();
assertEquals("sourceFolder", baseSourceFolder.getType());
assertFalse(baseSourceFolder.getIsTestSource());
assertEquals("com.example.base", baseSourceFolder.getPackagePrefix());
assertEquals("file://$MODULE_DIR$/../../java/com/example/base", baseSourceFolder.getUrl());
assertThat(
dataPreparer.getDependencies(baseModule),
contains(
allOf(
hasProperty("type", equalTo(IjDependencyListBuilder.Type.MODULE)),
hasProperty(
"data",
equalTo(
Optional.of(
DependencyEntryData.builder()
.setName("third_party_guava")
.setScope(IjDependencyListBuilder.Scope.COMPILE)
.setExported(false)
.build())))),
allOf(hasProperty("type", equalTo(IjDependencyListBuilder.Type.SOURCE_FOLDER)))));
}
@Test
public void testModuleIndex() throws Exception {
TargetNode<?> guavaTargetNode =
PrebuiltJarBuilder.createBuilder(
BuildTargetFactory.newInstance("//third-party/guava:guava"))
.setBinaryJar(Paths.get("third-party/guava/guava.jar"))
.build();
TargetNode<?> baseTargetNode =
JavaLibraryBuilder.createBuilder(
BuildTargetFactory.newInstance("//java/com/example/base:base"))
.addDep(guavaTargetNode.getBuildTarget())
.addSrc(Paths.get("java/com/example/base/Base.java"))
.build();
TargetNode<?> baseTestsTargetNode =
JavaTestBuilder.createBuilder(
BuildTargetFactory.newInstance("//javatests/com/example/base:base"))
.addDep(baseTargetNode.getBuildTarget())
.addSrc(Paths.get("javatests/com/example/base/Base.java"))
.build();
IjModuleGraph moduleGraph =
IjModuleGraphTest.createModuleGraph(
ImmutableSet.of(guavaTargetNode, baseTargetNode, baseTestsTargetNode));
IjProjectTemplateDataPreparer dataPreparer =
new IjProjectTemplateDataPreparer(javaPackageFinder, moduleGraph, filesystem);
// Libraries don't go into the index.
assertEquals(
ImmutableSet.of(
ModuleIndexEntry.builder()
.setFileUrl("file://$PROJECT_DIR$/.idea/modules/project_root.iml")
.setFilePath(Paths.get(".idea/modules/project_root.iml"))
.build(),
ModuleIndexEntry.builder()
.setGroup("modules")
.setFileUrl("file://$PROJECT_DIR$/.idea/modules/java_com_example_base.iml")
.setFilePath(Paths.get(".idea/modules/java_com_example_base.iml"))
.build(),
ModuleIndexEntry.builder()
.setGroup("modules")
.setFileUrl("file://$PROJECT_DIR$/.idea/modules/javatests_com_example_base.iml")
.setFilePath(Paths.get(".idea/modules/javatests_com_example_base.iml"))
.build()),
dataPreparer.getModuleIndexEntries());
}
@Test
public void testImplicitDepsAreAddedCorrectly() throws NoSuchBuildTargetException {
Description<GenruleDescription.Arg> genruleDescription = new GenruleDescription();
Map<String, Object> instance =
ImmutableMap.<String, Object>of(
"srcs", ImmutableList.of(":baz", "//biz:baz"),
"out", "AndroidManifest.xml",
"cmd", "$(exe //bin:executable) $(location :arg)");
ProjectFilesystem projectFilesystem = new AllExistingProjectFilesystem();
BuildRuleFactoryParams params =
new BuildRuleFactoryParams(projectFilesystem, BuildTargetFactory.newInstance("//foo:bar"));
ConstructorArgMarshaller marshaller =
new ConstructorArgMarshaller(
new DefaultTypeCoercerFactory(ObjectMappers.newDefaultInstance()));
ImmutableSet.Builder<BuildTarget> declaredDeps = ImmutableSet.builder();
ImmutableSet.Builder<VisibilityPattern> visibilityPatterns = ImmutableSet.builder();
GenruleDescription.Arg constructorArg = genruleDescription.createUnpopulatedConstructorArg();
try {
marshaller.populate(
createCellRoots(projectFilesystem),
projectFilesystem,
params,
constructorArg,
declaredDeps,
visibilityPatterns,
instance);
} catch (ConstructorArgMarshalException e) {
fail("Expected constructorArg to be correctly populated.");
}
TargetNode<GenruleDescription.Arg> targetNode =
new TargetNodeFactory(new DefaultTypeCoercerFactory(ObjectMappers.newDefaultInstance()))
.create(
Hashing.sha1().hashString(params.target.getFullyQualifiedName(), UTF_8),
genruleDescription,
constructorArg,
params,
declaredDeps.build(),
visibilityPatterns.build(),
createCellRoots(projectFilesystem));
assertEquals(
"SourcePaths and targets from cmd string should be extracted as extra deps.",
ImmutableSet.of("//foo:baz", "//biz:baz", "//bin:executable", "//foo:arg"),
FluentIterable.from(targetNode.getExtraDeps())
.transform(Functions.toStringFunction())
.toSet());
}
/** Print detailed report on all owners. */
private void printFullReport(CommandRunnerParams params, OwnersReport report) {
PrintStream out = params.getConsole().getStdOut();
Ansi ansi = params.getConsole().getAnsi();
if (report.owners.isEmpty()) {
out.println(ansi.asErrorText("No owners found"));
} else {
out.println(ansi.asSuccessText("Owners:"));
for (TargetNode<?> targetNode : report.owners.keySet()) {
out.println(targetNode.getBuildTarget().getFullyQualifiedName());
Set<Path> files = report.owners.get(targetNode);
for (Path input : files) {
out.println(FILE_INDENT + input);
}
}
}
if (!report.inputsWithNoOwners.isEmpty()) {
out.println();
out.println(ansi.asErrorText("Files without owners:"));
for (Path input : report.inputsWithNoOwners) {
out.println(FILE_INDENT + input);
}
}
if (!report.nonExistentInputs.isEmpty()) {
out.println();
out.println(ansi.asErrorText("Non existent files:"));
for (String input : report.nonExistentInputs) {
out.println(FILE_INDENT + input);
}
}
if (!report.nonFileInputs.isEmpty()) {
out.println();
out.println(ansi.asErrorText("Non-file inputs:"));
for (String input : report.nonFileInputs) {
out.println(FILE_INDENT + input);
}
}
}
@Test
public void testImplicitDepsAreAddedCorrectly()
throws NoSuchBuildTargetException, TargetNode.InvalidSourcePathInputException {
Description<GenruleDescription.Arg> genruleDescription = new GenruleDescription();
Map<String, Object> instance =
ImmutableMap.<String, Object>of(
"srcs", ImmutableList.of(":baz", "//biz:baz"),
"out", "AndroidManifest.xml",
"cmd", "$(exe //bin:executable) $(location :arg)");
ProjectFilesystem projectFilesystem = new AllExistingProjectFilesystem();
BuildRuleFactoryParams params =
new BuildRuleFactoryParams(
projectFilesystem,
BuildTargetFactory.newInstance("//foo:bar"),
new InMemoryBuildFileTree(ImmutableList.<BuildTarget>of()),
/* enforeBuckBoundaryCheck */ true);
ConstructorArgMarshaller marshaller = new ConstructorArgMarshaller();
ImmutableSet.Builder<BuildTarget> declaredDeps = ImmutableSet.builder();
ImmutableSet.Builder<BuildTargetPattern> visibilityPatterns = ImmutableSet.builder();
GenruleDescription.Arg constructorArg = genruleDescription.createUnpopulatedConstructorArg();
try {
marshaller.populate(
projectFilesystem, params, constructorArg, declaredDeps, visibilityPatterns, instance);
} catch (ConstructorArgMarshalException e) {
fail("Expected constructorArg to be correctly populated.");
}
TargetNode<GenruleDescription.Arg> targetNode =
new TargetNode<>(
genruleDescription,
constructorArg,
params,
declaredDeps.build(),
visibilityPatterns.build());
assertEquals(
"SourcePaths and targets from cmd string should be extracted as extra deps.",
ImmutableSet.of("//foo:baz", "//biz:baz", "//bin:executable", "//foo:arg"),
FluentIterable.from(targetNode.getExtraDeps())
.transform(Functions.toStringFunction())
.toSet());
}
@VisibleForTesting
static OwnersReport generateOwnersReport(
CommandRunnerParams params,
TargetNode<?> targetNode,
Iterable<String> filePaths,
boolean guessForDeletedEnabled) {
// Process arguments assuming they are all relative file paths.
Set<Path> inputs = Sets.newHashSet();
Set<String> nonExistentInputs = Sets.newHashSet();
Set<String> nonFileInputs = Sets.newHashSet();
for (String filePath : filePaths) {
File file = params.getCell().getFilesystem().getFileForRelativePath(filePath);
if (!file.exists()) {
nonExistentInputs.add(filePath);
} else if (!file.isFile()) {
nonFileInputs.add(filePath);
} else {
inputs.add(Paths.get(filePath));
}
}
// Try to find owners for each valid and existing file.
Set<Path> inputsWithNoOwners = Sets.newHashSet(inputs);
SetMultimap<TargetNode<?>, Path> owners = TreeMultimap.create();
for (final Path commandInput : inputs) {
Predicate<Path> startsWith =
new Predicate<Path>() {
@Override
public boolean apply(Path input) {
return !commandInput.equals(input) && commandInput.startsWith(input);
}
};
Set<Path> ruleInputs = targetNode.getInputs();
if (ruleInputs.contains(commandInput)
|| FluentIterable.from(ruleInputs).anyMatch(startsWith)) {
inputsWithNoOwners.remove(commandInput);
owners.put(targetNode, commandInput);
}
}
// Try to guess owners for nonexistent files.
if (guessForDeletedEnabled) {
for (String nonExistentInput : nonExistentInputs) {
owners.put(targetNode, new File(nonExistentInput).toPath());
}
}
return new OwnersReport(owners, inputsWithNoOwners, nonExistentInputs, nonFileInputs);
}
@Test
public void testCreatePackageLookupPahtSet() {
TargetNode<?> guavaTargetNode =
PrebuiltJarBuilder.createBuilder(BuildTargetFactory.newInstance("//lib:guava"))
.setBinaryJar(Paths.get("lib/guava.jar"))
.build();
Path sourcePath = Paths.get("java/com/src/Source.java");
Path subSourcePath = Paths.get("java/com/src/subpackage/SubSource.java");
TargetNode<?> srcTargetNode =
JavaLibraryBuilder.createBuilder(BuildTargetFactory.newInstance("//java/com/src:src"))
.addDep(guavaTargetNode.getBuildTarget())
.addSrc(sourcePath)
.addSrc(subSourcePath)
.build();
IjModuleGraph moduleGraph =
IjModuleGraphTest.createModuleGraph(ImmutableSet.of(guavaTargetNode, srcTargetNode));
assertThat(
IjProjectTemplateDataPreparer.createPackageLookupPathSet(moduleGraph),
containsInAnyOrder(subSourcePath, sourcePath));
}
/**
* Create all the modules we are capable of representing in IntelliJ from the supplied graph.
*
* @param targetGraph graph whose nodes will be converted to {@link IjModule}s.
* @return map which for every BuildTarget points to the corresponding IjModule. Multiple
* BuildTarget can point to one IjModule (many:one mapping), the BuildTargets which can't be
* prepresented in IntelliJ are missing from this mapping.
*/
private static ImmutableMap<BuildTarget, IjModule> createModules(
TargetGraph targetGraph,
IjModuleFactory moduleFactory,
final Function<Path, Path> basePathTransform) {
ImmutableSet<TargetNode<?>> supportedTargets =
FluentIterable.from(targetGraph.getNodes())
.filter(IjModuleFactory.SUPPORTED_MODULE_TYPES_PREDICATE)
.toSet();
ImmutableListMultimap<Path, TargetNode<?>> baseTargetPathMultimap =
FluentIterable.from(supportedTargets)
.index(
new Function<TargetNode<?>, Path>() {
@Override
public Path apply(TargetNode<?> input) {
if (!(input.getConstructorArg() instanceof AndroidResourceDescription.Arg)) {
return basePathTransform.apply(input.getBuildTarget().getBasePath());
}
return input.getBuildTarget().getBasePath();
}
});
ImmutableMap.Builder<BuildTarget, IjModule> moduleMapBuilder = new ImmutableMap.Builder<>();
for (Path baseTargetPath : baseTargetPathMultimap.keySet()) {
ImmutableSet<TargetNode<?>> targets =
FluentIterable.from(baseTargetPathMultimap.get(baseTargetPath)).toSet();
IjModule module = moduleFactory.createModule(baseTargetPath, targets);
for (TargetNode<?> target : targets) {
moduleMapBuilder.put(target.getBuildTarget(), module);
}
}
return moduleMapBuilder.build();
}
@Override
public void run() {
while (shouldWaitForWork()) {
try (BuildTargetProcessingScope processingScope = startProcessingBuildTarget()) {
// If we've already parsed this in another thread, we can skip doing any work.
if (permState.hasCachedTargetNodeForBuildTarget(processingScope.getBuildTarget())) {
continue;
}
TargetNode<?> node;
try (SimplePerfEvent.Scope scope =
Parser.getTargetNodeEventScope(eventBus, processingScope.getBuildTarget())) {
try {
node = getTargetNode(processingScope.getBuildTarget());
} catch (BuildFileParseException | BuildTargetException | IOException e) {
// It's okay to not raise this further up because in `Parser` we build the target
// graph and while doing so will hit the same error (the parsing will have been
// cached).
abortDoingMoreWork();
return;
}
}
processingScope.addDepsToProcess(
FluentIterable.from(node.getDeps())
.filter(permState.getHasCachedTargetNodeForBuildTargetPredicate())
.toSet());
} catch (TimeoutException e) {
// We timed out waiting to process something on the queue. This could mean we are done,
// so run through the while statement again.
continue;
} catch (InterruptedException e) {
abortDoingMoreWork();
return;
}
}
}
private <A extends Arg> BuildRule getFlavoredBinaryRule(
TargetGraph targetGraph,
final BuildRuleParams params,
final BuildRuleResolver resolver,
A args) {
// Cxx targets must have one Platform Flavor set otherwise nothing gets compiled.
ImmutableSet<Flavor> flavors =
params
.getBuildTarget()
.withoutFlavors(ImmutableSet.of(ReactNativeFlavors.DO_NOT_BUNDLE))
.getFlavors();
if (!cxxPlatformFlavorDomain.containsAnyOf(flavors)) {
flavors =
new ImmutableSet.Builder<Flavor>()
.addAll(flavors)
.add(defaultCxxPlatform.getFlavor())
.build();
}
final TargetNode<?> binaryTargetNode = Preconditions.checkNotNull(targetGraph.get(args.binary));
// If the binary target of the AppleBundle is an AppleLibrary then the build flavor
// must be specified.
if (binaryTargetNode.getDescription() instanceof AppleLibraryDescription
&& (Sets.intersection(
SUPPORTED_LIBRARY_FLAVORS, binaryTargetNode.getBuildTarget().getFlavors())
.size()
!= 1)) {
throw new HumanReadableException(
"AppleExtension bundle [%s] must have exactly one of these flavors: [%s].",
binaryTargetNode.getBuildTarget().toString(),
Joiner.on(", ").join(SUPPORTED_LIBRARY_FLAVORS));
}
BuildRuleParams binaryRuleParams =
new BuildRuleParams(
args.binary,
Suppliers.ofInstance(
BuildRules.toBuildRulesFor(
params.getBuildTarget(), resolver, binaryTargetNode.getDeclaredDeps())),
Suppliers.ofInstance(
BuildRules.toBuildRulesFor(
params.getBuildTarget(), resolver, binaryTargetNode.getExtraDeps())),
params.getProjectFilesystem(),
params.getCellRoots(),
params.getRuleKeyBuilderFactory());
return CxxDescriptionEnhancer.requireBuildRule(
targetGraph, binaryRuleParams, resolver, flavors.toArray(new Flavor[0]));
}
/** Print only targets which were identified as owners. */
private void printOwnersOnlyReport(CommandRunnerParams params, OwnersReport report) {
Set<TargetNode<?>> sortedTargetNodes = report.owners.keySet();
for (TargetNode<?> targetNode : sortedTargetNodes) {
params.getConsole().getStdOut().println(targetNode.getBuildTarget().getFullyQualifiedName());
}
}
@Test
public void testExcludePaths() throws Exception {
/**
* Fake filesystem structure .idea |- misc.xml .git |- HEAD java |- com |- BUCK |- data |-
* wsad.txt |- src |- BUCK |- Source.java |- foo |- Foo.java |- src2 |- Code.java |- org |- bar
* |- Bar.java lib |- BUCK |- guava.jar
*/
ImmutableSet<Path> paths =
ImmutableSet.of(
Paths.get(".idea/misc.xml"),
Paths.get(".git/HEAD"),
Paths.get("java/com/BUCK"),
Paths.get("java/com/data/wsad.txt"),
Paths.get("java/com/src/BUCK"),
Paths.get("java/com/src/Source.java"),
Paths.get("java/com/src/foo/Foo.java"),
Paths.get("java/org/bar/Bar.java"),
Paths.get("lib/BUCK"),
Paths.get("lib/guava.jar"));
FakeProjectFilesystem filesystemForExcludesTest =
new FakeProjectFilesystem(new FakeClock(0), Paths.get(".").toAbsolutePath(), paths);
TargetNode<?> guavaTargetNode =
PrebuiltJarBuilder.createBuilder(BuildTargetFactory.newInstance("//lib:guava"))
.setBinaryJar(Paths.get("lib/guava.jar"))
.build();
TargetNode<?> srcTargetNode =
JavaLibraryBuilder.createBuilder(BuildTargetFactory.newInstance("//java/com/src:src"))
.addDep(guavaTargetNode.getBuildTarget())
.addSrc(Paths.get("java/com/src/Source.java"))
.build();
TargetNode<?> src2TargetNode =
JavaLibraryBuilder.createBuilder(BuildTargetFactory.newInstance("//java/com:src2"))
.addDep(guavaTargetNode.getBuildTarget())
.addSrc(Paths.get("java/com/src2/Code.java"))
.build();
TargetNode<?> rootTargetNode =
JavaLibraryBuilder.createBuilder(BuildTargetFactory.newInstance("//:root")).build();
IjModuleGraph moduleGraph =
IjModuleGraphTest.createModuleGraph(
ImmutableSet.of(guavaTargetNode, srcTargetNode, src2TargetNode, rootTargetNode));
IjModule srcModule = IjModuleGraphTest.getModuleForTarget(moduleGraph, srcTargetNode);
IjModule src2Module = IjModuleGraphTest.getModuleForTarget(moduleGraph, src2TargetNode);
IjModule rootModule = IjModuleGraphTest.getModuleForTarget(moduleGraph, rootTargetNode);
IjProjectTemplateDataPreparer dataPreparer =
new IjProjectTemplateDataPreparer(
javaPackageFinder, moduleGraph, filesystemForExcludesTest);
assertEquals(
ImmutableSet.of(Paths.get("java/com/src/foo")),
distillExcludeFolders(dataPreparer.createExcludes(srcModule)));
assertEquals(
ImmutableSet.of(Paths.get("java/com/data")),
distillExcludeFolders(dataPreparer.createExcludes(src2Module)));
// In this case it's fine to exclude "lib" as there is no source code there.
assertEquals(
ImmutableSet.of(Paths.get(".git"), Paths.get("java/org"), Paths.get("lib")),
distillExcludeFolders(dataPreparer.createExcludes(rootModule)));
}
@Test
@SuppressWarnings("unchecked")
public void testDependencies() throws Exception {
TargetNode<?> hamcrestTargetNode =
PrebuiltJarBuilder.createBuilder(
BuildTargetFactory.newInstance("//third-party/hamcrest:hamcrest"))
.setBinaryJar(Paths.get("third-party/hamcrest/hamcrest.jar"))
.build();
TargetNode<?> guavaTargetNode =
PrebuiltJarBuilder.createBuilder(
BuildTargetFactory.newInstance("//third-party/guava:guava"))
.setBinaryJar(Paths.get("third-party/guava/guava.jar"))
.build();
TargetNode<?> baseTargetNode =
JavaLibraryBuilder.createBuilder(
BuildTargetFactory.newInstance("//java/com/example/base:base"))
.addDep(guavaTargetNode.getBuildTarget())
.addSrc(Paths.get("java/com/example/base/Base.java"))
.build();
TargetNode<?> baseGenruleTarget =
GenruleBuilder.newGenruleBuilder(
BuildTargetFactory.newInstance("//java/com/example/base:genrule"))
.build();
TargetNode<?> baseInlineTestsTargetNode =
JavaLibraryBuilder.createBuilder(
BuildTargetFactory.newInstance("//java/com/example/base:tests"))
.addDep(hamcrestTargetNode.getBuildTarget())
.addSrc(Paths.get("java/com/example/base/TestBase.java"))
.addSrcTarget(baseGenruleTarget.getBuildTarget())
.build();
TargetNode<?> baseTestsTargetNode =
JavaTestBuilder.createBuilder(
BuildTargetFactory.newInstance("//javatests/com/example/base:base"))
.addDep(baseTargetNode.getBuildTarget())
.addDep(hamcrestTargetNode.getBuildTarget())
.addSrc(Paths.get("javatests/com/example/base/Base.java"))
.build();
IjModuleGraph moduleGraph =
IjModuleGraphTest.createModuleGraph(
ImmutableSet.of(
hamcrestTargetNode,
guavaTargetNode,
baseTargetNode,
baseGenruleTarget,
baseInlineTestsTargetNode,
baseTestsTargetNode),
ImmutableMap.<TargetNode<?>, Path>of(
baseInlineTestsTargetNode, Paths.get("buck-out/baseInlineTests.jar")),
Functions.constant(Optional.<Path>absent()));
IjLibrary hamcrestLibrary =
IjModuleGraphTest.getLibraryForTarget(moduleGraph, hamcrestTargetNode);
IjLibrary guavaLibrary = IjModuleGraphTest.getLibraryForTarget(moduleGraph, guavaTargetNode);
IjModule baseModule = IjModuleGraphTest.getModuleForTarget(moduleGraph, baseTargetNode);
IjModule baseTestModule =
IjModuleGraphTest.getModuleForTarget(moduleGraph, baseTestsTargetNode);
IjProjectTemplateDataPreparer dataPreparer =
new IjProjectTemplateDataPreparer(javaPackageFinder, moduleGraph, filesystem);
assertEquals(
IjModuleGraphTest.getModuleForTarget(moduleGraph, baseInlineTestsTargetNode),
IjModuleGraphTest.getModuleForTarget(moduleGraph, baseTargetNode));
DependencyEntryData.Builder dependencyEntryBuilder =
DependencyEntryData.builder().setExported(false);
assertThat(
dataPreparer.getDependencies(baseModule),
contains(
allOf(
hasProperty("type", equalTo(IjDependencyListBuilder.Type.LIBRARY)),
hasProperty(
"data",
equalTo(
Optional.of(
dependencyEntryBuilder
.setName(guavaLibrary.getName())
.setScope(IjDependencyListBuilder.Scope.COMPILE)
.build())))),
allOf(
hasProperty("type", equalTo(IjDependencyListBuilder.Type.LIBRARY)),
hasProperty(
"data",
equalTo(
Optional.of(
dependencyEntryBuilder
.setName(hamcrestLibrary.getName())
.setScope(IjDependencyListBuilder.Scope.COMPILE)
.build())))),
allOf(hasProperty("type", equalTo(IjDependencyListBuilder.Type.SOURCE_FOLDER))),
allOf(
hasProperty("type", equalTo(IjDependencyListBuilder.Type.LIBRARY)),
hasProperty(
"data",
equalTo(
Optional.of(
DependencyEntryData.builder()
.setExported(true)
.setName("library_java_com_example_base_tests")
.setScope(IjDependencyListBuilder.Scope.PROVIDED)
.build()))))));
assertThat(
dataPreparer.getDependencies(baseTestModule),
contains(
allOf(
hasProperty("type", equalTo(IjDependencyListBuilder.Type.MODULE)),
hasProperty(
"data",
equalTo(
Optional.of(
dependencyEntryBuilder
.setName(baseModule.getName())
.setScope(IjDependencyListBuilder.Scope.TEST)
.build())))),
allOf(
hasProperty("type", equalTo(IjDependencyListBuilder.Type.LIBRARY)),
hasProperty(
"data",
equalTo(
Optional.of(
dependencyEntryBuilder
.setName(hamcrestLibrary.getName())
.setScope(IjDependencyListBuilder.Scope.TEST)
.build())))),
allOf(hasProperty("type", equalTo(IjDependencyListBuilder.Type.SOURCE_FOLDER)))));
}
@Override
public int runWithoutHelp(CommandRunnerParams params) throws IOException, InterruptedException {
LOG.debug("Running with arguments %s", getArguments());
try (CommandThreadManager pool =
new CommandThreadManager(
"Test",
params.getBuckConfig().getWorkQueueExecutionOrder(),
getConcurrencyLimit(params.getBuckConfig()))) {
// Post the build started event, setting it to the Parser recorded start time if appropriate.
BuildEvent.Started started = BuildEvent.started(getArguments());
if (params.getParser().getParseStartTime().isPresent()) {
params.getBuckEventBus().post(started, params.getParser().getParseStartTime().get());
} else {
params.getBuckEventBus().post(started);
}
// The first step is to parse all of the build files. This will populate the parser and find
// all of the test rules.
TargetGraph targetGraph;
ImmutableSet<BuildTarget> explicitBuildTargets;
try {
// If the user asked to run all of the tests, parse all of the build files looking for any
// test rules.
if (isRunAllTests()) {
targetGraph =
params
.getParser()
.buildTargetGraphForTargetNodeSpecs(
params.getBuckEventBus(),
params.getCell(),
getEnableProfiling(),
pool.getExecutor(),
ImmutableList.of(
TargetNodePredicateSpec.of(
new Predicate<TargetNode<?>>() {
@Override
public boolean apply(TargetNode<?> input) {
return input.getType().isTestRule();
}
},
BuildFileSpec.fromRecursivePath(Paths.get("")))))
.getSecond();
explicitBuildTargets = ImmutableSet.of();
// Otherwise, the user specified specific test targets to build and run, so build a graph
// around these.
} else {
LOG.debug("Parsing graph for arguments %s", getArguments());
Pair<ImmutableSet<BuildTarget>, TargetGraph> result =
params
.getParser()
.buildTargetGraphForTargetNodeSpecs(
params.getBuckEventBus(),
params.getCell(),
getEnableProfiling(),
pool.getExecutor(),
parseArgumentsAsTargetNodeSpecs(params.getBuckConfig(), getArguments()));
targetGraph = result.getSecond();
explicitBuildTargets = result.getFirst();
LOG.debug("Got explicit build targets %s", explicitBuildTargets);
ImmutableSet.Builder<BuildTarget> testTargetsBuilder = ImmutableSet.builder();
for (TargetNode<?> node : targetGraph.getAll(explicitBuildTargets)) {
ImmutableSortedSet<BuildTarget> nodeTests = TargetNodes.getTestTargetsForNode(node);
if (!nodeTests.isEmpty()) {
LOG.debug("Got tests for target %s: %s", node.getBuildTarget(), nodeTests);
testTargetsBuilder.addAll(nodeTests);
}
}
ImmutableSet<BuildTarget> testTargets = testTargetsBuilder.build();
if (!testTargets.isEmpty()) {
LOG.debug("Got related test targets %s, building new target graph...", testTargets);
targetGraph =
params
.getParser()
.buildTargetGraph(
params.getBuckEventBus(),
params.getCell(),
getEnableProfiling(),
pool.getExecutor(),
Iterables.concat(explicitBuildTargets, testTargets));
LOG.debug("Finished building new target graph with tests.");
}
}
} catch (BuildTargetException | BuildFileParseException e) {
params
.getBuckEventBus()
.post(ConsoleEvent.severe(MoreExceptions.getHumanReadableOrLocalizedMessage(e)));
return 1;
}
TargetGraphToActionGraph targetGraphToActionGraph =
new TargetGraphToActionGraph(
params.getBuckEventBus(), new BuildTargetNodeToBuildRuleTransformer());
Pair<ActionGraph, BuildRuleResolver> actionGraphAndResolver =
Preconditions.checkNotNull(targetGraphToActionGraph.apply(targetGraph));
// Look up all of the test rules in the action graph.
Iterable<TestRule> testRules =
Iterables.filter(actionGraphAndResolver.getFirst().getNodes(), TestRule.class);
// Unless the user requests that we build filtered tests, filter them out here, before
// the build.
if (!isBuildFiltered(params.getBuckConfig())) {
testRules = filterTestRules(params.getBuckConfig(), explicitBuildTargets, testRules);
}
if (isDryRun()) {
printMatchingTestRules(params.getConsole(), testRules);
}
CachingBuildEngine cachingBuildEngine =
new CachingBuildEngine(
pool.getExecutor(),
params.getFileHashCache(),
getBuildEngineMode().or(params.getBuckConfig().getBuildEngineMode()),
params.getBuckConfig().getDependencySchedulingOrder(),
params.getBuckConfig().getBuildDepFiles(),
params.getBuckConfig().getBuildMaxDepFileCacheEntries(),
actionGraphAndResolver.getSecond());
try (Build build =
createBuild(
params.getBuckConfig(),
actionGraphAndResolver.getFirst(),
actionGraphAndResolver.getSecond(),
params.getAndroidPlatformTargetSupplier(),
cachingBuildEngine,
params.getArtifactCache(),
params.getConsole(),
params.getBuckEventBus(),
getTargetDeviceOptional(),
params.getPlatform(),
params.getEnvironment(),
params.getObjectMapper(),
params.getClock(),
Optional.of(getAdbOptions(params.getBuckConfig())),
Optional.of(getTargetDeviceOptions()))) {
// Build all of the test rules.
int exitCode =
build.executeAndPrintFailuresToEventBus(
testRules,
isKeepGoing(),
params.getBuckEventBus(),
params.getConsole(),
getPathToBuildReport(params.getBuckConfig()));
params.getBuckEventBus().post(BuildEvent.finished(started, exitCode));
if (exitCode != 0) {
return exitCode;
}
// If the user requests that we build tests that we filter out, then we perform
// the filtering here, after we've done the build but before we run the tests.
if (isBuildFiltered(params.getBuckConfig())) {
testRules = filterTestRules(params.getBuckConfig(), explicitBuildTargets, testRules);
}
// Once all of the rules are built, then run the tests.
Optional<ImmutableList<String>> externalTestRunner =
params.getBuckConfig().getExternalTestRunner();
if (externalTestRunner.isPresent()) {
return runTestsExternal(params, build, externalTestRunner.get(), testRules);
}
return runTestsInternal(params, cachingBuildEngine, build, testRules);
}
}
}
