/**
* Returns the ShellCommand object that is supposed to generate a code coverage report from data
* obtained during the test run. This method will also generate a set of source paths to the class
* files tested during the test run.
*/
private static Step getReportCommand(
ImmutableSet<JavaLibrary> rulesUnderTest,
Optional<DefaultJavaPackageFinder> defaultJavaPackageFinderOptional,
ProjectFilesystem filesystem,
Path outputDirectory,
CoverageReportFormat format) {
ImmutableSet.Builder<String> srcDirectories = ImmutableSet.builder();
ImmutableSet.Builder<Path> pathsToClasses = ImmutableSet.builder();
// Add all source directories of java libraries that we are testing to -sourcepath.
for (JavaLibrary rule : rulesUnderTest) {
ImmutableSet<String> sourceFolderPath =
getPathToSourceFolders(rule, defaultJavaPackageFinderOptional, filesystem);
if (!sourceFolderPath.isEmpty()) {
srcDirectories.addAll(sourceFolderPath);
}
Path pathToOutput = rule.getPathToOutput();
if (pathToOutput == null) {
continue;
}
pathsToClasses.add(pathToOutput);
}
return new GenerateCodeCoverageReportStep(
srcDirectories.build(), pathsToClasses.build(), outputDirectory, format);
}
/**
* Creates/finds the set of build rules that correspond to pre-dex'd artifacts that should be
* merged to create the final classes.dex for the APK.
*
* <p>This method may modify {@code ruleResolver}, inserting new rules into its index.
*/
@VisibleForTesting
BuildRule createPreDexMergeRule(UberRDotJava uberRDotJava) {
ImmutableSet.Builder<DexProducedFromJavaLibrary> preDexDeps = ImmutableSet.builder();
ImmutableSet<JavaLibrary> transitiveJavaDeps =
Classpaths.getClasspathEntries(originalDeps).keySet();
for (JavaLibrary javaLibrary : transitiveJavaDeps) {
// If the rule has no output file (which happens when a java_library has no srcs or
// resources, but export_deps is true), then there will not be anything to dx.
if (javaLibrary.getPathToOutputFile() == null) {
continue;
}
// If the rule is in the no_dx list, then do not pre-dex it.
if (buildRulesToExcludeFromDex.contains(javaLibrary.getBuildTarget())) {
continue;
}
// See whether the corresponding IntermediateDexRule has already been added to the
// ruleResolver.
BuildTarget originalTarget = javaLibrary.getBuildTarget();
BuildTarget preDexTarget =
new BuildTarget(originalTarget.getBaseName(), originalTarget.getShortName(), DEX_FLAVOR);
BuildRule preDexRule = ruleResolver.get(preDexTarget);
if (preDexRule != null) {
preDexDeps.add((DexProducedFromJavaLibrary) preDexRule.getBuildable());
continue;
}
// Create the IntermediateDexRule and add it to both the ruleResolver and preDexDeps.
DexProducedFromJavaLibrary preDex = new DexProducedFromJavaLibrary(preDexTarget, javaLibrary);
buildRuleAndAddToIndex(
preDex,
BuildRuleType.PRE_DEX,
preDexTarget,
ImmutableSortedSet.of(ruleResolver.get(javaLibrary.getBuildTarget())));
preDexDeps.add(preDex);
}
ImmutableSet<DexProducedFromJavaLibrary> allPreDexDeps = preDexDeps.build();
BuildTarget buildTargetForDexMerge = createBuildTargetWithFlavor(DEX_MERGE_FLAVOR);
PreDexMerge preDexMerge =
new PreDexMerge(
buildTargetForDexMerge, primaryDexPath, dexSplitMode, allPreDexDeps, uberRDotJava);
BuildRule preDexMergeBuildRule =
buildRuleAndAddToIndex(
preDexMerge,
BuildRuleType.DEX_MERGE,
buildTargetForDexMerge,
getDexMergeDeps(uberRDotJava, allPreDexDeps));
return preDexMergeBuildRule;
}
@Test
public void testGetBuildStepsWhenThereAreNoClassesToDex()
throws IOException, InterruptedException {
JavaLibrary javaLibrary = createMock(JavaLibrary.class);
expect(javaLibrary.getClassNamesToHashes())
.andReturn(ImmutableSortedMap.<String, HashCode>of());
BuildContext context = createMock(BuildContext.class);
FakeBuildableContext buildableContext = new FakeBuildableContext();
ProjectFilesystem projectFilesystem = createMock(ProjectFilesystem.class);
replayAll();
BuildTarget buildTarget = BuildTargetFactory.newInstance("//foo:bar");
BuildRuleParams params =
new FakeBuildRuleParamsBuilder(buildTarget).setProjectFilesystem(projectFilesystem).build();
DexProducedFromJavaLibrary preDex =
new DexProducedFromJavaLibrary(
params, new SourcePathResolver(new BuildRuleResolver()), javaLibrary);
List<Step> steps = preDex.getBuildSteps(context, buildableContext);
verifyAll();
resetAll();
expect(projectFilesystem.resolve(Paths.get("buck-out/gen/foo")))
.andReturn(Paths.get("/home/user/buck-out/gen/foo"));
expect(projectFilesystem.resolve(Paths.get("buck-out/gen/foo/bar.dex.jar")))
.andReturn(Paths.get("/home/user/buck-out/gen/foo/bar.dex.jar"));
replayAll();
ExecutionContext executionContext = TestExecutionContext.newBuilder().build();
MoreAsserts.assertSteps(
"Do not generate a .dex.jar file.",
ImmutableList.of(
String.format("rm -f %s", Paths.get("/home/user/buck-out/gen/foo/bar.dex.jar")),
String.format("mkdir -p %s", Paths.get("/home/user/buck-out/gen/foo")),
"record_empty_dx"),
steps,
executionContext);
verifyAll();
resetAll();
replayAll();
Step recordArtifactAndMetadataStep = steps.get(2);
assertThat(recordArtifactAndMetadataStep.getShortName(), startsWith("record_"));
int exitCode = recordArtifactAndMetadataStep.execute(executionContext);
assertEquals(0, exitCode);
verifyAll();
}
@Test
public void testObserverMethods() {
JavaLibrary accumulateClassNames = createMock(JavaLibrary.class);
expect(accumulateClassNames.getClassNamesToHashes())
.andReturn(ImmutableSortedMap.of("com/example/Foo", HashCode.fromString("cafebabe")))
.anyTimes();
replayAll();
BuildTarget buildTarget = BuildTargetFactory.newInstance("//foo:bar");
BuildRuleParams params = new FakeBuildRuleParamsBuilder(buildTarget).build();
DexProducedFromJavaLibrary preDexWithClasses =
new DexProducedFromJavaLibrary(
params, new SourcePathResolver(new BuildRuleResolver()), accumulateClassNames);
assertNull(preDexWithClasses.getPathToOutput());
assertEquals(Paths.get("buck-out/gen/foo/bar.dex.jar"), preDexWithClasses.getPathToDex());
verifyAll();
}
private Module createModuleForProjectConfig(ProjectConfig projectConfig) throws IOException {
BuildRule projectRule = projectConfig.getProjectRule();
Buildable buildable = projectRule.getBuildable();
Preconditions.checkState(
projectRule instanceof JavaLibrary
|| buildable instanceof JavaLibrary
|| buildable instanceof JavaBinary
|| buildable instanceof AndroidLibrary
|| buildable instanceof AndroidResource
|| buildable instanceof AndroidBinary
|| buildable instanceof NdkLibrary,
"project_config() does not know how to process a src_target of type %s.",
projectRule.getType().getName());
LinkedHashSet<DependentModule> dependencies = Sets.newLinkedHashSet();
final BuildTarget target = projectConfig.getBuildTarget();
Module module = new Module(projectRule, target);
module.name = getIntellijNameForRule(projectRule);
module.isIntelliJPlugin = projectConfig.getIsIntelliJPlugin();
String relativePath = projectConfig.getBuildTarget().getBasePathWithSlash();
module.pathToImlFile = String.format("%s%s.iml", relativePath, module.name);
// List the module source as the first dependency.
boolean includeSourceFolder = true;
// Do the tests before the sources so they appear earlier in the classpath. When tests are run,
// their classpath entries may be deliberately shadowing production classpath entries.
// tests folder
boolean hasSourceFoldersForTestRule =
addSourceFolders(
module,
projectConfig.getTestRule(),
projectConfig.getTestsSourceRoots(),
true /* isTestSource */);
// test dependencies
BuildRule testRule = projectConfig.getTestRule();
if (testRule != null) {
walkRuleAndAdd(testRule, true /* isForTests */, dependencies, projectConfig.getSrcRule());
}
// src folder
boolean hasSourceFoldersForSrcRule =
addSourceFolders(
module,
projectConfig.getSrcRule(),
projectConfig.getSourceRoots(),
false /* isTestSource */);
addRootExcludes(module, projectConfig.getSrcRule(), projectFilesystem);
// At least one of src or tests should contribute a source folder unless this is an
// non-library Android project with no source roots specified.
if (!hasSourceFoldersForTestRule && !hasSourceFoldersForSrcRule) {
includeSourceFolder = false;
}
// IntelliJ expects all Android projects to have a gen/ folder, even if there is no src/
// directory specified.
boolean isAndroidRule = projectRule.getProperties().is(ANDROID);
if (isAndroidRule) {
boolean hasSourceFolders = !module.sourceFolders.isEmpty();
module.sourceFolders.add(SourceFolder.GEN);
if (!hasSourceFolders) {
includeSourceFolder = true;
}
}
// src dependencies
// Note that isForTests is false even if projectRule is the project_config's test_target.
walkRuleAndAdd(projectRule, false /* isForTests */, dependencies, projectConfig.getSrcRule());
String basePathWithSlash = projectConfig.getBuildTarget().getBasePathWithSlash();
// Specify another path for intellij to generate gen/ for each android module,
// so that it will not disturb our glob() rules.
// To specify the location of gen, Intellij requires the relative path from
// the base path of current build target.
module.moduleGenPath = generateRelativeGenPath(basePathWithSlash).toString();
DependentModule jdkDependency;
if (isAndroidRule) {
// android details
if (projectRule.getBuildable() instanceof NdkLibrary) {
NdkLibrary ndkLibrary = (NdkLibrary) projectRule.getBuildable();
module.isAndroidLibraryProject = true;
module.keystorePath = null;
module.nativeLibs =
Paths.get(relativePath).relativize(ndkLibrary.getLibraryPath()).toString();
} else if (projectRule.getBuildable() instanceof AndroidResource) {
AndroidResource androidResource = (AndroidResource) projectRule.getBuildable();
module.resFolder = createRelativePath(androidResource.getRes(), target);
module.isAndroidLibraryProject = true;
module.keystorePath = null;
} else if (projectRule.getBuildable() instanceof AndroidBinary) {
AndroidBinary androidBinary = (AndroidBinary) projectRule.getBuildable();
module.resFolder = null;
module.isAndroidLibraryProject = false;
KeystoreProperties keystoreProperties =
KeystoreProperties.createFromPropertiesFile(
androidBinary.getKeystore().getPathToStore(),
androidBinary.getKeystore().getPathToPropertiesFile(),
projectFilesystem);
// getKeystore() returns a path relative to the project root, but an IntelliJ module
// expects the path to the keystore to be relative to the module root.
module.keystorePath =
Paths.get(relativePath).relativize(keystoreProperties.getKeystore()).toString();
} else {
module.isAndroidLibraryProject = true;
module.keystorePath = null;
}
module.hasAndroidFacet = true;
module.proguardConfigPath = null;
// If there is a default AndroidManifest.xml specified in .buckconfig, use it if
// AndroidManifest.xml is not present in the root of the [Android] IntelliJ module.
if (pathToDefaultAndroidManifest.isPresent()) {
String androidManifest = basePathWithSlash + "AndroidManifest.xml";
if (!projectFilesystem.exists(androidManifest)) {
String manifestPath = this.pathToDefaultAndroidManifest.get();
String rootPrefix = "//";
Preconditions.checkState(
manifestPath.startsWith(rootPrefix),
"Currently, we expect this option to start with '%s', "
+ "indicating that it is relative to the root of the repository.",
rootPrefix);
manifestPath = manifestPath.substring(rootPrefix.length());
String relativePathToManifest =
Paths.get(basePathWithSlash).relativize(Paths.get(manifestPath)).toString();
// IntelliJ requires that the path start with a slash to indicate that it is relative to
// the module.
module.androidManifest = "/" + relativePathToManifest;
}
}
// List this last so that classes from modules can shadow classes in the JDK.
jdkDependency = DependentModule.newInheritedJdk();
} else {
module.hasAndroidFacet = false;
if (module.isIntelliJPlugin()) {
jdkDependency = DependentModule.newIntelliJPluginJdk();
} else {
jdkDependency = DependentModule.newStandardJdk();
}
}
// Assign the dependencies.
module.dependencies =
createDependenciesInOrder(includeSourceFolder, dependencies, jdkDependency);
// Annotation processing generates sources for IntelliJ to consume, but does so outside
// the module directory to avoid messing up globbing.
JavaLibrary javaLibrary = null;
if (projectRule.getBuildable() instanceof JavaLibrary) {
javaLibrary = (JavaLibrary) projectRule.getBuildable();
} else if (projectRule instanceof JavaLibrary) {
javaLibrary = (JavaLibrary) projectRule;
}
if (javaLibrary != null) {
AnnotationProcessingData processingData = javaLibrary.getAnnotationProcessingData();
Path annotationGenSrc = processingData.getGeneratedSourceFolderName();
if (annotationGenSrc != null) {
module.annotationGenPath =
"/" + Paths.get(basePathWithSlash).relativize(annotationGenSrc).toString();
module.annotationGenIsForTest = !hasSourceFoldersForSrcRule;
}
}
return module;
}
/** Returns a set of source folders of the java files of a library. */
@VisibleForTesting
static ImmutableSet<String> getPathToSourceFolders(
JavaLibrary rule,
Optional<DefaultJavaPackageFinder> defaultJavaPackageFinderOptional,
ProjectFilesystem filesystem) {
ImmutableSet<Path> javaSrcs = rule.getJavaSrcs();
// A Java library rule with just resource files has an empty javaSrcs.
if (javaSrcs.isEmpty()) {
return ImmutableSet.of();
}
// If defaultJavaPackageFinderOptional is not present, then it could mean that there was an
// error reading from the buck configuration file.
if (!defaultJavaPackageFinderOptional.isPresent()) {
throw new HumanReadableException(
"Please include a [java] section with src_root property in the .buckconfig file.");
}
DefaultJavaPackageFinder defaultJavaPackageFinder = defaultJavaPackageFinderOptional.get();
// Iterate through all source paths to make sure we are generating a complete set of source
// folders for the source paths.
Set<String> srcFolders = Sets.newHashSet();
loopThroughSourcePath:
for (Path javaSrcPath : javaSrcs) {
if (!MorePaths.isGeneratedFile(javaSrcPath)) {
// If the source path is already under a known source folder, then we can skip this
// source path.
for (String srcFolder : srcFolders) {
if (javaSrcPath.startsWith(srcFolder)) {
continue loopThroughSourcePath;
}
}
// If the source path is under one of the source roots, then we can just add the source
// root.
ImmutableSortedSet<String> pathsFromRoot = defaultJavaPackageFinder.getPathsFromRoot();
for (String root : pathsFromRoot) {
if (javaSrcPath.startsWith(root)) {
srcFolders.add(root);
continue loopThroughSourcePath;
}
}
// Traverse the file system from the parent directory of the java file until we hit the
// parent of the src root directory.
ImmutableSet<String> pathElements = defaultJavaPackageFinder.getPathElements();
File directory = filesystem.getFileForRelativePath(javaSrcPath.getParent());
while (directory != null && !pathElements.contains(directory.getName())) {
directory = directory.getParentFile();
}
if (directory != null) {
String directoryPath = directory.getPath();
if (!directoryPath.endsWith("/")) {
directoryPath += "/";
}
srcFolders.add(directoryPath);
}
}
}
return ImmutableSet.copyOf(srcFolders);
}
/**
* Creates/finds the set of build rules that correspond to pre-dex'd artifacts that should be
* merged to create the final classes.dex for the APK.
*
* <p>This method may modify {@code ruleResolver}, inserting new rules into its index.
*/
@VisibleForTesting
PreDexMerge createPreDexMergeRule(
AaptPackageResources aaptPackageResources,
Iterable<DexProducedFromJavaLibrary> preDexRulesNotInThePackageableCollection,
AndroidPackageableCollection packageableCollection) {
ImmutableSortedSet.Builder<JavaLibrary> javaLibraryDepsBuilder =
ImmutableSortedSet.naturalOrder();
ImmutableSet.Builder<DexProducedFromJavaLibrary> preDexDeps = ImmutableSet.builder();
preDexDeps.addAll(preDexRulesNotInThePackageableCollection);
for (BuildTarget buildTarget : packageableCollection.getJavaLibrariesToDex()) {
Preconditions.checkState(
!buildTargetsToExcludeFromDex.contains(buildTarget),
"JavaLibrary should have been excluded from target to dex: %s",
buildTarget);
BuildRule libraryRule = ruleResolver.getRule(buildTarget);
// Skip uber R.java since AaptPackageResources takes care of dexing.
if (libraryRule.equals(aaptPackageResources)) {
continue;
}
Preconditions.checkState(libraryRule instanceof JavaLibrary);
JavaLibrary javaLibrary = (JavaLibrary) libraryRule;
// If the rule has no output file (which happens when a java_library has no srcs or
// resources, but export_deps is true), then there will not be anything to dx.
if (javaLibrary.getPathToOutput() == null) {
continue;
}
// Take note of the rule so we add it to the enhanced deps.
javaLibraryDepsBuilder.add(javaLibrary);
// See whether the corresponding IntermediateDexRule has already been added to the
// ruleResolver.
BuildTarget originalTarget = javaLibrary.getBuildTarget();
BuildTarget preDexTarget = BuildTarget.builder(originalTarget).addFlavors(DEX_FLAVOR).build();
Optional<BuildRule> preDexRule = ruleResolver.getRuleOptional(preDexTarget);
if (preDexRule.isPresent()) {
preDexDeps.add((DexProducedFromJavaLibrary) preDexRule.get());
continue;
}
// Create the IntermediateDexRule and add it to both the ruleResolver and preDexDeps.
BuildRuleParams paramsForPreDex =
buildRuleParams.copyWithChanges(
preDexTarget,
Suppliers.ofInstance(
ImmutableSortedSet.of(ruleResolver.getRule(javaLibrary.getBuildTarget()))),
/* extraDeps */ Suppliers.ofInstance(ImmutableSortedSet.<BuildRule>of()));
DexProducedFromJavaLibrary preDex =
new DexProducedFromJavaLibrary(paramsForPreDex, pathResolver, javaLibrary);
ruleResolver.addToIndex(preDex);
preDexDeps.add(preDex);
}
ImmutableSet<DexProducedFromJavaLibrary> allPreDexDeps = preDexDeps.build();
BuildRuleParams paramsForPreDexMerge =
buildRuleParams.copyWithChanges(
createBuildTargetWithFlavor(DEX_MERGE_FLAVOR),
Suppliers.ofInstance(
ImmutableSortedSet.<BuildRule>naturalOrder()
.addAll(getDexMergeDeps(aaptPackageResources, allPreDexDeps))
.addAll(javaLibraryDepsBuilder.build())
.build()),
/* extraDeps */ Suppliers.ofInstance(ImmutableSortedSet.<BuildRule>of()));
PreDexMerge preDexMerge =
new PreDexMerge(
paramsForPreDexMerge,
pathResolver,
primaryDexPath,
dexSplitMode,
allPreDexDeps,
aaptPackageResources,
dxExecutorService,
xzCompressionLevel);
ruleResolver.addToIndex(preDexMerge);
return preDexMerge;
}
/**
* If the user specified any android_build_config() rules, then we must add some build rules to
* generate the production {@code BuildConfig.class} files and ensure that they are included in
* the list of {@link AndroidPackageableCollection#getClasspathEntriesToDex}.
*/
private void addBuildConfigDeps(
boolean shouldPreDex,
AndroidPackageableCollection packageableCollection,
ImmutableSortedSet.Builder<BuildRule> enhancedDeps,
ImmutableList.Builder<DexProducedFromJavaLibrary> preDexRules,
ImmutableList.Builder<Path> buildConfigJarFilesBuilder) {
BuildConfigFields buildConfigConstants =
BuildConfigFields.fromFields(
ImmutableList.<BuildConfigFields.Field>of(
BuildConfigFields.Field.of(
"boolean",
BuildConfigs.DEBUG_CONSTANT,
String.valueOf(packageType != AndroidBinary.PackageType.RELEASE)),
BuildConfigFields.Field.of(
"boolean",
BuildConfigs.IS_EXO_CONSTANT,
String.valueOf(!exopackageModes.isEmpty())),
BuildConfigFields.Field.of(
"int",
BuildConfigs.EXOPACKAGE_FLAGS,
String.valueOf(ExopackageMode.toBitmask(exopackageModes)))));
for (Map.Entry<String, BuildConfigFields> entry :
packageableCollection.getBuildConfigs().entrySet()) {
// Merge the user-defined constants with the APK-specific overrides.
BuildConfigFields totalBuildConfigValues =
BuildConfigFields.empty()
.putAll(entry.getValue())
.putAll(buildConfigValues)
.putAll(buildConfigConstants);
// Each enhanced dep needs a unique build target, so we parameterize the build target by the
// Java package.
String javaPackage = entry.getKey();
Flavor flavor = ImmutableFlavor.of("buildconfig_" + javaPackage.replace('.', '_'));
BuildRuleParams buildConfigParams =
new BuildRuleParams(
createBuildTargetWithFlavor(flavor),
/* declaredDeps */ Suppliers.ofInstance(ImmutableSortedSet.<BuildRule>of()),
/* extraDeps */ Suppliers.ofInstance(ImmutableSortedSet.<BuildRule>of()),
buildRuleParams.getProjectFilesystem(),
buildRuleParams.getRuleKeyBuilderFactory());
JavaLibrary buildConfigJavaLibrary =
AndroidBuildConfigDescription.createBuildRule(
buildConfigParams,
javaPackage,
totalBuildConfigValues,
buildConfigValuesFile,
/* useConstantExpressions */ true,
javacOptions,
ruleResolver);
ruleResolver.addToIndex(buildConfigJavaLibrary);
enhancedDeps.add(buildConfigJavaLibrary);
Path buildConfigJar = buildConfigJavaLibrary.getPathToOutput();
Preconditions.checkNotNull(
buildConfigJar, "%s must have an output file.", buildConfigJavaLibrary);
buildConfigJarFilesBuilder.add(buildConfigJar);
if (shouldPreDex) {
DexProducedFromJavaLibrary buildConfigDex =
new DexProducedFromJavaLibrary(
buildConfigParams.copyWithChanges(
createBuildTargetWithFlavor(ImmutableFlavor.of("dex_" + flavor.getName())),
Suppliers.ofInstance(ImmutableSortedSet.<BuildRule>of(buildConfigJavaLibrary)),
/* extraDeps */ Suppliers.ofInstance(ImmutableSortedSet.<BuildRule>of())),
pathResolver,
buildConfigJavaLibrary);
ruleResolver.addToIndex(buildConfigDex);
enhancedDeps.add(buildConfigDex);
preDexRules.add(buildConfigDex);
}
}
}