/**
* Calculates the set of {@link Package} objects, represented as source file targets, that depend
* on the given list of BUILD files and subincludes (other files are filtered out).
*/
@Nullable
Set<Target> getRBuildFiles(Collection<PathFragment> fileIdentifiers) {
Collection<SkyKey> files = getSkyKeysForFileFragments(fileIdentifiers);
Collection<SkyKey> current = graph.getSuccessfulValues(files).keySet();
Set<SkyKey> resultKeys = CompactHashSet.create();
while (!current.isEmpty()) {
Collection<Iterable<SkyKey>> reverseDeps = graph.getReverseDeps(current).values();
current = new HashSet<>();
for (SkyKey rdep : Iterables.concat(reverseDeps)) {
if (rdep.functionName().equals(SkyFunctions.PACKAGE)) {
resultKeys.add(rdep);
} else if (!rdep.functionName().equals(SkyFunctions.PACKAGE_LOOKUP)) {
// Packages may depend on subpackages for existence, but we don't report them as rdeps.
current.add(rdep);
}
}
}
Map<SkyKey, SkyValue> packageValues = graph.getSuccessfulValues(resultKeys);
ImmutableSet.Builder<Target> result = ImmutableSet.builder();
for (SkyValue value : packageValues.values()) {
Package pkg = ((PackageValue) value).getPackage();
if (!pkg.containsErrors()) {
result.add(pkg.getBuildFile());
}
}
return result.build();
}
@Override
public Set<Target> getBuildFiles(
QueryExpression caller,
Set<Target> nodes,
boolean buildFiles,
boolean subincludes,
boolean loads)
throws QueryException {
Set<Target> dependentFiles = new LinkedHashSet<>();
Set<Package> seenPackages = new HashSet<>();
// Keep track of seen labels, to avoid adding a fake subinclude label that also exists as a
// real target.
Set<Label> seenLabels = new HashSet<>();
// Adds all the package definition files (BUILD files and build
// extensions) for package "pkg", to "buildfiles".
for (Target x : nodes) {
Package pkg = x.getPackage();
if (seenPackages.add(pkg)) {
if (buildFiles) {
addIfUniqueLabel(pkg.getBuildFile(), seenLabels, dependentFiles);
}
List<Label> extensions = new ArrayList<>();
if (subincludes) {
extensions.addAll(pkg.getSubincludeLabels());
}
if (loads) {
extensions.addAll(pkg.getSkylarkFileDependencies());
}
for (Label subinclude : extensions) {
addIfUniqueLabel(getSubincludeTarget(subinclude, pkg), seenLabels, dependentFiles);
if (buildFiles) {
// Also add the BUILD file of the subinclude.
try {
addIfUniqueLabel(
getSubincludeTarget(subinclude.getLocalTargetLabel("BUILD"), pkg),
seenLabels,
dependentFiles);
} catch (LabelSyntaxException e) {
throw new AssertionError("BUILD should always parse as a target name", e);
}
}
}
}
}
return dependentFiles;
}
protected Rule createRule(String pkgPath, String ruleName, String... ruleDef) throws Exception {
Scratch scratch = new Scratch();
EventCollectionApparatus events = new EventCollectionApparatus();
PackageFactoryApparatus packages = new PackageFactoryApparatus(events, scratch);
Path buildFile = scratch.file(pkgPath + "/BUILD", ruleDef);
pkg = packages.createPackage(pkgPath, buildFile);
return pkg.getRule(ruleName);
}
@Override
public Target getTarget(Label label) throws TargetNotFoundException, QueryException {
SkyKey packageKey = getPackageKeyAndValidateLabel(label);
if (!graph.exists(packageKey)) {
throw new QueryException(packageKey + " does not exist in graph");
}
try {
PackageValue packageValue = (PackageValue) graph.getValue(packageKey);
if (packageValue != null) {
Package pkg = packageValue.getPackage();
if (pkg.containsErrors()) {
throw new BuildFileContainsErrorsException(label.getPackageIdentifier());
}
return packageValue.getPackage().getTarget(label.getName());
} else {
throw (NoSuchThingException)
Preconditions.checkNotNull(graph.getException(packageKey), label);
}
} catch (NoSuchThingException e) {
throw new TargetNotFoundException(e);
}
}
@Override
public SkyValue compute(SkyKey skyKey, Environment env)
throws WorkspaceFileFunctionException, InterruptedException {
RootedPath workspaceRoot = (RootedPath) skyKey.argument();
FileValue workspaceFileValue = (FileValue) env.getValue(FileValue.key(workspaceRoot));
if (workspaceFileValue == null) {
return null;
}
Path repoWorkspace = workspaceRoot.getRoot().getRelative(workspaceRoot.getRelativePath());
Builder builder =
com.google.devtools.build.lib.packages.Package.newExternalPackageBuilder(
repoWorkspace, packageFactory.getRuleClassProvider().getRunfilesPrefix());
try (Mutability mutability = Mutability.create("workspace %s", repoWorkspace)) {
WorkspaceFactory parser =
new WorkspaceFactory(
builder,
packageFactory.getRuleClassProvider(),
mutability,
installDir.getPathString());
parser.parse(
ParserInputSource.create(
ruleClassProvider.getDefaultWorkspaceFile(), new PathFragment("DEFAULT.WORKSPACE")));
if (!workspaceFileValue.exists()) {
return new PackageValue(builder.build());
}
try {
parser.parse(ParserInputSource.create(repoWorkspace, workspaceFileValue.getSize()));
} catch (IOException e) {
throw new WorkspaceFileFunctionException(e, Transience.TRANSIENT);
}
}
return new PackageValue(builder.build());
}
/**
* Looks in the directory specified by {@code recursivePkgKey} for a package, does some work as
* specified by {@link Visitor} if such a package exists, then recursively does work in each
* non-excluded subdirectory as specified by {@link #getSkyKeyForSubdirectory}, and finally
* aggregates the {@link Visitor} value along with values from each subdirectory as specified by
* {@link #aggregateWithSubdirectorySkyValues}, and returns that aggregation.
*
* <p>Returns null if {@code env.valuesMissing()} is true, checked after each call to one of
* {@link RecursiveDirectoryTraversalFunction}'s abstract methods that were given {@code env}.
* (And after each of {@code visitDirectory}'s own uses of {@code env}, of course.)
*/
TReturn visitDirectory(RecursivePkgKey recursivePkgKey, Environment env) {
RootedPath rootedPath = recursivePkgKey.getRootedPath();
ImmutableSet<PathFragment> excludedPaths = recursivePkgKey.getExcludedPaths();
Path root = rootedPath.getRoot();
PathFragment rootRelativePath = rootedPath.getRelativePath();
SkyKey fileKey = FileValue.key(rootedPath);
FileValue fileValue;
try {
fileValue =
(FileValue)
env.getValueOrThrow(
fileKey,
InconsistentFilesystemException.class,
FileSymlinkException.class,
IOException.class);
} catch (InconsistentFilesystemException | FileSymlinkException | IOException e) {
return reportErrorAndReturn(
"Failed to get information about path", e, rootRelativePath, env.getListener());
}
if (fileValue == null) {
return null;
}
if (!fileValue.isDirectory()) {
return getEmptyReturn();
}
PackageIdentifier packageId =
PackageIdentifier.create(recursivePkgKey.getRepository(), rootRelativePath);
if ((packageId.getRepository().isDefault() || packageId.getRepository().isMain())
&& fileValue.isSymlink()
&& fileValue
.getUnresolvedLinkTarget()
.startsWith(directories.getOutputBase().asFragment())) {
// Symlinks back to the output base are not traversed so that we avoid convenience symlinks.
// Note that it's not enough to just check for the convenience symlinks themselves, because
// if the value of --symlink_prefix changes, the old symlinks are left in place. This
// algorithm also covers more creative use cases where people create convenience symlinks
// somewhere in the directory tree manually.
return getEmptyReturn();
}
SkyKey pkgLookupKey = PackageLookupValue.key(packageId);
SkyKey dirListingKey = DirectoryListingValue.key(rootedPath);
Map<
SkyKey,
ValueOrException4<
NoSuchPackageException,
InconsistentFilesystemException,
FileSymlinkException,
IOException>>
pkgLookupAndDirectoryListingDeps =
env.getValuesOrThrow(
ImmutableList.of(pkgLookupKey, dirListingKey),
NoSuchPackageException.class,
InconsistentFilesystemException.class,
FileSymlinkException.class,
IOException.class);
if (env.valuesMissing()) {
return null;
}
PackageLookupValue pkgLookupValue;
try {
pkgLookupValue =
(PackageLookupValue)
Preconditions.checkNotNull(
pkgLookupAndDirectoryListingDeps.get(pkgLookupKey).get(),
"%s %s",
recursivePkgKey,
pkgLookupKey);
} catch (NoSuchPackageException | InconsistentFilesystemException e) {
return reportErrorAndReturn("Failed to load package", e, rootRelativePath, env.getListener());
} catch (IOException | FileSymlinkException e) {
throw new IllegalStateException(e);
}
TVisitor visitor = getInitialVisitor();
DirectoryListingValue dirListingValue;
try {
dirListingValue =
(DirectoryListingValue)
Preconditions.checkNotNull(
pkgLookupAndDirectoryListingDeps.get(dirListingKey).get(),
"%s %s",
recursivePkgKey,
dirListingKey);
} catch (InconsistentFilesystemException | IOException e) {
return reportErrorAndReturn(
"Failed to list directory contents", e, rootRelativePath, env.getListener());
} catch (FileSymlinkException e) {
// DirectoryListingFunction only throws FileSymlinkCycleException when FileFunction throws it,
// but FileFunction was evaluated for rootedPath above, and didn't throw there. It shouldn't
// be able to avoid throwing there but throw here.
throw new IllegalStateException(
"Symlink cycle found after not being found for \"" + rootedPath + "\"");
} catch (NoSuchPackageException e) {
throw new IllegalStateException(e);
}
boolean followSymlinks = shouldFollowSymlinksWhenTraversing(dirListingValue.getDirents());
List<SkyKey> childDeps = new ArrayList<>();
for (Dirent dirent : dirListingValue.getDirents()) {
Type type = dirent.getType();
if (type != Type.DIRECTORY
&& (type != Type.SYMLINK || (type == Type.SYMLINK && !followSymlinks))) {
// Non-directories can never host packages. Symlinks to non-directories are weeded out at
// the next level of recursion when we check if its FileValue is a directory. This is slower
// if there are a lot of symlinks in the tree, but faster if there are only a few, which is
// the case most of the time.
//
// We are not afraid of weird symlink structure here: both cyclical ones and ones that give
// rise to infinite directory trees are diagnosed by FileValue.
continue;
}
String basename = dirent.getName();
if (rootRelativePath.equals(PathFragment.EMPTY_FRAGMENT)
&& PathPackageLocator.DEFAULT_TOP_LEVEL_EXCLUDES.contains(basename)) {
continue;
}
PathFragment subdirectory = rootRelativePath.getRelative(basename);
// If this subdirectory is one of the excluded paths, don't recurse into it.
if (excludedPaths.contains(subdirectory)) {
continue;
}
// If we have an excluded path that isn't below this subdirectory, we shouldn't pass that
// excluded path to our evaluation of the subdirectory, because the exclusion can't
// possibly match anything beneath the subdirectory.
//
// For example, if we're currently evaluating directory "a", are looking at its subdirectory
// "a/b", and we have an excluded path "a/c/d", there's no need to pass the excluded path
// "a/c/d" to our evaluation of "a/b".
//
// This strategy should help to get more skyframe sharing. Consider the example above. A
// subsequent request of "a/b/...", without any excluded paths, will be a cache hit.
//
// TODO(bazel-team): Replace the excludedPaths set with a trie or a SortedSet for better
// efficiency.
ImmutableSet<PathFragment> excludedSubdirectoriesBeneathThisSubdirectory =
PathFragment.filterPathsStartingWith(excludedPaths, subdirectory);
RootedPath subdirectoryRootedPath = RootedPath.toRootedPath(root, subdirectory);
childDeps.add(
getSkyKeyForSubdirectory(
recursivePkgKey.getRepository(),
subdirectoryRootedPath,
excludedSubdirectoriesBeneathThisSubdirectory));
}
Map<SkyKey, SkyValue> subdirectorySkyValues;
if (pkgLookupValue.packageExists() && pkgLookupValue.getRoot().equals(root)) {
SkyKey packageKey = PackageValue.key(packageId);
Map<SkyKey, ValueOrException<NoSuchPackageException>> dependentSkyValues =
env.getValuesOrThrow(
Iterables.concat(childDeps, ImmutableList.of(packageKey)),
NoSuchPackageException.class);
if (env.valuesMissing()) {
return null;
}
Package pkg = null;
try {
PackageValue pkgValue = (PackageValue) dependentSkyValues.get(packageKey).get();
if (pkgValue == null) {
return null;
}
pkg = pkgValue.getPackage();
if (pkg.containsErrors()) {
env.getListener()
.handle(Event.error("package contains errors: " + rootRelativePath.getPathString()));
}
} catch (NoSuchPackageException e) {
// The package had errors, but don't fail-fast as there might be subpackages below the
// current directory.
env.getListener()
.handle(Event.error("package contains errors: " + rootRelativePath.getPathString()));
}
if (pkg != null) {
visitor.visitPackageValue(pkg, env);
if (env.valuesMissing()) {
return null;
}
}
ImmutableMap.Builder<SkyKey, SkyValue> subdirectoryBuilder = ImmutableMap.builder();
for (Map.Entry<SkyKey, ValueOrException<NoSuchPackageException>> entry :
Maps.filterKeys(dependentSkyValues, Predicates.not(Predicates.equalTo(packageKey)))
.entrySet()) {
try {
subdirectoryBuilder.put(entry.getKey(), entry.getValue().get());
} catch (NoSuchPackageException e) {
// ignored.
}
}
subdirectorySkyValues = subdirectoryBuilder.build();
} else {
subdirectorySkyValues = env.getValues(childDeps);
}
if (env.valuesMissing()) {
return null;
}
return aggregateWithSubdirectorySkyValues(visitor, subdirectorySkyValues);
}
public void testPackageDefaultProperties() throws Exception {
assertEquals(pkg.getDefaultHdrsCheck(), mapper.getPackageDefaultHdrsCheck());
assertEquals(pkg.getDefaultTestOnly(), mapper.getPackageDefaultTestOnly());
assertEquals(pkg.getDefaultDeprecation(), mapper.getPackageDefaultDeprecation());
}
private static ArtifactLocation makeArtifactLocation(Package pkg) {
Root root = Root.asSourceRoot(pkg.getSourceRoot());
PathFragment relativePath = pkg.getBuildFile().getPath().relativeTo(root.getPath());
return makeArtifactLocation(root, relativePath);
}
@Nullable
@Override
public SkyValue compute(SkyKey skyKey, Environment env)
throws AspectFunctionException, InterruptedException {
SkyframeBuildView view = buildViewProvider.getSkyframeBuildView();
NestedSetBuilder<Package> transitivePackages = NestedSetBuilder.stableOrder();
NestedSetBuilder<Label> transitiveRootCauses = NestedSetBuilder.stableOrder();
AspectKey key = (AspectKey) skyKey.argument();
ConfiguredAspectFactory aspectFactory;
if (key.getAspectClass() instanceof NativeAspectClass<?>) {
aspectFactory =
(ConfiguredAspectFactory) ((NativeAspectClass<?>) key.getAspectClass()).newInstance();
} else if (key.getAspectClass() instanceof SkylarkAspectClass) {
SkylarkAspectClass skylarkAspectClass = (SkylarkAspectClass) key.getAspectClass();
SkylarkAspect skylarkAspect;
try {
skylarkAspect =
loadSkylarkAspect(
env, skylarkAspectClass.getExtensionLabel(), skylarkAspectClass.getExportedName());
} catch (AspectCreationException e) {
throw new AspectFunctionException(e);
}
if (skylarkAspect == null) {
return null;
}
aspectFactory = new SkylarkAspectFactory(skylarkAspect.getName(), skylarkAspect);
} else {
throw new IllegalStateException();
}
// Keep this in sync with the same code in ConfiguredTargetFunction.
PackageValue packageValue =
(PackageValue) env.getValue(PackageValue.key(key.getLabel().getPackageIdentifier()));
if (packageValue == null) {
return null;
}
Package pkg = packageValue.getPackage();
if (pkg.containsErrors()) {
throw new AspectFunctionException(
new BuildFileContainsErrorsException(key.getLabel().getPackageIdentifier()));
}
Target target;
try {
target = pkg.getTarget(key.getLabel().getName());
} catch (NoSuchTargetException e) {
throw new AspectFunctionException(e);
}
if (!(target instanceof Rule)) {
throw new AspectFunctionException(
new AspectCreationException("aspects must be attached to rules"));
}
final ConfiguredTargetValue configuredTargetValue;
try {
configuredTargetValue =
(ConfiguredTargetValue)
env.getValueOrThrow(
ConfiguredTargetValue.key(key.getLabel(), key.getConfiguration()),
ConfiguredValueCreationException.class);
} catch (ConfiguredValueCreationException e) {
throw new AspectFunctionException(new AspectCreationException(e.getRootCauses()));
}
if (configuredTargetValue == null) {
// TODO(bazel-team): remove this check when top-level targets also use dynamic configurations.
// Right now the key configuration may be dynamic while the original target's configuration
// is static, resulting in a Skyframe cache miss even though the original target is, in fact,
// precomputed.
return null;
}
RuleConfiguredTarget associatedTarget =
(RuleConfiguredTarget) configuredTargetValue.getConfiguredTarget();
if (associatedTarget == null) {
return null;
}
SkyframeDependencyResolver resolver = view.createDependencyResolver(env);
TargetAndConfiguration ctgValue = new TargetAndConfiguration(target, key.getConfiguration());
try {
// Get the configuration targets that trigger this rule's configurable attributes.
Set<ConfigMatchingProvider> configConditions =
ConfiguredTargetFunction.getConfigConditions(
target, env, resolver, ctgValue, transitivePackages, transitiveRootCauses);
if (configConditions == null) {
// Those targets haven't yet been resolved.
return null;
}
ListMultimap<Attribute, ConfiguredTarget> depValueMap =
ConfiguredTargetFunction.computeDependencies(
env,
resolver,
ctgValue,
key.getAspect(),
configConditions,
ruleClassProvider,
view.getHostConfiguration(ctgValue.getConfiguration()),
transitivePackages,
transitiveRootCauses);
if (depValueMap == null) {
return null;
}
if (!transitiveRootCauses.isEmpty()) {
throw new AspectFunctionException(
new AspectCreationException("Loading failed", transitiveRootCauses.build()));
}
return createAspect(
env,
key,
aspectFactory,
associatedTarget,
configConditions,
depValueMap,
transitivePackages);
} catch (DependencyEvaluationException e) {
if (e.getCause() instanceof ConfiguredValueCreationException) {
ConfiguredValueCreationException cause = (ConfiguredValueCreationException) e.getCause();
throw new AspectFunctionException(
new AspectCreationException(cause.getMessage(), cause.getAnalysisRootCause()));
} else {
// Cast to InvalidConfigurationException as a consistency check. If you add any
// DependencyEvaluationException constructors, you may need to change this code, too.
InvalidConfigurationException cause = (InvalidConfigurationException) e.getCause();
throw new AspectFunctionException(new AspectCreationException(cause.getMessage()));
}
} catch (AspectCreationException e) {
throw new AspectFunctionException(e);
}
}