SkyValue computeInternal(
SkyKey skyKey, Environment env, @Nullable Set<SkyKey> visitedKeysForCycle)
throws SkyFunctionException, InterruptedException {
PackageIdentifier arg = (PackageIdentifier) skyKey.argument();
PathFragment file = arg.getPackageFragment();
ASTFileLookupValue astLookupValue = null;
try {
SkyKey astLookupKey = ASTFileLookupValue.key(arg);
astLookupValue =
(ASTFileLookupValue)
env.getValueOrThrow(
astLookupKey,
ErrorReadingSkylarkExtensionException.class,
InconsistentFilesystemException.class);
} catch (ErrorReadingSkylarkExtensionException e) {
throw new SkylarkImportLookupFunctionException(
SkylarkImportFailedException.errorReadingFile(file, e.getMessage()));
} catch (InconsistentFilesystemException e) {
throw new SkylarkImportLookupFunctionException(e, Transience.PERSISTENT);
}
if (astLookupValue == null) {
return null;
}
if (astLookupValue.getAST() == null) {
// Skylark import files have to exist.
throw new SkylarkImportLookupFunctionException(SkylarkImportFailedException.noFile(file));
}
BuildFileAST ast = astLookupValue.getAST();
if (ast.containsErrors()) {
throw new SkylarkImportLookupFunctionException(
SkylarkImportFailedException.skylarkErrors(file));
}
Label label = pathFragmentToLabel(arg.getRepository(), file, env);
if (label == null) {
Preconditions.checkState(env.valuesMissing(), "null label with no missing %s", file);
return null;
}
Map<Location, PathFragment> astImports = ast.getImports();
Map<PathFragment, Extension> importMap = Maps.newHashMapWithExpectedSize(astImports.size());
ImmutableList.Builder<SkylarkFileDependency> fileDependencies = ImmutableList.builder();
Map<SkyKey, PathFragment> skylarkImports = Maps.newHashMapWithExpectedSize(astImports.size());
for (Map.Entry<Location, PathFragment> entry : ast.getImports().entrySet()) {
try {
skylarkImports.put(
PackageFunction.getImportKey(entry, ruleClassProvider.getPreludePath(), file, arg),
entry.getValue());
} catch (ASTLookupInputException e) {
throw new SkylarkImportLookupFunctionException(e, Transience.PERSISTENT);
}
}
Map<SkyKey, SkyValue> skylarkImportMap;
boolean valuesMissing = false;
if (visitedKeysForCycle == null) {
// Not inlining.
skylarkImportMap = env.getValues(skylarkImports.keySet());
valuesMissing = env.valuesMissing();
} else {
// inlining calls to SkylarkImportLookupFunction.
if (!visitedKeysForCycle.add(skyKey)) {
ImmutableList<SkyKey> cycle =
CycleUtils.splitIntoPathAndChain(Predicates.equalTo(skyKey), visitedKeysForCycle)
.second;
if (env.getValue(SkylarkImportUniqueCycleValue.key(cycle)) == null) {
return null;
}
throw new SkylarkImportLookupFunctionException(
new SkylarkImportFailedException("Skylark import cycle"));
}
skylarkImportMap = Maps.newHashMapWithExpectedSize(astImports.size());
for (SkyKey skylarkImport : skylarkImports.keySet()) {
SkyValue skyValue = this.computeWithInlineCalls(skylarkImport, env, visitedKeysForCycle);
if (skyValue == null) {
Preconditions.checkState(
env.valuesMissing(), "no skylark import value for %s", skylarkImport);
// Don't give up on computing. This is against the Skyframe contract, but we don't want to
// pay the price of serializing all these calls, since they are fundamentally independent.
valuesMissing = true;
} else {
skylarkImportMap.put(skylarkImport, skyValue);
}
}
// All imports traversed, this key can no longer be part of a cycle.
visitedKeysForCycle.remove(skyKey);
}
if (valuesMissing) {
// This means some imports are unavailable.
return null;
}
for (Map.Entry<SkyKey, SkyValue> entry : skylarkImportMap.entrySet()) {
SkylarkImportLookupValue importLookupValue = (SkylarkImportLookupValue) entry.getValue();
importMap.put(
skylarkImports.get(entry.getKey()), importLookupValue.getEnvironmentExtension());
fileDependencies.add(importLookupValue.getDependency());
}
// Skylark UserDefinedFunction-s in that file will share this function definition Environment,
// which will be frozen by the time it is returned by createExtension.
Extension extension = createExtension(ast, file, importMap, env);
return new SkylarkImportLookupValue(
extension, new SkylarkFileDependency(label, fileDependencies.build()));
}
@Override
public SkyValue compute(SkyKey skyKey, Environment env)
throws SkyFunctionException, InterruptedException {
PackageIdentifier arg = (PackageIdentifier) skyKey.argument();
PathFragment file = arg.getPackageFragment();
ASTFileLookupValue astLookupValue = null;
try {
SkyKey astLookupKey = ASTFileLookupValue.key(arg);
astLookupValue =
(ASTFileLookupValue)
env.getValueOrThrow(
astLookupKey,
ErrorReadingSkylarkExtensionException.class,
InconsistentFilesystemException.class);
} catch (ErrorReadingSkylarkExtensionException e) {
throw new SkylarkImportLookupFunctionException(
SkylarkImportFailedException.errorReadingFile(file, e.getMessage()));
} catch (InconsistentFilesystemException e) {
throw new SkylarkImportLookupFunctionException(e, Transience.PERSISTENT);
}
if (astLookupValue == null) {
return null;
}
if (astLookupValue.getAST() == null) {
// Skylark import files have to exist.
throw new SkylarkImportLookupFunctionException(SkylarkImportFailedException.noFile(file));
}
BuildFileAST ast = astLookupValue.getAST();
if (ast.containsErrors()) {
throw new SkylarkImportLookupFunctionException(
SkylarkImportFailedException.skylarkErrors(file));
}
Map<Location, PathFragment> astImports = ast.getImports();
Map<PathFragment, Extension> importMap = Maps.newHashMapWithExpectedSize(astImports.size());
ImmutableList.Builder<SkylarkFileDependency> fileDependencies = ImmutableList.builder();
Map<SkyKey, PathFragment> skylarkImports = Maps.newHashMapWithExpectedSize(astImports.size());
for (Map.Entry<Location, PathFragment> entry : ast.getImports().entrySet()) {
try {
skylarkImports.put(
PackageFunction.getImportKey(entry, ruleClassProvider.getPreludePath(), file, arg),
entry.getValue());
} catch (ASTLookupInputException e) {
throw new SkylarkImportLookupFunctionException(e, Transience.PERSISTENT);
}
}
Map<SkyKey, SkyValue> skylarkImportMap = env.getValues(skylarkImports.keySet());
if (env.valuesMissing()) {
// This means some imports are unavailable.
return null;
}
for (Map.Entry<SkyKey, SkyValue> entry : skylarkImportMap.entrySet()) {
SkylarkImportLookupValue importLookupValue = (SkylarkImportLookupValue) entry.getValue();
importMap.put(
skylarkImports.get(entry.getKey()), importLookupValue.getEnvironmentExtension());
fileDependencies.add(importLookupValue.getDependency());
}
Label label = pathFragmentToLabel(arg.getRepository(), file, env);
if (label == null) {
Preconditions.checkState(env.valuesMissing(), "label null but no missing for %s", file);
return null;
}
// Skylark UserDefinedFunction-s in that file will share this function definition Environment,
// which will be frozen by the time it is returned by createExtension.
Extension extension = createExtension(ast, file, importMap, env);
return new SkylarkImportLookupValue(
extension, new SkylarkFileDependency(label, fileDependencies.build()));
}
@VisibleForTesting
static SkyKey key(PackageIdentifier pkgIdentifier) throws ASTLookupInputException {
return key(pkgIdentifier.getRepository(), pkgIdentifier.getPackageFragment());
}
/**
* Looks in the directory specified by {@code recursivePkgKey} for a package, does some work as
* specified by {@link Visitor} if such a package exists, then recursively does work in each
* non-excluded subdirectory as specified by {@link #getSkyKeyForSubdirectory}, and finally
* aggregates the {@link Visitor} value along with values from each subdirectory as specified by
* {@link #aggregateWithSubdirectorySkyValues}, and returns that aggregation.
*
* <p>Returns null if {@code env.valuesMissing()} is true, checked after each call to one of
* {@link RecursiveDirectoryTraversalFunction}'s abstract methods that were given {@code env}.
* (And after each of {@code visitDirectory}'s own uses of {@code env}, of course.)
*/
TReturn visitDirectory(RecursivePkgKey recursivePkgKey, Environment env) {
RootedPath rootedPath = recursivePkgKey.getRootedPath();
ImmutableSet<PathFragment> excludedPaths = recursivePkgKey.getExcludedPaths();
Path root = rootedPath.getRoot();
PathFragment rootRelativePath = rootedPath.getRelativePath();
SkyKey fileKey = FileValue.key(rootedPath);
FileValue fileValue;
try {
fileValue =
(FileValue)
env.getValueOrThrow(
fileKey,
InconsistentFilesystemException.class,
FileSymlinkException.class,
IOException.class);
} catch (InconsistentFilesystemException | FileSymlinkException | IOException e) {
return reportErrorAndReturn(
"Failed to get information about path", e, rootRelativePath, env.getListener());
}
if (fileValue == null) {
return null;
}
if (!fileValue.isDirectory()) {
return getEmptyReturn();
}
PackageIdentifier packageId =
PackageIdentifier.create(recursivePkgKey.getRepository(), rootRelativePath);
if ((packageId.getRepository().isDefault() || packageId.getRepository().isMain())
&& fileValue.isSymlink()
&& fileValue
.getUnresolvedLinkTarget()
.startsWith(directories.getOutputBase().asFragment())) {
// Symlinks back to the output base are not traversed so that we avoid convenience symlinks.
// Note that it's not enough to just check for the convenience symlinks themselves, because
// if the value of --symlink_prefix changes, the old symlinks are left in place. This
// algorithm also covers more creative use cases where people create convenience symlinks
// somewhere in the directory tree manually.
return getEmptyReturn();
}
SkyKey pkgLookupKey = PackageLookupValue.key(packageId);
SkyKey dirListingKey = DirectoryListingValue.key(rootedPath);
Map<
SkyKey,
ValueOrException4<
NoSuchPackageException,
InconsistentFilesystemException,
FileSymlinkException,
IOException>>
pkgLookupAndDirectoryListingDeps =
env.getValuesOrThrow(
ImmutableList.of(pkgLookupKey, dirListingKey),
NoSuchPackageException.class,
InconsistentFilesystemException.class,
FileSymlinkException.class,
IOException.class);
if (env.valuesMissing()) {
return null;
}
PackageLookupValue pkgLookupValue;
try {
pkgLookupValue =
(PackageLookupValue)
Preconditions.checkNotNull(
pkgLookupAndDirectoryListingDeps.get(pkgLookupKey).get(),
"%s %s",
recursivePkgKey,
pkgLookupKey);
} catch (NoSuchPackageException | InconsistentFilesystemException e) {
return reportErrorAndReturn("Failed to load package", e, rootRelativePath, env.getListener());
} catch (IOException | FileSymlinkException e) {
throw new IllegalStateException(e);
}
TVisitor visitor = getInitialVisitor();
DirectoryListingValue dirListingValue;
try {
dirListingValue =
(DirectoryListingValue)
Preconditions.checkNotNull(
pkgLookupAndDirectoryListingDeps.get(dirListingKey).get(),
"%s %s",
recursivePkgKey,
dirListingKey);
} catch (InconsistentFilesystemException | IOException e) {
return reportErrorAndReturn(
"Failed to list directory contents", e, rootRelativePath, env.getListener());
} catch (FileSymlinkException e) {
// DirectoryListingFunction only throws FileSymlinkCycleException when FileFunction throws it,
// but FileFunction was evaluated for rootedPath above, and didn't throw there. It shouldn't
// be able to avoid throwing there but throw here.
throw new IllegalStateException(
"Symlink cycle found after not being found for \"" + rootedPath + "\"");
} catch (NoSuchPackageException e) {
throw new IllegalStateException(e);
}
boolean followSymlinks = shouldFollowSymlinksWhenTraversing(dirListingValue.getDirents());
List<SkyKey> childDeps = new ArrayList<>();
for (Dirent dirent : dirListingValue.getDirents()) {
Type type = dirent.getType();
if (type != Type.DIRECTORY
&& (type != Type.SYMLINK || (type == Type.SYMLINK && !followSymlinks))) {
// Non-directories can never host packages. Symlinks to non-directories are weeded out at
// the next level of recursion when we check if its FileValue is a directory. This is slower
// if there are a lot of symlinks in the tree, but faster if there are only a few, which is
// the case most of the time.
//
// We are not afraid of weird symlink structure here: both cyclical ones and ones that give
// rise to infinite directory trees are diagnosed by FileValue.
continue;
}
String basename = dirent.getName();
if (rootRelativePath.equals(PathFragment.EMPTY_FRAGMENT)
&& PathPackageLocator.DEFAULT_TOP_LEVEL_EXCLUDES.contains(basename)) {
continue;
}
PathFragment subdirectory = rootRelativePath.getRelative(basename);
// If this subdirectory is one of the excluded paths, don't recurse into it.
if (excludedPaths.contains(subdirectory)) {
continue;
}
// If we have an excluded path that isn't below this subdirectory, we shouldn't pass that
// excluded path to our evaluation of the subdirectory, because the exclusion can't
// possibly match anything beneath the subdirectory.
//
// For example, if we're currently evaluating directory "a", are looking at its subdirectory
// "a/b", and we have an excluded path "a/c/d", there's no need to pass the excluded path
// "a/c/d" to our evaluation of "a/b".
//
// This strategy should help to get more skyframe sharing. Consider the example above. A
// subsequent request of "a/b/...", without any excluded paths, will be a cache hit.
//
// TODO(bazel-team): Replace the excludedPaths set with a trie or a SortedSet for better
// efficiency.
ImmutableSet<PathFragment> excludedSubdirectoriesBeneathThisSubdirectory =
PathFragment.filterPathsStartingWith(excludedPaths, subdirectory);
RootedPath subdirectoryRootedPath = RootedPath.toRootedPath(root, subdirectory);
childDeps.add(
getSkyKeyForSubdirectory(
recursivePkgKey.getRepository(),
subdirectoryRootedPath,
excludedSubdirectoriesBeneathThisSubdirectory));
}
Map<SkyKey, SkyValue> subdirectorySkyValues;
if (pkgLookupValue.packageExists() && pkgLookupValue.getRoot().equals(root)) {
SkyKey packageKey = PackageValue.key(packageId);
Map<SkyKey, ValueOrException<NoSuchPackageException>> dependentSkyValues =
env.getValuesOrThrow(
Iterables.concat(childDeps, ImmutableList.of(packageKey)),
NoSuchPackageException.class);
if (env.valuesMissing()) {
return null;
}
Package pkg = null;
try {
PackageValue pkgValue = (PackageValue) dependentSkyValues.get(packageKey).get();
if (pkgValue == null) {
return null;
}
pkg = pkgValue.getPackage();
if (pkg.containsErrors()) {
env.getListener()
.handle(Event.error("package contains errors: " + rootRelativePath.getPathString()));
}
} catch (NoSuchPackageException e) {
// The package had errors, but don't fail-fast as there might be subpackages below the
// current directory.
env.getListener()
.handle(Event.error("package contains errors: " + rootRelativePath.getPathString()));
}
if (pkg != null) {
visitor.visitPackageValue(pkg, env);
if (env.valuesMissing()) {
return null;
}
}
ImmutableMap.Builder<SkyKey, SkyValue> subdirectoryBuilder = ImmutableMap.builder();
for (Map.Entry<SkyKey, ValueOrException<NoSuchPackageException>> entry :
Maps.filterKeys(dependentSkyValues, Predicates.not(Predicates.equalTo(packageKey)))
.entrySet()) {
try {
subdirectoryBuilder.put(entry.getKey(), entry.getValue().get());
} catch (NoSuchPackageException e) {
// ignored.
}
}
subdirectorySkyValues = subdirectoryBuilder.build();
} else {
subdirectorySkyValues = env.getValues(childDeps);
}
if (env.valuesMissing()) {
return null;
}
return aggregateWithSubdirectorySkyValues(visitor, subdirectorySkyValues);
}