@Override
public SkyValue compute(SkyKey skyKey, Environment env)
throws WorkspaceFileFunctionException, InterruptedException {
RootedPath workspaceRoot = (RootedPath) skyKey.argument();
FileValue workspaceFileValue = (FileValue) env.getValue(FileValue.key(workspaceRoot));
if (workspaceFileValue == null) {
return null;
}
Path repoWorkspace = workspaceRoot.getRoot().getRelative(workspaceRoot.getRelativePath());
Builder builder =
new Builder(repoWorkspace, packageFactory.getRuleClassProvider().getRunfilesPrefix());
WorkspaceFactory parser =
new WorkspaceFactory(
builder, packageFactory.getRuleClassProvider(), installDir.getPathString());
parser.parse(
ParserInputSource.create(
ruleClassProvider.getDefaultWorkspaceFile(), new PathFragment("DEFAULT.WORKSPACE")));
if (!workspaceFileValue.exists()) {
return new PackageValue(builder.build());
}
try {
parser.parse(ParserInputSource.create(repoWorkspace));
} catch (IOException e) {
throw new WorkspaceFileFunctionException(e, Transience.TRANSIENT);
}
return new PackageValue(builder.build());
}
/**
* Symlinks a BUILD file from the local filesystem into the external repository's root.
*
* @param rule the rule that declares the build_file path.
* @param workspaceDirectory the workspace root for the build.
* @param directoryValue the FileValue corresponding to the external repository's root directory.
* @param env the Skyframe environment.
* @return the file value of the symlink created.
* @throws
* com.google.devtools.build.lib.bazel.repository.RepositoryFunction.RepositoryFunctionException
* if the BUILD file specified does not exist or cannot be linked.
*/
protected RepositoryValue symlinkBuildFile(
Rule rule, Path workspaceDirectory, FileValue directoryValue, Environment env)
throws RepositoryFunctionException {
AggregatingAttributeMapper mapper = AggregatingAttributeMapper.of(rule);
PathFragment buildFile = new PathFragment(mapper.get("build_file", Type.STRING));
Path buildFileTarget = workspaceDirectory.getRelative(buildFile);
if (!buildFileTarget.exists()) {
throw new RepositoryFunctionException(
new EvalException(
rule.getLocation(),
String.format(
"In %s the 'build_file' attribute does not specify an existing file "
+ "(%s does not exist)",
rule, buildFileTarget)),
Transience.PERSISTENT);
}
RootedPath rootedBuild;
if (buildFile.isAbsolute()) {
rootedBuild =
RootedPath.toRootedPath(
buildFileTarget.getParentDirectory(),
new PathFragment(buildFileTarget.getBaseName()));
} else {
rootedBuild = RootedPath.toRootedPath(workspaceDirectory, buildFile);
}
SkyKey buildFileKey = FileValue.key(rootedBuild);
FileValue buildFileValue;
try {
buildFileValue =
(FileValue)
env.getValueOrThrow(
buildFileKey,
IOException.class,
FileSymlinkException.class,
InconsistentFilesystemException.class);
if (buildFileValue == null) {
return null;
}
} catch (IOException | FileSymlinkException | InconsistentFilesystemException e) {
throw new RepositoryFunctionException(
new IOException("Cannot lookup " + buildFile + ": " + e.getMessage()),
Transience.TRANSIENT);
}
Path buildFilePath = directoryValue.realRootedPath().asPath().getRelative("BUILD");
if (createSymbolicLink(buildFilePath, buildFileTarget, env) == null) {
return null;
}
return RepositoryValue.createNew(directoryValue, buildFileValue);
}
@Override
public ResolvedTargets<Void> findTargetsBeneathDirectory(
String originalPattern,
String directory,
boolean rulesOnly,
ImmutableSet<String> excludedSubdirectories)
throws TargetParsingException, InterruptedException {
FilteringPolicy policy =
rulesOnly ? FilteringPolicies.RULES_ONLY : FilteringPolicies.NO_FILTER;
ImmutableSet<PathFragment> excludedPathFragments =
TargetPatternResolverUtil.getPathFragments(excludedSubdirectories);
PathFragment pathFragment = TargetPatternResolverUtil.getPathFragment(directory);
for (Path root : pkgPath.getPathEntries()) {
RootedPath rootedPath = RootedPath.toRootedPath(root, pathFragment);
SkyValue token =
env.getValue(
PrepareDepsOfTargetsUnderDirectoryValue.key(
rootedPath, excludedPathFragments, policy));
if (token == null) {
// A null token value means there is a missing dependency, because RecursivePkgFunction
// never throws.
throw new MissingDepException();
}
}
return ResolvedTargets.empty();
}
private static FileValue createSymbolicLink(Path from, Path to, Environment env)
throws RepositoryFunctionException {
try {
if (!from.exists()) {
from.createSymbolicLink(to);
}
} catch (IOException e) {
throw new RepositoryFunctionException(
new IOException(
String.format(
"Error creating symbolic link from %s to %s: %s", from, to, e.getMessage())),
Transience.TRANSIENT);
}
SkyKey outputDirectoryKey =
FileValue.key(RootedPath.toRootedPath(from, PathFragment.EMPTY_FRAGMENT));
try {
return (FileValue)
env.getValueOrThrow(
outputDirectoryKey,
IOException.class,
FileSymlinkException.class,
InconsistentFilesystemException.class);
} catch (IOException | FileSymlinkException | InconsistentFilesystemException e) {
throw new RepositoryFunctionException(
new IOException(String.format("Could not access %s: %s", from, e.getMessage())),
Transience.PERSISTENT);
}
}
private FileLookupResult getASTFile(Environment env, PathFragment astFilePathFragment)
throws ASTLookupFunctionException {
for (Path packagePathEntry : pkgLocator.get().getPathEntries()) {
RootedPath rootedPath = RootedPath.toRootedPath(packagePathEntry, astFilePathFragment);
SkyKey fileSkyKey = FileValue.key(rootedPath);
FileValue fileValue = null;
try {
fileValue =
(FileValue)
env.getValueOrThrow(
fileSkyKey,
IOException.class,
FileSymlinkCycleException.class,
InconsistentFilesystemException.class);
} catch (IOException | FileSymlinkCycleException e) {
throw new ASTLookupFunctionException(
new ErrorReadingSkylarkExtensionException(e.getMessage()), Transience.PERSISTENT);
} catch (InconsistentFilesystemException e) {
throw new ASTLookupFunctionException(e, Transience.PERSISTENT);
}
if (fileValue == null) {
return null;
}
if (fileValue.isFile()) {
return FileLookupResult.file(rootedPath);
}
}
return FileLookupResult.noFile();
}
private SkyKey buildRecursivePkgKey(
RepositoryName repository,
PathFragment rootRelativePath,
ImmutableSet<PathFragment> excludedPaths) {
RootedPath rootedPath = RootedPath.toRootedPath(rootDirectory, rootRelativePath);
return RecursivePkgValue.key(repository, rootedPath, excludedPaths);
}
static boolean isInternal(RootedPath rootedPath, PathPackageLocator packageLocator) {
// TODO(bazel-team): This is inefficient when there are a lot of package roots or there are a
// lot of external directories. Consider either explicitly preventing this case or using a more
// efficient approach here (e.g. use a trie for determining if a file is under an external
// directory).
return packageLocator.getPathEntries().contains(rootedPath.getRoot());
}
static FileStateValue create(RootedPath rootedPath, @Nullable TimestampGranularityMonitor tsgm)
throws InconsistentFilesystemException, IOException {
Path path = rootedPath.asPath();
// Stat, but don't throw an exception for the common case of a nonexistent file. This still
// throws an IOException in case any other IO error is encountered.
FileStatus stat = path.statIfFound(Symlinks.NOFOLLOW);
if (stat == null) {
return NONEXISTENT_FILE_STATE_NODE;
}
return createWithStatNoFollow(rootedPath, FileStatusWithDigestAdapter.adapt(stat), tsgm);
}
/**
* List the directory and create {@code SkyKey}s to request contents of its children recursively.
*
* <p>The returned keys are of type {@link SkyFunctions#RECURSIVE_FILESYSTEM_TRAVERSAL}.
*/
private static Collection<SkyKey> createRecursiveTraversalKeys(
Environment env, TraversalRequest traversal) throws MissingDepException {
// Use the traversal's path, even if it's a symlink. The contents of the directory, as listed
// in the result, must be relative to it.
DirectoryListingValue dirListing =
(DirectoryListingValue)
getDependentSkyValue(env, DirectoryListingValue.key(traversal.path));
List<SkyKey> result = new ArrayList<>();
for (Dirent dirent : dirListing.getDirents()) {
RootedPath childPath =
RootedPath.toRootedPath(
traversal.path.getRoot(),
traversal.path.getRelativePath().getRelative(dirent.getName()));
TraversalRequest childTraversal = traversal.forChildEntry(childPath);
result.add(RecursiveFilesystemTraversalValue.key(childTraversal));
}
return result;
}
/**
* Adds the repository's directory to the graph and, if it's a symlink, resolves it to an actual
* directory.
*/
@Nullable
public static FileValue getRepositoryDirectory(Path repositoryDirectory, Environment env)
throws RepositoryFunctionException {
SkyKey outputDirectoryKey =
FileValue.key(RootedPath.toRootedPath(repositoryDirectory, PathFragment.EMPTY_FRAGMENT));
FileValue value;
try {
value =
(FileValue)
env.getValueOrThrow(
outputDirectoryKey,
IOException.class,
FileSymlinkException.class,
InconsistentFilesystemException.class);
} catch (IOException | FileSymlinkException | InconsistentFilesystemException e) {
throw new RepositoryFunctionException(
new IOException("Could not access " + repositoryDirectory + ": " + e.getMessage()),
Transience.PERSISTENT);
}
return value;
}
static FileStateValue createWithStatNoFollow(
RootedPath rootedPath,
FileStatusWithDigest statNoFollow,
@Nullable TimestampGranularityMonitor tsgm)
throws InconsistentFilesystemException, IOException {
Path path = rootedPath.asPath();
if (statNoFollow.isFile()) {
return statNoFollow.isSpecialFile()
? SpecialFileStateValue.fromStat(statNoFollow, tsgm)
: RegularFileStateValue.fromPath(path, statNoFollow, tsgm);
} else if (statNoFollow.isDirectory()) {
return DIRECTORY_FILE_STATE_NODE;
} else if (statNoFollow.isSymbolicLink()) {
return new SymlinkFileStateValue(path.readSymbolicLinkUnchecked());
}
throw new InconsistentFilesystemException(
"according to stat, existing path "
+ path
+ " is "
+ "neither a file nor directory nor symlink.");
}
/**
* Get SkyKeys for the FileValues for the given {@code pathFragments}. To do this, we look for a
* package lookup node for each path fragment, since package lookup nodes contain the "root" of a
* package. The returned SkyKeys correspond to FileValues that may not exist in the graph.
*/
private Collection<SkyKey> getSkyKeysForFileFragments(Iterable<PathFragment> pathFragments) {
Set<SkyKey> result = new HashSet<>();
Multimap<PathFragment, PathFragment> currentToOriginal = ArrayListMultimap.create();
for (PathFragment pathFragment : pathFragments) {
currentToOriginal.put(pathFragment, pathFragment);
}
while (!currentToOriginal.isEmpty()) {
Map<SkyKey, PathFragment> keys = new HashMap<>();
for (PathFragment pathFragment : currentToOriginal.keySet()) {
keys.put(
PackageLookupValue.key(PackageIdentifier.createInDefaultRepo(pathFragment)),
pathFragment);
}
Map<SkyKey, SkyValue> lookupValues = graph.getSuccessfulValues(keys.keySet());
for (Map.Entry<SkyKey, SkyValue> entry : lookupValues.entrySet()) {
PackageLookupValue packageLookupValue = (PackageLookupValue) entry.getValue();
if (packageLookupValue.packageExists()) {
PathFragment dir = keys.get(entry.getKey());
Collection<PathFragment> originalFiles = currentToOriginal.get(dir);
Preconditions.checkState(!originalFiles.isEmpty(), entry);
for (PathFragment fileName : originalFiles) {
result.add(
FileValue.key(RootedPath.toRootedPath(packageLookupValue.getRoot(), fileName)));
}
currentToOriginal.removeAll(dir);
}
}
Multimap<PathFragment, PathFragment> newCurrentToOriginal = ArrayListMultimap.create();
for (PathFragment pathFragment : currentToOriginal.keySet()) {
PathFragment parent = pathFragment.getParentDirectory();
if (parent != null) {
newCurrentToOriginal.putAll(parent, currentToOriginal.get(pathFragment));
}
}
currentToOriginal = newCurrentToOriginal;
}
return result;
}
/**
* If this instance is configured with DEPEND_ON_EXTERNAL_PKG and rootedPath is a file that isn't
* under a package root then this adds a dependency on the //external package. If the action is
* ERROR_OUT, it will throw an error instead.
*/
public void maybeHandleExternalFile(RootedPath rootedPath, SkyFunction.Environment env)
throws FileOutsidePackageRootsException {
if (isInternal(rootedPath, pkgLocator.get())) {
return;
}
externalFileSeen = true;
if (externalFileAction == ExternalFileAction.ERROR_OUT) {
throw new FileOutsidePackageRootsException(rootedPath);
}
// The outputBase may be null if we're not actually running a build.
Path outputBase = pkgLocator.get().getOutputBase();
if (outputBase != null
&& !rootedPath
.asPath()
.startsWith(pkgLocator.get().getOutputBase().getRelative(Label.EXTERNAL_PATH_PREFIX))) {
return;
}
// For files that are under $OUTPUT_BASE/external, add a dependency on the //external package
// so that if the WORKSPACE file changes, the File/DirectoryStateValue will be re-evaluated.
//
// Note that:
// - We don't add a dependency on the parent directory at the package root boundary, so
// the only transitive dependencies from files inside the package roots to external files
// are through symlinks. So the upwards transitive closure of external files is small.
// - The only way other than external repositories for external source files to get into the
// skyframe graph in the first place is through symlinks outside the package roots, which we
// neither want to encourage nor optimize for since it is not common. So the set of external
// files is small.
PackageValue pkgValue =
(PackageValue) env.getValue(PackageValue.key(Label.EXTERNAL_PACKAGE_IDENTIFIER));
if (pkgValue == null) {
return;
}
Preconditions.checkState(!pkgValue.getPackage().containsErrors());
}
/**
* Looks in the directory specified by {@code recursivePkgKey} for a package, does some work as
* specified by {@link Visitor} if such a package exists, then recursively does work in each
* non-excluded subdirectory as specified by {@link #getSkyKeyForSubdirectory}, and finally
* aggregates the {@link Visitor} value along with values from each subdirectory as specified by
* {@link #aggregateWithSubdirectorySkyValues}, and returns that aggregation.
*
* <p>Returns null if {@code env.valuesMissing()} is true, checked after each call to one of
* {@link RecursiveDirectoryTraversalFunction}'s abstract methods that were given {@code env}.
* (And after each of {@code visitDirectory}'s own uses of {@code env}, of course.)
*/
TReturn visitDirectory(RecursivePkgKey recursivePkgKey, Environment env) {
RootedPath rootedPath = recursivePkgKey.getRootedPath();
ImmutableSet<PathFragment> excludedPaths = recursivePkgKey.getExcludedPaths();
Path root = rootedPath.getRoot();
PathFragment rootRelativePath = rootedPath.getRelativePath();
SkyKey fileKey = FileValue.key(rootedPath);
FileValue fileValue;
try {
fileValue =
(FileValue)
env.getValueOrThrow(
fileKey,
InconsistentFilesystemException.class,
FileSymlinkException.class,
IOException.class);
} catch (InconsistentFilesystemException | FileSymlinkException | IOException e) {
return reportErrorAndReturn(
"Failed to get information about path", e, rootRelativePath, env.getListener());
}
if (fileValue == null) {
return null;
}
if (!fileValue.isDirectory()) {
return getEmptyReturn();
}
PackageIdentifier packageId =
PackageIdentifier.create(recursivePkgKey.getRepository(), rootRelativePath);
if ((packageId.getRepository().isDefault() || packageId.getRepository().isMain())
&& fileValue.isSymlink()
&& fileValue
.getUnresolvedLinkTarget()
.startsWith(directories.getOutputBase().asFragment())) {
// Symlinks back to the output base are not traversed so that we avoid convenience symlinks.
// Note that it's not enough to just check for the convenience symlinks themselves, because
// if the value of --symlink_prefix changes, the old symlinks are left in place. This
// algorithm also covers more creative use cases where people create convenience symlinks
// somewhere in the directory tree manually.
return getEmptyReturn();
}
SkyKey pkgLookupKey = PackageLookupValue.key(packageId);
SkyKey dirListingKey = DirectoryListingValue.key(rootedPath);
Map<
SkyKey,
ValueOrException4<
NoSuchPackageException,
InconsistentFilesystemException,
FileSymlinkException,
IOException>>
pkgLookupAndDirectoryListingDeps =
env.getValuesOrThrow(
ImmutableList.of(pkgLookupKey, dirListingKey),
NoSuchPackageException.class,
InconsistentFilesystemException.class,
FileSymlinkException.class,
IOException.class);
if (env.valuesMissing()) {
return null;
}
PackageLookupValue pkgLookupValue;
try {
pkgLookupValue =
(PackageLookupValue)
Preconditions.checkNotNull(
pkgLookupAndDirectoryListingDeps.get(pkgLookupKey).get(),
"%s %s",
recursivePkgKey,
pkgLookupKey);
} catch (NoSuchPackageException | InconsistentFilesystemException e) {
return reportErrorAndReturn("Failed to load package", e, rootRelativePath, env.getListener());
} catch (IOException | FileSymlinkException e) {
throw new IllegalStateException(e);
}
TVisitor visitor = getInitialVisitor();
DirectoryListingValue dirListingValue;
try {
dirListingValue =
(DirectoryListingValue)
Preconditions.checkNotNull(
pkgLookupAndDirectoryListingDeps.get(dirListingKey).get(),
"%s %s",
recursivePkgKey,
dirListingKey);
} catch (InconsistentFilesystemException | IOException e) {
return reportErrorAndReturn(
"Failed to list directory contents", e, rootRelativePath, env.getListener());
} catch (FileSymlinkException e) {
// DirectoryListingFunction only throws FileSymlinkCycleException when FileFunction throws it,
// but FileFunction was evaluated for rootedPath above, and didn't throw there. It shouldn't
// be able to avoid throwing there but throw here.
throw new IllegalStateException(
"Symlink cycle found after not being found for \"" + rootedPath + "\"");
} catch (NoSuchPackageException e) {
throw new IllegalStateException(e);
}
boolean followSymlinks = shouldFollowSymlinksWhenTraversing(dirListingValue.getDirents());
List<SkyKey> childDeps = new ArrayList<>();
for (Dirent dirent : dirListingValue.getDirents()) {
Type type = dirent.getType();
if (type != Type.DIRECTORY
&& (type != Type.SYMLINK || (type == Type.SYMLINK && !followSymlinks))) {
// Non-directories can never host packages. Symlinks to non-directories are weeded out at
// the next level of recursion when we check if its FileValue is a directory. This is slower
// if there are a lot of symlinks in the tree, but faster if there are only a few, which is
// the case most of the time.
//
// We are not afraid of weird symlink structure here: both cyclical ones and ones that give
// rise to infinite directory trees are diagnosed by FileValue.
continue;
}
String basename = dirent.getName();
if (rootRelativePath.equals(PathFragment.EMPTY_FRAGMENT)
&& PathPackageLocator.DEFAULT_TOP_LEVEL_EXCLUDES.contains(basename)) {
continue;
}
PathFragment subdirectory = rootRelativePath.getRelative(basename);
// If this subdirectory is one of the excluded paths, don't recurse into it.
if (excludedPaths.contains(subdirectory)) {
continue;
}
// If we have an excluded path that isn't below this subdirectory, we shouldn't pass that
// excluded path to our evaluation of the subdirectory, because the exclusion can't
// possibly match anything beneath the subdirectory.
//
// For example, if we're currently evaluating directory "a", are looking at its subdirectory
// "a/b", and we have an excluded path "a/c/d", there's no need to pass the excluded path
// "a/c/d" to our evaluation of "a/b".
//
// This strategy should help to get more skyframe sharing. Consider the example above. A
// subsequent request of "a/b/...", without any excluded paths, will be a cache hit.
//
// TODO(bazel-team): Replace the excludedPaths set with a trie or a SortedSet for better
// efficiency.
ImmutableSet<PathFragment> excludedSubdirectoriesBeneathThisSubdirectory =
PathFragment.filterPathsStartingWith(excludedPaths, subdirectory);
RootedPath subdirectoryRootedPath = RootedPath.toRootedPath(root, subdirectory);
childDeps.add(
getSkyKeyForSubdirectory(
recursivePkgKey.getRepository(),
subdirectoryRootedPath,
excludedSubdirectoriesBeneathThisSubdirectory));
}
Map<SkyKey, SkyValue> subdirectorySkyValues;
if (pkgLookupValue.packageExists() && pkgLookupValue.getRoot().equals(root)) {
SkyKey packageKey = PackageValue.key(packageId);
Map<SkyKey, ValueOrException<NoSuchPackageException>> dependentSkyValues =
env.getValuesOrThrow(
Iterables.concat(childDeps, ImmutableList.of(packageKey)),
NoSuchPackageException.class);
if (env.valuesMissing()) {
return null;
}
Package pkg = null;
try {
PackageValue pkgValue = (PackageValue) dependentSkyValues.get(packageKey).get();
if (pkgValue == null) {
return null;
}
pkg = pkgValue.getPackage();
if (pkg.containsErrors()) {
env.getListener()
.handle(Event.error("package contains errors: " + rootRelativePath.getPathString()));
}
} catch (NoSuchPackageException e) {
// The package had errors, but don't fail-fast as there might be subpackages below the
// current directory.
env.getListener()
.handle(Event.error("package contains errors: " + rootRelativePath.getPathString()));
}
if (pkg != null) {
visitor.visitPackageValue(pkg, env);
if (env.valuesMissing()) {
return null;
}
}
ImmutableMap.Builder<SkyKey, SkyValue> subdirectoryBuilder = ImmutableMap.builder();
for (Map.Entry<SkyKey, ValueOrException<NoSuchPackageException>> entry :
Maps.filterKeys(dependentSkyValues, Predicates.not(Predicates.equalTo(packageKey)))
.entrySet()) {
try {
subdirectoryBuilder.put(entry.getKey(), entry.getValue().get());
} catch (NoSuchPackageException e) {
// ignored.
}
}
subdirectorySkyValues = subdirectoryBuilder.build();
} else {
subdirectorySkyValues = env.getValues(childDeps);
}
if (env.valuesMissing()) {
return null;
}
return aggregateWithSubdirectorySkyValues(visitor, subdirectorySkyValues);
}
