@Override public SkyValue compute(SkyKey skyKey, Environment env) throws WorkspaceFileFunctionException, InterruptedException { RootedPath workspaceRoot = (RootedPath) skyKey.argument(); FileValue workspaceFileValue = (FileValue) env.getValue(FileValue.key(workspaceRoot)); if (workspaceFileValue == null) { return null; } Path repoWorkspace = workspaceRoot.getRoot().getRelative(workspaceRoot.getRelativePath()); Builder builder = new Builder(repoWorkspace, packageFactory.getRuleClassProvider().getRunfilesPrefix()); WorkspaceFactory parser = new WorkspaceFactory( builder, packageFactory.getRuleClassProvider(), installDir.getPathString()); parser.parse( ParserInputSource.create( ruleClassProvider.getDefaultWorkspaceFile(), new PathFragment("DEFAULT.WORKSPACE"))); if (!workspaceFileValue.exists()) { return new PackageValue(builder.build()); } try { parser.parse(ParserInputSource.create(repoWorkspace)); } catch (IOException e) { throw new WorkspaceFileFunctionException(e, Transience.TRANSIENT); } return new PackageValue(builder.build()); }
/** * Symlinks a BUILD file from the local filesystem into the external repository's root. * * @param rule the rule that declares the build_file path. * @param workspaceDirectory the workspace root for the build. * @param directoryValue the FileValue corresponding to the external repository's root directory. * @param env the Skyframe environment. * @return the file value of the symlink created. * @throws * com.google.devtools.build.lib.bazel.repository.RepositoryFunction.RepositoryFunctionException * if the BUILD file specified does not exist or cannot be linked. */ protected RepositoryValue symlinkBuildFile( Rule rule, Path workspaceDirectory, FileValue directoryValue, Environment env) throws RepositoryFunctionException { AggregatingAttributeMapper mapper = AggregatingAttributeMapper.of(rule); PathFragment buildFile = new PathFragment(mapper.get("build_file", Type.STRING)); Path buildFileTarget = workspaceDirectory.getRelative(buildFile); if (!buildFileTarget.exists()) { throw new RepositoryFunctionException( new EvalException( rule.getLocation(), String.format( "In %s the 'build_file' attribute does not specify an existing file " + "(%s does not exist)", rule, buildFileTarget)), Transience.PERSISTENT); } RootedPath rootedBuild; if (buildFile.isAbsolute()) { rootedBuild = RootedPath.toRootedPath( buildFileTarget.getParentDirectory(), new PathFragment(buildFileTarget.getBaseName())); } else { rootedBuild = RootedPath.toRootedPath(workspaceDirectory, buildFile); } SkyKey buildFileKey = FileValue.key(rootedBuild); FileValue buildFileValue; try { buildFileValue = (FileValue) env.getValueOrThrow( buildFileKey, IOException.class, FileSymlinkException.class, InconsistentFilesystemException.class); if (buildFileValue == null) { return null; } } catch (IOException | FileSymlinkException | InconsistentFilesystemException e) { throw new RepositoryFunctionException( new IOException("Cannot lookup " + buildFile + ": " + e.getMessage()), Transience.TRANSIENT); } Path buildFilePath = directoryValue.realRootedPath().asPath().getRelative("BUILD"); if (createSymbolicLink(buildFilePath, buildFileTarget, env) == null) { return null; } return RepositoryValue.createNew(directoryValue, buildFileValue); }
@Override public ResolvedTargets<Void> findTargetsBeneathDirectory( String originalPattern, String directory, boolean rulesOnly, ImmutableSet<String> excludedSubdirectories) throws TargetParsingException, InterruptedException { FilteringPolicy policy = rulesOnly ? FilteringPolicies.RULES_ONLY : FilteringPolicies.NO_FILTER; ImmutableSet<PathFragment> excludedPathFragments = TargetPatternResolverUtil.getPathFragments(excludedSubdirectories); PathFragment pathFragment = TargetPatternResolverUtil.getPathFragment(directory); for (Path root : pkgPath.getPathEntries()) { RootedPath rootedPath = RootedPath.toRootedPath(root, pathFragment); SkyValue token = env.getValue( PrepareDepsOfTargetsUnderDirectoryValue.key( rootedPath, excludedPathFragments, policy)); if (token == null) { // A null token value means there is a missing dependency, because RecursivePkgFunction // never throws. throw new MissingDepException(); } } return ResolvedTargets.empty(); }
private static FileValue createSymbolicLink(Path from, Path to, Environment env) throws RepositoryFunctionException { try { if (!from.exists()) { from.createSymbolicLink(to); } } catch (IOException e) { throw new RepositoryFunctionException( new IOException( String.format( "Error creating symbolic link from %s to %s: %s", from, to, e.getMessage())), Transience.TRANSIENT); } SkyKey outputDirectoryKey = FileValue.key(RootedPath.toRootedPath(from, PathFragment.EMPTY_FRAGMENT)); try { return (FileValue) env.getValueOrThrow( outputDirectoryKey, IOException.class, FileSymlinkException.class, InconsistentFilesystemException.class); } catch (IOException | FileSymlinkException | InconsistentFilesystemException e) { throw new RepositoryFunctionException( new IOException(String.format("Could not access %s: %s", from, e.getMessage())), Transience.PERSISTENT); } }
private FileLookupResult getASTFile(Environment env, PathFragment astFilePathFragment) throws ASTLookupFunctionException { for (Path packagePathEntry : pkgLocator.get().getPathEntries()) { RootedPath rootedPath = RootedPath.toRootedPath(packagePathEntry, astFilePathFragment); SkyKey fileSkyKey = FileValue.key(rootedPath); FileValue fileValue = null; try { fileValue = (FileValue) env.getValueOrThrow( fileSkyKey, IOException.class, FileSymlinkCycleException.class, InconsistentFilesystemException.class); } catch (IOException | FileSymlinkCycleException e) { throw new ASTLookupFunctionException( new ErrorReadingSkylarkExtensionException(e.getMessage()), Transience.PERSISTENT); } catch (InconsistentFilesystemException e) { throw new ASTLookupFunctionException(e, Transience.PERSISTENT); } if (fileValue == null) { return null; } if (fileValue.isFile()) { return FileLookupResult.file(rootedPath); } } return FileLookupResult.noFile(); }
private SkyKey buildRecursivePkgKey( RepositoryName repository, PathFragment rootRelativePath, ImmutableSet<PathFragment> excludedPaths) { RootedPath rootedPath = RootedPath.toRootedPath(rootDirectory, rootRelativePath); return RecursivePkgValue.key(repository, rootedPath, excludedPaths); }
static boolean isInternal(RootedPath rootedPath, PathPackageLocator packageLocator) { // TODO(bazel-team): This is inefficient when there are a lot of package roots or there are a // lot of external directories. Consider either explicitly preventing this case or using a more // efficient approach here (e.g. use a trie for determining if a file is under an external // directory). return packageLocator.getPathEntries().contains(rootedPath.getRoot()); }
static FileStateValue create(RootedPath rootedPath, @Nullable TimestampGranularityMonitor tsgm) throws InconsistentFilesystemException, IOException { Path path = rootedPath.asPath(); // Stat, but don't throw an exception for the common case of a nonexistent file. This still // throws an IOException in case any other IO error is encountered. FileStatus stat = path.statIfFound(Symlinks.NOFOLLOW); if (stat == null) { return NONEXISTENT_FILE_STATE_NODE; } return createWithStatNoFollow(rootedPath, FileStatusWithDigestAdapter.adapt(stat), tsgm); }
/** * List the directory and create {@code SkyKey}s to request contents of its children recursively. * * <p>The returned keys are of type {@link SkyFunctions#RECURSIVE_FILESYSTEM_TRAVERSAL}. */ private static Collection<SkyKey> createRecursiveTraversalKeys( Environment env, TraversalRequest traversal) throws MissingDepException { // Use the traversal's path, even if it's a symlink. The contents of the directory, as listed // in the result, must be relative to it. DirectoryListingValue dirListing = (DirectoryListingValue) getDependentSkyValue(env, DirectoryListingValue.key(traversal.path)); List<SkyKey> result = new ArrayList<>(); for (Dirent dirent : dirListing.getDirents()) { RootedPath childPath = RootedPath.toRootedPath( traversal.path.getRoot(), traversal.path.getRelativePath().getRelative(dirent.getName())); TraversalRequest childTraversal = traversal.forChildEntry(childPath); result.add(RecursiveFilesystemTraversalValue.key(childTraversal)); } return result; }
/** * Adds the repository's directory to the graph and, if it's a symlink, resolves it to an actual * directory. */ @Nullable public static FileValue getRepositoryDirectory(Path repositoryDirectory, Environment env) throws RepositoryFunctionException { SkyKey outputDirectoryKey = FileValue.key(RootedPath.toRootedPath(repositoryDirectory, PathFragment.EMPTY_FRAGMENT)); FileValue value; try { value = (FileValue) env.getValueOrThrow( outputDirectoryKey, IOException.class, FileSymlinkException.class, InconsistentFilesystemException.class); } catch (IOException | FileSymlinkException | InconsistentFilesystemException e) { throw new RepositoryFunctionException( new IOException("Could not access " + repositoryDirectory + ": " + e.getMessage()), Transience.PERSISTENT); } return value; }
static FileStateValue createWithStatNoFollow( RootedPath rootedPath, FileStatusWithDigest statNoFollow, @Nullable TimestampGranularityMonitor tsgm) throws InconsistentFilesystemException, IOException { Path path = rootedPath.asPath(); if (statNoFollow.isFile()) { return statNoFollow.isSpecialFile() ? SpecialFileStateValue.fromStat(statNoFollow, tsgm) : RegularFileStateValue.fromPath(path, statNoFollow, tsgm); } else if (statNoFollow.isDirectory()) { return DIRECTORY_FILE_STATE_NODE; } else if (statNoFollow.isSymbolicLink()) { return new SymlinkFileStateValue(path.readSymbolicLinkUnchecked()); } throw new InconsistentFilesystemException( "according to stat, existing path " + path + " is " + "neither a file nor directory nor symlink."); }
/** * Get SkyKeys for the FileValues for the given {@code pathFragments}. To do this, we look for a * package lookup node for each path fragment, since package lookup nodes contain the "root" of a * package. The returned SkyKeys correspond to FileValues that may not exist in the graph. */ private Collection<SkyKey> getSkyKeysForFileFragments(Iterable<PathFragment> pathFragments) { Set<SkyKey> result = new HashSet<>(); Multimap<PathFragment, PathFragment> currentToOriginal = ArrayListMultimap.create(); for (PathFragment pathFragment : pathFragments) { currentToOriginal.put(pathFragment, pathFragment); } while (!currentToOriginal.isEmpty()) { Map<SkyKey, PathFragment> keys = new HashMap<>(); for (PathFragment pathFragment : currentToOriginal.keySet()) { keys.put( PackageLookupValue.key(PackageIdentifier.createInDefaultRepo(pathFragment)), pathFragment); } Map<SkyKey, SkyValue> lookupValues = graph.getSuccessfulValues(keys.keySet()); for (Map.Entry<SkyKey, SkyValue> entry : lookupValues.entrySet()) { PackageLookupValue packageLookupValue = (PackageLookupValue) entry.getValue(); if (packageLookupValue.packageExists()) { PathFragment dir = keys.get(entry.getKey()); Collection<PathFragment> originalFiles = currentToOriginal.get(dir); Preconditions.checkState(!originalFiles.isEmpty(), entry); for (PathFragment fileName : originalFiles) { result.add( FileValue.key(RootedPath.toRootedPath(packageLookupValue.getRoot(), fileName))); } currentToOriginal.removeAll(dir); } } Multimap<PathFragment, PathFragment> newCurrentToOriginal = ArrayListMultimap.create(); for (PathFragment pathFragment : currentToOriginal.keySet()) { PathFragment parent = pathFragment.getParentDirectory(); if (parent != null) { newCurrentToOriginal.putAll(parent, currentToOriginal.get(pathFragment)); } } currentToOriginal = newCurrentToOriginal; } return result; }
/** * If this instance is configured with DEPEND_ON_EXTERNAL_PKG and rootedPath is a file that isn't * under a package root then this adds a dependency on the //external package. If the action is * ERROR_OUT, it will throw an error instead. */ public void maybeHandleExternalFile(RootedPath rootedPath, SkyFunction.Environment env) throws FileOutsidePackageRootsException { if (isInternal(rootedPath, pkgLocator.get())) { return; } externalFileSeen = true; if (externalFileAction == ExternalFileAction.ERROR_OUT) { throw new FileOutsidePackageRootsException(rootedPath); } // The outputBase may be null if we're not actually running a build. Path outputBase = pkgLocator.get().getOutputBase(); if (outputBase != null && !rootedPath .asPath() .startsWith(pkgLocator.get().getOutputBase().getRelative(Label.EXTERNAL_PATH_PREFIX))) { return; } // For files that are under $OUTPUT_BASE/external, add a dependency on the //external package // so that if the WORKSPACE file changes, the File/DirectoryStateValue will be re-evaluated. // // Note that: // - We don't add a dependency on the parent directory at the package root boundary, so // the only transitive dependencies from files inside the package roots to external files // are through symlinks. So the upwards transitive closure of external files is small. // - The only way other than external repositories for external source files to get into the // skyframe graph in the first place is through symlinks outside the package roots, which we // neither want to encourage nor optimize for since it is not common. So the set of external // files is small. PackageValue pkgValue = (PackageValue) env.getValue(PackageValue.key(Label.EXTERNAL_PACKAGE_IDENTIFIER)); if (pkgValue == null) { return; } Preconditions.checkState(!pkgValue.getPackage().containsErrors()); }
/** * Looks in the directory specified by {@code recursivePkgKey} for a package, does some work as * specified by {@link Visitor} if such a package exists, then recursively does work in each * non-excluded subdirectory as specified by {@link #getSkyKeyForSubdirectory}, and finally * aggregates the {@link Visitor} value along with values from each subdirectory as specified by * {@link #aggregateWithSubdirectorySkyValues}, and returns that aggregation. * * <p>Returns null if {@code env.valuesMissing()} is true, checked after each call to one of * {@link RecursiveDirectoryTraversalFunction}'s abstract methods that were given {@code env}. * (And after each of {@code visitDirectory}'s own uses of {@code env}, of course.) */ TReturn visitDirectory(RecursivePkgKey recursivePkgKey, Environment env) { RootedPath rootedPath = recursivePkgKey.getRootedPath(); ImmutableSet<PathFragment> excludedPaths = recursivePkgKey.getExcludedPaths(); Path root = rootedPath.getRoot(); PathFragment rootRelativePath = rootedPath.getRelativePath(); SkyKey fileKey = FileValue.key(rootedPath); FileValue fileValue; try { fileValue = (FileValue) env.getValueOrThrow( fileKey, InconsistentFilesystemException.class, FileSymlinkException.class, IOException.class); } catch (InconsistentFilesystemException | FileSymlinkException | IOException e) { return reportErrorAndReturn( "Failed to get information about path", e, rootRelativePath, env.getListener()); } if (fileValue == null) { return null; } if (!fileValue.isDirectory()) { return getEmptyReturn(); } PackageIdentifier packageId = PackageIdentifier.create(recursivePkgKey.getRepository(), rootRelativePath); if ((packageId.getRepository().isDefault() || packageId.getRepository().isMain()) && fileValue.isSymlink() && fileValue .getUnresolvedLinkTarget() .startsWith(directories.getOutputBase().asFragment())) { // Symlinks back to the output base are not traversed so that we avoid convenience symlinks. // Note that it's not enough to just check for the convenience symlinks themselves, because // if the value of --symlink_prefix changes, the old symlinks are left in place. This // algorithm also covers more creative use cases where people create convenience symlinks // somewhere in the directory tree manually. return getEmptyReturn(); } SkyKey pkgLookupKey = PackageLookupValue.key(packageId); SkyKey dirListingKey = DirectoryListingValue.key(rootedPath); Map< SkyKey, ValueOrException4< NoSuchPackageException, InconsistentFilesystemException, FileSymlinkException, IOException>> pkgLookupAndDirectoryListingDeps = env.getValuesOrThrow( ImmutableList.of(pkgLookupKey, dirListingKey), NoSuchPackageException.class, InconsistentFilesystemException.class, FileSymlinkException.class, IOException.class); if (env.valuesMissing()) { return null; } PackageLookupValue pkgLookupValue; try { pkgLookupValue = (PackageLookupValue) Preconditions.checkNotNull( pkgLookupAndDirectoryListingDeps.get(pkgLookupKey).get(), "%s %s", recursivePkgKey, pkgLookupKey); } catch (NoSuchPackageException | InconsistentFilesystemException e) { return reportErrorAndReturn("Failed to load package", e, rootRelativePath, env.getListener()); } catch (IOException | FileSymlinkException e) { throw new IllegalStateException(e); } TVisitor visitor = getInitialVisitor(); DirectoryListingValue dirListingValue; try { dirListingValue = (DirectoryListingValue) Preconditions.checkNotNull( pkgLookupAndDirectoryListingDeps.get(dirListingKey).get(), "%s %s", recursivePkgKey, dirListingKey); } catch (InconsistentFilesystemException | IOException e) { return reportErrorAndReturn( "Failed to list directory contents", e, rootRelativePath, env.getListener()); } catch (FileSymlinkException e) { // DirectoryListingFunction only throws FileSymlinkCycleException when FileFunction throws it, // but FileFunction was evaluated for rootedPath above, and didn't throw there. It shouldn't // be able to avoid throwing there but throw here. throw new IllegalStateException( "Symlink cycle found after not being found for \"" + rootedPath + "\""); } catch (NoSuchPackageException e) { throw new IllegalStateException(e); } boolean followSymlinks = shouldFollowSymlinksWhenTraversing(dirListingValue.getDirents()); List<SkyKey> childDeps = new ArrayList<>(); for (Dirent dirent : dirListingValue.getDirents()) { Type type = dirent.getType(); if (type != Type.DIRECTORY && (type != Type.SYMLINK || (type == Type.SYMLINK && !followSymlinks))) { // Non-directories can never host packages. Symlinks to non-directories are weeded out at // the next level of recursion when we check if its FileValue is a directory. This is slower // if there are a lot of symlinks in the tree, but faster if there are only a few, which is // the case most of the time. // // We are not afraid of weird symlink structure here: both cyclical ones and ones that give // rise to infinite directory trees are diagnosed by FileValue. continue; } String basename = dirent.getName(); if (rootRelativePath.equals(PathFragment.EMPTY_FRAGMENT) && PathPackageLocator.DEFAULT_TOP_LEVEL_EXCLUDES.contains(basename)) { continue; } PathFragment subdirectory = rootRelativePath.getRelative(basename); // If this subdirectory is one of the excluded paths, don't recurse into it. if (excludedPaths.contains(subdirectory)) { continue; } // If we have an excluded path that isn't below this subdirectory, we shouldn't pass that // excluded path to our evaluation of the subdirectory, because the exclusion can't // possibly match anything beneath the subdirectory. // // For example, if we're currently evaluating directory "a", are looking at its subdirectory // "a/b", and we have an excluded path "a/c/d", there's no need to pass the excluded path // "a/c/d" to our evaluation of "a/b". // // This strategy should help to get more skyframe sharing. Consider the example above. A // subsequent request of "a/b/...", without any excluded paths, will be a cache hit. // // TODO(bazel-team): Replace the excludedPaths set with a trie or a SortedSet for better // efficiency. ImmutableSet<PathFragment> excludedSubdirectoriesBeneathThisSubdirectory = PathFragment.filterPathsStartingWith(excludedPaths, subdirectory); RootedPath subdirectoryRootedPath = RootedPath.toRootedPath(root, subdirectory); childDeps.add( getSkyKeyForSubdirectory( recursivePkgKey.getRepository(), subdirectoryRootedPath, excludedSubdirectoriesBeneathThisSubdirectory)); } Map<SkyKey, SkyValue> subdirectorySkyValues; if (pkgLookupValue.packageExists() && pkgLookupValue.getRoot().equals(root)) { SkyKey packageKey = PackageValue.key(packageId); Map<SkyKey, ValueOrException<NoSuchPackageException>> dependentSkyValues = env.getValuesOrThrow( Iterables.concat(childDeps, ImmutableList.of(packageKey)), NoSuchPackageException.class); if (env.valuesMissing()) { return null; } Package pkg = null; try { PackageValue pkgValue = (PackageValue) dependentSkyValues.get(packageKey).get(); if (pkgValue == null) { return null; } pkg = pkgValue.getPackage(); if (pkg.containsErrors()) { env.getListener() .handle(Event.error("package contains errors: " + rootRelativePath.getPathString())); } } catch (NoSuchPackageException e) { // The package had errors, but don't fail-fast as there might be subpackages below the // current directory. env.getListener() .handle(Event.error("package contains errors: " + rootRelativePath.getPathString())); } if (pkg != null) { visitor.visitPackageValue(pkg, env); if (env.valuesMissing()) { return null; } } ImmutableMap.Builder<SkyKey, SkyValue> subdirectoryBuilder = ImmutableMap.builder(); for (Map.Entry<SkyKey, ValueOrException<NoSuchPackageException>> entry : Maps.filterKeys(dependentSkyValues, Predicates.not(Predicates.equalTo(packageKey))) .entrySet()) { try { subdirectoryBuilder.put(entry.getKey(), entry.getValue().get()); } catch (NoSuchPackageException e) { // ignored. } } subdirectorySkyValues = subdirectoryBuilder.build(); } else { subdirectorySkyValues = env.getValues(childDeps); } if (env.valuesMissing()) { return null; } return aggregateWithSubdirectorySkyValues(visitor, subdirectorySkyValues); }