private void assertSkipsFoo(ImmutableList<String> patternSequence) throws Exception { // When PrepareDepsOfPatternsFunction completes evaluation (successfully), WalkableGraph walkableGraph = getGraphFromPatternsEvaluation( patternSequence, /*successExpected=*/ true, /*keepGoing=*/ true); // Then the graph contains a package value for "@//foo", assertTrue(walkableGraph.exists(PackageValue.key(PackageIdentifier.parse("@//foo")))); // But no package value for "@//foo/foo", assertFalse(walkableGraph.exists(PackageValue.key(PackageIdentifier.parse("@//foo/foo")))); // And the graph does not contain a value for the target "@//foo/foo:foofoo". Label label = Label.create("@//foo/foo", "foofoo"); assertFalse(walkableGraph.exists(getKeyForLabel(label))); }
@Test public void testRecursiveEvaluationFailsOnBadBuildFile() throws Exception { // Given a well-formed package "@//foo" and a malformed package "@//foo/foo", createFooAndFooFoo(); // Given a target pattern sequence consisting of a recursive pattern for "//foo/...", ImmutableList<String> patternSequence = ImmutableList.of("//foo/..."); // When PrepareDepsOfPatternsFunction completes evaluation (with no error because it was // recovered from), WalkableGraph walkableGraph = getGraphFromPatternsEvaluation( patternSequence, /*successExpected=*/ true, /*keepGoing=*/ true); // Then the graph contains package values for "@//foo" and "@//foo/foo", assertTrue(walkableGraph.exists(PackageValue.key(PackageIdentifier.parse("@//foo")))); assertTrue(walkableGraph.exists(PackageValue.key(PackageIdentifier.parse("@//foo/foo")))); // But the graph does not contain a value for the target "@//foo/foo:foofoo". assertFalse(walkableGraph.exists(getKeyForLabel(Label.create("@//foo/foo", "foofoo")))); }
/** * If this instance is configured with DEPEND_ON_EXTERNAL_PKG and rootedPath is a file that isn't * under a package root then this adds a dependency on the //external package. If the action is * ERROR_OUT, it will throw an error instead. */ public void maybeHandleExternalFile(RootedPath rootedPath, SkyFunction.Environment env) throws FileOutsidePackageRootsException { if (isInternal(rootedPath, pkgLocator.get())) { return; } externalFileSeen = true; if (externalFileAction == ExternalFileAction.ERROR_OUT) { throw new FileOutsidePackageRootsException(rootedPath); } // The outputBase may be null if we're not actually running a build. Path outputBase = pkgLocator.get().getOutputBase(); if (outputBase != null && !rootedPath .asPath() .startsWith(pkgLocator.get().getOutputBase().getRelative(Label.EXTERNAL_PATH_PREFIX))) { return; } // For files that are under $OUTPUT_BASE/external, add a dependency on the //external package // so that if the WORKSPACE file changes, the File/DirectoryStateValue will be re-evaluated. // // Note that: // - We don't add a dependency on the parent directory at the package root boundary, so // the only transitive dependencies from files inside the package roots to external files // are through symlinks. So the upwards transitive closure of external files is small. // - The only way other than external repositories for external source files to get into the // skyframe graph in the first place is through symlinks outside the package roots, which we // neither want to encourage nor optimize for since it is not common. So the set of external // files is small. PackageValue pkgValue = (PackageValue) env.getValue(PackageValue.key(Label.EXTERNAL_PACKAGE_IDENTIFIER)); if (pkgValue == null) { return; } Preconditions.checkState(!pkgValue.getPackage().containsErrors()); }
/** * Looks in the directory specified by {@code recursivePkgKey} for a package, does some work as * specified by {@link Visitor} if such a package exists, then recursively does work in each * non-excluded subdirectory as specified by {@link #getSkyKeyForSubdirectory}, and finally * aggregates the {@link Visitor} value along with values from each subdirectory as specified by * {@link #aggregateWithSubdirectorySkyValues}, and returns that aggregation. * * <p>Returns null if {@code env.valuesMissing()} is true, checked after each call to one of * {@link RecursiveDirectoryTraversalFunction}'s abstract methods that were given {@code env}. * (And after each of {@code visitDirectory}'s own uses of {@code env}, of course.) */ TReturn visitDirectory(RecursivePkgKey recursivePkgKey, Environment env) { RootedPath rootedPath = recursivePkgKey.getRootedPath(); ImmutableSet<PathFragment> excludedPaths = recursivePkgKey.getExcludedPaths(); Path root = rootedPath.getRoot(); PathFragment rootRelativePath = rootedPath.getRelativePath(); SkyKey fileKey = FileValue.key(rootedPath); FileValue fileValue; try { fileValue = (FileValue) env.getValueOrThrow( fileKey, InconsistentFilesystemException.class, FileSymlinkException.class, IOException.class); } catch (InconsistentFilesystemException | FileSymlinkException | IOException e) { return reportErrorAndReturn( "Failed to get information about path", e, rootRelativePath, env.getListener()); } if (fileValue == null) { return null; } if (!fileValue.isDirectory()) { return getEmptyReturn(); } PackageIdentifier packageId = PackageIdentifier.create(recursivePkgKey.getRepository(), rootRelativePath); if ((packageId.getRepository().isDefault() || packageId.getRepository().isMain()) && fileValue.isSymlink() && fileValue .getUnresolvedLinkTarget() .startsWith(directories.getOutputBase().asFragment())) { // Symlinks back to the output base are not traversed so that we avoid convenience symlinks. // Note that it's not enough to just check for the convenience symlinks themselves, because // if the value of --symlink_prefix changes, the old symlinks are left in place. This // algorithm also covers more creative use cases where people create convenience symlinks // somewhere in the directory tree manually. return getEmptyReturn(); } SkyKey pkgLookupKey = PackageLookupValue.key(packageId); SkyKey dirListingKey = DirectoryListingValue.key(rootedPath); Map< SkyKey, ValueOrException4< NoSuchPackageException, InconsistentFilesystemException, FileSymlinkException, IOException>> pkgLookupAndDirectoryListingDeps = env.getValuesOrThrow( ImmutableList.of(pkgLookupKey, dirListingKey), NoSuchPackageException.class, InconsistentFilesystemException.class, FileSymlinkException.class, IOException.class); if (env.valuesMissing()) { return null; } PackageLookupValue pkgLookupValue; try { pkgLookupValue = (PackageLookupValue) Preconditions.checkNotNull( pkgLookupAndDirectoryListingDeps.get(pkgLookupKey).get(), "%s %s", recursivePkgKey, pkgLookupKey); } catch (NoSuchPackageException | InconsistentFilesystemException e) { return reportErrorAndReturn("Failed to load package", e, rootRelativePath, env.getListener()); } catch (IOException | FileSymlinkException e) { throw new IllegalStateException(e); } TVisitor visitor = getInitialVisitor(); DirectoryListingValue dirListingValue; try { dirListingValue = (DirectoryListingValue) Preconditions.checkNotNull( pkgLookupAndDirectoryListingDeps.get(dirListingKey).get(), "%s %s", recursivePkgKey, dirListingKey); } catch (InconsistentFilesystemException | IOException e) { return reportErrorAndReturn( "Failed to list directory contents", e, rootRelativePath, env.getListener()); } catch (FileSymlinkException e) { // DirectoryListingFunction only throws FileSymlinkCycleException when FileFunction throws it, // but FileFunction was evaluated for rootedPath above, and didn't throw there. It shouldn't // be able to avoid throwing there but throw here. throw new IllegalStateException( "Symlink cycle found after not being found for \"" + rootedPath + "\""); } catch (NoSuchPackageException e) { throw new IllegalStateException(e); } boolean followSymlinks = shouldFollowSymlinksWhenTraversing(dirListingValue.getDirents()); List<SkyKey> childDeps = new ArrayList<>(); for (Dirent dirent : dirListingValue.getDirents()) { Type type = dirent.getType(); if (type != Type.DIRECTORY && (type != Type.SYMLINK || (type == Type.SYMLINK && !followSymlinks))) { // Non-directories can never host packages. Symlinks to non-directories are weeded out at // the next level of recursion when we check if its FileValue is a directory. This is slower // if there are a lot of symlinks in the tree, but faster if there are only a few, which is // the case most of the time. // // We are not afraid of weird symlink structure here: both cyclical ones and ones that give // rise to infinite directory trees are diagnosed by FileValue. continue; } String basename = dirent.getName(); if (rootRelativePath.equals(PathFragment.EMPTY_FRAGMENT) && PathPackageLocator.DEFAULT_TOP_LEVEL_EXCLUDES.contains(basename)) { continue; } PathFragment subdirectory = rootRelativePath.getRelative(basename); // If this subdirectory is one of the excluded paths, don't recurse into it. if (excludedPaths.contains(subdirectory)) { continue; } // If we have an excluded path that isn't below this subdirectory, we shouldn't pass that // excluded path to our evaluation of the subdirectory, because the exclusion can't // possibly match anything beneath the subdirectory. // // For example, if we're currently evaluating directory "a", are looking at its subdirectory // "a/b", and we have an excluded path "a/c/d", there's no need to pass the excluded path // "a/c/d" to our evaluation of "a/b". // // This strategy should help to get more skyframe sharing. Consider the example above. A // subsequent request of "a/b/...", without any excluded paths, will be a cache hit. // // TODO(bazel-team): Replace the excludedPaths set with a trie or a SortedSet for better // efficiency. ImmutableSet<PathFragment> excludedSubdirectoriesBeneathThisSubdirectory = PathFragment.filterPathsStartingWith(excludedPaths, subdirectory); RootedPath subdirectoryRootedPath = RootedPath.toRootedPath(root, subdirectory); childDeps.add( getSkyKeyForSubdirectory( recursivePkgKey.getRepository(), subdirectoryRootedPath, excludedSubdirectoriesBeneathThisSubdirectory)); } Map<SkyKey, SkyValue> subdirectorySkyValues; if (pkgLookupValue.packageExists() && pkgLookupValue.getRoot().equals(root)) { SkyKey packageKey = PackageValue.key(packageId); Map<SkyKey, ValueOrException<NoSuchPackageException>> dependentSkyValues = env.getValuesOrThrow( Iterables.concat(childDeps, ImmutableList.of(packageKey)), NoSuchPackageException.class); if (env.valuesMissing()) { return null; } Package pkg = null; try { PackageValue pkgValue = (PackageValue) dependentSkyValues.get(packageKey).get(); if (pkgValue == null) { return null; } pkg = pkgValue.getPackage(); if (pkg.containsErrors()) { env.getListener() .handle(Event.error("package contains errors: " + rootRelativePath.getPathString())); } } catch (NoSuchPackageException e) { // The package had errors, but don't fail-fast as there might be subpackages below the // current directory. env.getListener() .handle(Event.error("package contains errors: " + rootRelativePath.getPathString())); } if (pkg != null) { visitor.visitPackageValue(pkg, env); if (env.valuesMissing()) { return null; } } ImmutableMap.Builder<SkyKey, SkyValue> subdirectoryBuilder = ImmutableMap.builder(); for (Map.Entry<SkyKey, ValueOrException<NoSuchPackageException>> entry : Maps.filterKeys(dependentSkyValues, Predicates.not(Predicates.equalTo(packageKey))) .entrySet()) { try { subdirectoryBuilder.put(entry.getKey(), entry.getValue().get()); } catch (NoSuchPackageException e) { // ignored. } } subdirectorySkyValues = subdirectoryBuilder.build(); } else { subdirectorySkyValues = env.getValues(childDeps); } if (env.valuesMissing()) { return null; } return aggregateWithSubdirectorySkyValues(visitor, subdirectorySkyValues); }
@Nullable @Override public SkyValue compute(SkyKey skyKey, Environment env) throws AspectFunctionException, InterruptedException { SkyframeBuildView view = buildViewProvider.getSkyframeBuildView(); NestedSetBuilder<Package> transitivePackages = NestedSetBuilder.stableOrder(); NestedSetBuilder<Label> transitiveRootCauses = NestedSetBuilder.stableOrder(); AspectKey key = (AspectKey) skyKey.argument(); ConfiguredAspectFactory aspectFactory; if (key.getAspectClass() instanceof NativeAspectClass<?>) { aspectFactory = (ConfiguredAspectFactory) ((NativeAspectClass<?>) key.getAspectClass()).newInstance(); } else if (key.getAspectClass() instanceof SkylarkAspectClass) { SkylarkAspectClass skylarkAspectClass = (SkylarkAspectClass) key.getAspectClass(); SkylarkAspect skylarkAspect; try { skylarkAspect = loadSkylarkAspect( env, skylarkAspectClass.getExtensionLabel(), skylarkAspectClass.getExportedName()); } catch (AspectCreationException e) { throw new AspectFunctionException(e); } if (skylarkAspect == null) { return null; } aspectFactory = new SkylarkAspectFactory(skylarkAspect.getName(), skylarkAspect); } else { throw new IllegalStateException(); } // Keep this in sync with the same code in ConfiguredTargetFunction. PackageValue packageValue = (PackageValue) env.getValue(PackageValue.key(key.getLabel().getPackageIdentifier())); if (packageValue == null) { return null; } Package pkg = packageValue.getPackage(); if (pkg.containsErrors()) { throw new AspectFunctionException( new BuildFileContainsErrorsException(key.getLabel().getPackageIdentifier())); } Target target; try { target = pkg.getTarget(key.getLabel().getName()); } catch (NoSuchTargetException e) { throw new AspectFunctionException(e); } if (!(target instanceof Rule)) { throw new AspectFunctionException( new AspectCreationException("aspects must be attached to rules")); } final ConfiguredTargetValue configuredTargetValue; try { configuredTargetValue = (ConfiguredTargetValue) env.getValueOrThrow( ConfiguredTargetValue.key(key.getLabel(), key.getConfiguration()), ConfiguredValueCreationException.class); } catch (ConfiguredValueCreationException e) { throw new AspectFunctionException(new AspectCreationException(e.getRootCauses())); } if (configuredTargetValue == null) { // TODO(bazel-team): remove this check when top-level targets also use dynamic configurations. // Right now the key configuration may be dynamic while the original target's configuration // is static, resulting in a Skyframe cache miss even though the original target is, in fact, // precomputed. return null; } RuleConfiguredTarget associatedTarget = (RuleConfiguredTarget) configuredTargetValue.getConfiguredTarget(); if (associatedTarget == null) { return null; } SkyframeDependencyResolver resolver = view.createDependencyResolver(env); TargetAndConfiguration ctgValue = new TargetAndConfiguration(target, key.getConfiguration()); try { // Get the configuration targets that trigger this rule's configurable attributes. Set<ConfigMatchingProvider> configConditions = ConfiguredTargetFunction.getConfigConditions( target, env, resolver, ctgValue, transitivePackages, transitiveRootCauses); if (configConditions == null) { // Those targets haven't yet been resolved. return null; } ListMultimap<Attribute, ConfiguredTarget> depValueMap = ConfiguredTargetFunction.computeDependencies( env, resolver, ctgValue, key.getAspect(), configConditions, ruleClassProvider, view.getHostConfiguration(ctgValue.getConfiguration()), transitivePackages, transitiveRootCauses); if (depValueMap == null) { return null; } if (!transitiveRootCauses.isEmpty()) { throw new AspectFunctionException( new AspectCreationException("Loading failed", transitiveRootCauses.build())); } return createAspect( env, key, aspectFactory, associatedTarget, configConditions, depValueMap, transitivePackages); } catch (DependencyEvaluationException e) { if (e.getCause() instanceof ConfiguredValueCreationException) { ConfiguredValueCreationException cause = (ConfiguredValueCreationException) e.getCause(); throw new AspectFunctionException( new AspectCreationException(cause.getMessage(), cause.getAnalysisRootCause())); } else { // Cast to InvalidConfigurationException as a consistency check. If you add any // DependencyEvaluationException constructors, you may need to change this code, too. InvalidConfigurationException cause = (InvalidConfigurationException) e.getCause(); throw new AspectFunctionException(new AspectCreationException(cause.getMessage())); } } catch (AspectCreationException e) { throw new AspectFunctionException(e); } }