예제 #1
0
  @Override
  public SkyValue compute(SkyKey skyKey, Environment env)
      throws WorkspaceFileFunctionException, InterruptedException {
    RootedPath workspaceRoot = (RootedPath) skyKey.argument();
    FileValue workspaceFileValue = (FileValue) env.getValue(FileValue.key(workspaceRoot));
    if (workspaceFileValue == null) {
      return null;
    }

    Path repoWorkspace = workspaceRoot.getRoot().getRelative(workspaceRoot.getRelativePath());
    Builder builder =
        new Builder(repoWorkspace, packageFactory.getRuleClassProvider().getRunfilesPrefix());
    WorkspaceFactory parser =
        new WorkspaceFactory(
            builder, packageFactory.getRuleClassProvider(), installDir.getPathString());
    parser.parse(
        ParserInputSource.create(
            ruleClassProvider.getDefaultWorkspaceFile(), new PathFragment("DEFAULT.WORKSPACE")));
    if (!workspaceFileValue.exists()) {
      return new PackageValue(builder.build());
    }

    try {
      parser.parse(ParserInputSource.create(repoWorkspace));
    } catch (IOException e) {
      throw new WorkspaceFileFunctionException(e, Transience.TRANSIENT);
    }

    return new PackageValue(builder.build());
  }
예제 #2
0
  /**
   * Symlinks a BUILD file from the local filesystem into the external repository's root.
   *
   * @param rule the rule that declares the build_file path.
   * @param workspaceDirectory the workspace root for the build.
   * @param directoryValue the FileValue corresponding to the external repository's root directory.
   * @param env the Skyframe environment.
   * @return the file value of the symlink created.
   * @throws
   *     com.google.devtools.build.lib.bazel.repository.RepositoryFunction.RepositoryFunctionException
   *     if the BUILD file specified does not exist or cannot be linked.
   */
  protected RepositoryValue symlinkBuildFile(
      Rule rule, Path workspaceDirectory, FileValue directoryValue, Environment env)
      throws RepositoryFunctionException {
    AggregatingAttributeMapper mapper = AggregatingAttributeMapper.of(rule);
    PathFragment buildFile = new PathFragment(mapper.get("build_file", Type.STRING));
    Path buildFileTarget = workspaceDirectory.getRelative(buildFile);
    if (!buildFileTarget.exists()) {
      throw new RepositoryFunctionException(
          new EvalException(
              rule.getLocation(),
              String.format(
                  "In %s the 'build_file' attribute does not specify an existing file "
                      + "(%s does not exist)",
                  rule, buildFileTarget)),
          Transience.PERSISTENT);
    }

    RootedPath rootedBuild;
    if (buildFile.isAbsolute()) {
      rootedBuild =
          RootedPath.toRootedPath(
              buildFileTarget.getParentDirectory(),
              new PathFragment(buildFileTarget.getBaseName()));
    } else {
      rootedBuild = RootedPath.toRootedPath(workspaceDirectory, buildFile);
    }
    SkyKey buildFileKey = FileValue.key(rootedBuild);
    FileValue buildFileValue;
    try {
      buildFileValue =
          (FileValue)
              env.getValueOrThrow(
                  buildFileKey,
                  IOException.class,
                  FileSymlinkException.class,
                  InconsistentFilesystemException.class);
      if (buildFileValue == null) {
        return null;
      }
    } catch (IOException | FileSymlinkException | InconsistentFilesystemException e) {
      throw new RepositoryFunctionException(
          new IOException("Cannot lookup " + buildFile + ": " + e.getMessage()),
          Transience.TRANSIENT);
    }

    Path buildFilePath = directoryValue.realRootedPath().asPath().getRelative("BUILD");
    if (createSymbolicLink(buildFilePath, buildFileTarget, env) == null) {
      return null;
    }
    return RepositoryValue.createNew(directoryValue, buildFileValue);
  }
 @Override
 public ResolvedTargets<Void> findTargetsBeneathDirectory(
     String originalPattern,
     String directory,
     boolean rulesOnly,
     ImmutableSet<String> excludedSubdirectories)
     throws TargetParsingException, InterruptedException {
   FilteringPolicy policy =
       rulesOnly ? FilteringPolicies.RULES_ONLY : FilteringPolicies.NO_FILTER;
   ImmutableSet<PathFragment> excludedPathFragments =
       TargetPatternResolverUtil.getPathFragments(excludedSubdirectories);
   PathFragment pathFragment = TargetPatternResolverUtil.getPathFragment(directory);
   for (Path root : pkgPath.getPathEntries()) {
     RootedPath rootedPath = RootedPath.toRootedPath(root, pathFragment);
     SkyValue token =
         env.getValue(
             PrepareDepsOfTargetsUnderDirectoryValue.key(
                 rootedPath, excludedPathFragments, policy));
     if (token == null) {
       // A null token value means there is a missing dependency, because RecursivePkgFunction
       // never throws.
       throw new MissingDepException();
     }
   }
   return ResolvedTargets.empty();
 }
예제 #4
0
  private static FileValue createSymbolicLink(Path from, Path to, Environment env)
      throws RepositoryFunctionException {
    try {
      if (!from.exists()) {
        from.createSymbolicLink(to);
      }
    } catch (IOException e) {
      throw new RepositoryFunctionException(
          new IOException(
              String.format(
                  "Error creating symbolic link from %s to %s: %s", from, to, e.getMessage())),
          Transience.TRANSIENT);
    }

    SkyKey outputDirectoryKey =
        FileValue.key(RootedPath.toRootedPath(from, PathFragment.EMPTY_FRAGMENT));
    try {
      return (FileValue)
          env.getValueOrThrow(
              outputDirectoryKey,
              IOException.class,
              FileSymlinkException.class,
              InconsistentFilesystemException.class);
    } catch (IOException | FileSymlinkException | InconsistentFilesystemException e) {
      throw new RepositoryFunctionException(
          new IOException(String.format("Could not access %s: %s", from, e.getMessage())),
          Transience.PERSISTENT);
    }
  }
예제 #5
0
 private FileLookupResult getASTFile(Environment env, PathFragment astFilePathFragment)
     throws ASTLookupFunctionException {
   for (Path packagePathEntry : pkgLocator.get().getPathEntries()) {
     RootedPath rootedPath = RootedPath.toRootedPath(packagePathEntry, astFilePathFragment);
     SkyKey fileSkyKey = FileValue.key(rootedPath);
     FileValue fileValue = null;
     try {
       fileValue =
           (FileValue)
               env.getValueOrThrow(
                   fileSkyKey,
                   IOException.class,
                   FileSymlinkCycleException.class,
                   InconsistentFilesystemException.class);
     } catch (IOException | FileSymlinkCycleException e) {
       throw new ASTLookupFunctionException(
           new ErrorReadingSkylarkExtensionException(e.getMessage()), Transience.PERSISTENT);
     } catch (InconsistentFilesystemException e) {
       throw new ASTLookupFunctionException(e, Transience.PERSISTENT);
     }
     if (fileValue == null) {
       return null;
     }
     if (fileValue.isFile()) {
       return FileLookupResult.file(rootedPath);
     }
   }
   return FileLookupResult.noFile();
 }
예제 #6
0
 private SkyKey buildRecursivePkgKey(
     RepositoryName repository,
     PathFragment rootRelativePath,
     ImmutableSet<PathFragment> excludedPaths) {
   RootedPath rootedPath = RootedPath.toRootedPath(rootDirectory, rootRelativePath);
   return RecursivePkgValue.key(repository, rootedPath, excludedPaths);
 }
예제 #7
0
 static boolean isInternal(RootedPath rootedPath, PathPackageLocator packageLocator) {
   // TODO(bazel-team): This is inefficient when there are a lot of package roots or there are a
   // lot of external directories. Consider either explicitly preventing this case or using a more
   // efficient approach here (e.g. use a trie for determining if a file is under an external
   // directory).
   return packageLocator.getPathEntries().contains(rootedPath.getRoot());
 }
예제 #8
0
 static FileStateValue create(RootedPath rootedPath, @Nullable TimestampGranularityMonitor tsgm)
     throws InconsistentFilesystemException, IOException {
   Path path = rootedPath.asPath();
   // Stat, but don't throw an exception for the common case of a nonexistent file. This still
   // throws an IOException in case any other IO error is encountered.
   FileStatus stat = path.statIfFound(Symlinks.NOFOLLOW);
   if (stat == null) {
     return NONEXISTENT_FILE_STATE_NODE;
   }
   return createWithStatNoFollow(rootedPath, FileStatusWithDigestAdapter.adapt(stat), tsgm);
 }
  /**
   * List the directory and create {@code SkyKey}s to request contents of its children recursively.
   *
   * <p>The returned keys are of type {@link SkyFunctions#RECURSIVE_FILESYSTEM_TRAVERSAL}.
   */
  private static Collection<SkyKey> createRecursiveTraversalKeys(
      Environment env, TraversalRequest traversal) throws MissingDepException {
    // Use the traversal's path, even if it's a symlink. The contents of the directory, as listed
    // in the result, must be relative to it.
    DirectoryListingValue dirListing =
        (DirectoryListingValue)
            getDependentSkyValue(env, DirectoryListingValue.key(traversal.path));

    List<SkyKey> result = new ArrayList<>();
    for (Dirent dirent : dirListing.getDirents()) {
      RootedPath childPath =
          RootedPath.toRootedPath(
              traversal.path.getRoot(),
              traversal.path.getRelativePath().getRelative(dirent.getName()));
      TraversalRequest childTraversal = traversal.forChildEntry(childPath);
      result.add(RecursiveFilesystemTraversalValue.key(childTraversal));
    }
    return result;
  }
예제 #10
0
 /**
  * Adds the repository's directory to the graph and, if it's a symlink, resolves it to an actual
  * directory.
  */
 @Nullable
 public static FileValue getRepositoryDirectory(Path repositoryDirectory, Environment env)
     throws RepositoryFunctionException {
   SkyKey outputDirectoryKey =
       FileValue.key(RootedPath.toRootedPath(repositoryDirectory, PathFragment.EMPTY_FRAGMENT));
   FileValue value;
   try {
     value =
         (FileValue)
             env.getValueOrThrow(
                 outputDirectoryKey,
                 IOException.class,
                 FileSymlinkException.class,
                 InconsistentFilesystemException.class);
   } catch (IOException | FileSymlinkException | InconsistentFilesystemException e) {
     throw new RepositoryFunctionException(
         new IOException("Could not access " + repositoryDirectory + ": " + e.getMessage()),
         Transience.PERSISTENT);
   }
   return value;
 }
예제 #11
0
 static FileStateValue createWithStatNoFollow(
     RootedPath rootedPath,
     FileStatusWithDigest statNoFollow,
     @Nullable TimestampGranularityMonitor tsgm)
     throws InconsistentFilesystemException, IOException {
   Path path = rootedPath.asPath();
   if (statNoFollow.isFile()) {
     return statNoFollow.isSpecialFile()
         ? SpecialFileStateValue.fromStat(statNoFollow, tsgm)
         : RegularFileStateValue.fromPath(path, statNoFollow, tsgm);
   } else if (statNoFollow.isDirectory()) {
     return DIRECTORY_FILE_STATE_NODE;
   } else if (statNoFollow.isSymbolicLink()) {
     return new SymlinkFileStateValue(path.readSymbolicLinkUnchecked());
   }
   throw new InconsistentFilesystemException(
       "according to stat, existing path "
           + path
           + " is "
           + "neither a file nor directory nor symlink.");
 }
예제 #12
0
 /**
  * Get SkyKeys for the FileValues for the given {@code pathFragments}. To do this, we look for a
  * package lookup node for each path fragment, since package lookup nodes contain the "root" of a
  * package. The returned SkyKeys correspond to FileValues that may not exist in the graph.
  */
 private Collection<SkyKey> getSkyKeysForFileFragments(Iterable<PathFragment> pathFragments) {
   Set<SkyKey> result = new HashSet<>();
   Multimap<PathFragment, PathFragment> currentToOriginal = ArrayListMultimap.create();
   for (PathFragment pathFragment : pathFragments) {
     currentToOriginal.put(pathFragment, pathFragment);
   }
   while (!currentToOriginal.isEmpty()) {
     Map<SkyKey, PathFragment> keys = new HashMap<>();
     for (PathFragment pathFragment : currentToOriginal.keySet()) {
       keys.put(
           PackageLookupValue.key(PackageIdentifier.createInDefaultRepo(pathFragment)),
           pathFragment);
     }
     Map<SkyKey, SkyValue> lookupValues = graph.getSuccessfulValues(keys.keySet());
     for (Map.Entry<SkyKey, SkyValue> entry : lookupValues.entrySet()) {
       PackageLookupValue packageLookupValue = (PackageLookupValue) entry.getValue();
       if (packageLookupValue.packageExists()) {
         PathFragment dir = keys.get(entry.getKey());
         Collection<PathFragment> originalFiles = currentToOriginal.get(dir);
         Preconditions.checkState(!originalFiles.isEmpty(), entry);
         for (PathFragment fileName : originalFiles) {
           result.add(
               FileValue.key(RootedPath.toRootedPath(packageLookupValue.getRoot(), fileName)));
         }
         currentToOriginal.removeAll(dir);
       }
     }
     Multimap<PathFragment, PathFragment> newCurrentToOriginal = ArrayListMultimap.create();
     for (PathFragment pathFragment : currentToOriginal.keySet()) {
       PathFragment parent = pathFragment.getParentDirectory();
       if (parent != null) {
         newCurrentToOriginal.putAll(parent, currentToOriginal.get(pathFragment));
       }
     }
     currentToOriginal = newCurrentToOriginal;
   }
   return result;
 }
예제 #13
0
  /**
   * If this instance is configured with DEPEND_ON_EXTERNAL_PKG and rootedPath is a file that isn't
   * under a package root then this adds a dependency on the //external package. If the action is
   * ERROR_OUT, it will throw an error instead.
   */
  public void maybeHandleExternalFile(RootedPath rootedPath, SkyFunction.Environment env)
      throws FileOutsidePackageRootsException {
    if (isInternal(rootedPath, pkgLocator.get())) {
      return;
    }

    externalFileSeen = true;
    if (externalFileAction == ExternalFileAction.ERROR_OUT) {
      throw new FileOutsidePackageRootsException(rootedPath);
    }

    // The outputBase may be null if we're not actually running a build.
    Path outputBase = pkgLocator.get().getOutputBase();
    if (outputBase != null
        && !rootedPath
            .asPath()
            .startsWith(pkgLocator.get().getOutputBase().getRelative(Label.EXTERNAL_PATH_PREFIX))) {
      return;
    }

    // For files that are under $OUTPUT_BASE/external, add a dependency on the //external package
    // so that if the WORKSPACE file changes, the File/DirectoryStateValue will be re-evaluated.
    //
    // Note that:
    // - We don't add a dependency on the parent directory at the package root boundary, so
    // the only transitive dependencies from files inside the package roots to external files
    // are through symlinks. So the upwards transitive closure of external files is small.
    // - The only way other than external repositories for external source files to get into the
    // skyframe graph in the first place is through symlinks outside the package roots, which we
    // neither want to encourage nor optimize for since it is not common. So the set of external
    // files is small.
    PackageValue pkgValue =
        (PackageValue) env.getValue(PackageValue.key(Label.EXTERNAL_PACKAGE_IDENTIFIER));
    if (pkgValue == null) {
      return;
    }
    Preconditions.checkState(!pkgValue.getPackage().containsErrors());
  }
  /**
   * Looks in the directory specified by {@code recursivePkgKey} for a package, does some work as
   * specified by {@link Visitor} if such a package exists, then recursively does work in each
   * non-excluded subdirectory as specified by {@link #getSkyKeyForSubdirectory}, and finally
   * aggregates the {@link Visitor} value along with values from each subdirectory as specified by
   * {@link #aggregateWithSubdirectorySkyValues}, and returns that aggregation.
   *
   * <p>Returns null if {@code env.valuesMissing()} is true, checked after each call to one of
   * {@link RecursiveDirectoryTraversalFunction}'s abstract methods that were given {@code env}.
   * (And after each of {@code visitDirectory}'s own uses of {@code env}, of course.)
   */
  TReturn visitDirectory(RecursivePkgKey recursivePkgKey, Environment env) {
    RootedPath rootedPath = recursivePkgKey.getRootedPath();
    ImmutableSet<PathFragment> excludedPaths = recursivePkgKey.getExcludedPaths();
    Path root = rootedPath.getRoot();
    PathFragment rootRelativePath = rootedPath.getRelativePath();

    SkyKey fileKey = FileValue.key(rootedPath);
    FileValue fileValue;
    try {
      fileValue =
          (FileValue)
              env.getValueOrThrow(
                  fileKey,
                  InconsistentFilesystemException.class,
                  FileSymlinkException.class,
                  IOException.class);
    } catch (InconsistentFilesystemException | FileSymlinkException | IOException e) {
      return reportErrorAndReturn(
          "Failed to get information about path", e, rootRelativePath, env.getListener());
    }
    if (fileValue == null) {
      return null;
    }

    if (!fileValue.isDirectory()) {
      return getEmptyReturn();
    }

    PackageIdentifier packageId =
        PackageIdentifier.create(recursivePkgKey.getRepository(), rootRelativePath);

    if ((packageId.getRepository().isDefault() || packageId.getRepository().isMain())
        && fileValue.isSymlink()
        && fileValue
            .getUnresolvedLinkTarget()
            .startsWith(directories.getOutputBase().asFragment())) {
      // Symlinks back to the output base are not traversed so that we avoid convenience symlinks.
      // Note that it's not enough to just check for the convenience symlinks themselves, because
      // if the value of --symlink_prefix changes, the old symlinks are left in place. This
      // algorithm also covers more creative use cases where people create convenience symlinks
      // somewhere in the directory tree manually.
      return getEmptyReturn();
    }

    SkyKey pkgLookupKey = PackageLookupValue.key(packageId);
    SkyKey dirListingKey = DirectoryListingValue.key(rootedPath);
    Map<
            SkyKey,
            ValueOrException4<
                NoSuchPackageException,
                InconsistentFilesystemException,
                FileSymlinkException,
                IOException>>
        pkgLookupAndDirectoryListingDeps =
            env.getValuesOrThrow(
                ImmutableList.of(pkgLookupKey, dirListingKey),
                NoSuchPackageException.class,
                InconsistentFilesystemException.class,
                FileSymlinkException.class,
                IOException.class);
    if (env.valuesMissing()) {
      return null;
    }
    PackageLookupValue pkgLookupValue;
    try {
      pkgLookupValue =
          (PackageLookupValue)
              Preconditions.checkNotNull(
                  pkgLookupAndDirectoryListingDeps.get(pkgLookupKey).get(),
                  "%s %s",
                  recursivePkgKey,
                  pkgLookupKey);
    } catch (NoSuchPackageException | InconsistentFilesystemException e) {
      return reportErrorAndReturn("Failed to load package", e, rootRelativePath, env.getListener());
    } catch (IOException | FileSymlinkException e) {
      throw new IllegalStateException(e);
    }

    TVisitor visitor = getInitialVisitor();

    DirectoryListingValue dirListingValue;
    try {
      dirListingValue =
          (DirectoryListingValue)
              Preconditions.checkNotNull(
                  pkgLookupAndDirectoryListingDeps.get(dirListingKey).get(),
                  "%s %s",
                  recursivePkgKey,
                  dirListingKey);
    } catch (InconsistentFilesystemException | IOException e) {
      return reportErrorAndReturn(
          "Failed to list directory contents", e, rootRelativePath, env.getListener());
    } catch (FileSymlinkException e) {
      // DirectoryListingFunction only throws FileSymlinkCycleException when FileFunction throws it,
      // but FileFunction was evaluated for rootedPath above, and didn't throw there. It shouldn't
      // be able to avoid throwing there but throw here.
      throw new IllegalStateException(
          "Symlink cycle found after not being found for \"" + rootedPath + "\"");
    } catch (NoSuchPackageException e) {
      throw new IllegalStateException(e);
    }

    boolean followSymlinks = shouldFollowSymlinksWhenTraversing(dirListingValue.getDirents());
    List<SkyKey> childDeps = new ArrayList<>();
    for (Dirent dirent : dirListingValue.getDirents()) {
      Type type = dirent.getType();
      if (type != Type.DIRECTORY
          && (type != Type.SYMLINK || (type == Type.SYMLINK && !followSymlinks))) {
        // Non-directories can never host packages. Symlinks to non-directories are weeded out at
        // the next level of recursion when we check if its FileValue is a directory. This is slower
        // if there are a lot of symlinks in the tree, but faster if there are only a few, which is
        // the case most of the time.
        //
        // We are not afraid of weird symlink structure here: both cyclical ones and ones that give
        // rise to infinite directory trees are diagnosed by FileValue.
        continue;
      }
      String basename = dirent.getName();
      if (rootRelativePath.equals(PathFragment.EMPTY_FRAGMENT)
          && PathPackageLocator.DEFAULT_TOP_LEVEL_EXCLUDES.contains(basename)) {
        continue;
      }
      PathFragment subdirectory = rootRelativePath.getRelative(basename);

      // If this subdirectory is one of the excluded paths, don't recurse into it.
      if (excludedPaths.contains(subdirectory)) {
        continue;
      }

      // If we have an excluded path that isn't below this subdirectory, we shouldn't pass that
      // excluded path to our evaluation of the subdirectory, because the exclusion can't
      // possibly match anything beneath the subdirectory.
      //
      // For example, if we're currently evaluating directory "a", are looking at its subdirectory
      // "a/b", and we have an excluded path "a/c/d", there's no need to pass the excluded path
      // "a/c/d" to our evaluation of "a/b".
      //
      // This strategy should help to get more skyframe sharing. Consider the example above. A
      // subsequent request of "a/b/...", without any excluded paths, will be a cache hit.
      //
      // TODO(bazel-team): Replace the excludedPaths set with a trie or a SortedSet for better
      // efficiency.
      ImmutableSet<PathFragment> excludedSubdirectoriesBeneathThisSubdirectory =
          PathFragment.filterPathsStartingWith(excludedPaths, subdirectory);
      RootedPath subdirectoryRootedPath = RootedPath.toRootedPath(root, subdirectory);
      childDeps.add(
          getSkyKeyForSubdirectory(
              recursivePkgKey.getRepository(),
              subdirectoryRootedPath,
              excludedSubdirectoriesBeneathThisSubdirectory));
    }

    Map<SkyKey, SkyValue> subdirectorySkyValues;
    if (pkgLookupValue.packageExists() && pkgLookupValue.getRoot().equals(root)) {
      SkyKey packageKey = PackageValue.key(packageId);
      Map<SkyKey, ValueOrException<NoSuchPackageException>> dependentSkyValues =
          env.getValuesOrThrow(
              Iterables.concat(childDeps, ImmutableList.of(packageKey)),
              NoSuchPackageException.class);
      if (env.valuesMissing()) {
        return null;
      }
      Package pkg = null;
      try {
        PackageValue pkgValue = (PackageValue) dependentSkyValues.get(packageKey).get();
        if (pkgValue == null) {
          return null;
        }
        pkg = pkgValue.getPackage();
        if (pkg.containsErrors()) {
          env.getListener()
              .handle(Event.error("package contains errors: " + rootRelativePath.getPathString()));
        }
      } catch (NoSuchPackageException e) {
        // The package had errors, but don't fail-fast as there might be subpackages below the
        // current directory.
        env.getListener()
            .handle(Event.error("package contains errors: " + rootRelativePath.getPathString()));
      }
      if (pkg != null) {
        visitor.visitPackageValue(pkg, env);
        if (env.valuesMissing()) {
          return null;
        }
      }
      ImmutableMap.Builder<SkyKey, SkyValue> subdirectoryBuilder = ImmutableMap.builder();
      for (Map.Entry<SkyKey, ValueOrException<NoSuchPackageException>> entry :
          Maps.filterKeys(dependentSkyValues, Predicates.not(Predicates.equalTo(packageKey)))
              .entrySet()) {
        try {
          subdirectoryBuilder.put(entry.getKey(), entry.getValue().get());
        } catch (NoSuchPackageException e) {
          // ignored.
        }
      }
      subdirectorySkyValues = subdirectoryBuilder.build();
    } else {
      subdirectorySkyValues = env.getValues(childDeps);
    }
    if (env.valuesMissing()) {
      return null;
    }
    return aggregateWithSubdirectorySkyValues(visitor, subdirectorySkyValues);
  }