Esempio n. 1
0
  /**
   * Symlinks a BUILD file from the local filesystem into the external repository's root.
   *
   * @param rule the rule that declares the build_file path.
   * @param workspaceDirectory the workspace root for the build.
   * @param directoryValue the FileValue corresponding to the external repository's root directory.
   * @param env the Skyframe environment.
   * @return the file value of the symlink created.
   * @throws
   *     com.google.devtools.build.lib.bazel.repository.RepositoryFunction.RepositoryFunctionException
   *     if the BUILD file specified does not exist or cannot be linked.
   */
  protected RepositoryValue symlinkBuildFile(
      Rule rule, Path workspaceDirectory, FileValue directoryValue, Environment env)
      throws RepositoryFunctionException {
    AggregatingAttributeMapper mapper = AggregatingAttributeMapper.of(rule);
    PathFragment buildFile = new PathFragment(mapper.get("build_file", Type.STRING));
    Path buildFileTarget = workspaceDirectory.getRelative(buildFile);
    if (!buildFileTarget.exists()) {
      throw new RepositoryFunctionException(
          new EvalException(
              rule.getLocation(),
              String.format(
                  "In %s the 'build_file' attribute does not specify an existing file "
                      + "(%s does not exist)",
                  rule, buildFileTarget)),
          Transience.PERSISTENT);
    }

    RootedPath rootedBuild;
    if (buildFile.isAbsolute()) {
      rootedBuild =
          RootedPath.toRootedPath(
              buildFileTarget.getParentDirectory(),
              new PathFragment(buildFileTarget.getBaseName()));
    } else {
      rootedBuild = RootedPath.toRootedPath(workspaceDirectory, buildFile);
    }
    SkyKey buildFileKey = FileValue.key(rootedBuild);
    FileValue buildFileValue;
    try {
      buildFileValue =
          (FileValue)
              env.getValueOrThrow(
                  buildFileKey,
                  IOException.class,
                  FileSymlinkException.class,
                  InconsistentFilesystemException.class);
      if (buildFileValue == null) {
        return null;
      }
    } catch (IOException | FileSymlinkException | InconsistentFilesystemException e) {
      throw new RepositoryFunctionException(
          new IOException("Cannot lookup " + buildFile + ": " + e.getMessage()),
          Transience.TRANSIENT);
    }

    Path buildFilePath = directoryValue.realRootedPath().asPath().getRelative("BUILD");
    if (createSymbolicLink(buildFilePath, buildFileTarget, env) == null) {
      return null;
    }
    return RepositoryValue.createNew(directoryValue, buildFileValue);
  }
Esempio n. 2
0
  /**
   * Given a targetDirectory /some/path/to/y that contains files z, w, and v, create the following
   * directory structure:
   *
   * <pre>
   * .external-repository/
   *   x/
   *     WORKSPACE
   *     BUILD -> &lt;build_root&gt;/x.BUILD
   *     z -> /some/path/to/y/z
   *     w -> /some/path/to/y/w
   *     v -> /some/path/to/y/v
   * </pre>
   */
  public static boolean symlinkLocalRepositoryContents(
      Path repositoryDirectory, Path targetDirectory, Environment env)
      throws RepositoryFunctionException {
    try {
      for (Path target : targetDirectory.getDirectoryEntries()) {
        Path symlinkPath = repositoryDirectory.getRelative(target.getBaseName());
        if (createSymbolicLink(symlinkPath, target, env) == null) {
          return false;
        }
      }
    } catch (IOException e) {
      throw new RepositoryFunctionException(e, Transience.TRANSIENT);
    }

    return true;
  }
Esempio n. 3
0
  /**
   * Returns the runfiles directory associated with the test executable, creating/updating it if
   * necessary and --build_runfile_links is specified.
   */
  protected static Path getLocalRunfilesDirectory(
      TestRunnerAction testAction,
      ActionExecutionContext actionExecutionContext,
      BinTools binTools,
      ImmutableMap<String, String> shellEnvironment,
      boolean enableRunfiles)
      throws ExecException, InterruptedException {
    TestTargetExecutionSettings execSettings = testAction.getExecutionSettings();

    // If the symlink farm is already created then return the existing directory. If not we
    // need to explicitly build it. This can happen when --nobuild_runfile_links is supplied
    // as a flag to the build.
    if (execSettings.getRunfilesSymlinksCreated()) {
      return execSettings.getRunfilesDir();
    }

    // TODO(bazel-team): Should we be using TestTargetExecutionSettings#getRunfilesDir() here over
    // generating the directory ourselves?
    Path program = execSettings.getExecutable().getPath();
    Path runfilesDir = program.getParentDirectory().getChild(program.getBaseName() + ".runfiles");

    // Synchronize runfiles tree generation on the runfiles manifest artifact.
    // This is necessary, because we might end up with multiple test runner actions
    // trying to generate same runfiles tree in case of --runs_per_test > 1 or
    // local test sharding.
    long startTime = Profiler.nanoTimeMaybe();
    synchronized (execSettings.getInputManifest()) {
      Profiler.instance().logSimpleTask(startTime, ProfilerTask.WAIT, testAction);
      updateLocalRunfilesDirectory(
          testAction,
          runfilesDir,
          actionExecutionContext,
          binTools,
          shellEnvironment,
          enableRunfiles);
    }

    return runfilesDir;
  }
Esempio n. 4
0
  public static SkyKey key(
      String targetKind,
      String targetName,
      Path archivePath,
      Path repositoryPath,
      @Nullable String prefix)
      throws IOException {
    String baseName = archivePath.getBaseName();

    DecompressorDescriptor descriptor =
        new DecompressorDescriptor(targetKind, targetName, archivePath, repositoryPath, prefix);

    if (baseName.endsWith(".zip") || baseName.endsWith(".jar") || baseName.endsWith(".war")) {
      return new SkyKey(ZipFunction.NAME, descriptor);
    } else if (baseName.endsWith(".tar.gz") || baseName.endsWith(".tgz")) {
      return new SkyKey(TarGzFunction.NAME, descriptor);
    } else {
      throw new IOException(
          String.format(
              "Expected %s %s to create file with a .zip, .jar, .war, .tar.gz, or .tgz"
                  + " suffix (got %s)",
              targetKind, targetName, archivePath));
    }
  }
Esempio n. 5
0
  private void actuallyClean(
      CommandEnvironment env, Path outputBase, Options cleanOptions, String symlinkPrefix)
      throws IOException, ShutdownBlazeServerException, CommandException, ExecException,
          InterruptedException {
    BlazeRuntime runtime = env.getRuntime();
    if (env.getOutputService() != null) {
      env.getOutputService().clean();
    }
    if (cleanOptions.expunge) {
      LOG.info("Expunging...");
      // Delete the big subdirectories with the important content first--this
      // will take the most time. Then quickly delete the little locks, logs
      // and links right before we exit. Once the lock file is gone there will
      // be a small possibility of a server race if a client is waiting, but
      // all significant files will be gone by then.
      FileSystemUtils.deleteTreesBelow(outputBase);
      FileSystemUtils.deleteTree(outputBase);
    } else if (cleanOptions.expunge_async) {
      LOG.info("Expunging asynchronously...");
      String tempBaseName = outputBase.getBaseName() + "_tmp_" + ProcessUtils.getpid();

      // Keeping tempOutputBase in the same directory ensures it remains in the
      // same file system, and therefore the mv will be atomic and fast.
      Path tempOutputBase = outputBase.getParentDirectory().getChild(tempBaseName);
      outputBase.renameTo(tempOutputBase);
      env.getReporter()
          .handle(Event.info(null, "Output base moved to " + tempOutputBase + " for deletion"));

      // Daemonize the shell and use the double-fork idiom to ensure that the shell
      // exits even while the "rm -rf" command continues.
      String command =
          String.format(
              "exec >&- 2>&- <&- && (/usr/bin/setsid /bin/rm -rf %s &)&",
              ShellEscaper.escapeString(tempOutputBase.getPathString()));

      LOG.info("Executing shell commmand " + ShellEscaper.escapeString(command));

      // Doesn't throw iff command exited and was successful.
      new CommandBuilder()
          .addArg(command)
          .useShell(true)
          .setWorkingDir(tempOutputBase.getParentDirectory())
          .build()
          .execute();
    } else {
      LOG.info("Output cleaning...");
      runtime.clearCaches();
      // In order to be sure that we delete everything, delete the workspace directory both for
      // --deep_execroot and for --nodeep_execroot.
      for (String directory :
          new String[] {runtime.getWorkspaceName(), "execroot/" + runtime.getWorkspaceName()}) {
        Path child = outputBase.getRelative(directory);
        if (child.exists()) {
          LOG.finest("Cleaning " + child);
          FileSystemUtils.deleteTreesBelow(child);
        }
      }
    }
    // remove convenience links
    OutputDirectoryLinksUtils.removeOutputDirectoryLinks(
        runtime.getWorkspaceName(), runtime.getWorkspace(), env.getReporter(), symlinkPrefix);
    // shutdown on expunge cleans
    if (cleanOptions.expunge || cleanOptions.expunge_async) {
      throw new ShutdownBlazeServerException(0);
    }
  }