Esempio n. 1
0
  /**
   * Generates a command line to run for the test action, taking into account coverage and {@code
   * --run_under} settings.
   *
   * @param testScript the setup script that invokes the test
   * @param coverageScript a script interjected between setup script and rest of command line to
   *     collect coverage data. If this is an empty string, it is ignored.
   * @param testAction The test action.
   * @return the command line as string list.
   */
  protected List<String> getArgs(
      String testScript, String coverageScript, TestRunnerAction testAction) {
    List<String> args = Lists.newArrayList();
    if (OS.getCurrent() == OS.WINDOWS) {
      args.add(testAction.getShExecutable().getPathString());
      args.add("-c");
      args.add("$0 $*");
    }
    args.add(testScript);
    TestTargetExecutionSettings execSettings = testAction.getExecutionSettings();

    List<String> execArgs = new ArrayList<>();
    if (!coverageScript.isEmpty() && isCoverageMode(testAction)) {
      execArgs.add(coverageScript);
    }

    // Execute the test using the alias in the runfiles tree, as mandated by
    // the Test Encyclopedia.
    execArgs.add(execSettings.getExecutable().getRootRelativePath().getPathString());
    execArgs.addAll(execSettings.getArgs());

    // Insert the command prefix specified by the "--run_under=<command-prefix>" option,
    // if any.
    if (execSettings.getRunUnder() == null) {
      args.addAll(execArgs);
    } else if (execSettings.getRunUnderExecutable() != null) {
      args.add(execSettings.getRunUnderExecutable().getRootRelativePath().getPathString());
      args.addAll(execSettings.getRunUnder().getOptions());
      args.addAll(execArgs);
    } else {
      args.add(testAction.getConfiguration().getShellExecutable().getPathString());
      args.add("-c");

      String runUnderCommand = ShellEscaper.escapeString(execSettings.getRunUnder().getCommand());

      Path fullySpecified =
          SearchPath.which(
              SearchPath.parse(
                  testAction.getTestLog().getPath().getFileSystem(), clientEnv.get("PATH")),
              runUnderCommand);

      if (fullySpecified != null) {
        runUnderCommand = fullySpecified.toString();
      }

      args.add(
          runUnderCommand
              + ' '
              + ShellEscaper.escapeJoinAll(
                  Iterables.concat(execSettings.getRunUnder().getOptions(), execArgs)));
    }
    return args;
  }
 /**
  * Adds linkstamp compilation to the (otherwise) fully specified link command if {@link
  * #getLinkstamps} is non-empty.
  *
  * <p>Linkstamps were historically compiled implicitly as part of the link command, but implicit
  * compilation doesn't guarantee consistent outputs. For example, the command "gcc input.o input.o
  * foo/linkstamp.cc -o myapp" causes gcc to implicitly run "gcc foo/linkstamp.cc -o
  * /tmp/ccEtJHDB.o", for some internally decided output path /tmp/ccEtJHDB.o, then add that path
  * to the linker's command line options. The name of this path can change even between
  * equivalently specified gcc invocations.
  *
  * <p>So now we explicitly compile these files in their own command invocations before running the
  * link command, thus giving us direct control over the naming of their outputs. This method adds
  * those extra steps as necessary.
  *
  * @param linkstampCommands individual linkstamp compilation commands
  * @param linkCommand the complete list of link command arguments (after .params file compacting)
  *     for an invocation
  * @param escapeArgs if true, linkCommand arguments are shell escaped. if false, arguments are
  *     returned as-is
  * @return The original argument list if no linkstamps compilation commands are given, otherwise
  *     an expanded list that adds the linkstamp compilation commands and funnels their outputs
  *     into the link step. Note that these outputs only need to persist for the duration of the
  *     link step.
  */
 private static List<String> addLinkstampingToCommand(
     List<String> linkstampCommands, List<String> linkCommand, boolean escapeArgs) {
   if (linkstampCommands.isEmpty()) {
     return linkCommand;
   } else {
     List<String> batchCommand = Lists.newArrayListWithCapacity(3);
     batchCommand.add("/bin/bash");
     batchCommand.add("-c");
     batchCommand.add(
         Joiner.on(" && ").join(linkstampCommands)
             + " && "
             + (escapeArgs
                 ? ShellEscaper.escapeJoinAll(linkCommand)
                 : Joiner.on(" ").join(linkCommand)));
     return ImmutableList.copyOf(batchCommand);
   }
 }
Esempio n. 3
0
  private void actuallyClean(
      CommandEnvironment env, Path outputBase, Options cleanOptions, String symlinkPrefix)
      throws IOException, ShutdownBlazeServerException, CommandException, ExecException,
          InterruptedException {
    BlazeRuntime runtime = env.getRuntime();
    if (env.getOutputService() != null) {
      env.getOutputService().clean();
    }
    if (cleanOptions.expunge) {
      LOG.info("Expunging...");
      // Delete the big subdirectories with the important content first--this
      // will take the most time. Then quickly delete the little locks, logs
      // and links right before we exit. Once the lock file is gone there will
      // be a small possibility of a server race if a client is waiting, but
      // all significant files will be gone by then.
      FileSystemUtils.deleteTreesBelow(outputBase);
      FileSystemUtils.deleteTree(outputBase);
    } else if (cleanOptions.expunge_async) {
      LOG.info("Expunging asynchronously...");
      String tempBaseName = outputBase.getBaseName() + "_tmp_" + ProcessUtils.getpid();

      // Keeping tempOutputBase in the same directory ensures it remains in the
      // same file system, and therefore the mv will be atomic and fast.
      Path tempOutputBase = outputBase.getParentDirectory().getChild(tempBaseName);
      outputBase.renameTo(tempOutputBase);
      env.getReporter()
          .handle(Event.info(null, "Output base moved to " + tempOutputBase + " for deletion"));

      // Daemonize the shell and use the double-fork idiom to ensure that the shell
      // exits even while the "rm -rf" command continues.
      String command =
          String.format(
              "exec >&- 2>&- <&- && (/usr/bin/setsid /bin/rm -rf %s &)&",
              ShellEscaper.escapeString(tempOutputBase.getPathString()));

      LOG.info("Executing shell commmand " + ShellEscaper.escapeString(command));

      // Doesn't throw iff command exited and was successful.
      new CommandBuilder()
          .addArg(command)
          .useShell(true)
          .setWorkingDir(tempOutputBase.getParentDirectory())
          .build()
          .execute();
    } else {
      LOG.info("Output cleaning...");
      runtime.clearCaches();
      // In order to be sure that we delete everything, delete the workspace directory both for
      // --deep_execroot and for --nodeep_execroot.
      for (String directory :
          new String[] {runtime.getWorkspaceName(), "execroot/" + runtime.getWorkspaceName()}) {
        Path child = outputBase.getRelative(directory);
        if (child.exists()) {
          LOG.finest("Cleaning " + child);
          FileSystemUtils.deleteTreesBelow(child);
        }
      }
    }
    // remove convenience links
    OutputDirectoryLinksUtils.removeOutputDirectoryLinks(
        runtime.getWorkspaceName(), runtime.getWorkspace(), env.getReporter(), symlinkPrefix);
    // shutdown on expunge cleans
    if (cleanOptions.expunge || cleanOptions.expunge_async) {
      throw new ShutdownBlazeServerException(0);
    }
  }
  /**
   * Computes, for each C++ source file in {@link #getLinkstamps}, the command necessary to compile
   * that file such that the output is correctly fed into the link command.
   *
   * <p>As these options (as well as all others) are taken into account when computing the action
   * key, they do not directly contain volatile build information to avoid unnecessary relinking.
   * Instead this information is passed as an additional header generated by {@link
   * com.google.devtools.build.lib.rules.cpp.WriteBuildInfoHeaderAction}.
   *
   * @param outputPrefix prefix to add before the linkstamp outputs' exec paths
   * @return a list of shell-escaped compiler commmands, one for each entry in {@link
   *     #getLinkstamps}
   */
  public List<String> getLinkstampCompileCommands(String outputPrefix) {
    if (linkstamps.isEmpty()) {
      return ImmutableList.of();
    }

    String compilerCommand = cppConfiguration.getCppExecutable().getPathString();
    List<String> commands = Lists.newArrayListWithCapacity(linkstamps.size());

    for (Map.Entry<Artifact, Artifact> linkstamp : linkstamps.entrySet()) {
      List<String> optionList = new ArrayList<>();

      // Defines related to the build info are read from generated headers.
      for (Artifact header : buildInfoHeaderArtifacts) {
        optionList.add("-include");
        optionList.add(header.getExecPathString());
      }

      String labelReplacement =
          Matcher.quoteReplacement(
              isSharedNativeLibrary() ? output.getExecPathString() : Label.print(owner.getLabel()));
      String outputPathReplacement = Matcher.quoteReplacement(output.getExecPathString());
      for (String option : linkstampCompileOptions) {
        optionList.add(
            option
                .replaceAll(Pattern.quote("${LABEL}"), labelReplacement)
                .replaceAll(Pattern.quote("${OUTPUT_PATH}"), outputPathReplacement));
      }

      optionList.add("-DGPLATFORM=\"" + cppConfiguration + "\"");

      // Needed to find headers included from linkstamps.
      optionList.add("-I.");

      // Add sysroot.
      PathFragment sysroot = cppConfiguration.getSysroot();
      if (sysroot != null) {
        optionList.add("--sysroot=" + sysroot.getPathString());
      }

      // Add toolchain compiler options.
      optionList.addAll(cppConfiguration.getCompilerOptions(features));
      optionList.addAll(cppConfiguration.getCOptions());
      optionList.addAll(cppConfiguration.getUnfilteredCompilerOptions(features));
      if (CppFileTypes.CPP_SOURCE.matches(linkstamp.getKey().getExecPath())) {
        optionList.addAll(cppConfiguration.getCxxOptions(features));
      }

      // For dynamic libraries, produce position independent code.
      if (linkTargetType == LinkTargetType.DYNAMIC_LIBRARY
          && cppConfiguration.toolchainNeedsPic()) {
        optionList.add("-fPIC");
      }

      // Stamp FDO builds with FDO subtype string
      if (fdoBuildStamp != null) {
        optionList.add("-D" + CppConfiguration.FDO_STAMP_MACRO + "=\"" + fdoBuildStamp + "\"");
      }

      // Add the compilation target.
      optionList.add("-c");
      optionList.add(linkstamp.getKey().getExecPathString());

      // Assemble the final command, exempting outputPrefix from shell escaping.
      commands.add(
          compilerCommand
              + " "
              + ShellEscaper.escapeJoinAll(optionList)
              + " -o "
              + outputPrefix
              + ShellEscaper.escapeString(linkstamp.getValue().getExecPathString()));
    }

    return commands;
  }
Esempio n. 5
0
/**
 * Utility class to escape strings for use with shell commands.
 *
 * <p>Escaped strings may safely be inserted into shell commands. Escaping is only done if
 * necessary. Strings containing only shell-neutral characters will not be escaped.
 *
 * <p>This is a replacement for {@code ShellUtils.shellEscape(String)} and {@code
 * ShellUtils.prettyPrintArgv(java.util.List)} (see {@link
 * com.google.devtools.build.lib.shell.ShellUtils}). Its advantage is the use of standard building
 * blocks from the {@code com.google.common.base} package, such as {@link Joiner} and {@link
 * CharMatcher}, making this class more efficient and reliable than {@code ShellUtils}.
 *
 * <p>The behavior is slightly different though: this implementation will defensively escape
 * non-ASCII letters and digits, whereas {@code shellEscape} does not.
 */
@Immutable
public final class ShellEscaper extends Escaper {
  // Note: extending Escaper may seem desirable, but is in fact harmful.
  // The class would then need to implement escape(Appendable), returning an Appendable
  // that escapes everything it receives. In case of shell escaping, we most often join
  // string parts on spaces, using a Joiner. Spaces are escaped characters. Using the
  // Appendable returned by escape(Appendable) would escape these spaces too, which
  // is unwanted.

  public static final ShellEscaper INSTANCE = new ShellEscaper();

  private static final Function<String, String> AS_FUNCTION = INSTANCE.asFunction();

  private static final Joiner SPACE_JOINER = Joiner.on(' ');
  private static final Escaper STRONGQUOTE_ESCAPER =
      new CharEscaperBuilder().addEscape('\'', "'\\''").toEscaper();
  private static final CharMatcher SAFECHAR_MATCHER =
      CharMatcher.anyOf("@%-_+:,./")
          .or(CharMatcher.inRange('0', '9')) // We can't use CharMatcher.javaLetterOrDigit(),
          .or(CharMatcher.inRange('a', 'z')) // that would also accept non-ASCII digits and
          .or(CharMatcher.inRange('A', 'Z')) // letters.
          .precomputed();

  /**
   * Escapes a string by adding strong (single) quotes around it if necessary.
   *
   * <p>A string is not escaped iff it only contains safe characters. The following characters are
   * safe:
   *
   * <ul>
   *   <li>ASCII letters and digits: [a-zA-Z0-9]
   *   <li>shell-neutral characters: at symbol (@), percent symbol (%), dash/minus sign (-),
   *       underscore (_), plus sign (+), colon (:), comma(,), period (.) and slash (/).
   * </ul>
   *
   * <p>A string is escaped iff it contains at least one non-safe character. Escaped strings are
   * created by replacing every occurrence of single quotes with the string '\'' and enclosing the
   * result in a pair of single quotes.
   *
   * <p>Examples:
   *
   * <ul>
   *   <li>"{@code foo}" becomes "{@code foo}" (remains the same)
   *   <li>"{@code +bar}" becomes "{@code +bar}" (remains the same)
   *   <li>"" becomes "{@code''}" (empty string becomes a pair of strong quotes)
   *   <li>"{@code $BAZ}" becomes "{@code '$BAZ'}"
   *   <li>"{@code quote'd}" becomes "{@code 'quote'\''d'}"
   * </ul>
   */
  @Override
  public String escape(String unescaped) {
    final String s = unescaped.toString();
    if (s.isEmpty()) {
      // Empty string is a special case: needs to be quoted to ensure that it
      // gets treated as a separate argument.
      return "''";
    } else {
      return SAFECHAR_MATCHER.matchesAllOf(s) ? s : "'" + STRONGQUOTE_ESCAPER.escape(s) + "'";
    }
  }

  public static String escapeString(String unescaped) {
    return INSTANCE.escape(unescaped);
  }

  /**
   * Transforms the input {@code Iterable} of unescaped strings to an {@code Iterable} of escaped
   * ones. The escaping is done lazily.
   */
  public static Iterable<String> escapeAll(Iterable<? extends String> unescaped) {
    return Iterables.transform(unescaped, AS_FUNCTION);
  }

  /**
   * Escapes all strings in {@code argv} individually and joins them on single spaces into {@code
   * out}. The result is appended directly into {@code out}, without adding a separator.
   *
   * <p>This method works as if by invoking {@link #escapeJoinAll(Appendable, Iterable, Joiner)}
   * with {@code Joiner.on(' ')}.
   *
   * @param out what the result will be appended to
   * @param argv the strings to escape and join
   * @return the same reference as {@code out}, now containing the the joined, escaped fragments
   * @throws IOException if an I/O error occurs while appending
   */
  public static Appendable escapeJoinAll(Appendable out, Iterable<? extends String> argv)
      throws IOException {
    return SPACE_JOINER.appendTo(out, escapeAll(argv));
  }

  /**
   * Escapes all strings in {@code argv} individually and joins them into {@code out} using the
   * specified {@link Joiner}. The result is appended directly into {@code out}, without adding a
   * separator.
   *
   * <p>The resulting strings are the same as if escaped one by one using {@link
   * #escapeString(String)}.
   *
   * <p>Example: if the joiner is {@code Joiner.on('|')}, then the input {@code ["abc", "de'f"]}
   * will be escaped as "{@code abc|'de'\''f'}". If {@code out} initially contains "{@code 123}",
   * then the returned {@code Appendable} will contain "{@code 123abc|'de'\''f'}".
   *
   * @param out what the result will be appended to
   * @param argv the strings to escape and join
   * @param joiner the {@link Joiner} to use to join the escaped strings
   * @return the same reference as {@code out}, now containing the the joined, escaped fragments
   * @throws IOException if an I/O error occurs while appending
   */
  public static Appendable escapeJoinAll(
      Appendable out, Iterable<? extends String> argv, Joiner joiner) throws IOException {
    return joiner.appendTo(out, escapeAll(argv));
  }

  /**
   * Escapes all strings in {@code argv} individually and joins them on single spaces, then returns
   * the resulting string.
   *
   * <p>This method works as if by invoking {@link #escapeJoinAll(Iterable, Joiner)} with {@code
   * Joiner.on(' ')}.
   *
   * <p>Example: {@code ["abc", "de'f"]} will be escaped and joined as "abc 'de'\''f'".
   *
   * @param argv the strings to escape and join
   * @return the string of escaped and joined input elements
   */
  public static String escapeJoinAll(Iterable<? extends String> argv) {
    return SPACE_JOINER.join(escapeAll(argv));
  }

  /**
   * Escapes all strings in {@code argv} individually and joins them using the specified {@link
   * Joiner}, then returns the resulting string.
   *
   * <p>The resulting strings are the same as if escaped one by one using {@link
   * #escapeString(String)}.
   *
   * <p>Example: if the joiner is {@code Joiner.on('|')}, then the input {@code ["abc", "de'f"]}
   * will be escaped and joined as "abc|'de'\''f'".
   *
   * @param argv the strings to escape and join
   * @param joiner the {@link Joiner} to use to join the escaped strings
   * @return the string of escaped and joined input elements
   */
  public static String escapeJoinAll(Iterable<? extends String> argv, Joiner joiner) {
    return joiner.join(escapeAll(argv));
  }

  private ShellEscaper() {
    // Utility class - do not instantiate.
  }
}
Esempio n. 6
0
 public static String escapeString(String unescaped) {
   return INSTANCE.escape(unescaped);
 }