/**
* Generates a command line to run for the test action, taking into account coverage and {@code
* --run_under} settings.
*
* @param testScript the setup script that invokes the test
* @param coverageScript a script interjected between setup script and rest of command line to
* collect coverage data. If this is an empty string, it is ignored.
* @param testAction The test action.
* @return the command line as string list.
*/
protected List<String> getArgs(
String testScript, String coverageScript, TestRunnerAction testAction) {
List<String> args = Lists.newArrayList();
if (OS.getCurrent() == OS.WINDOWS) {
args.add(testAction.getShExecutable().getPathString());
args.add("-c");
args.add("$0 $*");
}
args.add(testScript);
TestTargetExecutionSettings execSettings = testAction.getExecutionSettings();
List<String> execArgs = new ArrayList<>();
if (!coverageScript.isEmpty() && isCoverageMode(testAction)) {
execArgs.add(coverageScript);
}
// Execute the test using the alias in the runfiles tree, as mandated by
// the Test Encyclopedia.
execArgs.add(execSettings.getExecutable().getRootRelativePath().getPathString());
execArgs.addAll(execSettings.getArgs());
// Insert the command prefix specified by the "--run_under=<command-prefix>" option,
// if any.
if (execSettings.getRunUnder() == null) {
args.addAll(execArgs);
} else if (execSettings.getRunUnderExecutable() != null) {
args.add(execSettings.getRunUnderExecutable().getRootRelativePath().getPathString());
args.addAll(execSettings.getRunUnder().getOptions());
args.addAll(execArgs);
} else {
args.add(testAction.getConfiguration().getShellExecutable().getPathString());
args.add("-c");
String runUnderCommand = ShellEscaper.escapeString(execSettings.getRunUnder().getCommand());
Path fullySpecified =
SearchPath.which(
SearchPath.parse(
testAction.getTestLog().getPath().getFileSystem(), clientEnv.get("PATH")),
runUnderCommand);
if (fullySpecified != null) {
runUnderCommand = fullySpecified.toString();
}
args.add(
runUnderCommand
+ ' '
+ ShellEscaper.escapeJoinAll(
Iterables.concat(execSettings.getRunUnder().getOptions(), execArgs)));
}
return args;
}
/**
* Adds linkstamp compilation to the (otherwise) fully specified link command if {@link
* #getLinkstamps} is non-empty.
*
* <p>Linkstamps were historically compiled implicitly as part of the link command, but implicit
* compilation doesn't guarantee consistent outputs. For example, the command "gcc input.o input.o
* foo/linkstamp.cc -o myapp" causes gcc to implicitly run "gcc foo/linkstamp.cc -o
* /tmp/ccEtJHDB.o", for some internally decided output path /tmp/ccEtJHDB.o, then add that path
* to the linker's command line options. The name of this path can change even between
* equivalently specified gcc invocations.
*
* <p>So now we explicitly compile these files in their own command invocations before running the
* link command, thus giving us direct control over the naming of their outputs. This method adds
* those extra steps as necessary.
*
* @param linkstampCommands individual linkstamp compilation commands
* @param linkCommand the complete list of link command arguments (after .params file compacting)
* for an invocation
* @param escapeArgs if true, linkCommand arguments are shell escaped. if false, arguments are
* returned as-is
* @return The original argument list if no linkstamps compilation commands are given, otherwise
* an expanded list that adds the linkstamp compilation commands and funnels their outputs
* into the link step. Note that these outputs only need to persist for the duration of the
* link step.
*/
private static List<String> addLinkstampingToCommand(
List<String> linkstampCommands, List<String> linkCommand, boolean escapeArgs) {
if (linkstampCommands.isEmpty()) {
return linkCommand;
} else {
List<String> batchCommand = Lists.newArrayListWithCapacity(3);
batchCommand.add("/bin/bash");
batchCommand.add("-c");
batchCommand.add(
Joiner.on(" && ").join(linkstampCommands)
+ " && "
+ (escapeArgs
? ShellEscaper.escapeJoinAll(linkCommand)
: Joiner.on(" ").join(linkCommand)));
return ImmutableList.copyOf(batchCommand);
}
}
private void actuallyClean(
CommandEnvironment env, Path outputBase, Options cleanOptions, String symlinkPrefix)
throws IOException, ShutdownBlazeServerException, CommandException, ExecException,
InterruptedException {
BlazeRuntime runtime = env.getRuntime();
if (env.getOutputService() != null) {
env.getOutputService().clean();
}
if (cleanOptions.expunge) {
LOG.info("Expunging...");
// Delete the big subdirectories with the important content first--this
// will take the most time. Then quickly delete the little locks, logs
// and links right before we exit. Once the lock file is gone there will
// be a small possibility of a server race if a client is waiting, but
// all significant files will be gone by then.
FileSystemUtils.deleteTreesBelow(outputBase);
FileSystemUtils.deleteTree(outputBase);
} else if (cleanOptions.expunge_async) {
LOG.info("Expunging asynchronously...");
String tempBaseName = outputBase.getBaseName() + "_tmp_" + ProcessUtils.getpid();
// Keeping tempOutputBase in the same directory ensures it remains in the
// same file system, and therefore the mv will be atomic and fast.
Path tempOutputBase = outputBase.getParentDirectory().getChild(tempBaseName);
outputBase.renameTo(tempOutputBase);
env.getReporter()
.handle(Event.info(null, "Output base moved to " + tempOutputBase + " for deletion"));
// Daemonize the shell and use the double-fork idiom to ensure that the shell
// exits even while the "rm -rf" command continues.
String command =
String.format(
"exec >&- 2>&- <&- && (/usr/bin/setsid /bin/rm -rf %s &)&",
ShellEscaper.escapeString(tempOutputBase.getPathString()));
LOG.info("Executing shell commmand " + ShellEscaper.escapeString(command));
// Doesn't throw iff command exited and was successful.
new CommandBuilder()
.addArg(command)
.useShell(true)
.setWorkingDir(tempOutputBase.getParentDirectory())
.build()
.execute();
} else {
LOG.info("Output cleaning...");
runtime.clearCaches();
// In order to be sure that we delete everything, delete the workspace directory both for
// --deep_execroot and for --nodeep_execroot.
for (String directory :
new String[] {runtime.getWorkspaceName(), "execroot/" + runtime.getWorkspaceName()}) {
Path child = outputBase.getRelative(directory);
if (child.exists()) {
LOG.finest("Cleaning " + child);
FileSystemUtils.deleteTreesBelow(child);
}
}
}
// remove convenience links
OutputDirectoryLinksUtils.removeOutputDirectoryLinks(
runtime.getWorkspaceName(), runtime.getWorkspace(), env.getReporter(), symlinkPrefix);
// shutdown on expunge cleans
if (cleanOptions.expunge || cleanOptions.expunge_async) {
throw new ShutdownBlazeServerException(0);
}
}
/**
* Computes, for each C++ source file in {@link #getLinkstamps}, the command necessary to compile
* that file such that the output is correctly fed into the link command.
*
* <p>As these options (as well as all others) are taken into account when computing the action
* key, they do not directly contain volatile build information to avoid unnecessary relinking.
* Instead this information is passed as an additional header generated by {@link
* com.google.devtools.build.lib.rules.cpp.WriteBuildInfoHeaderAction}.
*
* @param outputPrefix prefix to add before the linkstamp outputs' exec paths
* @return a list of shell-escaped compiler commmands, one for each entry in {@link
* #getLinkstamps}
*/
public List<String> getLinkstampCompileCommands(String outputPrefix) {
if (linkstamps.isEmpty()) {
return ImmutableList.of();
}
String compilerCommand = cppConfiguration.getCppExecutable().getPathString();
List<String> commands = Lists.newArrayListWithCapacity(linkstamps.size());
for (Map.Entry<Artifact, Artifact> linkstamp : linkstamps.entrySet()) {
List<String> optionList = new ArrayList<>();
// Defines related to the build info are read from generated headers.
for (Artifact header : buildInfoHeaderArtifacts) {
optionList.add("-include");
optionList.add(header.getExecPathString());
}
String labelReplacement =
Matcher.quoteReplacement(
isSharedNativeLibrary() ? output.getExecPathString() : Label.print(owner.getLabel()));
String outputPathReplacement = Matcher.quoteReplacement(output.getExecPathString());
for (String option : linkstampCompileOptions) {
optionList.add(
option
.replaceAll(Pattern.quote("${LABEL}"), labelReplacement)
.replaceAll(Pattern.quote("${OUTPUT_PATH}"), outputPathReplacement));
}
optionList.add("-DGPLATFORM=\"" + cppConfiguration + "\"");
// Needed to find headers included from linkstamps.
optionList.add("-I.");
// Add sysroot.
PathFragment sysroot = cppConfiguration.getSysroot();
if (sysroot != null) {
optionList.add("--sysroot=" + sysroot.getPathString());
}
// Add toolchain compiler options.
optionList.addAll(cppConfiguration.getCompilerOptions(features));
optionList.addAll(cppConfiguration.getCOptions());
optionList.addAll(cppConfiguration.getUnfilteredCompilerOptions(features));
if (CppFileTypes.CPP_SOURCE.matches(linkstamp.getKey().getExecPath())) {
optionList.addAll(cppConfiguration.getCxxOptions(features));
}
// For dynamic libraries, produce position independent code.
if (linkTargetType == LinkTargetType.DYNAMIC_LIBRARY
&& cppConfiguration.toolchainNeedsPic()) {
optionList.add("-fPIC");
}
// Stamp FDO builds with FDO subtype string
if (fdoBuildStamp != null) {
optionList.add("-D" + CppConfiguration.FDO_STAMP_MACRO + "=\"" + fdoBuildStamp + "\"");
}
// Add the compilation target.
optionList.add("-c");
optionList.add(linkstamp.getKey().getExecPathString());
// Assemble the final command, exempting outputPrefix from shell escaping.
commands.add(
compilerCommand
+ " "
+ ShellEscaper.escapeJoinAll(optionList)
+ " -o "
+ outputPrefix
+ ShellEscaper.escapeString(linkstamp.getValue().getExecPathString()));
}
return commands;
}
/**
* Utility class to escape strings for use with shell commands.
*
* <p>Escaped strings may safely be inserted into shell commands. Escaping is only done if
* necessary. Strings containing only shell-neutral characters will not be escaped.
*
* <p>This is a replacement for {@code ShellUtils.shellEscape(String)} and {@code
* ShellUtils.prettyPrintArgv(java.util.List)} (see {@link
* com.google.devtools.build.lib.shell.ShellUtils}). Its advantage is the use of standard building
* blocks from the {@code com.google.common.base} package, such as {@link Joiner} and {@link
* CharMatcher}, making this class more efficient and reliable than {@code ShellUtils}.
*
* <p>The behavior is slightly different though: this implementation will defensively escape
* non-ASCII letters and digits, whereas {@code shellEscape} does not.
*/
@Immutable
public final class ShellEscaper extends Escaper {
// Note: extending Escaper may seem desirable, but is in fact harmful.
// The class would then need to implement escape(Appendable), returning an Appendable
// that escapes everything it receives. In case of shell escaping, we most often join
// string parts on spaces, using a Joiner. Spaces are escaped characters. Using the
// Appendable returned by escape(Appendable) would escape these spaces too, which
// is unwanted.
public static final ShellEscaper INSTANCE = new ShellEscaper();
private static final Function<String, String> AS_FUNCTION = INSTANCE.asFunction();
private static final Joiner SPACE_JOINER = Joiner.on(' ');
private static final Escaper STRONGQUOTE_ESCAPER =
new CharEscaperBuilder().addEscape('\'', "'\\''").toEscaper();
private static final CharMatcher SAFECHAR_MATCHER =
CharMatcher.anyOf("@%-_+:,./")
.or(CharMatcher.inRange('0', '9')) // We can't use CharMatcher.javaLetterOrDigit(),
.or(CharMatcher.inRange('a', 'z')) // that would also accept non-ASCII digits and
.or(CharMatcher.inRange('A', 'Z')) // letters.
.precomputed();
/**
* Escapes a string by adding strong (single) quotes around it if necessary.
*
* <p>A string is not escaped iff it only contains safe characters. The following characters are
* safe:
*
* <ul>
* <li>ASCII letters and digits: [a-zA-Z0-9]
* <li>shell-neutral characters: at symbol (@), percent symbol (%), dash/minus sign (-),
* underscore (_), plus sign (+), colon (:), comma(,), period (.) and slash (/).
* </ul>
*
* <p>A string is escaped iff it contains at least one non-safe character. Escaped strings are
* created by replacing every occurrence of single quotes with the string '\'' and enclosing the
* result in a pair of single quotes.
*
* <p>Examples:
*
* <ul>
* <li>"{@code foo}" becomes "{@code foo}" (remains the same)
* <li>"{@code +bar}" becomes "{@code +bar}" (remains the same)
* <li>"" becomes "{@code''}" (empty string becomes a pair of strong quotes)
* <li>"{@code $BAZ}" becomes "{@code '$BAZ'}"
* <li>"{@code quote'd}" becomes "{@code 'quote'\''d'}"
* </ul>
*/
@Override
public String escape(String unescaped) {
final String s = unescaped.toString();
if (s.isEmpty()) {
// Empty string is a special case: needs to be quoted to ensure that it
// gets treated as a separate argument.
return "''";
} else {
return SAFECHAR_MATCHER.matchesAllOf(s) ? s : "'" + STRONGQUOTE_ESCAPER.escape(s) + "'";
}
}
public static String escapeString(String unescaped) {
return INSTANCE.escape(unescaped);
}
/**
* Transforms the input {@code Iterable} of unescaped strings to an {@code Iterable} of escaped
* ones. The escaping is done lazily.
*/
public static Iterable<String> escapeAll(Iterable<? extends String> unescaped) {
return Iterables.transform(unescaped, AS_FUNCTION);
}
/**
* Escapes all strings in {@code argv} individually and joins them on single spaces into {@code
* out}. The result is appended directly into {@code out}, without adding a separator.
*
* <p>This method works as if by invoking {@link #escapeJoinAll(Appendable, Iterable, Joiner)}
* with {@code Joiner.on(' ')}.
*
* @param out what the result will be appended to
* @param argv the strings to escape and join
* @return the same reference as {@code out}, now containing the the joined, escaped fragments
* @throws IOException if an I/O error occurs while appending
*/
public static Appendable escapeJoinAll(Appendable out, Iterable<? extends String> argv)
throws IOException {
return SPACE_JOINER.appendTo(out, escapeAll(argv));
}
/**
* Escapes all strings in {@code argv} individually and joins them into {@code out} using the
* specified {@link Joiner}. The result is appended directly into {@code out}, without adding a
* separator.
*
* <p>The resulting strings are the same as if escaped one by one using {@link
* #escapeString(String)}.
*
* <p>Example: if the joiner is {@code Joiner.on('|')}, then the input {@code ["abc", "de'f"]}
* will be escaped as "{@code abc|'de'\''f'}". If {@code out} initially contains "{@code 123}",
* then the returned {@code Appendable} will contain "{@code 123abc|'de'\''f'}".
*
* @param out what the result will be appended to
* @param argv the strings to escape and join
* @param joiner the {@link Joiner} to use to join the escaped strings
* @return the same reference as {@code out}, now containing the the joined, escaped fragments
* @throws IOException if an I/O error occurs while appending
*/
public static Appendable escapeJoinAll(
Appendable out, Iterable<? extends String> argv, Joiner joiner) throws IOException {
return joiner.appendTo(out, escapeAll(argv));
}
/**
* Escapes all strings in {@code argv} individually and joins them on single spaces, then returns
* the resulting string.
*
* <p>This method works as if by invoking {@link #escapeJoinAll(Iterable, Joiner)} with {@code
* Joiner.on(' ')}.
*
* <p>Example: {@code ["abc", "de'f"]} will be escaped and joined as "abc 'de'\''f'".
*
* @param argv the strings to escape and join
* @return the string of escaped and joined input elements
*/
public static String escapeJoinAll(Iterable<? extends String> argv) {
return SPACE_JOINER.join(escapeAll(argv));
}
/**
* Escapes all strings in {@code argv} individually and joins them using the specified {@link
* Joiner}, then returns the resulting string.
*
* <p>The resulting strings are the same as if escaped one by one using {@link
* #escapeString(String)}.
*
* <p>Example: if the joiner is {@code Joiner.on('|')}, then the input {@code ["abc", "de'f"]}
* will be escaped and joined as "abc|'de'\''f'".
*
* @param argv the strings to escape and join
* @param joiner the {@link Joiner} to use to join the escaped strings
* @return the string of escaped and joined input elements
*/
public static String escapeJoinAll(Iterable<? extends String> argv, Joiner joiner) {
return joiner.join(escapeAll(argv));
}
private ShellEscaper() {
// Utility class - do not instantiate.
}
}
public static String escapeString(String unescaped) {
return INSTANCE.escape(unescaped);
}