/**
* Creates an action that converts {@code jarToDex} to a dex file. The output will be stored in
* the {@link com.google.devtools.build.lib.actions.Artifact} {@code dxJar}.
*/
public static void createDexAction(
RuleContext ruleContext,
Artifact jarToDex,
Artifact classesDex,
List<String> dexOptions,
boolean multidex,
Artifact mainDexList) {
List<String> args = new ArrayList<>();
args.add("--dex");
// Add --no-locals to coverage builds. Older coverage tools don't correctly preserve local
// variable information in stack frame maps that are required since Java 7, so to avoid runtime
// errors we just don't add local variable info in the first place. This may no longer be
// necessary, however, as long as we use a coverage tool that generates stack frame maps.
if (ruleContext.getConfiguration().isCodeCoverageEnabled()) {
args.add("--no-locals"); // TODO(bazel-team): Is this still needed?
}
// Multithreaded dex does not work when using --multi-dex.
if (!multidex) {
// Multithreaded dex tends to run faster, but only up to about 5 threads (at which point the
// law of diminishing returns kicks in). This was determined experimentally, with 5-thread dex
// performing about 25% faster than 1-thread dex.
args.add("--num-threads=5");
}
args.addAll(dexOptions);
if (multidex) {
args.add("--multi-dex");
if (mainDexList != null) {
args.add("--main-dex-list=" + mainDexList.getExecPathString());
}
}
args.add("--output=" + classesDex.getExecPathString());
args.add(jarToDex.getExecPathString());
SpawnAction.Builder builder =
new SpawnAction.Builder()
.setExecutable(AndroidSdkProvider.fromRuleContext(ruleContext).getDx())
.addInput(jarToDex)
.addOutput(classesDex)
.addArguments(args)
.setProgressMessage("Converting " + jarToDex.getExecPathString() + " to dex format")
.setMnemonic("AndroidDexer")
.setResources(ResourceSet.createWithRamCpuIo(4096.0, 5.0, 0.0));
if (mainDexList != null) {
builder.addInput(mainDexList);
}
ruleContext.registerAction(builder.build(ruleContext));
}
/**
* A spawn action for genrules. Genrules are handled specially in that inputs and outputs are
* checked for directories.
*/
public final class GenRuleAction extends SpawnAction {
private static final ResourceSet GENRULE_RESOURCES =
// Not chosen scientifically/carefully. 300MB memory, 100% CPU, no I/O.
ResourceSet.createWithRamCpuIo(300, 1.0, 0.0);
public GenRuleAction(
ActionOwner owner,
Iterable<Artifact> tools,
Iterable<Artifact> inputs,
Iterable<Artifact> outputs,
List<String> argv,
ImmutableMap<String, String> environment,
ImmutableSet<String> clientEnvironmentVariables,
ImmutableMap<String, String> executionInfo,
ImmutableMap<PathFragment, Artifact> runfilesManifests,
String progressMessage) {
super(
owner,
tools,
inputs,
outputs,
GENRULE_RESOURCES,
CommandLine.of(argv, false),
environment,
clientEnvironmentVariables,
executionInfo,
progressMessage,
runfilesManifests,
"Genrule",
false,
null);
}
@Override
protected void internalExecute(ActionExecutionContext actionExecutionContext)
throws ExecException, InterruptedException {
EventHandler reporter = actionExecutionContext.getExecutor().getEventHandler();
checkInputsForDirectories(reporter, actionExecutionContext.getMetadataHandler());
super.internalExecute(actionExecutionContext);
checkOutputsForDirectories(reporter);
}
}
@Override
public ResourceSet estimateResourceConsumptionLocal() {
return ResourceSet.createWithRamCpuIo(/*memoryMb=*/ 1, /*cpuUsage=*/ 0.1, /*ioUsage=*/ 0.0);
}
/** Builds the action as configured. */
public void build() throws InterruptedException {
ImmutableList<Artifact> classpathResources = attributes.getClassPathResources();
Set<String> classPathResourceNames = new HashSet<>();
for (Artifact artifact : classpathResources) {
String name = artifact.getExecPath().getBaseName();
if (!classPathResourceNames.add(name)) {
ruleContext.attributeError(
"classpath_resources",
"entries must have different file names (duplicate: " + name + ")");
return;
}
}
IterablesChain<Artifact> runtimeJars = runtimeJarsBuilder.build();
// TODO(kmb): Consider not using getArchiveInputs, specifically because we don't want/need to
// transform anything but the runtimeClasspath and b/c we currently do it twice here and below
IterablesChain.Builder<Artifact> inputs = IterablesChain.builder();
inputs.add(getArchiveInputs(attributes, derivedJars));
inputs.add(ImmutableList.copyOf(Iterables.transform(runtimeJars, derivedJars)));
if (runfilesMiddleman != null) {
inputs.addElement(runfilesMiddleman);
}
ImmutableList<Artifact> buildInfoArtifacts = ruleContext.getBuildInfo(JavaBuildInfoFactory.KEY);
inputs.add(buildInfoArtifacts);
Iterable<Artifact> runtimeClasspath =
Iterables.transform(
Iterables.concat(runtimeJars, attributes.getRuntimeClassPathForArchive()), derivedJars);
if (launcher != null) {
inputs.addElement(launcher);
}
CommandLine commandLine =
semantics.buildSingleJarCommandLine(
ruleContext.getConfiguration(),
outputJar,
javaStartClass,
deployManifestLines,
buildInfoArtifacts,
classpathResources,
runtimeClasspath,
includeBuildData,
compression,
launcher);
List<String> jvmArgs = ImmutableList.of("-client", SINGLEJAR_MAX_MEMORY);
ResourceSet resourceSet =
ResourceSet.createWithRamCpuIo(/*memoryMb = */ 200.0, /*cpuUsage = */ .2, /*ioUsage=*/ .2);
// If singlejar's name ends with .jar, it is Java application, otherwise it is native.
// TODO(asmundak): once b/28640279 is fixed (that is, the native singlejar is released),
// eliminate this check, allowing only native singlejar.
Artifact singlejar = getSingleJar(ruleContext);
if (singlejar.getFilename().endsWith(".jar")) {
ruleContext.registerAction(
new SpawnAction.Builder()
.addInputs(inputs.build())
.addTransitiveInputs(JavaHelper.getHostJavabaseInputs(ruleContext))
.addOutput(outputJar)
.setResources(resourceSet)
.setJarExecutable(
ruleContext.getHostConfiguration().getFragment(Jvm.class).getJavaExecutable(),
singlejar,
jvmArgs)
.setCommandLine(commandLine)
.alwaysUseParameterFile(ParameterFileType.SHELL_QUOTED)
.setProgressMessage("Building deploy jar " + outputJar.prettyPrint())
.setMnemonic("JavaDeployJar")
.setExecutionInfo(ImmutableMap.of("supports-workers", "1"))
.build(ruleContext));
} else {
ruleContext.registerAction(
new SpawnAction.Builder()
.addInputs(inputs.build())
.addTransitiveInputs(JavaHelper.getHostJavabaseInputs(ruleContext))
.addOutput(outputJar)
.setResources(resourceSet)
.setExecutable(singlejar)
.setCommandLine(commandLine)
.alwaysUseParameterFile(ParameterFileType.SHELL_QUOTED)
.setProgressMessage("Building deploy jar " + outputJar.prettyPrint())
.setMnemonic("JavaDeployJar")
.build(ruleContext));
}
}
/**
* Abstract implementation of Action which implements basic functionality: the inputs, outputs, and
* toString method. Both input and output sets are immutable.
*/
@Immutable
@ThreadSafe
public abstract class AbstractAction implements Action, SkylarkValue {
/** An arbitrary default resource set. Currently 250MB of memory, 50% CPU and 0% of total I/O. */
public static final ResourceSet DEFAULT_RESOURCE_SET =
ResourceSet.createWithRamCpuIo(250, 0.5, 0);
/**
* The owner/inputs/outputs attributes below should never be directly accessed even within
* AbstractAction itself. The appropriate getter methods should be used instead. This has to be
* done due to the fact that the getter methods can be overridden in subclasses.
*/
private final ActionOwner owner;
/**
* Tools are a subset of inputs and used by the WorkerSpawnStrategy to determine whether a
* compiler has changed since the last time it was used. This should include all artifacts that
* the tool does not dynamically reload / check on each unit of work - e.g. its own binary, the
* JDK for Java binaries, shared libraries, ... but not a configuration file, if it reloads that
* when it has changed.
*
* <p>If the "tools" set does not contain exactly the right set of artifacts, the following can
* happen: If an artifact that should be included is missing, the tool might not be restarted when
* it should, and builds can become incorrect (example: The compiler binary is not part of this
* set, then the compiler gets upgraded, but the worker strategy still reuses the old version). If
* an artifact that should *not* be included is accidentally part of this set, the worker process
* will be restarted more often that is necessary - e.g. if a file that is unique to each unit of
* work, e.g. the source code that a compiler should compile for a compile action, is part of this
* set, then the worker will never be reused and will be restarted for each unit of work.
*/
private final Iterable<Artifact> tools;
// The variable inputs is non-final only so that actions that discover their inputs can modify it.
private Iterable<Artifact> inputs;
private final RunfilesSupplier runfilesSupplier;
private final ImmutableSet<Artifact> outputs;
private String cachedKey;
/** Construct an abstract action with the specified inputs and outputs; */
protected AbstractAction(
ActionOwner owner, Iterable<Artifact> inputs, Iterable<Artifact> outputs) {
this(owner, ImmutableList.<Artifact>of(), inputs, EmptyRunfilesSupplier.INSTANCE, outputs);
}
/** Construct an abstract action with the specified tools, inputs and outputs; */
protected AbstractAction(
ActionOwner owner,
Iterable<Artifact> tools,
Iterable<Artifact> inputs,
Iterable<Artifact> outputs) {
this(owner, tools, inputs, EmptyRunfilesSupplier.INSTANCE, outputs);
}
protected AbstractAction(
ActionOwner owner,
Iterable<Artifact> inputs,
RunfilesSupplier runfilesSupplier,
Iterable<Artifact> outputs) {
this(owner, ImmutableList.<Artifact>of(), inputs, runfilesSupplier, outputs);
}
protected AbstractAction(
ActionOwner owner,
Iterable<Artifact> tools,
Iterable<Artifact> inputs,
RunfilesSupplier runfilesSupplier,
Iterable<Artifact> outputs) {
Preconditions.checkNotNull(owner);
// TODO(bazel-team): Use RuleContext.actionOwner here instead
this.owner = new ActionOwnerDescription(owner);
this.tools = CollectionUtils.makeImmutable(tools);
this.inputs = CollectionUtils.makeImmutable(inputs);
this.outputs = ImmutableSet.copyOf(outputs);
this.runfilesSupplier =
Preconditions.checkNotNull(runfilesSupplier, "runfilesSupplier may not be null");
Preconditions.checkArgument(!this.outputs.isEmpty(), "action outputs may not be empty");
}
@Override
public final ActionOwner getOwner() {
return owner;
}
@Override
public boolean inputsKnown() {
return true;
}
@Override
public boolean discoversInputs() {
return false;
}
@Override
public Collection<Artifact> discoverInputs(ActionExecutionContext actionExecutionContext)
throws ActionExecutionException, InterruptedException {
throw new IllegalStateException(
"discoverInputs cannot be called for "
+ this.prettyPrint()
+ " since it does not discover inputs");
}
@Nullable
@Override
public Iterable<Artifact> resolveInputsFromCache(
ArtifactResolver artifactResolver,
PackageRootResolver resolver,
Collection<PathFragment> inputPaths)
throws PackageRootResolutionException {
throw new IllegalStateException(
"Method must be overridden for actions that may have unknown inputs.");
}
@Override
public void updateInputs(Iterable<Artifact> inputs) {
throw new IllegalStateException(
"Method must be overridden for actions that may have unknown inputs.");
}
@Override
public Iterable<Artifact> getTools() {
return tools;
}
/**
* Should only be overridden by actions that need to optionally insert inputs. Actions that
* discover their inputs should use {@link #setInputs} to set the new iterable of inputs when they
* know it.
*/
@Override
public Iterable<Artifact> getInputs() {
return inputs;
}
@Override
public RunfilesSupplier getRunfilesSupplier() {
return runfilesSupplier;
}
/**
* Set the inputs of the action. May only be used by an action that {@link #discoversInputs()}.
* The iterable passed in is automatically made immutable.
*/
protected void setInputs(Iterable<Artifact> inputs) {
Preconditions.checkState(discoversInputs(), this);
this.inputs = CollectionUtils.makeImmutable(inputs);
}
@Override
public ImmutableSet<Artifact> getOutputs() {
return outputs;
}
@Override
public Artifact getPrimaryInput() {
// The default behavior is to return the first input artifact.
// Call through the method, not the field, because it may be overridden.
return Iterables.getFirst(getInputs(), null);
}
@Override
public Artifact getPrimaryOutput() {
// Default behavior is to return the first output artifact.
// Use the method rather than field in case of overriding in subclasses.
return Iterables.getFirst(getOutputs(), null);
}
@Override
public Iterable<Artifact> getMandatoryInputs() {
return getInputs();
}
@Override
public String toString() {
return prettyPrint()
+ " ("
+ getMnemonic()
+ "["
+ ImmutableList.copyOf(getInputs())
+ (inputsKnown() ? " -> " : ", unknown inputs -> ")
+ getOutputs()
+ "]"
+ ")";
}
@Override
public abstract String getMnemonic();
protected abstract String computeKey();
@Override
public final synchronized String getKey() {
if (cachedKey == null) {
cachedKey = computeKey();
}
return cachedKey;
}
@Override
public String describeKey() {
return null;
}
@Override
public boolean executeUnconditionally() {
return false;
}
@Override
public boolean isVolatile() {
return false;
}
@Override
public boolean showsOutputUnconditionally() {
return false;
}
@Override
public final String getProgressMessage() {
String message = getRawProgressMessage();
if (message == null) {
return null;
}
String additionalInfo = getOwner().getAdditionalProgressInfo();
return additionalInfo == null ? message : message + " [" + additionalInfo + "]";
}
/**
* Returns a progress message string that is specific for this action. This is then annotated with
* additional information, currently the string '[for host]' for actions in the host
* configurations.
*
* <p>A return value of null indicates no message should be reported.
*/
protected String getRawProgressMessage() {
// A cheesy default implementation. Subclasses are invited to do something
// more meaningful.
return defaultProgressMessage();
}
private String defaultProgressMessage() {
return getMnemonic() + " " + getPrimaryOutput().prettyPrint();
}
@Override
public String prettyPrint() {
return "action '" + describe() + "'";
}
@Override
public boolean isImmutable() {
return false;
}
@Override
public void write(Appendable buffer, char quotationMark) {
Printer.append(buffer, prettyPrint()); // TODO(bazel-team): implement a readable representation
}
/**
* Deletes all of the action's output files, if they exist. If any of the Artifacts refers to a
* directory recursively removes the contents of the directory.
*
* @param execRoot the exec root in which this action is executed
*/
protected void deleteOutputs(Path execRoot) throws IOException {
for (Artifact output : getOutputs()) {
deleteOutput(output);
}
}
/**
* Helper method to remove an Artifact. If the Artifact refers to a directory recursively removes
* the contents of the directory.
*/
protected void deleteOutput(Artifact output) throws IOException {
Path path = output.getPath();
try {
// Optimize for the common case: output artifacts are files.
path.delete();
} catch (IOException e) {
// Only try to recursively delete a directory if the output root is known. This is just a
// sanity check so that we do not start deleting random files on disk.
// TODO(bazel-team): Strengthen this test by making sure that the output is part of the
// output tree.
if (path.isDirectory(Symlinks.NOFOLLOW) && output.getRoot() != null) {
FileSystemUtils.deleteTree(path);
} else {
throw e;
}
}
}
/**
* If the action might read directories as inputs in a way that is unsound wrt dependency
* checking, this method must be called.
*/
protected void checkInputsForDirectories(
EventHandler eventHandler, MetadataHandler metadataHandler) {
// Report "directory dependency checking" warning only for non-generated directories (generated
// ones will be reported earlier).
for (Artifact input : getMandatoryInputs()) {
// Assume that if the file did not exist, we would not have gotten here.
if (input.isSourceArtifact() && !metadataHandler.isRegularFile(input)) {
eventHandler.handle(
Event.warn(
getOwner().getLocation(),
"input '"
+ input.prettyPrint()
+ "' to "
+ getOwner().getLabel()
+ " is a directory; dependency checking of directories is unsound"));
}
}
}
@Override
public MiddlemanType getActionType() {
return MiddlemanType.NORMAL;
}
/** If the action might create directories as outputs this method must be called. */
protected void checkOutputsForDirectories(EventHandler eventHandler) {
for (Artifact output : getOutputs()) {
Path path = output.getPath();
String ownerString = Label.print(getOwner().getLabel());
if (path.isDirectory()) {
eventHandler.handle(
new Event(
EventKind.WARNING,
getOwner().getLocation(),
"output '"
+ output.prettyPrint()
+ "' of "
+ ownerString
+ " is a directory; dependency checking of directories is unsound",
ownerString));
}
}
}
@Override
public void prepare(Path execRoot) throws IOException {
deleteOutputs(execRoot);
}
@Override
public String describe() {
String progressMessage = getProgressMessage();
return progressMessage != null ? progressMessage : defaultProgressMessage();
}
@Override
public abstract ResourceSet estimateResourceConsumption(Executor executor);
@Override
public boolean shouldReportPathPrefixConflict(Action action) {
return this != action;
}
@Override
public ExtraActionInfo.Builder getExtraActionInfo() {
return ExtraActionInfo.newBuilder()
.setOwner(getOwner().getLabel().toString())
.setId(getKey())
.setMnemonic(getMnemonic());
}
@Override
public ImmutableSet<Artifact> getMandatoryOutputs() {
return ImmutableSet.of();
}
/**
* Returns input files that need to be present to allow extra_action rules to shadow this action
* correctly when run remotely. This is at least the normal inputs of the action, but may include
* other files as well. For example C(++) compilation may perform include file header scanning.
* This needs to be mirrored by the extra_action rule. Called by {@link
* com.google.devtools.build.lib.rules.extra.ExtraAction} at execution time.
*
* <p>As this method is called from the ExtraAction, make sure it is ok to call this method from a
* different thread than the one this action is executed on.
*
* @param actionExecutionContext Services in the scope of the action, like the Out/Err streams.
* @throws ActionExecutionException only when code called from this method throws that exception.
* @throws InterruptedException if interrupted
*/
public Iterable<Artifact> getInputFilesForExtraAction(
ActionExecutionContext actionExecutionContext)
throws ActionExecutionException, InterruptedException {
return getInputs();
}
/**
* A copying implementation of {@link ActionOwner}.
*
* <p>ConfiguredTargets implement ActionOwner themselves, but we do not want actions to keep
* direct references to configured targets just for a label and a few strings.
*/
@Immutable
private static class ActionOwnerDescription implements ActionOwner {
private final Location location;
private final Label label;
private final String configurationMnemonic;
private final String configurationChecksum;
private final String targetKind;
private final String additionalProgressInfo;
private ActionOwnerDescription(ActionOwner originalOwner) {
this.location = originalOwner.getLocation();
this.label = originalOwner.getLabel();
this.configurationMnemonic = originalOwner.getConfigurationMnemonic();
this.configurationChecksum = originalOwner.getConfigurationChecksum();
this.targetKind = originalOwner.getTargetKind();
this.additionalProgressInfo = originalOwner.getAdditionalProgressInfo();
}
@Override
public Location getLocation() {
return location;
}
@Override
public Label getLabel() {
return label;
}
@Override
public String getConfigurationMnemonic() {
return configurationMnemonic;
}
@Override
public String getConfigurationChecksum() {
return configurationChecksum;
}
@Override
public String getTargetKind() {
return targetKind;
}
@Override
public String getAdditionalProgressInfo() {
return additionalProgressInfo;
}
}
}
