Esempio n. 1
0
  /**
   * Creates an action that converts {@code jarToDex} to a dex file. The output will be stored in
   * the {@link com.google.devtools.build.lib.actions.Artifact} {@code dxJar}.
   */
  public static void createDexAction(
      RuleContext ruleContext,
      Artifact jarToDex,
      Artifact classesDex,
      List<String> dexOptions,
      boolean multidex,
      Artifact mainDexList) {
    List<String> args = new ArrayList<>();
    args.add("--dex");
    // Add --no-locals to coverage builds.  Older coverage tools don't correctly preserve local
    // variable information in stack frame maps that are required since Java 7, so to avoid runtime
    // errors we just don't add local variable info in the first place.  This may no longer be
    // necessary, however, as long as we use a coverage tool that generates stack frame maps.
    if (ruleContext.getConfiguration().isCodeCoverageEnabled()) {
      args.add("--no-locals"); // TODO(bazel-team): Is this still needed?
    }

    // Multithreaded dex does not work when using --multi-dex.
    if (!multidex) {
      // Multithreaded dex tends to run faster, but only up to about 5 threads (at which point the
      // law of diminishing returns kicks in). This was determined experimentally, with 5-thread dex
      // performing about 25% faster than 1-thread dex.
      args.add("--num-threads=5");
    }

    args.addAll(dexOptions);
    if (multidex) {
      args.add("--multi-dex");
      if (mainDexList != null) {
        args.add("--main-dex-list=" + mainDexList.getExecPathString());
      }
    }
    args.add("--output=" + classesDex.getExecPathString());
    args.add(jarToDex.getExecPathString());

    SpawnAction.Builder builder =
        new SpawnAction.Builder()
            .setExecutable(AndroidSdkProvider.fromRuleContext(ruleContext).getDx())
            .addInput(jarToDex)
            .addOutput(classesDex)
            .addArguments(args)
            .setProgressMessage("Converting " + jarToDex.getExecPathString() + " to dex format")
            .setMnemonic("AndroidDexer")
            .setResources(ResourceSet.createWithRamCpuIo(4096.0, 5.0, 0.0));
    if (mainDexList != null) {
      builder.addInput(mainDexList);
    }
    ruleContext.registerAction(builder.build(ruleContext));
  }
Esempio n. 2
0
/**
 * A spawn action for genrules. Genrules are handled specially in that inputs and outputs are
 * checked for directories.
 */
public final class GenRuleAction extends SpawnAction {

  private static final ResourceSet GENRULE_RESOURCES =
      // Not chosen scientifically/carefully.  300MB memory, 100% CPU, no I/O.
      ResourceSet.createWithRamCpuIo(300, 1.0, 0.0);

  public GenRuleAction(
      ActionOwner owner,
      Iterable<Artifact> tools,
      Iterable<Artifact> inputs,
      Iterable<Artifact> outputs,
      List<String> argv,
      ImmutableMap<String, String> environment,
      ImmutableSet<String> clientEnvironmentVariables,
      ImmutableMap<String, String> executionInfo,
      ImmutableMap<PathFragment, Artifact> runfilesManifests,
      String progressMessage) {
    super(
        owner,
        tools,
        inputs,
        outputs,
        GENRULE_RESOURCES,
        CommandLine.of(argv, false),
        environment,
        clientEnvironmentVariables,
        executionInfo,
        progressMessage,
        runfilesManifests,
        "Genrule",
        false,
        null);
  }

  @Override
  protected void internalExecute(ActionExecutionContext actionExecutionContext)
      throws ExecException, InterruptedException {
    EventHandler reporter = actionExecutionContext.getExecutor().getEventHandler();
    checkInputsForDirectories(reporter, actionExecutionContext.getMetadataHandler());
    super.internalExecute(actionExecutionContext);
    checkOutputsForDirectories(reporter);
  }
}
Esempio n. 3
0
 @Override
 public ResourceSet estimateResourceConsumptionLocal() {
   return ResourceSet.createWithRamCpuIo(/*memoryMb=*/ 1, /*cpuUsage=*/ 0.1, /*ioUsage=*/ 0.0);
 }
Esempio n. 4
0
  /** Builds the action as configured. */
  public void build() throws InterruptedException {
    ImmutableList<Artifact> classpathResources = attributes.getClassPathResources();
    Set<String> classPathResourceNames = new HashSet<>();
    for (Artifact artifact : classpathResources) {
      String name = artifact.getExecPath().getBaseName();
      if (!classPathResourceNames.add(name)) {
        ruleContext.attributeError(
            "classpath_resources",
            "entries must have different file names (duplicate: " + name + ")");
        return;
      }
    }

    IterablesChain<Artifact> runtimeJars = runtimeJarsBuilder.build();

    // TODO(kmb): Consider not using getArchiveInputs, specifically because we don't want/need to
    // transform anything but the runtimeClasspath and b/c we currently do it twice here and below
    IterablesChain.Builder<Artifact> inputs = IterablesChain.builder();
    inputs.add(getArchiveInputs(attributes, derivedJars));

    inputs.add(ImmutableList.copyOf(Iterables.transform(runtimeJars, derivedJars)));
    if (runfilesMiddleman != null) {
      inputs.addElement(runfilesMiddleman);
    }

    ImmutableList<Artifact> buildInfoArtifacts = ruleContext.getBuildInfo(JavaBuildInfoFactory.KEY);
    inputs.add(buildInfoArtifacts);

    Iterable<Artifact> runtimeClasspath =
        Iterables.transform(
            Iterables.concat(runtimeJars, attributes.getRuntimeClassPathForArchive()), derivedJars);

    if (launcher != null) {
      inputs.addElement(launcher);
    }

    CommandLine commandLine =
        semantics.buildSingleJarCommandLine(
            ruleContext.getConfiguration(),
            outputJar,
            javaStartClass,
            deployManifestLines,
            buildInfoArtifacts,
            classpathResources,
            runtimeClasspath,
            includeBuildData,
            compression,
            launcher);

    List<String> jvmArgs = ImmutableList.of("-client", SINGLEJAR_MAX_MEMORY);
    ResourceSet resourceSet =
        ResourceSet.createWithRamCpuIo(/*memoryMb = */ 200.0, /*cpuUsage = */ .2, /*ioUsage=*/ .2);

    // If singlejar's name ends with .jar, it is Java application, otherwise it is native.
    // TODO(asmundak): once b/28640279 is fixed (that is, the native singlejar is released),
    // eliminate this check, allowing only native singlejar.
    Artifact singlejar = getSingleJar(ruleContext);
    if (singlejar.getFilename().endsWith(".jar")) {
      ruleContext.registerAction(
          new SpawnAction.Builder()
              .addInputs(inputs.build())
              .addTransitiveInputs(JavaHelper.getHostJavabaseInputs(ruleContext))
              .addOutput(outputJar)
              .setResources(resourceSet)
              .setJarExecutable(
                  ruleContext.getHostConfiguration().getFragment(Jvm.class).getJavaExecutable(),
                  singlejar,
                  jvmArgs)
              .setCommandLine(commandLine)
              .alwaysUseParameterFile(ParameterFileType.SHELL_QUOTED)
              .setProgressMessage("Building deploy jar " + outputJar.prettyPrint())
              .setMnemonic("JavaDeployJar")
              .setExecutionInfo(ImmutableMap.of("supports-workers", "1"))
              .build(ruleContext));
    } else {
      ruleContext.registerAction(
          new SpawnAction.Builder()
              .addInputs(inputs.build())
              .addTransitiveInputs(JavaHelper.getHostJavabaseInputs(ruleContext))
              .addOutput(outputJar)
              .setResources(resourceSet)
              .setExecutable(singlejar)
              .setCommandLine(commandLine)
              .alwaysUseParameterFile(ParameterFileType.SHELL_QUOTED)
              .setProgressMessage("Building deploy jar " + outputJar.prettyPrint())
              .setMnemonic("JavaDeployJar")
              .build(ruleContext));
    }
  }
Esempio n. 5
0
/**
 * Abstract implementation of Action which implements basic functionality: the inputs, outputs, and
 * toString method. Both input and output sets are immutable.
 */
@Immutable
@ThreadSafe
public abstract class AbstractAction implements Action, SkylarkValue {

  /** An arbitrary default resource set. Currently 250MB of memory, 50% CPU and 0% of total I/O. */
  public static final ResourceSet DEFAULT_RESOURCE_SET =
      ResourceSet.createWithRamCpuIo(250, 0.5, 0);

  /**
   * The owner/inputs/outputs attributes below should never be directly accessed even within
   * AbstractAction itself. The appropriate getter methods should be used instead. This has to be
   * done due to the fact that the getter methods can be overridden in subclasses.
   */
  private final ActionOwner owner;

  /**
   * Tools are a subset of inputs and used by the WorkerSpawnStrategy to determine whether a
   * compiler has changed since the last time it was used. This should include all artifacts that
   * the tool does not dynamically reload / check on each unit of work - e.g. its own binary, the
   * JDK for Java binaries, shared libraries, ... but not a configuration file, if it reloads that
   * when it has changed.
   *
   * <p>If the "tools" set does not contain exactly the right set of artifacts, the following can
   * happen: If an artifact that should be included is missing, the tool might not be restarted when
   * it should, and builds can become incorrect (example: The compiler binary is not part of this
   * set, then the compiler gets upgraded, but the worker strategy still reuses the old version). If
   * an artifact that should *not* be included is accidentally part of this set, the worker process
   * will be restarted more often that is necessary - e.g. if a file that is unique to each unit of
   * work, e.g. the source code that a compiler should compile for a compile action, is part of this
   * set, then the worker will never be reused and will be restarted for each unit of work.
   */
  private final Iterable<Artifact> tools;

  // The variable inputs is non-final only so that actions that discover their inputs can modify it.
  private Iterable<Artifact> inputs;
  private final RunfilesSupplier runfilesSupplier;
  private final ImmutableSet<Artifact> outputs;

  private String cachedKey;

  /** Construct an abstract action with the specified inputs and outputs; */
  protected AbstractAction(
      ActionOwner owner, Iterable<Artifact> inputs, Iterable<Artifact> outputs) {
    this(owner, ImmutableList.<Artifact>of(), inputs, EmptyRunfilesSupplier.INSTANCE, outputs);
  }

  /** Construct an abstract action with the specified tools, inputs and outputs; */
  protected AbstractAction(
      ActionOwner owner,
      Iterable<Artifact> tools,
      Iterable<Artifact> inputs,
      Iterable<Artifact> outputs) {
    this(owner, tools, inputs, EmptyRunfilesSupplier.INSTANCE, outputs);
  }

  protected AbstractAction(
      ActionOwner owner,
      Iterable<Artifact> inputs,
      RunfilesSupplier runfilesSupplier,
      Iterable<Artifact> outputs) {
    this(owner, ImmutableList.<Artifact>of(), inputs, runfilesSupplier, outputs);
  }

  protected AbstractAction(
      ActionOwner owner,
      Iterable<Artifact> tools,
      Iterable<Artifact> inputs,
      RunfilesSupplier runfilesSupplier,
      Iterable<Artifact> outputs) {
    Preconditions.checkNotNull(owner);
    // TODO(bazel-team): Use RuleContext.actionOwner here instead
    this.owner = new ActionOwnerDescription(owner);
    this.tools = CollectionUtils.makeImmutable(tools);
    this.inputs = CollectionUtils.makeImmutable(inputs);
    this.outputs = ImmutableSet.copyOf(outputs);
    this.runfilesSupplier =
        Preconditions.checkNotNull(runfilesSupplier, "runfilesSupplier may not be null");
    Preconditions.checkArgument(!this.outputs.isEmpty(), "action outputs may not be empty");
  }

  @Override
  public final ActionOwner getOwner() {
    return owner;
  }

  @Override
  public boolean inputsKnown() {
    return true;
  }

  @Override
  public boolean discoversInputs() {
    return false;
  }

  @Override
  public Collection<Artifact> discoverInputs(ActionExecutionContext actionExecutionContext)
      throws ActionExecutionException, InterruptedException {
    throw new IllegalStateException(
        "discoverInputs cannot be called for "
            + this.prettyPrint()
            + " since it does not discover inputs");
  }

  @Nullable
  @Override
  public Iterable<Artifact> resolveInputsFromCache(
      ArtifactResolver artifactResolver,
      PackageRootResolver resolver,
      Collection<PathFragment> inputPaths)
      throws PackageRootResolutionException {
    throw new IllegalStateException(
        "Method must be overridden for actions that may have unknown inputs.");
  }

  @Override
  public void updateInputs(Iterable<Artifact> inputs) {
    throw new IllegalStateException(
        "Method must be overridden for actions that may have unknown inputs.");
  }

  @Override
  public Iterable<Artifact> getTools() {
    return tools;
  }

  /**
   * Should only be overridden by actions that need to optionally insert inputs. Actions that
   * discover their inputs should use {@link #setInputs} to set the new iterable of inputs when they
   * know it.
   */
  @Override
  public Iterable<Artifact> getInputs() {
    return inputs;
  }

  @Override
  public RunfilesSupplier getRunfilesSupplier() {
    return runfilesSupplier;
  }

  /**
   * Set the inputs of the action. May only be used by an action that {@link #discoversInputs()}.
   * The iterable passed in is automatically made immutable.
   */
  protected void setInputs(Iterable<Artifact> inputs) {
    Preconditions.checkState(discoversInputs(), this);
    this.inputs = CollectionUtils.makeImmutable(inputs);
  }

  @Override
  public ImmutableSet<Artifact> getOutputs() {
    return outputs;
  }

  @Override
  public Artifact getPrimaryInput() {
    // The default behavior is to return the first input artifact.
    // Call through the method, not the field, because it may be overridden.
    return Iterables.getFirst(getInputs(), null);
  }

  @Override
  public Artifact getPrimaryOutput() {
    // Default behavior is to return the first output artifact.
    // Use the method rather than field in case of overriding in subclasses.
    return Iterables.getFirst(getOutputs(), null);
  }

  @Override
  public Iterable<Artifact> getMandatoryInputs() {
    return getInputs();
  }

  @Override
  public String toString() {
    return prettyPrint()
        + " ("
        + getMnemonic()
        + "["
        + ImmutableList.copyOf(getInputs())
        + (inputsKnown() ? " -> " : ", unknown inputs -> ")
        + getOutputs()
        + "]"
        + ")";
  }

  @Override
  public abstract String getMnemonic();

  protected abstract String computeKey();

  @Override
  public final synchronized String getKey() {
    if (cachedKey == null) {
      cachedKey = computeKey();
    }
    return cachedKey;
  }

  @Override
  public String describeKey() {
    return null;
  }

  @Override
  public boolean executeUnconditionally() {
    return false;
  }

  @Override
  public boolean isVolatile() {
    return false;
  }

  @Override
  public boolean showsOutputUnconditionally() {
    return false;
  }

  @Override
  public final String getProgressMessage() {
    String message = getRawProgressMessage();
    if (message == null) {
      return null;
    }
    String additionalInfo = getOwner().getAdditionalProgressInfo();
    return additionalInfo == null ? message : message + " [" + additionalInfo + "]";
  }

  /**
   * Returns a progress message string that is specific for this action. This is then annotated with
   * additional information, currently the string '[for host]' for actions in the host
   * configurations.
   *
   * <p>A return value of null indicates no message should be reported.
   */
  protected String getRawProgressMessage() {
    // A cheesy default implementation.  Subclasses are invited to do something
    // more meaningful.
    return defaultProgressMessage();
  }

  private String defaultProgressMessage() {
    return getMnemonic() + " " + getPrimaryOutput().prettyPrint();
  }

  @Override
  public String prettyPrint() {
    return "action '" + describe() + "'";
  }

  @Override
  public boolean isImmutable() {
    return false;
  }

  @Override
  public void write(Appendable buffer, char quotationMark) {
    Printer.append(buffer, prettyPrint()); // TODO(bazel-team): implement a readable representation
  }

  /**
   * Deletes all of the action's output files, if they exist. If any of the Artifacts refers to a
   * directory recursively removes the contents of the directory.
   *
   * @param execRoot the exec root in which this action is executed
   */
  protected void deleteOutputs(Path execRoot) throws IOException {
    for (Artifact output : getOutputs()) {
      deleteOutput(output);
    }
  }

  /**
   * Helper method to remove an Artifact. If the Artifact refers to a directory recursively removes
   * the contents of the directory.
   */
  protected void deleteOutput(Artifact output) throws IOException {
    Path path = output.getPath();
    try {
      // Optimize for the common case: output artifacts are files.
      path.delete();
    } catch (IOException e) {
      // Only try to recursively delete a directory if the output root is known. This is just a
      // sanity check so that we do not start deleting random files on disk.
      // TODO(bazel-team): Strengthen this test by making sure that the output is part of the
      // output tree.
      if (path.isDirectory(Symlinks.NOFOLLOW) && output.getRoot() != null) {
        FileSystemUtils.deleteTree(path);
      } else {
        throw e;
      }
    }
  }

  /**
   * If the action might read directories as inputs in a way that is unsound wrt dependency
   * checking, this method must be called.
   */
  protected void checkInputsForDirectories(
      EventHandler eventHandler, MetadataHandler metadataHandler) {
    // Report "directory dependency checking" warning only for non-generated directories (generated
    // ones will be reported earlier).
    for (Artifact input : getMandatoryInputs()) {
      // Assume that if the file did not exist, we would not have gotten here.
      if (input.isSourceArtifact() && !metadataHandler.isRegularFile(input)) {
        eventHandler.handle(
            Event.warn(
                getOwner().getLocation(),
                "input '"
                    + input.prettyPrint()
                    + "' to "
                    + getOwner().getLabel()
                    + " is a directory; dependency checking of directories is unsound"));
      }
    }
  }

  @Override
  public MiddlemanType getActionType() {
    return MiddlemanType.NORMAL;
  }

  /** If the action might create directories as outputs this method must be called. */
  protected void checkOutputsForDirectories(EventHandler eventHandler) {
    for (Artifact output : getOutputs()) {
      Path path = output.getPath();
      String ownerString = Label.print(getOwner().getLabel());
      if (path.isDirectory()) {
        eventHandler.handle(
            new Event(
                EventKind.WARNING,
                getOwner().getLocation(),
                "output '"
                    + output.prettyPrint()
                    + "' of "
                    + ownerString
                    + " is a directory; dependency checking of directories is unsound",
                ownerString));
      }
    }
  }

  @Override
  public void prepare(Path execRoot) throws IOException {
    deleteOutputs(execRoot);
  }

  @Override
  public String describe() {
    String progressMessage = getProgressMessage();
    return progressMessage != null ? progressMessage : defaultProgressMessage();
  }

  @Override
  public abstract ResourceSet estimateResourceConsumption(Executor executor);

  @Override
  public boolean shouldReportPathPrefixConflict(Action action) {
    return this != action;
  }

  @Override
  public ExtraActionInfo.Builder getExtraActionInfo() {
    return ExtraActionInfo.newBuilder()
        .setOwner(getOwner().getLabel().toString())
        .setId(getKey())
        .setMnemonic(getMnemonic());
  }

  @Override
  public ImmutableSet<Artifact> getMandatoryOutputs() {
    return ImmutableSet.of();
  }

  /**
   * Returns input files that need to be present to allow extra_action rules to shadow this action
   * correctly when run remotely. This is at least the normal inputs of the action, but may include
   * other files as well. For example C(++) compilation may perform include file header scanning.
   * This needs to be mirrored by the extra_action rule. Called by {@link
   * com.google.devtools.build.lib.rules.extra.ExtraAction} at execution time.
   *
   * <p>As this method is called from the ExtraAction, make sure it is ok to call this method from a
   * different thread than the one this action is executed on.
   *
   * @param actionExecutionContext Services in the scope of the action, like the Out/Err streams.
   * @throws ActionExecutionException only when code called from this method throws that exception.
   * @throws InterruptedException if interrupted
   */
  public Iterable<Artifact> getInputFilesForExtraAction(
      ActionExecutionContext actionExecutionContext)
      throws ActionExecutionException, InterruptedException {
    return getInputs();
  }

  /**
   * A copying implementation of {@link ActionOwner}.
   *
   * <p>ConfiguredTargets implement ActionOwner themselves, but we do not want actions to keep
   * direct references to configured targets just for a label and a few strings.
   */
  @Immutable
  private static class ActionOwnerDescription implements ActionOwner {

    private final Location location;
    private final Label label;
    private final String configurationMnemonic;
    private final String configurationChecksum;
    private final String targetKind;
    private final String additionalProgressInfo;

    private ActionOwnerDescription(ActionOwner originalOwner) {
      this.location = originalOwner.getLocation();
      this.label = originalOwner.getLabel();
      this.configurationMnemonic = originalOwner.getConfigurationMnemonic();
      this.configurationChecksum = originalOwner.getConfigurationChecksum();
      this.targetKind = originalOwner.getTargetKind();
      this.additionalProgressInfo = originalOwner.getAdditionalProgressInfo();
    }

    @Override
    public Location getLocation() {
      return location;
    }

    @Override
    public Label getLabel() {
      return label;
    }

    @Override
    public String getConfigurationMnemonic() {
      return configurationMnemonic;
    }

    @Override
    public String getConfigurationChecksum() {
      return configurationChecksum;
    }

    @Override
    public String getTargetKind() {
      return targetKind;
    }

    @Override
    public String getAdditionalProgressInfo() {
      return additionalProgressInfo;
    }
  }
}