Ejemplo n.º 1
0
  // Provide a future that resolve to the result of executing this rule and its runtime
  // dependencies.
  private ListenableFuture<BuildResult> getBuildRuleResultWithRuntimeDeps(
      final BuildRule rule,
      final BuildContext context,
      final ConcurrentLinkedQueue<ListenableFuture<Void>> asyncCallbacks) {

    // Get the future holding the result for this rule and, if we have no additional runtime deps
    // to attach, return it.
    final ListenableFuture<BuildResult> result = getBuildRuleResult(rule, context, asyncCallbacks);
    if (!(rule instanceof HasRuntimeDeps)) {
      return result;
    }

    // Collect any runtime deps we have into a list of futures.
    ImmutableSortedSet<BuildRule> runtimeDeps = ((HasRuntimeDeps) rule).getRuntimeDeps();
    List<ListenableFuture<BuildResult>> runtimeDepResults =
        Lists.newArrayListWithExpectedSize(runtimeDeps.size());
    for (BuildRule dep : runtimeDeps) {
      runtimeDepResults.add(getBuildRuleResultWithRuntimeDeps(dep, context, asyncCallbacks));
    }

    // Create a new combined future, which runs the original rule and all the runtime deps in
    // parallel, but which propagates an error if any one of them fails.
    return MoreFutures.chainExceptions(Futures.allAsList(runtimeDepResults), result);
  }
Ejemplo n.º 2
0
  private ListenableFuture<BuildResult> processBuildRule(
      final BuildRule rule,
      final BuildContext context,
      ConcurrentLinkedQueue<ListenableFuture<Void>> asyncCallbacks)
      throws InterruptedException {

    // Log to the event bus.
    context.getEventBus().logVerboseAndPost(LOG, BuildRuleEvent.resumed(rule));

    final OnDiskBuildInfo onDiskBuildInfo = context.createOnDiskBuildInfoFor(rule.getBuildTarget());
    final BuildInfoRecorder buildInfoRecorder =
        context
            .createBuildInfoRecorder(rule.getBuildTarget())
            .addBuildMetadata(BuildInfo.METADATA_KEY_FOR_RULE_KEY, rule.getRuleKey().toString())
            .addBuildMetadata(
                BuildInfo.METADATA_KEY_FOR_RULE_KEY_WITHOUT_DEPS,
                rule.getRuleKeyWithoutDeps().toString());
    final BuildableContext buildableContext = new DefaultBuildableContext(buildInfoRecorder);

    // Dispatch the build job for this rule.
    ListenableFuture<BuildResult> buildResult =
        processBuildRule(
            rule, context, onDiskBuildInfo, buildInfoRecorder, buildableContext, asyncCallbacks);

    // If we're performing a deep build, guarantee that all dependencies will *always* get
    // materialized locally by chaining up to our result future.
    if (buildMode == BuildMode.DEEP) {
      buildResult =
          MoreFutures.chainExceptions(getDepResults(rule, context, asyncCallbacks), buildResult);
    }

    // Setup a callback to handle either the cached or built locally cases.
    AsyncFunction<BuildResult, BuildResult> callback =
        new AsyncFunction<BuildResult, BuildResult>() {
          @Override
          public ListenableFuture<BuildResult> apply(@Nonnull BuildResult input) throws Exception {

            // If we weren't successful, exit now.
            if (input.getStatus() != BuildRuleStatus.SUCCESS) {
              return Futures.immediateFuture(input);
            }

            // We shouldn't see any build fail result at this point.
            BuildRuleSuccessType success = Preconditions.checkNotNull(input.getSuccess());

            // If we didn't build the rule locally, reload the recorded paths from the build
            // metadata.
            if (success != BuildRuleSuccessType.BUILT_LOCALLY) {
              for (String str :
                  onDiskBuildInfo.getValues(BuildInfo.METADATA_KEY_FOR_RECORDED_PATHS).get()) {
                buildInfoRecorder.recordArtifact(Paths.get(str));
              }
            }

            // If the success type means the rule has potentially changed it's outputs...
            if (success.outputsHaveChanged()) {

              // The build has succeeded, whether we've fetched from cache, or built locally.
              // So run the post-build steps.
              if (rule instanceof HasPostBuildSteps) {
                executePostBuildSteps(
                    rule,
                    ((HasPostBuildSteps) rule).getPostBuildSteps(context, buildableContext),
                    context);
              }

              // Invalidate any cached hashes for the output paths, since we've updated them.
              for (Path path : buildInfoRecorder.getRecordedPaths()) {
                fileHashCache.invalidate(path);
              }
            }

            // If this rule uses dep files and we built locally, make sure we store the new dep file
            // list and re-calculate the dep file rule key.
            if (useDependencyFileRuleKey(rule) && success == BuildRuleSuccessType.BUILT_LOCALLY) {

              // Query the rule for the actual inputs it used, and verify these are relative.
              ImmutableList<Path> inputs =
                  ((SupportsDependencyFileRuleKey) rule).getInputsAfterBuildingLocally();
              for (Path path : inputs) {
                Preconditions.checkState(
                    !path.isAbsolute(),
                    String.format(
                        "%s: reported absolute path as an input: %s", rule.getBuildTarget(), path));
              }

              // Record the inputs into our metadata for next time.
              ImmutableList<String> inputStrings =
                  FluentIterable.from(inputs).transform(Functions.toStringFunction()).toList();
              buildInfoRecorder.addMetadata(BuildInfo.METADATA_KEY_FOR_DEP_FILE, inputStrings);

              // Re-calculate and store the depfile rule key for next time.
              Optional<RuleKey> depFileRuleKey =
                  calculateDepFileRuleKey(
                      rule, Optional.of(inputStrings), /* allowMissingInputs */ false);
              Preconditions.checkState(depFileRuleKey.isPresent());
              buildInfoRecorder.addBuildMetadata(
                  BuildInfo.METADATA_KEY_FOR_DEP_FILE_RULE_KEY, depFileRuleKey.get().toString());
            }

            // Make sure that all of the local files have the same values they would as if the
            // rule had been built locally.
            buildInfoRecorder.addBuildMetadata(
                BuildInfo.METADATA_KEY_FOR_TARGET, rule.getBuildTarget().toString());
            buildInfoRecorder.addMetadata(
                BuildInfo.METADATA_KEY_FOR_RECORDED_PATHS,
                FluentIterable.from(buildInfoRecorder.getRecordedPaths())
                    .transform(Functions.toStringFunction()));
            if (success.shouldWriteRecordedMetadataToDiskAfterBuilding()) {
              try {
                boolean clearExistingMetadata = success.shouldClearAndOverwriteMetadataOnDisk();
                buildInfoRecorder.writeMetadataToDisk(clearExistingMetadata);
              } catch (IOException e) {
                throw new IOException(
                    String.format("Failed to write metadata to disk for %s.", rule), e);
              }
            }

            // Give the rule a chance to populate its internal data structures now that all of
            // the files should be in a valid state.
            try {
              if (rule instanceof InitializableFromDisk) {
                doInitializeFromDisk((InitializableFromDisk<?>) rule, onDiskBuildInfo);
              }
            } catch (IOException e) {
              throw new IOException(String.format("Error initializing %s from disk.", rule), e);
            }

            return Futures.immediateFuture(input);
          }
        };
    buildResult = Futures.transform(buildResult, callback);

    // Handle either build success or failure.
    final SettableFuture<BuildResult> result = SettableFuture.create();
    asyncCallbacks.add(
        MoreFutures.addListenableCallback(
            buildResult,
            new FutureCallback<BuildResult>() {

              // TODO(mbolin): Delete all files produced by the rule, as they are not guaranteed
              // to be valid at this point?
              private void cleanupAfterError() {
                try {
                  onDiskBuildInfo.deleteExistingMetadata();
                } catch (Throwable t) {
                  context
                      .getEventBus()
                      .post(
                          ThrowableConsoleEvent.create(
                              t, "Error when deleting metadata for %s.", rule));
                }
              }

              private void uploadToCache(BuildRuleSuccessType success) {

                // Collect up all the rule keys we have index the artifact in the cache with.
                Set<RuleKey> ruleKeys = Sets.newHashSet();

                // If the rule key has changed (and is not already in the cache), we need to push
                // the artifact to cache using the new key.
                if (success.shouldUploadResultingArtifact()) {
                  ruleKeys.add(rule.getRuleKey());
                }

                // If the input-based rule key has changed, we need to push the artifact to cache
                // using the new key.
                if (rule instanceof SupportsInputBasedRuleKey
                    && success.shouldUploadResultingArtifactInputBased()) {
                  ruleKeys.add(
                      onDiskBuildInfo
                          .getRuleKey(BuildInfo.METADATA_KEY_FOR_INPUT_BASED_RULE_KEY)
                          .get());
                }

                // If we have any rule keys to push to the cache with, do the upload now.
                if (!ruleKeys.isEmpty()) {
                  try {
                    buildInfoRecorder.performUploadToArtifactCache(
                        ImmutableSet.copyOf(ruleKeys),
                        context.getArtifactCache(),
                        context.getEventBus());
                  } catch (Throwable t) {
                    context
                        .getEventBus()
                        .post(
                            ThrowableConsoleEvent.create(
                                t, "Error uploading to cache for %s.", rule));
                  }
                }
              }

              private void handleResult(BuildResult input) {
                Optional<Long> outputSize = Optional.absent();
                Optional<HashCode> outputHash = Optional.absent();
                Optional<BuildRuleSuccessType> successType = Optional.absent();

                if (input.getStatus() == BuildRuleStatus.FAIL) {

                  // Make this failure visible for other rules, so that they can stop early.
                  firstFailure = input.getFailure();

                  // If we failed, cleanup the state of this rule.
                  cleanupAfterError();
                }

                // Unblock dependents.
                result.set(input);

                if (input.getStatus() == BuildRuleStatus.SUCCESS) {
                  BuildRuleSuccessType success = Preconditions.checkNotNull(input.getSuccess());
                  successType = Optional.of(success);
                  uploadToCache(success);

                  // Calculate the hash and size of the rule outputs.
                  try {
                    outputSize = Optional.of(buildInfoRecorder.getOutputSize());
                    outputHash = Optional.of(buildInfoRecorder.getOutputHash(fileHashCache));
                  } catch (IOException e) {
                    context
                        .getEventBus()
                        .post(
                            ThrowableConsoleEvent.create(
                                e, "Error getting output hash and size for %s.", rule));
                  }
                }

                // Log the result to the event bus.
                context
                    .getEventBus()
                    .logVerboseAndPost(
                        LOG,
                        BuildRuleEvent.finished(
                            rule,
                            input.getStatus(),
                            input.getCacheResult(),
                            successType,
                            outputHash,
                            outputSize));
              }

              @Override
              public void onSuccess(BuildResult input) {
                handleResult(input);
              }

              @Override
              public void onFailure(@Nonnull Throwable thrown) {
                handleResult(BuildResult.failure(rule, thrown));

                // Reset interrupted flag once failure has been recorded.
                if (thrown instanceof InterruptedException) {
                  Thread.currentThread().interrupt();
                }
              }
            }));
    return result;
  }