private ListenableFuture<BuildResult> processBuildRule(
final BuildRule rule,
final BuildContext context,
ConcurrentLinkedQueue<ListenableFuture<Void>> asyncCallbacks)
throws InterruptedException {
// Log to the event bus.
context.getEventBus().logVerboseAndPost(LOG, BuildRuleEvent.resumed(rule));
final OnDiskBuildInfo onDiskBuildInfo = context.createOnDiskBuildInfoFor(rule.getBuildTarget());
final BuildInfoRecorder buildInfoRecorder =
context
.createBuildInfoRecorder(rule.getBuildTarget())
.addBuildMetadata(BuildInfo.METADATA_KEY_FOR_RULE_KEY, rule.getRuleKey().toString())
.addBuildMetadata(
BuildInfo.METADATA_KEY_FOR_RULE_KEY_WITHOUT_DEPS,
rule.getRuleKeyWithoutDeps().toString());
final BuildableContext buildableContext = new DefaultBuildableContext(buildInfoRecorder);
// Dispatch the build job for this rule.
ListenableFuture<BuildResult> buildResult =
processBuildRule(
rule, context, onDiskBuildInfo, buildInfoRecorder, buildableContext, asyncCallbacks);
// If we're performing a deep build, guarantee that all dependencies will *always* get
// materialized locally by chaining up to our result future.
if (buildMode == BuildMode.DEEP) {
buildResult =
MoreFutures.chainExceptions(getDepResults(rule, context, asyncCallbacks), buildResult);
}
// Setup a callback to handle either the cached or built locally cases.
AsyncFunction<BuildResult, BuildResult> callback =
new AsyncFunction<BuildResult, BuildResult>() {
@Override
public ListenableFuture<BuildResult> apply(@Nonnull BuildResult input) throws Exception {
// If we weren't successful, exit now.
if (input.getStatus() != BuildRuleStatus.SUCCESS) {
return Futures.immediateFuture(input);
}
// We shouldn't see any build fail result at this point.
BuildRuleSuccessType success = Preconditions.checkNotNull(input.getSuccess());
// If we didn't build the rule locally, reload the recorded paths from the build
// metadata.
if (success != BuildRuleSuccessType.BUILT_LOCALLY) {
for (String str :
onDiskBuildInfo.getValues(BuildInfo.METADATA_KEY_FOR_RECORDED_PATHS).get()) {
buildInfoRecorder.recordArtifact(Paths.get(str));
}
}
// If the success type means the rule has potentially changed it's outputs...
if (success.outputsHaveChanged()) {
// The build has succeeded, whether we've fetched from cache, or built locally.
// So run the post-build steps.
if (rule instanceof HasPostBuildSteps) {
executePostBuildSteps(
rule,
((HasPostBuildSteps) rule).getPostBuildSteps(context, buildableContext),
context);
}
// Invalidate any cached hashes for the output paths, since we've updated them.
for (Path path : buildInfoRecorder.getRecordedPaths()) {
fileHashCache.invalidate(path);
}
}
// If this rule uses dep files and we built locally, make sure we store the new dep file
// list and re-calculate the dep file rule key.
if (useDependencyFileRuleKey(rule) && success == BuildRuleSuccessType.BUILT_LOCALLY) {
// Query the rule for the actual inputs it used, and verify these are relative.
ImmutableList<Path> inputs =
((SupportsDependencyFileRuleKey) rule).getInputsAfterBuildingLocally();
for (Path path : inputs) {
Preconditions.checkState(
!path.isAbsolute(),
String.format(
"%s: reported absolute path as an input: %s", rule.getBuildTarget(), path));
}
// Record the inputs into our metadata for next time.
ImmutableList<String> inputStrings =
FluentIterable.from(inputs).transform(Functions.toStringFunction()).toList();
buildInfoRecorder.addMetadata(BuildInfo.METADATA_KEY_FOR_DEP_FILE, inputStrings);
// Re-calculate and store the depfile rule key for next time.
Optional<RuleKey> depFileRuleKey =
calculateDepFileRuleKey(
rule, Optional.of(inputStrings), /* allowMissingInputs */ false);
Preconditions.checkState(depFileRuleKey.isPresent());
buildInfoRecorder.addBuildMetadata(
BuildInfo.METADATA_KEY_FOR_DEP_FILE_RULE_KEY, depFileRuleKey.get().toString());
}
// Make sure that all of the local files have the same values they would as if the
// rule had been built locally.
buildInfoRecorder.addBuildMetadata(
BuildInfo.METADATA_KEY_FOR_TARGET, rule.getBuildTarget().toString());
buildInfoRecorder.addMetadata(
BuildInfo.METADATA_KEY_FOR_RECORDED_PATHS,
FluentIterable.from(buildInfoRecorder.getRecordedPaths())
.transform(Functions.toStringFunction()));
if (success.shouldWriteRecordedMetadataToDiskAfterBuilding()) {
try {
boolean clearExistingMetadata = success.shouldClearAndOverwriteMetadataOnDisk();
buildInfoRecorder.writeMetadataToDisk(clearExistingMetadata);
} catch (IOException e) {
throw new IOException(
String.format("Failed to write metadata to disk for %s.", rule), e);
}
}
// Give the rule a chance to populate its internal data structures now that all of
// the files should be in a valid state.
try {
if (rule instanceof InitializableFromDisk) {
doInitializeFromDisk((InitializableFromDisk<?>) rule, onDiskBuildInfo);
}
} catch (IOException e) {
throw new IOException(String.format("Error initializing %s from disk.", rule), e);
}
return Futures.immediateFuture(input);
}
};
buildResult = Futures.transform(buildResult, callback);
// Handle either build success or failure.
final SettableFuture<BuildResult> result = SettableFuture.create();
asyncCallbacks.add(
MoreFutures.addListenableCallback(
buildResult,
new FutureCallback<BuildResult>() {
// TODO(mbolin): Delete all files produced by the rule, as they are not guaranteed
// to be valid at this point?
private void cleanupAfterError() {
try {
onDiskBuildInfo.deleteExistingMetadata();
} catch (Throwable t) {
context
.getEventBus()
.post(
ThrowableConsoleEvent.create(
t, "Error when deleting metadata for %s.", rule));
}
}
private void uploadToCache(BuildRuleSuccessType success) {
// Collect up all the rule keys we have index the artifact in the cache with.
Set<RuleKey> ruleKeys = Sets.newHashSet();
// If the rule key has changed (and is not already in the cache), we need to push
// the artifact to cache using the new key.
if (success.shouldUploadResultingArtifact()) {
ruleKeys.add(rule.getRuleKey());
}
// If the input-based rule key has changed, we need to push the artifact to cache
// using the new key.
if (rule instanceof SupportsInputBasedRuleKey
&& success.shouldUploadResultingArtifactInputBased()) {
ruleKeys.add(
onDiskBuildInfo
.getRuleKey(BuildInfo.METADATA_KEY_FOR_INPUT_BASED_RULE_KEY)
.get());
}
// If we have any rule keys to push to the cache with, do the upload now.
if (!ruleKeys.isEmpty()) {
try {
buildInfoRecorder.performUploadToArtifactCache(
ImmutableSet.copyOf(ruleKeys),
context.getArtifactCache(),
context.getEventBus());
} catch (Throwable t) {
context
.getEventBus()
.post(
ThrowableConsoleEvent.create(
t, "Error uploading to cache for %s.", rule));
}
}
}
private void handleResult(BuildResult input) {
Optional<Long> outputSize = Optional.absent();
Optional<HashCode> outputHash = Optional.absent();
Optional<BuildRuleSuccessType> successType = Optional.absent();
if (input.getStatus() == BuildRuleStatus.FAIL) {
// Make this failure visible for other rules, so that they can stop early.
firstFailure = input.getFailure();
// If we failed, cleanup the state of this rule.
cleanupAfterError();
}
// Unblock dependents.
result.set(input);
if (input.getStatus() == BuildRuleStatus.SUCCESS) {
BuildRuleSuccessType success = Preconditions.checkNotNull(input.getSuccess());
successType = Optional.of(success);
uploadToCache(success);
// Calculate the hash and size of the rule outputs.
try {
outputSize = Optional.of(buildInfoRecorder.getOutputSize());
outputHash = Optional.of(buildInfoRecorder.getOutputHash(fileHashCache));
} catch (IOException e) {
context
.getEventBus()
.post(
ThrowableConsoleEvent.create(
e, "Error getting output hash and size for %s.", rule));
}
}
// Log the result to the event bus.
context
.getEventBus()
.logVerboseAndPost(
LOG,
BuildRuleEvent.finished(
rule,
input.getStatus(),
input.getCacheResult(),
successType,
outputHash,
outputSize));
}
@Override
public void onSuccess(BuildResult input) {
handleResult(input);
}
@Override
public void onFailure(@Nonnull Throwable thrown) {
handleResult(BuildResult.failure(rule, thrown));
// Reset interrupted flag once failure has been recorded.
if (thrown instanceof InterruptedException) {
Thread.currentThread().interrupt();
}
}
}));
return result;
}
private ListenableFuture<BuildResult> processBuildRule(
final BuildRule rule,
final BuildContext context,
final OnDiskBuildInfo onDiskBuildInfo,
final BuildInfoRecorder buildInfoRecorder,
final BuildableContext buildableContext,
ConcurrentLinkedQueue<ListenableFuture<Void>> asyncCallbacks)
throws InterruptedException {
// If we've already seen a failure, exit early.
if (!context.isKeepGoing() && firstFailure != null) {
return Futures.immediateFuture(BuildResult.canceled(rule, firstFailure));
}
// 1. Check if it's already built.
Optional<RuleKey> cachedRuleKey =
onDiskBuildInfo.getRuleKey(BuildInfo.METADATA_KEY_FOR_RULE_KEY);
if (cachedRuleKey.isPresent() && rule.getRuleKey().equals(cachedRuleKey.get())) {
return Futures.immediateFuture(
BuildResult.success(
rule, BuildRuleSuccessType.MATCHING_RULE_KEY, CacheResult.localKeyUnchangedHit()));
}
// 2. Rule key cache lookup.
final CacheResult cacheResult =
tryToFetchArtifactFromBuildCacheAndOverlayOnTopOfProjectFilesystem(
rule,
rule.getRuleKey(),
buildInfoRecorder,
context.getArtifactCache(),
context.getProjectFilesystem(),
context);
if (cacheResult.getType().isSuccess()) {
return Futures.immediateFuture(
BuildResult.success(rule, BuildRuleSuccessType.FETCHED_FROM_CACHE, cacheResult));
}
// Log to the event bus.
context.getEventBus().logVerboseAndPost(LOG, BuildRuleEvent.suspended(rule));
// 3. Build deps.
return Futures.transform(
getDepResults(rule, context, asyncCallbacks),
new AsyncFunction<List<BuildResult>, BuildResult>() {
@Override
public ListenableFuture<BuildResult> apply(@Nonnull List<BuildResult> depResults)
throws Exception {
// Log to the event bus.
context.getEventBus().logVerboseAndPost(LOG, BuildRuleEvent.resumed(rule));
// If any dependency wasn't successful, cancel ourselves.
for (BuildResult depResult : depResults) {
if (depResult.getStatus() != BuildRuleStatus.SUCCESS) {
return Futures.immediateFuture(
BuildResult.canceled(rule, Preconditions.checkNotNull(depResult.getFailure())));
}
}
// If we've already seen a failure, exit early.
if (!context.isKeepGoing() && firstFailure != null) {
return Futures.immediateFuture(BuildResult.canceled(rule, firstFailure));
}
// Dep-file rule keys.
if (useDependencyFileRuleKey(rule)) {
// Try to get the current dep-file rule key.
Optional<RuleKey> depFileRuleKey =
calculateDepFileRuleKey(
rule,
onDiskBuildInfo.getValues(BuildInfo.METADATA_KEY_FOR_DEP_FILE),
/* allowMissingInputs */ true);
if (depFileRuleKey.isPresent()) {
// Check the input-based rule key says we're already built.
Optional<RuleKey> lastDepFileRuleKey =
onDiskBuildInfo.getRuleKey(BuildInfo.METADATA_KEY_FOR_DEP_FILE_RULE_KEY);
if (lastDepFileRuleKey.isPresent()
&& lastDepFileRuleKey.get().equals(depFileRuleKey.get())) {
return Futures.immediateFuture(
BuildResult.success(
rule,
BuildRuleSuccessType.MATCHING_DEP_FILE_RULE_KEY,
CacheResult.localKeyUnchangedHit()));
}
}
}
// Input-based rule keys.
if (rule instanceof SupportsInputBasedRuleKey) {
// Calculate the input-based rule key and record it in the metadata.
RuleKey inputRuleKey = inputBasedRuleKeyBuilderFactory.newInstance(rule).build();
buildInfoRecorder.addBuildMetadata(
BuildInfo.METADATA_KEY_FOR_INPUT_BASED_RULE_KEY, inputRuleKey.toString());
// Check the input-based rule key says we're already built.
Optional<RuleKey> lastInputRuleKey =
onDiskBuildInfo.getRuleKey(BuildInfo.METADATA_KEY_FOR_INPUT_BASED_RULE_KEY);
if (lastInputRuleKey.isPresent() && lastInputRuleKey.get().equals(inputRuleKey)) {
return Futures.immediateFuture(
BuildResult.success(
rule,
BuildRuleSuccessType.MATCHING_INPUT_BASED_RULE_KEY,
CacheResult.localKeyUnchangedHit()));
}
// Try to fetch the artifact using the input-based rule key.
CacheResult cacheResult =
tryToFetchArtifactFromBuildCacheAndOverlayOnTopOfProjectFilesystem(
rule,
inputRuleKey,
buildInfoRecorder,
context.getArtifactCache(),
context.getProjectFilesystem(),
context);
if (cacheResult.getType().isSuccess()) {
return Futures.immediateFuture(
BuildResult.success(
rule, BuildRuleSuccessType.FETCHED_FROM_CACHE_INPUT_BASED, cacheResult));
}
}
// 4. ABI check
// Deciding whether we need to rebuild is tricky business. We want to rebuild as little
// as possible while always being sound.
//
// For java_library rules that depend only on their first-order deps,
// they only need to rebuild themselves if any of the following conditions hold:
// (1) The definition of the build rule has changed.
// (2) Any of the input files (which includes resources as well as .java files) have
// changed.
// (3) The ABI of any of its dependent java_library rules has changed.
//
// For other types of build rules, we have to be more conservative when rebuilding. In
// those cases, we rebuild if any of the following conditions hold:
// (1) The definition of the build rule has changed.
// (2) Any of the input files have changed.
// (3) Any of the RuleKeys of this rule's deps have changed.
//
// Because a RuleKey for a rule will change if any of its transitive deps have changed,
// that means a change in one of the leaves can result in almost all rules being
// rebuilt, which is slow. Fortunately, we limit the effects of this when building Java
// code when checking the ABI of deps instead of the RuleKey for deps.
AbiRule abiRule = checkIfRuleOrBuildableIsAbiRule(rule);
if (abiRule != null) {
RuleKey ruleKeyNoDeps = rule.getRuleKeyWithoutDeps();
Optional<RuleKey> cachedRuleKeyNoDeps =
onDiskBuildInfo.getRuleKey(BuildInfo.METADATA_KEY_FOR_RULE_KEY_WITHOUT_DEPS);
if (ruleKeyNoDeps.equals(cachedRuleKeyNoDeps.orNull())) {
// The RuleKey for the definition of this build rule and its input files has not
// changed. Therefore, if the ABI of its deps has not changed, there is nothing to
// rebuild.
Sha1HashCode abiKeyForDeps = abiRule.getAbiKeyForDeps();
Optional<Sha1HashCode> cachedAbiKeyForDeps =
onDiskBuildInfo.getHash(ABI_KEY_FOR_DEPS_ON_DISK_METADATA);
if (abiKeyForDeps.equals(cachedAbiKeyForDeps.orNull())) {
return Futures.immediateFuture(
BuildResult.success(
rule,
BuildRuleSuccessType.MATCHING_DEPS_ABI_AND_RULE_KEY_NO_DEPS,
AbstractCacheResult.localKeyUnchangedHit()));
}
}
}
// 5. build the rule
executeCommandsNowThatDepsAreBuilt(rule, context, buildableContext, buildInfoRecorder);
return Futures.immediateFuture(
BuildResult.success(rule, BuildRuleSuccessType.BUILT_LOCALLY, cacheResult));
}
},
service);
}
