// Provide a future that resolve to the result of executing this rule and its runtime
// dependencies.
private ListenableFuture<BuildResult> getBuildRuleResultWithRuntimeDeps(
final BuildRule rule,
final BuildContext context,
final ConcurrentLinkedQueue<ListenableFuture<Void>> asyncCallbacks) {
// Get the future holding the result for this rule and, if we have no additional runtime deps
// to attach, return it.
final ListenableFuture<BuildResult> result = getBuildRuleResult(rule, context, asyncCallbacks);
if (!(rule instanceof HasRuntimeDeps)) {
return result;
}
// Collect any runtime deps we have into a list of futures.
ImmutableSortedSet<BuildRule> runtimeDeps = ((HasRuntimeDeps) rule).getRuntimeDeps();
List<ListenableFuture<BuildResult>> runtimeDepResults =
Lists.newArrayListWithExpectedSize(runtimeDeps.size());
for (BuildRule dep : runtimeDeps) {
runtimeDepResults.add(getBuildRuleResultWithRuntimeDeps(dep, context, asyncCallbacks));
}
// Create a new combined future, which runs the original rule and all the runtime deps in
// parallel, but which propagates an error if any one of them fails.
return MoreFutures.chainExceptions(Futures.allAsList(runtimeDepResults), result);
}
private ListenableFuture<BuildResult> processBuildRule(
final BuildRule rule,
final BuildContext context,
ConcurrentLinkedQueue<ListenableFuture<Void>> asyncCallbacks)
throws InterruptedException {
// Log to the event bus.
context.getEventBus().logVerboseAndPost(LOG, BuildRuleEvent.resumed(rule));
final OnDiskBuildInfo onDiskBuildInfo = context.createOnDiskBuildInfoFor(rule.getBuildTarget());
final BuildInfoRecorder buildInfoRecorder =
context
.createBuildInfoRecorder(rule.getBuildTarget())
.addBuildMetadata(BuildInfo.METADATA_KEY_FOR_RULE_KEY, rule.getRuleKey().toString())
.addBuildMetadata(
BuildInfo.METADATA_KEY_FOR_RULE_KEY_WITHOUT_DEPS,
rule.getRuleKeyWithoutDeps().toString());
final BuildableContext buildableContext = new DefaultBuildableContext(buildInfoRecorder);
// Dispatch the build job for this rule.
ListenableFuture<BuildResult> buildResult =
processBuildRule(
rule, context, onDiskBuildInfo, buildInfoRecorder, buildableContext, asyncCallbacks);
// If we're performing a deep build, guarantee that all dependencies will *always* get
// materialized locally by chaining up to our result future.
if (buildMode == BuildMode.DEEP) {
buildResult =
MoreFutures.chainExceptions(getDepResults(rule, context, asyncCallbacks), buildResult);
}
// Setup a callback to handle either the cached or built locally cases.
AsyncFunction<BuildResult, BuildResult> callback =
new AsyncFunction<BuildResult, BuildResult>() {
@Override
public ListenableFuture<BuildResult> apply(@Nonnull BuildResult input) throws Exception {
// If we weren't successful, exit now.
if (input.getStatus() != BuildRuleStatus.SUCCESS) {
return Futures.immediateFuture(input);
}
// We shouldn't see any build fail result at this point.
BuildRuleSuccessType success = Preconditions.checkNotNull(input.getSuccess());
// If we didn't build the rule locally, reload the recorded paths from the build
// metadata.
if (success != BuildRuleSuccessType.BUILT_LOCALLY) {
for (String str :
onDiskBuildInfo.getValues(BuildInfo.METADATA_KEY_FOR_RECORDED_PATHS).get()) {
buildInfoRecorder.recordArtifact(Paths.get(str));
}
}
// If the success type means the rule has potentially changed it's outputs...
if (success.outputsHaveChanged()) {
// The build has succeeded, whether we've fetched from cache, or built locally.
// So run the post-build steps.
if (rule instanceof HasPostBuildSteps) {
executePostBuildSteps(
rule,
((HasPostBuildSteps) rule).getPostBuildSteps(context, buildableContext),
context);
}
// Invalidate any cached hashes for the output paths, since we've updated them.
for (Path path : buildInfoRecorder.getRecordedPaths()) {
fileHashCache.invalidate(path);
}
}
// If this rule uses dep files and we built locally, make sure we store the new dep file
// list and re-calculate the dep file rule key.
if (useDependencyFileRuleKey(rule) && success == BuildRuleSuccessType.BUILT_LOCALLY) {
// Query the rule for the actual inputs it used, and verify these are relative.
ImmutableList<Path> inputs =
((SupportsDependencyFileRuleKey) rule).getInputsAfterBuildingLocally();
for (Path path : inputs) {
Preconditions.checkState(
!path.isAbsolute(),
String.format(
"%s: reported absolute path as an input: %s", rule.getBuildTarget(), path));
}
// Record the inputs into our metadata for next time.
ImmutableList<String> inputStrings =
FluentIterable.from(inputs).transform(Functions.toStringFunction()).toList();
buildInfoRecorder.addMetadata(BuildInfo.METADATA_KEY_FOR_DEP_FILE, inputStrings);
// Re-calculate and store the depfile rule key for next time.
Optional<RuleKey> depFileRuleKey =
calculateDepFileRuleKey(
rule, Optional.of(inputStrings), /* allowMissingInputs */ false);
Preconditions.checkState(depFileRuleKey.isPresent());
buildInfoRecorder.addBuildMetadata(
BuildInfo.METADATA_KEY_FOR_DEP_FILE_RULE_KEY, depFileRuleKey.get().toString());
}
// Make sure that all of the local files have the same values they would as if the
// rule had been built locally.
buildInfoRecorder.addBuildMetadata(
BuildInfo.METADATA_KEY_FOR_TARGET, rule.getBuildTarget().toString());
buildInfoRecorder.addMetadata(
BuildInfo.METADATA_KEY_FOR_RECORDED_PATHS,
FluentIterable.from(buildInfoRecorder.getRecordedPaths())
.transform(Functions.toStringFunction()));
if (success.shouldWriteRecordedMetadataToDiskAfterBuilding()) {
try {
boolean clearExistingMetadata = success.shouldClearAndOverwriteMetadataOnDisk();
buildInfoRecorder.writeMetadataToDisk(clearExistingMetadata);
} catch (IOException e) {
throw new IOException(
String.format("Failed to write metadata to disk for %s.", rule), e);
}
}
// Give the rule a chance to populate its internal data structures now that all of
// the files should be in a valid state.
try {
if (rule instanceof InitializableFromDisk) {
doInitializeFromDisk((InitializableFromDisk<?>) rule, onDiskBuildInfo);
}
} catch (IOException e) {
throw new IOException(String.format("Error initializing %s from disk.", rule), e);
}
return Futures.immediateFuture(input);
}
};
buildResult = Futures.transform(buildResult, callback);
// Handle either build success or failure.
final SettableFuture<BuildResult> result = SettableFuture.create();
asyncCallbacks.add(
MoreFutures.addListenableCallback(
buildResult,
new FutureCallback<BuildResult>() {
// TODO(mbolin): Delete all files produced by the rule, as they are not guaranteed
// to be valid at this point?
private void cleanupAfterError() {
try {
onDiskBuildInfo.deleteExistingMetadata();
} catch (Throwable t) {
context
.getEventBus()
.post(
ThrowableConsoleEvent.create(
t, "Error when deleting metadata for %s.", rule));
}
}
private void uploadToCache(BuildRuleSuccessType success) {
// Collect up all the rule keys we have index the artifact in the cache with.
Set<RuleKey> ruleKeys = Sets.newHashSet();
// If the rule key has changed (and is not already in the cache), we need to push
// the artifact to cache using the new key.
if (success.shouldUploadResultingArtifact()) {
ruleKeys.add(rule.getRuleKey());
}
// If the input-based rule key has changed, we need to push the artifact to cache
// using the new key.
if (rule instanceof SupportsInputBasedRuleKey
&& success.shouldUploadResultingArtifactInputBased()) {
ruleKeys.add(
onDiskBuildInfo
.getRuleKey(BuildInfo.METADATA_KEY_FOR_INPUT_BASED_RULE_KEY)
.get());
}
// If we have any rule keys to push to the cache with, do the upload now.
if (!ruleKeys.isEmpty()) {
try {
buildInfoRecorder.performUploadToArtifactCache(
ImmutableSet.copyOf(ruleKeys),
context.getArtifactCache(),
context.getEventBus());
} catch (Throwable t) {
context
.getEventBus()
.post(
ThrowableConsoleEvent.create(
t, "Error uploading to cache for %s.", rule));
}
}
}
private void handleResult(BuildResult input) {
Optional<Long> outputSize = Optional.absent();
Optional<HashCode> outputHash = Optional.absent();
Optional<BuildRuleSuccessType> successType = Optional.absent();
if (input.getStatus() == BuildRuleStatus.FAIL) {
// Make this failure visible for other rules, so that they can stop early.
firstFailure = input.getFailure();
// If we failed, cleanup the state of this rule.
cleanupAfterError();
}
// Unblock dependents.
result.set(input);
if (input.getStatus() == BuildRuleStatus.SUCCESS) {
BuildRuleSuccessType success = Preconditions.checkNotNull(input.getSuccess());
successType = Optional.of(success);
uploadToCache(success);
// Calculate the hash and size of the rule outputs.
try {
outputSize = Optional.of(buildInfoRecorder.getOutputSize());
outputHash = Optional.of(buildInfoRecorder.getOutputHash(fileHashCache));
} catch (IOException e) {
context
.getEventBus()
.post(
ThrowableConsoleEvent.create(
e, "Error getting output hash and size for %s.", rule));
}
}
// Log the result to the event bus.
context
.getEventBus()
.logVerboseAndPost(
LOG,
BuildRuleEvent.finished(
rule,
input.getStatus(),
input.getCacheResult(),
successType,
outputHash,
outputSize));
}
@Override
public void onSuccess(BuildResult input) {
handleResult(input);
}
@Override
public void onFailure(@Nonnull Throwable thrown) {
handleResult(BuildResult.failure(rule, thrown));
// Reset interrupted flag once failure has been recorded.
if (thrown instanceof InterruptedException) {
Thread.currentThread().interrupt();
}
}
}));
return result;
}
@Override
public boolean isRuleBuilt(BuildTarget buildTarget) throws InterruptedException {
ListenableFuture<BuildResult> resultFuture = results.get(buildTarget);
return resultFuture != null && MoreFutures.isSuccess(resultFuture);
}