private ListenableFuture<BuildResult> processBuildRule( final BuildRule rule, final BuildContext context, ConcurrentLinkedQueue<ListenableFuture<Void>> asyncCallbacks) throws InterruptedException { // Log to the event bus. context.getEventBus().logVerboseAndPost(LOG, BuildRuleEvent.resumed(rule)); final OnDiskBuildInfo onDiskBuildInfo = context.createOnDiskBuildInfoFor(rule.getBuildTarget()); final BuildInfoRecorder buildInfoRecorder = context .createBuildInfoRecorder(rule.getBuildTarget()) .addBuildMetadata(BuildInfo.METADATA_KEY_FOR_RULE_KEY, rule.getRuleKey().toString()) .addBuildMetadata( BuildInfo.METADATA_KEY_FOR_RULE_KEY_WITHOUT_DEPS, rule.getRuleKeyWithoutDeps().toString()); final BuildableContext buildableContext = new DefaultBuildableContext(buildInfoRecorder); // Dispatch the build job for this rule. ListenableFuture<BuildResult> buildResult = processBuildRule( rule, context, onDiskBuildInfo, buildInfoRecorder, buildableContext, asyncCallbacks); // If we're performing a deep build, guarantee that all dependencies will *always* get // materialized locally by chaining up to our result future. if (buildMode == BuildMode.DEEP) { buildResult = MoreFutures.chainExceptions(getDepResults(rule, context, asyncCallbacks), buildResult); } // Setup a callback to handle either the cached or built locally cases. AsyncFunction<BuildResult, BuildResult> callback = new AsyncFunction<BuildResult, BuildResult>() { @Override public ListenableFuture<BuildResult> apply(@Nonnull BuildResult input) throws Exception { // If we weren't successful, exit now. if (input.getStatus() != BuildRuleStatus.SUCCESS) { return Futures.immediateFuture(input); } // We shouldn't see any build fail result at this point. BuildRuleSuccessType success = Preconditions.checkNotNull(input.getSuccess()); // If we didn't build the rule locally, reload the recorded paths from the build // metadata. if (success != BuildRuleSuccessType.BUILT_LOCALLY) { for (String str : onDiskBuildInfo.getValues(BuildInfo.METADATA_KEY_FOR_RECORDED_PATHS).get()) { buildInfoRecorder.recordArtifact(Paths.get(str)); } } // If the success type means the rule has potentially changed it's outputs... if (success.outputsHaveChanged()) { // The build has succeeded, whether we've fetched from cache, or built locally. // So run the post-build steps. if (rule instanceof HasPostBuildSteps) { executePostBuildSteps( rule, ((HasPostBuildSteps) rule).getPostBuildSteps(context, buildableContext), context); } // Invalidate any cached hashes for the output paths, since we've updated them. for (Path path : buildInfoRecorder.getRecordedPaths()) { fileHashCache.invalidate(path); } } // If this rule uses dep files and we built locally, make sure we store the new dep file // list and re-calculate the dep file rule key. if (useDependencyFileRuleKey(rule) && success == BuildRuleSuccessType.BUILT_LOCALLY) { // Query the rule for the actual inputs it used, and verify these are relative. ImmutableList<Path> inputs = ((SupportsDependencyFileRuleKey) rule).getInputsAfterBuildingLocally(); for (Path path : inputs) { Preconditions.checkState( !path.isAbsolute(), String.format( "%s: reported absolute path as an input: %s", rule.getBuildTarget(), path)); } // Record the inputs into our metadata for next time. ImmutableList<String> inputStrings = FluentIterable.from(inputs).transform(Functions.toStringFunction()).toList(); buildInfoRecorder.addMetadata(BuildInfo.METADATA_KEY_FOR_DEP_FILE, inputStrings); // Re-calculate and store the depfile rule key for next time. Optional<RuleKey> depFileRuleKey = calculateDepFileRuleKey( rule, Optional.of(inputStrings), /* allowMissingInputs */ false); Preconditions.checkState(depFileRuleKey.isPresent()); buildInfoRecorder.addBuildMetadata( BuildInfo.METADATA_KEY_FOR_DEP_FILE_RULE_KEY, depFileRuleKey.get().toString()); } // Make sure that all of the local files have the same values they would as if the // rule had been built locally. buildInfoRecorder.addBuildMetadata( BuildInfo.METADATA_KEY_FOR_TARGET, rule.getBuildTarget().toString()); buildInfoRecorder.addMetadata( BuildInfo.METADATA_KEY_FOR_RECORDED_PATHS, FluentIterable.from(buildInfoRecorder.getRecordedPaths()) .transform(Functions.toStringFunction())); if (success.shouldWriteRecordedMetadataToDiskAfterBuilding()) { try { boolean clearExistingMetadata = success.shouldClearAndOverwriteMetadataOnDisk(); buildInfoRecorder.writeMetadataToDisk(clearExistingMetadata); } catch (IOException e) { throw new IOException( String.format("Failed to write metadata to disk for %s.", rule), e); } } // Give the rule a chance to populate its internal data structures now that all of // the files should be in a valid state. try { if (rule instanceof InitializableFromDisk) { doInitializeFromDisk((InitializableFromDisk<?>) rule, onDiskBuildInfo); } } catch (IOException e) { throw new IOException(String.format("Error initializing %s from disk.", rule), e); } return Futures.immediateFuture(input); } }; buildResult = Futures.transform(buildResult, callback); // Handle either build success or failure. final SettableFuture<BuildResult> result = SettableFuture.create(); asyncCallbacks.add( MoreFutures.addListenableCallback( buildResult, new FutureCallback<BuildResult>() { // TODO(mbolin): Delete all files produced by the rule, as they are not guaranteed // to be valid at this point? private void cleanupAfterError() { try { onDiskBuildInfo.deleteExistingMetadata(); } catch (Throwable t) { context .getEventBus() .post( ThrowableConsoleEvent.create( t, "Error when deleting metadata for %s.", rule)); } } private void uploadToCache(BuildRuleSuccessType success) { // Collect up all the rule keys we have index the artifact in the cache with. Set<RuleKey> ruleKeys = Sets.newHashSet(); // If the rule key has changed (and is not already in the cache), we need to push // the artifact to cache using the new key. if (success.shouldUploadResultingArtifact()) { ruleKeys.add(rule.getRuleKey()); } // If the input-based rule key has changed, we need to push the artifact to cache // using the new key. if (rule instanceof SupportsInputBasedRuleKey && success.shouldUploadResultingArtifactInputBased()) { ruleKeys.add( onDiskBuildInfo .getRuleKey(BuildInfo.METADATA_KEY_FOR_INPUT_BASED_RULE_KEY) .get()); } // If we have any rule keys to push to the cache with, do the upload now. if (!ruleKeys.isEmpty()) { try { buildInfoRecorder.performUploadToArtifactCache( ImmutableSet.copyOf(ruleKeys), context.getArtifactCache(), context.getEventBus()); } catch (Throwable t) { context .getEventBus() .post( ThrowableConsoleEvent.create( t, "Error uploading to cache for %s.", rule)); } } } private void handleResult(BuildResult input) { Optional<Long> outputSize = Optional.absent(); Optional<HashCode> outputHash = Optional.absent(); Optional<BuildRuleSuccessType> successType = Optional.absent(); if (input.getStatus() == BuildRuleStatus.FAIL) { // Make this failure visible for other rules, so that they can stop early. firstFailure = input.getFailure(); // If we failed, cleanup the state of this rule. cleanupAfterError(); } // Unblock dependents. result.set(input); if (input.getStatus() == BuildRuleStatus.SUCCESS) { BuildRuleSuccessType success = Preconditions.checkNotNull(input.getSuccess()); successType = Optional.of(success); uploadToCache(success); // Calculate the hash and size of the rule outputs. try { outputSize = Optional.of(buildInfoRecorder.getOutputSize()); outputHash = Optional.of(buildInfoRecorder.getOutputHash(fileHashCache)); } catch (IOException e) { context .getEventBus() .post( ThrowableConsoleEvent.create( e, "Error getting output hash and size for %s.", rule)); } } // Log the result to the event bus. context .getEventBus() .logVerboseAndPost( LOG, BuildRuleEvent.finished( rule, input.getStatus(), input.getCacheResult(), successType, outputHash, outputSize)); } @Override public void onSuccess(BuildResult input) { handleResult(input); } @Override public void onFailure(@Nonnull Throwable thrown) { handleResult(BuildResult.failure(rule, thrown)); // Reset interrupted flag once failure has been recorded. if (thrown instanceof InterruptedException) { Thread.currentThread().interrupt(); } } })); return result; }
private ListenableFuture<BuildResult> processBuildRule( final BuildRule rule, final BuildContext context, final OnDiskBuildInfo onDiskBuildInfo, final BuildInfoRecorder buildInfoRecorder, final BuildableContext buildableContext, ConcurrentLinkedQueue<ListenableFuture<Void>> asyncCallbacks) throws InterruptedException { // If we've already seen a failure, exit early. if (!context.isKeepGoing() && firstFailure != null) { return Futures.immediateFuture(BuildResult.canceled(rule, firstFailure)); } // 1. Check if it's already built. Optional<RuleKey> cachedRuleKey = onDiskBuildInfo.getRuleKey(BuildInfo.METADATA_KEY_FOR_RULE_KEY); if (cachedRuleKey.isPresent() && rule.getRuleKey().equals(cachedRuleKey.get())) { return Futures.immediateFuture( BuildResult.success( rule, BuildRuleSuccessType.MATCHING_RULE_KEY, CacheResult.localKeyUnchangedHit())); } // 2. Rule key cache lookup. final CacheResult cacheResult = tryToFetchArtifactFromBuildCacheAndOverlayOnTopOfProjectFilesystem( rule, rule.getRuleKey(), buildInfoRecorder, context.getArtifactCache(), context.getProjectFilesystem(), context); if (cacheResult.getType().isSuccess()) { return Futures.immediateFuture( BuildResult.success(rule, BuildRuleSuccessType.FETCHED_FROM_CACHE, cacheResult)); } // Log to the event bus. context.getEventBus().logVerboseAndPost(LOG, BuildRuleEvent.suspended(rule)); // 3. Build deps. return Futures.transform( getDepResults(rule, context, asyncCallbacks), new AsyncFunction<List<BuildResult>, BuildResult>() { @Override public ListenableFuture<BuildResult> apply(@Nonnull List<BuildResult> depResults) throws Exception { // Log to the event bus. context.getEventBus().logVerboseAndPost(LOG, BuildRuleEvent.resumed(rule)); // If any dependency wasn't successful, cancel ourselves. for (BuildResult depResult : depResults) { if (depResult.getStatus() != BuildRuleStatus.SUCCESS) { return Futures.immediateFuture( BuildResult.canceled(rule, Preconditions.checkNotNull(depResult.getFailure()))); } } // If we've already seen a failure, exit early. if (!context.isKeepGoing() && firstFailure != null) { return Futures.immediateFuture(BuildResult.canceled(rule, firstFailure)); } // Dep-file rule keys. if (useDependencyFileRuleKey(rule)) { // Try to get the current dep-file rule key. Optional<RuleKey> depFileRuleKey = calculateDepFileRuleKey( rule, onDiskBuildInfo.getValues(BuildInfo.METADATA_KEY_FOR_DEP_FILE), /* allowMissingInputs */ true); if (depFileRuleKey.isPresent()) { // Check the input-based rule key says we're already built. Optional<RuleKey> lastDepFileRuleKey = onDiskBuildInfo.getRuleKey(BuildInfo.METADATA_KEY_FOR_DEP_FILE_RULE_KEY); if (lastDepFileRuleKey.isPresent() && lastDepFileRuleKey.get().equals(depFileRuleKey.get())) { return Futures.immediateFuture( BuildResult.success( rule, BuildRuleSuccessType.MATCHING_DEP_FILE_RULE_KEY, CacheResult.localKeyUnchangedHit())); } } } // Input-based rule keys. if (rule instanceof SupportsInputBasedRuleKey) { // Calculate the input-based rule key and record it in the metadata. RuleKey inputRuleKey = inputBasedRuleKeyBuilderFactory.newInstance(rule).build(); buildInfoRecorder.addBuildMetadata( BuildInfo.METADATA_KEY_FOR_INPUT_BASED_RULE_KEY, inputRuleKey.toString()); // Check the input-based rule key says we're already built. Optional<RuleKey> lastInputRuleKey = onDiskBuildInfo.getRuleKey(BuildInfo.METADATA_KEY_FOR_INPUT_BASED_RULE_KEY); if (lastInputRuleKey.isPresent() && lastInputRuleKey.get().equals(inputRuleKey)) { return Futures.immediateFuture( BuildResult.success( rule, BuildRuleSuccessType.MATCHING_INPUT_BASED_RULE_KEY, CacheResult.localKeyUnchangedHit())); } // Try to fetch the artifact using the input-based rule key. CacheResult cacheResult = tryToFetchArtifactFromBuildCacheAndOverlayOnTopOfProjectFilesystem( rule, inputRuleKey, buildInfoRecorder, context.getArtifactCache(), context.getProjectFilesystem(), context); if (cacheResult.getType().isSuccess()) { return Futures.immediateFuture( BuildResult.success( rule, BuildRuleSuccessType.FETCHED_FROM_CACHE_INPUT_BASED, cacheResult)); } } // 4. ABI check // Deciding whether we need to rebuild is tricky business. We want to rebuild as little // as possible while always being sound. // // For java_library rules that depend only on their first-order deps, // they only need to rebuild themselves if any of the following conditions hold: // (1) The definition of the build rule has changed. // (2) Any of the input files (which includes resources as well as .java files) have // changed. // (3) The ABI of any of its dependent java_library rules has changed. // // For other types of build rules, we have to be more conservative when rebuilding. In // those cases, we rebuild if any of the following conditions hold: // (1) The definition of the build rule has changed. // (2) Any of the input files have changed. // (3) Any of the RuleKeys of this rule's deps have changed. // // Because a RuleKey for a rule will change if any of its transitive deps have changed, // that means a change in one of the leaves can result in almost all rules being // rebuilt, which is slow. Fortunately, we limit the effects of this when building Java // code when checking the ABI of deps instead of the RuleKey for deps. AbiRule abiRule = checkIfRuleOrBuildableIsAbiRule(rule); if (abiRule != null) { RuleKey ruleKeyNoDeps = rule.getRuleKeyWithoutDeps(); Optional<RuleKey> cachedRuleKeyNoDeps = onDiskBuildInfo.getRuleKey(BuildInfo.METADATA_KEY_FOR_RULE_KEY_WITHOUT_DEPS); if (ruleKeyNoDeps.equals(cachedRuleKeyNoDeps.orNull())) { // The RuleKey for the definition of this build rule and its input files has not // changed. Therefore, if the ABI of its deps has not changed, there is nothing to // rebuild. Sha1HashCode abiKeyForDeps = abiRule.getAbiKeyForDeps(); Optional<Sha1HashCode> cachedAbiKeyForDeps = onDiskBuildInfo.getHash(ABI_KEY_FOR_DEPS_ON_DISK_METADATA); if (abiKeyForDeps.equals(cachedAbiKeyForDeps.orNull())) { return Futures.immediateFuture( BuildResult.success( rule, BuildRuleSuccessType.MATCHING_DEPS_ABI_AND_RULE_KEY_NO_DEPS, AbstractCacheResult.localKeyUnchangedHit())); } } } // 5. build the rule executeCommandsNowThatDepsAreBuilt(rule, context, buildableContext, buildInfoRecorder); return Futures.immediateFuture( BuildResult.success(rule, BuildRuleSuccessType.BUILT_LOCALLY, cacheResult)); } }, service); }