private void reportTargets(AnalysisResult analysisResult) {
Collection<ConfiguredTarget> targetsToBuild = analysisResult.getTargetsToBuild();
Collection<ConfiguredTarget> targetsToTest = analysisResult.getTargetsToTest();
if (targetsToTest != null) {
int testCount = targetsToTest.size();
int targetCount = targetsToBuild.size() - testCount;
if (targetCount == 0) {
getReporter()
.handle(
Event.info(
"Found "
+ testCount
+ (testCount == 1 ? " test target..." : " test targets...")));
} else {
getReporter()
.handle(
Event.info(
"Found "
+ targetCount
+ (targetCount == 1 ? " target and " : " targets and ")
+ testCount
+ (testCount == 1 ? " test target..." : " test targets...")));
}
} else {
int targetCount = targetsToBuild.size();
getReporter()
.handle(
Event.info(
"Found " + targetCount + (targetCount == 1 ? " target..." : " targets...")));
}
}
Event toEvent() {
if (getCause() != null) {
return Event.error(getMessage() + ": " + getCause().getMessage());
} else {
return Event.error(getMessage());
}
}
private ExitCode doTest(
CommandEnvironment env, OptionsProvider options, AggregatingTestListener testListener) {
BlazeRuntime runtime = env.getRuntime();
// Run simultaneous build and test.
List<String> targets = ProjectFileSupport.getTargets(runtime, options);
BuildRequest request =
BuildRequest.create(
getClass().getAnnotation(Command.class).name(),
options,
runtime.getStartupOptionsProvider(),
targets,
env.getReporter().getOutErr(),
env.getCommandId(),
env.getCommandStartTime());
request.setRunTests();
BuildResult buildResult = new BuildTool(env).processRequest(request, null);
Collection<ConfiguredTarget> testTargets = buildResult.getTestTargets();
// TODO(bazel-team): don't handle isEmpty here or fix up a bunch of tests
if (buildResult.getSuccessfulTargets() == null) {
// This can happen if there were errors in the target parsing or loading phase
// (original exitcode=BUILD_FAILURE) or if there weren't but --noanalyze was given
// (original exitcode=SUCCESS).
env.getReporter().handle(Event.error("Couldn't start the build. Unable to run tests"));
return buildResult.getSuccess() ? ExitCode.PARSING_FAILURE : buildResult.getExitCondition();
}
// TODO(bazel-team): the check above shadows NO_TESTS_FOUND, but switching the conditions breaks
// more tests
if (testTargets.isEmpty()) {
env.getReporter()
.handle(Event.error(null, "No test targets were found, yet testing was requested"));
return buildResult.getSuccess() ? ExitCode.NO_TESTS_FOUND : buildResult.getExitCondition();
}
boolean buildSuccess = buildResult.getSuccess();
boolean testSuccess = analyzeTestResults(testTargets, testListener, options);
if (testSuccess && !buildSuccess) {
// If all tests run successfully, test summary should include warning if
// there were build errors not associated with the test targets.
printer.printLn(
AnsiTerminalPrinter.Mode.ERROR
+ "One or more non-test targets failed to build.\n"
+ AnsiTerminalPrinter.Mode.DEFAULT);
}
return buildSuccess
? (testSuccess ? ExitCode.SUCCESS : ExitCode.TESTS_FAILED)
: buildResult.getExitCondition();
}
/**
* Performs the initial phases 0-2 of the build: Setup, Loading and Analysis.
*
* <p>Postcondition: On success, populates the BuildRequest's set of targets to build.
*
* @return null if loading / analysis phases were successful; a useful error message if loading or
* analysis phase errors were encountered and request.keepGoing.
* @throws InterruptedException if the current thread was interrupted.
* @throws ViewCreationFailedException if analysis failed for any reason.
*/
private AnalysisResult runAnalysisPhase(
BuildRequest request,
LoadingResult loadingResult,
BuildConfigurationCollection configurations)
throws InterruptedException, ViewCreationFailedException {
Stopwatch timer = Stopwatch.createStarted();
if (!request.getBuildOptions().performAnalysisPhase) {
getReporter().handle(Event.progress("Loading complete."));
LOG.info("No analysis requested, so finished");
return AnalysisResult.EMPTY;
}
getReporter().handle(Event.progress("Loading complete. Analyzing..."));
Profiler.instance().markPhase(ProfilePhase.ANALYZE);
AnalysisResult analysisResult =
getView()
.update(
loadingResult,
configurations,
request.getAspects(),
request.getViewOptions(),
request.getTopLevelArtifactContext(),
getReporter(),
getEventBus());
// TODO(bazel-team): Merge these into one event.
getEventBus()
.post(
new AnalysisPhaseCompleteEvent(
analysisResult.getTargetsToBuild(),
getView().getTargetsVisited(),
timer.stop().elapsed(TimeUnit.MILLISECONDS)));
getEventBus()
.post(
new TestFilteringCompleteEvent(
analysisResult.getTargetsToBuild(), analysisResult.getTargetsToTest()));
// Check licenses.
// We check licenses if the first target configuration has license checking enabled. Right now,
// it is not possible to have multiple target configurations with different settings for this
// flag, which allows us to take this short cut.
boolean checkLicenses = configurations.getTargetConfigurations().get(0).checkLicenses();
if (checkLicenses) {
Profiler.instance().markPhase(ProfilePhase.LICENSE);
validateLicensingForTargets(
analysisResult.getTargetsToBuild(), request.getViewOptions().keepGoing);
}
return analysisResult;
}
/**
* Load Skylark aspect from an extension file. Is to be called from a SkyFunction.
*
* @return {@code null} if dependencies cannot be satisfied.
*/
@Nullable
public static SkylarkAspect loadSkylarkAspect(
Environment env, Label extensionLabel, String skylarkValueName)
throws AspectCreationException {
SkyKey importFileKey = SkylarkImportLookupValue.key(extensionLabel, false);
try {
SkylarkImportLookupValue skylarkImportLookupValue =
(SkylarkImportLookupValue)
env.getValueOrThrow(importFileKey, SkylarkImportFailedException.class);
if (skylarkImportLookupValue == null) {
return null;
}
Object skylarkValue =
skylarkImportLookupValue.getEnvironmentExtension().get(skylarkValueName);
if (!(skylarkValue instanceof SkylarkAspect)) {
throw new ConversionException(
skylarkValueName + " from " + extensionLabel.toString() + " is not an aspect");
}
return (SkylarkAspect) skylarkValue;
} catch (SkylarkImportFailedException | ConversionException e) {
env.getListener().handle(Event.error(e.getMessage()));
throw new AspectCreationException(e.getMessage());
}
}
@ThreadSafety.ThreadSafe
@Override
public void missingEdge(Target target, Label label, NoSuchThingException e) {
eventHandler.handle(
Event.error(
TargetUtils.getLocationMaybe(target), TargetUtils.formatMissingEdge(target, label, e)));
super.missingEdge(target, label, e);
}
@Override
public void reportInvalidOptions(EventHandler reporter, BuildOptions buildOptions) {
if (generateDebugSymbols && !iosMultiCpus.isEmpty()) {
reporter.handle(
Event.error(
"--objc_generate_debug_symbols is not supported when --ios_multi_cpus is set"));
}
}
private static void deserializeEvent(StoredEventHandler eventHandler, Build.Event event) {
String message = event.getMessage();
switch (event.getKind()) {
case ERROR:
eventHandler.handle(Event.error(message));
break;
case WARNING:
eventHandler.handle(Event.warn(message));
break;
case INFO:
eventHandler.handle(Event.info(message));
break;
case PROGRESS:
eventHandler.handle(Event.progress(message));
break;
default:
break; // Ignore
}
}
// Kill workers on Ctrl-C to quickly end the interrupted build.
// TODO(philwo) - make sure that this actually *kills* the workers and not just politely waits
// for them to finish.
@Subscribe
public void buildInterrupted(BuildInterruptedEvent event) {
if (workers != null) {
if (verbose) {
env.getReporter().handle(Event.info("Build interrupted, shutting down worker pool..."));
}
workers.close();
workers = null;
}
}
@Subscribe
public void buildComplete(BuildCompleteEvent event) {
if (workers != null && buildRequest.getOptions(WorkerOptions.class).workerQuitAfterBuild) {
if (verbose) {
env.getReporter().handle(Event.info("Build completed, shutting down worker pool..."));
}
workers.close();
workers = null;
}
}
@Override
public void reportInvalidOptions(EventHandler reporter, BuildOptions buildOptions) {
if (!Collections.disjoint(translationFlags, J2OBJC_BLACKLISTED_TRANSLATION_FLAGS)) {
String errorMsg =
String.format(
INVALID_TRANSLATION_FLAGS_MSG_TEMPLATE,
Joiner.on(",").join(translationFlags),
Joiner.on(",").join(J2OBJC_BLACKLISTED_TRANSLATION_FLAGS));
reporter.handle(Event.error(errorMsg));
}
}
@Override
public ExitCode exec(CommandEnvironment env, OptionsProvider options)
throws ShutdownBlazeServerException {
BlazeRuntime runtime = env.getRuntime();
Options cleanOptions = options.getOptions(Options.class);
cleanOptions.expunge_async = cleanOptions.cleanStyle.equals("expunge_async");
cleanOptions.expunge = cleanOptions.cleanStyle.equals("expunge");
if (!cleanOptions.expunge
&& !cleanOptions.expunge_async
&& !cleanOptions.cleanStyle.isEmpty()) {
env.getReporter()
.handle(Event.error(null, "Invalid clean_style value '" + cleanOptions.cleanStyle + "'"));
return ExitCode.COMMAND_LINE_ERROR;
}
String cleanBanner =
cleanOptions.expunge_async
? "Starting clean."
: "Starting clean (this may take a while). "
+ "Consider using --expunge_async if the clean takes more than several minutes.";
env.getReporter().handle(Event.info(null /*location*/, cleanBanner));
try {
String symlinkPrefix =
options.getOptions(BuildRequest.BuildRequestOptions.class).getSymlinkPrefix();
actuallyClean(env, runtime.getOutputBase(), cleanOptions, symlinkPrefix);
return ExitCode.SUCCESS;
} catch (IOException e) {
env.getReporter().handle(Event.error(e.getMessage()));
return ExitCode.LOCAL_ENVIRONMENTAL_ERROR;
} catch (CommandException | ExecException e) {
env.getReporter().handle(Event.error(e.getMessage()));
return ExitCode.RUN_FAILURE;
} catch (InterruptedException e) {
env.getReporter().handle(Event.error("clean interrupted"));
return ExitCode.INTERRUPTED;
}
}
/**
* Ensure the runfiles tree exists and is consistent with the TestAction's manifest
* ($0.runfiles_manifest), bringing it into consistency if not. The contents of the output file
* $0.runfiles/MANIFEST, if it exists, are used a proxy for the set of existing symlinks, to avoid
* the need for recursion.
*/
private static void updateLocalRunfilesDirectory(
TestRunnerAction testAction,
Path runfilesDir,
ActionExecutionContext actionExecutionContext,
BinTools binTools,
ImmutableMap<String, String> shellEnvironment,
boolean enableRunfiles)
throws ExecException, InterruptedException {
Executor executor = actionExecutionContext.getExecutor();
TestTargetExecutionSettings execSettings = testAction.getExecutionSettings();
try {
// Avoid rebuilding the runfiles directory if the manifest in it matches the input manifest,
// implying the symlinks exist and are already up to date.
if (Arrays.equals(
runfilesDir.getRelative("MANIFEST").getMD5Digest(),
execSettings.getInputManifest().getPath().getMD5Digest())) {
return;
}
} catch (IOException e1) {
// Ignore it - we will just try to create runfiles directory.
}
executor
.getEventHandler()
.handle(
Event.progress(
"Building runfiles directory for '"
+ execSettings.getExecutable().prettyPrint()
+ "'."));
new SymlinkTreeHelper(execSettings.getInputManifest().getPath(), runfilesDir, false)
.createSymlinks(
testAction, actionExecutionContext, binTools, shellEnvironment, enableRunfiles);
executor.getEventHandler().handle(Event.progress(testAction.getProgressMessage()));
}
public Sender(
String url, String secret, CommandEnvironment env, ExecutorService executorService)
throws SenderException {
this.reporter = env.getReporter();
this.secret = readSecret(secret, reporter);
try {
this.url = new URL(url);
if (!this.secret.isEmpty()) {
if (!(this.url.getProtocol().equals("https")
|| this.url.getHost().equals("localhost")
|| this.url.getHost().matches("^127.0.0.[0-9]+$"))) {
reporter.handle(
Event.warn(
"Using authentication over unsecure channel, " + "consider using HTTPS."));
}
}
} catch (MalformedURLException e) {
throw new SenderException("Invalid server url " + url, e);
}
this.buildId = env.getCommandId().toString();
this.executorService = executorService;
sendMessage("test", null); // test connecting to the server.
reporter.handle(Event.info("Results are being streamed to " + url + "/result/" + buildId));
}
/**
* The crux of the build system. Builds the targets specified in the request using the specified
* Executor.
*
* <p>Performs loading, analysis and execution for the specified set of targets, honoring the
* configuration options in the BuildRequest. Returns normally iff successful, throws an exception
* otherwise.
*
* <p>The caller is responsible for setting up and syncing the package cache.
*
* <p>During this function's execution, the actualTargets and successfulTargets fields of the
* request object are set.
*
* @param request the build request that this build tool is servicing, which specifies various
* options; during this method's execution, the actualTargets and successfulTargets fields of
* the request object are populated
* @param validator target validator
* @return the result as a {@link BuildResult} object
*/
public BuildResult processRequest(BuildRequest request, TargetValidator validator) {
BuildResult result = new BuildResult(request.getStartTime());
runtime.getEventBus().register(result);
Throwable catastrophe = null;
ExitCode exitCode = ExitCode.BLAZE_INTERNAL_ERROR;
try {
buildTargets(request, result, validator);
exitCode = ExitCode.SUCCESS;
} catch (BuildFailedException e) {
if (e.isErrorAlreadyShown()) {
// The actual error has already been reported by the Builder.
} else {
reportExceptionError(e);
}
if (e.isCatastrophic()) {
result.setCatastrophe();
}
exitCode = ExitCode.BUILD_FAILURE;
} catch (InterruptedException e) {
exitCode = ExitCode.INTERRUPTED;
getReporter().handle(Event.error("build interrupted"));
getEventBus().post(new BuildInterruptedEvent());
} catch (TargetParsingException | LoadingFailedException | ViewCreationFailedException e) {
exitCode = ExitCode.PARSING_FAILURE;
reportExceptionError(e);
} catch (TestExecException e) {
// ExitCode.SUCCESS means that build was successful. Real return code of program
// is going to be calculated in TestCommand.doTest().
exitCode = ExitCode.SUCCESS;
reportExceptionError(e);
} catch (InvalidConfigurationException e) {
exitCode = ExitCode.COMMAND_LINE_ERROR;
reportExceptionError(e);
} catch (AbruptExitException e) {
exitCode = e.getExitCode();
reportExceptionError(e);
result.setCatastrophe();
} catch (Throwable throwable) {
catastrophe = throwable;
Throwables.propagate(throwable);
} finally {
stopRequest(request, result, catastrophe, exitCode);
}
return result;
}
/**
* If the action might read directories as inputs in a way that is unsound wrt dependency
* checking, this method must be called.
*/
protected void checkInputsForDirectories(
EventHandler eventHandler, MetadataHandler metadataHandler) {
// Report "directory dependency checking" warning only for non-generated directories (generated
// ones will be reported earlier).
for (Artifact input : getMandatoryInputs()) {
// Assume that if the file did not exist, we would not have gotten here.
if (input.isSourceArtifact() && !metadataHandler.isRegularFile(input)) {
eventHandler.handle(
Event.warn(
getOwner().getLocation(),
"input '"
+ input.prettyPrint()
+ "' to "
+ getOwner().getLabel()
+ " is a directory; dependency checking of directories is unsound"));
}
}
}
@Override
public void buildTransitiveClosure(QueryExpression caller, Set<Target> targets, int maxDepth)
throws QueryException {
// Everything has already been loaded, so here we just check for errors so that we can
// pre-emptively throw/report if needed.
Iterable<SkyKey> transitiveTraversalKeys = makeTransitiveTraversalKeys(targets);
ImmutableList.Builder<String> errorMessagesBuilder = ImmutableList.builder();
// First, look for errors in the successfully evaluated TransitiveTraversalValues. They may
// have encountered errors that they were able to recover from.
Set<Entry<SkyKey, SkyValue>> successfulEntries =
graph.getSuccessfulValues(transitiveTraversalKeys).entrySet();
Builder<SkyKey> successfulKeysBuilder = ImmutableSet.builder();
for (Entry<SkyKey, SkyValue> successfulEntry : successfulEntries) {
successfulKeysBuilder.add(successfulEntry.getKey());
TransitiveTraversalValue value = (TransitiveTraversalValue) successfulEntry.getValue();
String firstErrorMessage = value.getFirstErrorMessage();
if (firstErrorMessage != null) {
errorMessagesBuilder.add(firstErrorMessage);
}
}
ImmutableSet<SkyKey> successfulKeys = successfulKeysBuilder.build();
// Next, look for errors from the unsuccessfully evaluated TransitiveTraversal skyfunctions.
Iterable<SkyKey> unsuccessfulKeys =
Iterables.filter(transitiveTraversalKeys, Predicates.not(Predicates.in(successfulKeys)));
Set<Entry<SkyKey, Exception>> errorEntries =
graph.getMissingAndExceptions(unsuccessfulKeys).entrySet();
for (Map.Entry<SkyKey, Exception> entry : errorEntries) {
if (entry.getValue() == null) {
// Targets may be in the graph because they are not in the universe or depend on cycles.
eventHandler.handle(Event.warn(entry.getKey().argument() + " does not exist in graph"));
} else {
errorMessagesBuilder.add(entry.getValue().getMessage());
}
}
// Lastly, report all found errors.
ImmutableList<String> errorMessages = errorMessagesBuilder.build();
for (String errorMessage : errorMessages) {
reportBuildFileError(caller, errorMessage);
}
}
@Override
public void beforeCommand(Command command, CommandEnvironment env) {
this.env = env;
env.getEventBus().register(this);
if (workers == null) {
Path logDir = env.getRuntime().getOutputBase().getRelative("worker-logs");
try {
logDir.createDirectory();
} catch (IOException e) {
env.getReporter()
.handle(Event.error("Could not create directory for worker logs: " + logDir));
}
GenericKeyedObjectPoolConfig config = new GenericKeyedObjectPoolConfig();
// It's better to re-use a worker as often as possible and keep it hot, in order to profit
// from JIT optimizations as much as possible.
config.setLifo(true);
// Check for & deal with idle workers every 5 seconds.
config.setTimeBetweenEvictionRunsMillis(5 * 1000);
// Always test the liveliness of worker processes.
config.setTestOnBorrow(true);
config.setTestOnCreate(true);
config.setTestOnReturn(true);
config.setTestWhileIdle(true);
// Don't limit the total number of worker processes, as otherwise the pool might be full of
// e.g. Java workers and could never accommodate another request for a different kind of
// worker.
config.setMaxTotal(-1);
workers = new WorkerPool(new WorkerFactory(), config);
workers.setReporter(env.getReporter());
workers.setLogDirectory(logDir);
}
}
private static String readSecret(String secretFile, Reporter reporter) throws SenderException {
if (secretFile.isEmpty()) {
return "";
}
Path secretPath = new File(secretFile).toPath();
if (!Files.isReadable(secretPath)) {
throw new SenderException("Secret file " + secretFile + " doesn't exists or is unreadable");
}
try {
if (Files.getPosixFilePermissions(secretPath).contains(PosixFilePermission.OTHERS_READ)
|| Files.getPosixFilePermissions(secretPath).contains(PosixFilePermission.GROUP_READ)) {
reporter.handle(
Event.warn(
"Secret file "
+ secretFile
+ " is readable by non-owner. "
+ "It is recommended to set its permission to 0600 (read-write only by the owner)."));
}
return new String(Files.readAllBytes(secretPath), StandardCharsets.UTF_8).trim();
} catch (IOException e) {
throw new SenderException("Invalid secret file " + secretFile, e);
}
}
@Override
public void editOptions(CommandEnvironment env, OptionsParser optionsParser)
throws AbruptExitException {
ProjectFileSupport.handleProjectFiles(env, optionsParser, commandName());
TestOutputFormat testOutput = optionsParser.getOptions(ExecutionOptions.class).testOutput;
try {
if (testOutput == TestStrategy.TestOutputFormat.STREAMED) {
env.getReporter()
.handle(
Event.warn(
"Streamed test output requested so all tests will be run locally, without sharding, "
+ "one at a time"));
optionsParser.parse(
OptionPriority.SOFTWARE_REQUIREMENT,
"streamed output requires locally run tests, without sharding",
ImmutableList.of("--test_sharding_strategy=disabled", "--test_strategy=exclusive"));
}
} catch (OptionsParsingException e) {
throw new IllegalStateException("Known options failed to parse", e);
}
}
/** Creates the Extension to be imported. */
private Extension createExtension(
BuildFileAST ast, PathFragment file, Map<PathFragment, Extension> importMap, Environment env)
throws InterruptedException, SkylarkImportLookupFunctionException {
StoredEventHandler eventHandler = new StoredEventHandler();
// TODO(bazel-team): this method overestimates the changes which can affect the
// Skylark RuleClass. For example changes to comments or unused functions can modify the hash.
// A more accurate - however much more complicated - way would be to calculate a hash based on
// the transitive closure of the accessible AST nodes.
try (Mutability mutability = Mutability.create("importing %s", file)) {
com.google.devtools.build.lib.syntax.Environment extensionEnv =
ruleClassProvider
.createSkylarkRuleClassEnvironment(
mutability, eventHandler, ast.getContentHashCode(), importMap)
.setupOverride("native", packageFactory.getNativeModule());
ast.exec(extensionEnv, eventHandler);
SkylarkRuleClassFunctions.exportRuleFunctions(extensionEnv, file);
Event.replayEventsOn(env.getListener(), eventHandler.getEvents());
if (eventHandler.hasErrors()) {
throw new SkylarkImportLookupFunctionException(SkylarkImportFailedException.errors(file));
}
return new Extension(extensionEnv);
}
}
private void actuallyClean(
CommandEnvironment env, Path outputBase, Options cleanOptions, String symlinkPrefix)
throws IOException, ShutdownBlazeServerException, CommandException, ExecException,
InterruptedException {
BlazeRuntime runtime = env.getRuntime();
if (env.getOutputService() != null) {
env.getOutputService().clean();
}
if (cleanOptions.expunge) {
LOG.info("Expunging...");
// Delete the big subdirectories with the important content first--this
// will take the most time. Then quickly delete the little locks, logs
// and links right before we exit. Once the lock file is gone there will
// be a small possibility of a server race if a client is waiting, but
// all significant files will be gone by then.
FileSystemUtils.deleteTreesBelow(outputBase);
FileSystemUtils.deleteTree(outputBase);
} else if (cleanOptions.expunge_async) {
LOG.info("Expunging asynchronously...");
String tempBaseName = outputBase.getBaseName() + "_tmp_" + ProcessUtils.getpid();
// Keeping tempOutputBase in the same directory ensures it remains in the
// same file system, and therefore the mv will be atomic and fast.
Path tempOutputBase = outputBase.getParentDirectory().getChild(tempBaseName);
outputBase.renameTo(tempOutputBase);
env.getReporter()
.handle(Event.info(null, "Output base moved to " + tempOutputBase + " for deletion"));
// Daemonize the shell and use the double-fork idiom to ensure that the shell
// exits even while the "rm -rf" command continues.
String command =
String.format(
"exec >&- 2>&- <&- && (/usr/bin/setsid /bin/rm -rf %s &)&",
ShellEscaper.escapeString(tempOutputBase.getPathString()));
LOG.info("Executing shell commmand " + ShellEscaper.escapeString(command));
// Doesn't throw iff command exited and was successful.
new CommandBuilder()
.addArg(command)
.useShell(true)
.setWorkingDir(tempOutputBase.getParentDirectory())
.build()
.execute();
} else {
LOG.info("Output cleaning...");
runtime.clearCaches();
// In order to be sure that we delete everything, delete the workspace directory both for
// --deep_execroot and for --nodeep_execroot.
for (String directory :
new String[] {runtime.getWorkspaceName(), "execroot/" + runtime.getWorkspaceName()}) {
Path child = outputBase.getRelative(directory);
if (child.exists()) {
LOG.finest("Cleaning " + child);
FileSystemUtils.deleteTreesBelow(child);
}
}
}
// remove convenience links
OutputDirectoryLinksUtils.removeOutputDirectoryLinks(
runtime.getWorkspaceName(), runtime.getWorkspace(), env.getReporter(), symlinkPrefix);
// shutdown on expunge cleans
if (cleanOptions.expunge || cleanOptions.expunge_async) {
throw new ShutdownBlazeServerException(0);
}
}
@Override
@Nullable
public BuildConfiguration createConfigurations(
ConfigurationFactory configurationFactory,
Cache<String, BuildConfiguration> cache,
PackageProviderForConfigurations packageProvider,
BuildOptions buildOptions,
EventHandler eventHandler,
boolean performSanityCheck)
throws InvalidConfigurationException {
// Target configuration
BuildConfiguration targetConfiguration =
configurationFactory.getConfiguration(packageProvider, buildOptions, false, cache);
if (targetConfiguration == null) {
return null;
}
BuildConfiguration dataConfiguration = targetConfiguration;
// Host configuration
// Note that this passes in the dataConfiguration, not the target
// configuration. This is intentional.
BuildConfiguration hostConfiguration =
getHostConfigurationFromRequest(
configurationFactory, packageProvider, dataConfiguration, buildOptions, cache);
if (hostConfiguration == null) {
return null;
}
ListMultimap<SplitTransition<?>, BuildConfiguration> splitTransitionsTable =
ArrayListMultimap.create();
for (SplitTransition<BuildOptions> transition : buildOptions.getPotentialSplitTransitions()) {
List<BuildOptions> splitOptionsList = transition.split(buildOptions);
// While it'd be clearer to condition the below on "if (!splitOptionsList.empty())",
// IosExtension.ExtensionSplitArchTransition defaults to a single-value split. If we failed
// that case then no builds would work, whether or not they're iOS builds (since iOS
// configurations are unconditionally loaded). Once we have dynamic configuraiton support
// for split transitions, this will all go away.
if (splitOptionsList.size() > 1 && targetConfiguration.useDynamicConfigurations()) {
throw new InvalidConfigurationException(
"dynamic configurations don't yet support split transitions");
}
for (BuildOptions splitOptions : splitOptionsList) {
BuildConfiguration splitConfig =
configurationFactory.getConfiguration(packageProvider, splitOptions, false, cache);
splitTransitionsTable.put(transition, splitConfig);
}
}
if (packageProvider.valuesMissing()) {
return null;
}
// Sanity check that the implicit labels are all in the transitive closure of explicit ones.
// This also registers all targets in the cache entry and validates them on subsequent requests.
Set<Label> reachableLabels = new HashSet<>();
if (performSanityCheck) {
// We allow the package provider to be null for testing.
for (Map.Entry<String, Label> entry : buildOptions.getAllLabels().entries()) {
Label label = entry.getValue();
try {
collectTransitiveClosure(packageProvider, reachableLabels, label);
} catch (NoSuchThingException e) {
eventHandler.handle(Event.error(e.getMessage()));
throw new InvalidConfigurationException(
String.format("Failed to load required %s target: '%s'", entry.getKey(), label));
}
}
if (packageProvider.valuesMissing()) {
return null;
}
sanityCheckImplicitLabels(reachableLabels, targetConfiguration);
sanityCheckImplicitLabels(reachableLabels, hostConfiguration);
}
BuildConfiguration result =
setupTransitions(
targetConfiguration, dataConfiguration, hostConfiguration, splitTransitionsTable);
result.reportInvalidOptions(eventHandler);
return result;
}
@Override
@ThreadCompatible
public void execute(ActionExecutionContext actionExecutionContext)
throws ActionExecutionException, InterruptedException {
Executor executor = actionExecutionContext.getExecutor();
// First, do a normal compilation, to generate the ".d" file. The generated object file is built
// to a temporary location (tempOutputFile) and ignored afterwards.
LOG.info("Generating " + getDotdFile());
CppCompileActionContext context = executor.getContext(actionContext);
CppCompileActionContext.Reply reply = null;
try {
// We delegate stdout/stderr to nowhere, i.e. same as redirecting to /dev/null.
reply = context.execWithReply(this, actionExecutionContext.withFileOutErr(new FileOutErr()));
} catch (ExecException e) {
// We ignore failures here (other than capturing the Distributor reply).
// The compilation may well fail (that's the whole point of negative compilation tests).
// We execute it here just for the side effect of generating the ".d" file.
reply = context.getReplyFromException(e, this);
if (reply == null) {
// This can only happen if the ExecException does not come from remote execution.
throw e.toActionExecutionException(
"Fake C++ Compilation of rule '" + getOwner().getLabel() + "'",
executor.getVerboseFailures(),
this);
}
}
IncludeScanningContext scanningContext = executor.getContext(IncludeScanningContext.class);
NestedSet<Artifact> discoveredInputs =
discoverInputsFromDotdFiles(
executor.getExecRoot(), scanningContext.getArtifactResolver(), reply);
reply = null; // Clear in-memory .d files early.
// Even cc_fake_binary rules need to properly declare their dependencies...
// In fact, they need to declare their dependencies even more than cc_binary rules do.
// CcCommonConfiguredTarget passes in an empty set of declaredIncludeDirs,
// so this check below will only allow inclusion of header files that are explicitly
// listed in the "srcs" of the cc_fake_binary or in the "srcs" of a cc_library that it
// depends on.
try {
validateInclusions(
discoveredInputs,
actionExecutionContext.getArtifactExpander(),
executor.getEventHandler());
} catch (ActionExecutionException e) {
// TODO(bazel-team): (2009) make this into an error, once most of the current warnings
// are fixed.
executor
.getEventHandler()
.handle(
Event.warn(
getOwner().getLocation(),
e.getMessage() + ";\n this warning may eventually become an error"));
}
updateActionInputs(discoveredInputs);
// Generate a fake ".o" file containing the command line needed to generate
// the real object file.
LOG.info("Generating " + outputFile);
// A cc_fake_binary rule generates fake .o files and a fake target file,
// which merely contain instructions on building the real target. We need to
// be careful to use a new set of output file names in the instructions, as
// to not overwrite the fake output files when someone tries to follow the
// instructions. As the real compilation is executed by the test from its
// runfiles directory (where writing is forbidden), we patch the command
// line to write to $TEST_TMPDIR instead.
final String outputPrefix = "$TEST_TMPDIR/";
String argv =
Joiner.on(' ')
.join(
Iterables.transform(
getArgv(outputFile.getExecPath()),
new Function<String, String>() {
@Override
public String apply(String input) {
String result = ShellEscaper.escapeString(input);
// Once -c and -o options are added into action_config, the argument of
// getArgv(outputFile.getExecPath()) won't be used anymore. There will
// always be
// -c <tempOutputFile>, but here it has to be outputFile, so we replace it.
if (input.equals(tempOutputFile.getPathString())) {
result =
outputPrefix
+ ShellEscaper.escapeString(outputFile.getExecPathString());
}
if (input.equals(outputFile.getExecPathString())
|| input.equals(getDotdFile().getSafeExecPath().getPathString())) {
result = outputPrefix + ShellEscaper.escapeString(input);
}
return result;
}
}));
// Write the command needed to build the real .o file to the fake .o file.
// Generate a command to ensure that the output directory exists; otherwise
// the compilation would fail.
try {
// Ensure that the .d file and .o file are siblings, so that the "mkdir" below works for
// both.
Preconditions.checkState(
outputFile
.getExecPath()
.getParentDirectory()
.equals(getDotdFile().getSafeExecPath().getParentDirectory()));
FileSystemUtils.writeContent(
outputFile.getPath(),
ISO_8859_1,
outputFile.getPath().getBaseName()
+ ": "
+ "mkdir -p "
+ outputPrefix
+ "$(dirname "
+ outputFile.getExecPath()
+ ")"
+ " && "
+ argv
+ "\n");
} catch (IOException e) {
throw new ActionExecutionException(
"failed to create fake compile command for rule '"
+ getOwner().getLabel()
+ ": "
+ e.getMessage(),
this,
false);
}
}
/**
* Validates the options for this BuildRequest.
*
* <p>Issues warnings for the use of deprecated options, and warnings or errors for any option
* settings that conflict.
*/
@VisibleForTesting
public void validateOptions(BuildRequest request) throws InvalidConfigurationException {
for (String issue : request.validateOptions()) {
getReporter().handle(Event.warn(issue));
}
}
@Override
public void handle(Event event) {
Preconditions.checkArgument(
!EventKind.ERRORS_AND_WARNINGS.contains(event.getKind()), event);
}
/**
* The crux of the build system. Builds the targets specified in the request using the specified
* Executor.
*
* <p>Performs loading, analysis and execution for the specified set of targets, honoring the
* configuration options in the BuildRequest. Returns normally iff successful, throws an exception
* otherwise.
*
* <p>Callers must ensure that {@link #stopRequest} is called after this method, even if it
* throws.
*
* <p>The caller is responsible for setting up and syncing the package cache.
*
* <p>During this function's execution, the actualTargets and successfulTargets fields of the
* request object are set.
*
* @param request the build request that this build tool is servicing, which specifies various
* options; during this method's execution, the actualTargets and successfulTargets fields of
* the request object are populated
* @param result the build result that is the mutable result of this build
* @param validator target validator
*/
public void buildTargets(BuildRequest request, BuildResult result, TargetValidator validator)
throws BuildFailedException, LocalEnvironmentException, InterruptedException,
ViewCreationFailedException, TargetParsingException, LoadingFailedException,
ExecutorInitException, AbruptExitException, InvalidConfigurationException,
TestExecException {
validateOptions(request);
BuildOptions buildOptions = runtime.createBuildOptions(request);
// Sync the package manager before sending the BuildStartingEvent in runLoadingPhase()
runtime.setupPackageCache(
request.getPackageCacheOptions(), DefaultsPackage.getDefaultsPackageContent(buildOptions));
ExecutionTool executionTool = null;
LoadingResult loadingResult = null;
BuildConfigurationCollection configurations = null;
try {
getEventBus().post(new BuildStartingEvent(runtime.getOutputFileSystem(), request));
LOG.info("Build identifier: " + request.getId());
executionTool = new ExecutionTool(runtime, request);
if (needsExecutionPhase(request.getBuildOptions())) {
// Initialize the execution tool early if we need it. This hides the latency of setting up
// the execution backends.
executionTool.init();
}
// Loading phase.
loadingResult = runLoadingPhase(request, validator);
// Create the build configurations.
if (!request.getMultiCpus().isEmpty()) {
getReporter()
.handle(
Event.warn(
"The --experimental_multi_cpu option is _very_ experimental and only intended for "
+ "internal testing at this time. If you do not work on the build tool, then you "
+ "should stop now!"));
if (!"build".equals(request.getCommandName()) && !"test".equals(request.getCommandName())) {
throw new InvalidConfigurationException(
"The experimental setting to select multiple CPUs is only supported for 'build' and "
+ "'test' right now!");
}
}
configurations =
getConfigurations(
buildOptions, request.getMultiCpus(), request.getViewOptions().keepGoing);
getEventBus().post(new ConfigurationsCreatedEvent(configurations));
runtime.throwPendingException();
if (configurations.getTargetConfigurations().size() == 1) {
// TODO(bazel-team): This is not optimal - we retain backwards compatibility in the case
// where there's only a single configuration, but we don't send an event in the multi-config
// case. Can we do better? [multi-config]
getEventBus()
.post(
new MakeEnvironmentEvent(
configurations.getTargetConfigurations().get(0).getMakeEnvironment()));
}
LOG.info("Configurations created");
// Analysis phase.
AnalysisResult analysisResult = runAnalysisPhase(request, loadingResult, configurations);
result.setActualTargets(analysisResult.getTargetsToBuild());
result.setTestTargets(analysisResult.getTargetsToTest());
checkTargetEnvironmentRestrictions(
analysisResult.getTargetsToBuild(), runtime.getPackageManager());
reportTargets(analysisResult);
// Execution phase.
if (needsExecutionPhase(request.getBuildOptions())) {
runtime.getSkyframeExecutor().injectTopLevelContext(request.getTopLevelArtifactContext());
executionTool.executeBuild(
request.getId(),
analysisResult,
result,
runtime.getSkyframeExecutor(),
configurations,
transformPackageRoots(loadingResult.getPackageRoots()));
}
String delayedErrorMsg = analysisResult.getError();
if (delayedErrorMsg != null) {
throw new BuildFailedException(delayedErrorMsg);
}
} catch (RuntimeException e) {
// Print an error message for unchecked runtime exceptions. This does not concern Error
// subclasses such as OutOfMemoryError.
request
.getOutErr()
.printErrLn("Unhandled exception thrown during build; message: " + e.getMessage());
throw e;
} finally {
// Delete dirty nodes to ensure that they do not accumulate indefinitely.
long versionWindow = request.getViewOptions().versionWindowForDirtyNodeGc;
if (versionWindow != -1) {
runtime.getSkyframeExecutor().deleteOldNodes(versionWindow);
}
if (executionTool != null) {
executionTool.shutdown();
}
// The workspace status actions will not run with certain flags, or if an error
// occurs early in the build. Tell a lie so that the event is not missing.
// If multiple build_info events are sent, only the first is kept, so this does not harm
// successful runs (which use the workspace status action).
getEventBus()
.post(
new BuildInfoEvent(
runtime.getworkspaceStatusActionFactory().createDummyWorkspaceStatus()));
}
if (loadingResult != null && loadingResult.hasTargetPatternError()) {
throw new BuildFailedException(
"execution phase successful, but there were errors " + "parsing the target pattern");
}
}
private void reportExceptionError(Exception e) {
if (e.getMessage() != null) {
getReporter().handle(Event.error(e.getMessage()));
}
}
/**
* Analyzes the specified targets using Skyframe as the driving framework.
*
* @return the configured targets that should be built along with a WalkableGraph of the analysis.
*/
public SkyframeAnalysisResult configureTargets(
EventHandler eventHandler,
List<ConfiguredTargetKey> values,
List<AspectValueKey> aspectKeys,
EventBus eventBus,
boolean keepGoing)
throws InterruptedException, ViewCreationFailedException {
enableAnalysis(true);
EvaluationResult<ActionLookupValue> result;
try {
result = skyframeExecutor.configureTargets(eventHandler, values, aspectKeys, keepGoing);
} finally {
enableAnalysis(false);
}
ImmutableMap<Action, ConflictException> badActions = skyframeExecutor.findArtifactConflicts();
Collection<AspectValue> goodAspects = Lists.newArrayListWithCapacity(values.size());
NestedSetBuilder<Package> packages = NestedSetBuilder.stableOrder();
for (AspectValueKey aspectKey : aspectKeys) {
AspectValue value = (AspectValue) result.get(AspectValue.key(aspectKey));
if (value == null) {
// Skip aspects that couldn't be applied to targets.
continue;
}
goodAspects.add(value);
packages.addTransitive(value.getTransitivePackages());
}
// Filter out all CTs that have a bad action and convert to a list of configured targets. This
// code ensures that the resulting list of configured targets has the same order as the incoming
// list of values, i.e., that the order is deterministic.
Collection<ConfiguredTarget> goodCts = Lists.newArrayListWithCapacity(values.size());
for (ConfiguredTargetKey value : values) {
ConfiguredTargetValue ctValue =
(ConfiguredTargetValue) result.get(ConfiguredTargetValue.key(value));
if (ctValue == null) {
continue;
}
goodCts.add(ctValue.getConfiguredTarget());
packages.addTransitive(ctValue.getTransitivePackages());
}
if (!result.hasError() && badActions.isEmpty()) {
setDeserializedArtifactOwners();
return new SkyframeAnalysisResult(
/*hasLoadingError=*/ false,
/*hasAnalysisError=*/ false,
ImmutableList.copyOf(goodCts),
result.getWalkableGraph(),
ImmutableList.copyOf(goodAspects),
LoadingPhaseRunner.collectPackageRoots(packages.build().toCollection()));
}
// --nokeep_going so we fail with an exception for the first error.
// TODO(bazel-team): We might want to report the other errors through the event bus but
// for keeping this code in parity with legacy we just report the first error for now.
if (!keepGoing) {
for (Map.Entry<Action, ConflictException> bad : badActions.entrySet()) {
ConflictException ex = bad.getValue();
try {
ex.rethrowTyped();
} catch (MutableActionGraph.ActionConflictException ace) {
ace.reportTo(eventHandler);
String errorMsg =
"Analysis of target '"
+ bad.getKey().getOwner().getLabel()
+ "' failed; build aborted";
throw new ViewCreationFailedException(errorMsg);
} catch (ArtifactPrefixConflictException apce) {
eventHandler.handle(Event.error(apce.getMessage()));
}
throw new ViewCreationFailedException(ex.getMessage());
}
Map.Entry<SkyKey, ErrorInfo> error = result.errorMap().entrySet().iterator().next();
SkyKey topLevel = error.getKey();
ErrorInfo errorInfo = error.getValue();
assertSaneAnalysisError(errorInfo, topLevel);
skyframeExecutor
.getCyclesReporter()
.reportCycles(errorInfo.getCycleInfo(), topLevel, eventHandler);
Throwable cause = errorInfo.getException();
Preconditions.checkState(
cause != null || !Iterables.isEmpty(errorInfo.getCycleInfo()), errorInfo);
String errorMsg = null;
if (topLevel.argument() instanceof ConfiguredTargetKey) {
errorMsg =
"Analysis of target '"
+ ConfiguredTargetValue.extractLabel(topLevel)
+ "' failed; build aborted";
} else if (topLevel.argument() instanceof AspectValueKey) {
AspectValueKey aspectKey = (AspectValueKey) topLevel.argument();
errorMsg = "Analysis of aspect '" + aspectKey.getDescription() + "' failed; build aborted";
} else {
assert false;
}
if (cause instanceof ActionConflictException) {
((ActionConflictException) cause).reportTo(eventHandler);
}
throw new ViewCreationFailedException(errorMsg);
}
boolean hasLoadingError = false;
// --keep_going : We notify the error and return a ConfiguredTargetValue
for (Map.Entry<SkyKey, ErrorInfo> errorEntry : result.errorMap().entrySet()) {
// Only handle errors of configured targets, not errors of top-level aspects.
// TODO(ulfjack): this is quadratic - if there are a lot of CTs, this could be rather slow.
if (!values.contains(errorEntry.getKey().argument())) {
continue;
}
SkyKey errorKey = errorEntry.getKey();
ConfiguredTargetKey label = (ConfiguredTargetKey) errorKey.argument();
Label topLevelLabel = label.getLabel();
ErrorInfo errorInfo = errorEntry.getValue();
assertSaneAnalysisError(errorInfo, errorKey);
skyframeExecutor
.getCyclesReporter()
.reportCycles(errorInfo.getCycleInfo(), errorKey, eventHandler);
Exception cause = errorInfo.getException();
Label analysisRootCause = null;
if (cause instanceof ConfiguredValueCreationException) {
ConfiguredValueCreationException ctCause = (ConfiguredValueCreationException) cause;
for (Label rootCause : ctCause.getRootCauses()) {
hasLoadingError = true;
eventBus.post(new LoadingFailureEvent(topLevelLabel, rootCause));
}
analysisRootCause = ctCause.getAnalysisRootCause();
} else if (!Iterables.isEmpty(errorInfo.getCycleInfo())) {
analysisRootCause =
maybeGetConfiguredTargetCycleCulprit(topLevelLabel, errorInfo.getCycleInfo());
} else if (cause instanceof ActionConflictException) {
((ActionConflictException) cause).reportTo(eventHandler);
}
eventHandler.handle(
Event.warn(
"errors encountered while analyzing target '"
+ topLevelLabel
+ "': it will not be built"));
if (analysisRootCause != null) {
eventBus.post(
new AnalysisFailureEvent(
LabelAndConfiguration.of(topLevelLabel, label.getConfiguration()),
analysisRootCause));
}
}
Collection<Exception> reportedExceptions = Sets.newHashSet();
for (Map.Entry<Action, ConflictException> bad : badActions.entrySet()) {
ConflictException ex = bad.getValue();
try {
ex.rethrowTyped();
} catch (MutableActionGraph.ActionConflictException ace) {
ace.reportTo(eventHandler);
eventHandler.handle(
Event.warn(
"errors encountered while analyzing target '"
+ bad.getKey().getOwner().getLabel()
+ "': it will not be built"));
} catch (ArtifactPrefixConflictException apce) {
if (reportedExceptions.add(apce)) {
eventHandler.handle(Event.error(apce.getMessage()));
}
}
}
if (!badActions.isEmpty()) {
// In order to determine the set of configured targets transitively error free from action
// conflict issues, we run a post-processing update() that uses the bad action map.
EvaluationResult<PostConfiguredTargetValue> actionConflictResult =
skyframeExecutor.postConfigureTargets(eventHandler, values, keepGoing, badActions);
goodCts = Lists.newArrayListWithCapacity(values.size());
for (ConfiguredTargetKey value : values) {
PostConfiguredTargetValue postCt =
actionConflictResult.get(PostConfiguredTargetValue.key(value));
if (postCt != null) {
goodCts.add(postCt.getCt());
}
}
}
setDeserializedArtifactOwners();
return new SkyframeAnalysisResult(
hasLoadingError,
result.hasError() || !badActions.isEmpty(),
ImmutableList.copyOf(goodCts),
result.getWalkableGraph(),
ImmutableList.copyOf(goodAspects),
LoadingPhaseRunner.collectPackageRoots(packages.build().toCollection()));
}
private void error(String message, int start, int end) {
this.containsErrors = true;
eventHandler.handle(Event.error(createLocation(start, end), message));
}