private ExitCode doTest(
CommandEnvironment env, OptionsProvider options, AggregatingTestListener testListener) {
BlazeRuntime runtime = env.getRuntime();
// Run simultaneous build and test.
List<String> targets = ProjectFileSupport.getTargets(runtime, options);
BuildRequest request =
BuildRequest.create(
getClass().getAnnotation(Command.class).name(),
options,
runtime.getStartupOptionsProvider(),
targets,
env.getReporter().getOutErr(),
env.getCommandId(),
env.getCommandStartTime());
request.setRunTests();
BuildResult buildResult = new BuildTool(env).processRequest(request, null);
Collection<ConfiguredTarget> testTargets = buildResult.getTestTargets();
// TODO(bazel-team): don't handle isEmpty here or fix up a bunch of tests
if (buildResult.getSuccessfulTargets() == null) {
// This can happen if there were errors in the target parsing or loading phase
// (original exitcode=BUILD_FAILURE) or if there weren't but --noanalyze was given
// (original exitcode=SUCCESS).
env.getReporter().handle(Event.error("Couldn't start the build. Unable to run tests"));
return buildResult.getSuccess() ? ExitCode.PARSING_FAILURE : buildResult.getExitCondition();
}
// TODO(bazel-team): the check above shadows NO_TESTS_FOUND, but switching the conditions breaks
// more tests
if (testTargets.isEmpty()) {
env.getReporter()
.handle(Event.error(null, "No test targets were found, yet testing was requested"));
return buildResult.getSuccess() ? ExitCode.NO_TESTS_FOUND : buildResult.getExitCondition();
}
boolean buildSuccess = buildResult.getSuccess();
boolean testSuccess = analyzeTestResults(testTargets, testListener, options);
if (testSuccess && !buildSuccess) {
// If all tests run successfully, test summary should include warning if
// there were build errors not associated with the test targets.
printer.printLn(
AnsiTerminalPrinter.Mode.ERROR
+ "One or more non-test targets failed to build.\n"
+ AnsiTerminalPrinter.Mode.DEFAULT);
}
return buildSuccess
? (testSuccess ? ExitCode.SUCCESS : ExitCode.TESTS_FAILED)
: buildResult.getExitCondition();
}
// Kill workers on Ctrl-C to quickly end the interrupted build.
// TODO(philwo) - make sure that this actually *kills* the workers and not just politely waits
// for them to finish.
@Subscribe
public void buildInterrupted(BuildInterruptedEvent event) {
if (workers != null) {
if (verbose) {
env.getReporter().handle(Event.info("Build interrupted, shutting down worker pool..."));
}
workers.close();
workers = null;
}
}
@Subscribe
public void buildComplete(BuildCompleteEvent event) {
if (workers != null && buildRequest.getOptions(WorkerOptions.class).workerQuitAfterBuild) {
if (verbose) {
env.getReporter().handle(Event.info("Build completed, shutting down worker pool..."));
}
workers.close();
workers = null;
}
}
@Override
public ExitCode exec(CommandEnvironment env, OptionsProvider options) {
TestResultAnalyzer resultAnalyzer =
new TestResultAnalyzer(
env.getDirectories().getExecRoot(),
options.getOptions(TestSummaryOptions.class),
options.getOptions(ExecutionOptions.class),
env.getEventBus());
printer =
new AnsiTerminalPrinter(
env.getReporter().getOutErr().getOutputStream(),
options.getOptions(BlazeCommandEventHandler.Options.class).useColor());
// Initialize test handler.
AggregatingTestListener testListener =
new AggregatingTestListener(resultAnalyzer, env.getEventBus(), env.getReporter());
env.getEventBus().register(testListener);
return doTest(env, options, testListener);
}
@Override
public ExitCode exec(CommandEnvironment env, OptionsProvider options)
throws ShutdownBlazeServerException {
BlazeRuntime runtime = env.getRuntime();
Options cleanOptions = options.getOptions(Options.class);
cleanOptions.expunge_async = cleanOptions.cleanStyle.equals("expunge_async");
cleanOptions.expunge = cleanOptions.cleanStyle.equals("expunge");
if (!cleanOptions.expunge
&& !cleanOptions.expunge_async
&& !cleanOptions.cleanStyle.isEmpty()) {
env.getReporter()
.handle(Event.error(null, "Invalid clean_style value '" + cleanOptions.cleanStyle + "'"));
return ExitCode.COMMAND_LINE_ERROR;
}
String cleanBanner =
cleanOptions.expunge_async
? "Starting clean."
: "Starting clean (this may take a while). "
+ "Consider using --expunge_async if the clean takes more than several minutes.";
env.getReporter().handle(Event.info(null /*location*/, cleanBanner));
try {
String symlinkPrefix =
options.getOptions(BuildRequest.BuildRequestOptions.class).getSymlinkPrefix();
actuallyClean(env, runtime.getOutputBase(), cleanOptions, symlinkPrefix);
return ExitCode.SUCCESS;
} catch (IOException e) {
env.getReporter().handle(Event.error(e.getMessage()));
return ExitCode.LOCAL_ENVIRONMENTAL_ERROR;
} catch (CommandException | ExecException e) {
env.getReporter().handle(Event.error(e.getMessage()));
return ExitCode.RUN_FAILURE;
} catch (InterruptedException e) {
env.getReporter().handle(Event.error("clean interrupted"));
return ExitCode.INTERRUPTED;
}
}
@Override
public void beforeCommand(Command command, CommandEnvironment env) {
this.env = env;
env.getEventBus().register(this);
if (workers == null) {
Path logDir = env.getRuntime().getOutputBase().getRelative("worker-logs");
try {
logDir.createDirectory();
} catch (IOException e) {
env.getReporter()
.handle(Event.error("Could not create directory for worker logs: " + logDir));
}
GenericKeyedObjectPoolConfig config = new GenericKeyedObjectPoolConfig();
// It's better to re-use a worker as often as possible and keep it hot, in order to profit
// from JIT optimizations as much as possible.
config.setLifo(true);
// Check for & deal with idle workers every 5 seconds.
config.setTimeBetweenEvictionRunsMillis(5 * 1000);
// Always test the liveliness of worker processes.
config.setTestOnBorrow(true);
config.setTestOnCreate(true);
config.setTestOnReturn(true);
config.setTestWhileIdle(true);
// Don't limit the total number of worker processes, as otherwise the pool might be full of
// e.g. Java workers and could never accommodate another request for a different kind of
// worker.
config.setMaxTotal(-1);
workers = new WorkerPool(new WorkerFactory(), config);
workers.setReporter(env.getReporter());
workers.setLogDirectory(logDir);
}
}
@Override
public void handleOptions(OptionsProvider optionsProvider) {
DashOptions options = optionsProvider.getOptions(DashOptions.class);
try {
sender =
(options == null || !options.useDash)
? NO_OP_SENDER
: new Sender(options.url, options.secret, env, executorService);
} catch (SenderException e) {
env.getReporter().handle(e.toEvent());
sender = NO_OP_SENDER;
}
if (optionsBuildData != null) {
sender.send("options", optionsBuildData);
}
optionsBuildData = null;
}
private Log getLog(String logPath) {
Log.Builder builder = Log.newBuilder().setPath(logPath);
File log = new File(logPath);
try {
long fileSize = Files.size(log.toPath());
if (fileSize > ONE_MB) {
fileSize = ONE_MB;
builder.setTruncated(true);
}
byte buffer[] = new byte[(int) fileSize];
try (FileInputStream in = new FileInputStream(log)) {
ByteStreams.readFully(in, buffer);
}
builder.setContents(ByteString.copyFrom(buffer));
} catch (IOException e) {
env.getReporter()
.getOutErr()
.printOutLn("Error reading log file " + logPath + ": " + e.getMessage());
// TODO(kchodorow): add this info to the proto and send.
}
return builder.build();
}
@Override
public void editOptions(CommandEnvironment env, OptionsParser optionsParser)
throws AbruptExitException {
ProjectFileSupport.handleProjectFiles(env, optionsParser, commandName());
TestOutputFormat testOutput = optionsParser.getOptions(ExecutionOptions.class).testOutput;
try {
if (testOutput == TestStrategy.TestOutputFormat.STREAMED) {
env.getReporter()
.handle(
Event.warn(
"Streamed test output requested so all tests will be run locally, without sharding, "
+ "one at a time"));
optionsParser.parse(
OptionPriority.SOFTWARE_REQUIREMENT,
"streamed output requires locally run tests, without sharding",
ImmutableList.of("--test_sharding_strategy=disabled", "--test_strategy=exclusive"));
}
} catch (OptionsParsingException e) {
throw new IllegalStateException("Known options failed to parse", e);
}
}
public Sender(
String url, String secret, CommandEnvironment env, ExecutorService executorService)
throws SenderException {
this.reporter = env.getReporter();
this.secret = readSecret(secret, reporter);
try {
this.url = new URL(url);
if (!this.secret.isEmpty()) {
if (!(this.url.getProtocol().equals("https")
|| this.url.getHost().equals("localhost")
|| this.url.getHost().matches("^127.0.0.[0-9]+$"))) {
reporter.handle(
Event.warn(
"Using authentication over unsecure channel, " + "consider using HTTPS."));
}
}
} catch (MalformedURLException e) {
throw new SenderException("Invalid server url " + url, e);
}
this.buildId = env.getCommandId().toString();
this.executorService = executorService;
sendMessage("test", null); // test connecting to the server.
reporter.handle(Event.info("Results are being streamed to " + url + "/result/" + buildId));
}
private void actuallyClean(
CommandEnvironment env, Path outputBase, Options cleanOptions, String symlinkPrefix)
throws IOException, ShutdownBlazeServerException, CommandException, ExecException,
InterruptedException {
BlazeRuntime runtime = env.getRuntime();
if (env.getOutputService() != null) {
env.getOutputService().clean();
}
if (cleanOptions.expunge) {
LOG.info("Expunging...");
// Delete the big subdirectories with the important content first--this
// will take the most time. Then quickly delete the little locks, logs
// and links right before we exit. Once the lock file is gone there will
// be a small possibility of a server race if a client is waiting, but
// all significant files will be gone by then.
FileSystemUtils.deleteTreesBelow(outputBase);
FileSystemUtils.deleteTree(outputBase);
} else if (cleanOptions.expunge_async) {
LOG.info("Expunging asynchronously...");
String tempBaseName = outputBase.getBaseName() + "_tmp_" + ProcessUtils.getpid();
// Keeping tempOutputBase in the same directory ensures it remains in the
// same file system, and therefore the mv will be atomic and fast.
Path tempOutputBase = outputBase.getParentDirectory().getChild(tempBaseName);
outputBase.renameTo(tempOutputBase);
env.getReporter()
.handle(Event.info(null, "Output base moved to " + tempOutputBase + " for deletion"));
// Daemonize the shell and use the double-fork idiom to ensure that the shell
// exits even while the "rm -rf" command continues.
String command =
String.format(
"exec >&- 2>&- <&- && (/usr/bin/setsid /bin/rm -rf %s &)&",
ShellEscaper.escapeString(tempOutputBase.getPathString()));
LOG.info("Executing shell commmand " + ShellEscaper.escapeString(command));
// Doesn't throw iff command exited and was successful.
new CommandBuilder()
.addArg(command)
.useShell(true)
.setWorkingDir(tempOutputBase.getParentDirectory())
.build()
.execute();
} else {
LOG.info("Output cleaning...");
runtime.clearCaches();
// In order to be sure that we delete everything, delete the workspace directory both for
// --deep_execroot and for --nodeep_execroot.
for (String directory :
new String[] {runtime.getWorkspaceName(), "execroot/" + runtime.getWorkspaceName()}) {
Path child = outputBase.getRelative(directory);
if (child.exists()) {
LOG.finest("Cleaning " + child);
FileSystemUtils.deleteTreesBelow(child);
}
}
}
// remove convenience links
OutputDirectoryLinksUtils.removeOutputDirectoryLinks(
runtime.getWorkspaceName(), runtime.getWorkspace(), env.getReporter(), symlinkPrefix);
// shutdown on expunge cleans
if (cleanOptions.expunge || cleanOptions.expunge_async) {
throw new ShutdownBlazeServerException(0);
}
}