Exemple #1
0
  @Override
  public Iterable<ActionContextProvider> getActionContextProviders() {
    Preconditions.checkNotNull(env);
    Preconditions.checkNotNull(buildRequest);
    Preconditions.checkNotNull(workers);

    return ImmutableList.<ActionContextProvider>of(
        new WorkerActionContextProvider(
            env.getRuntime(), buildRequest, workers, env.getEventBus()));
  }
Exemple #2
0
 // Kill workers on Ctrl-C to quickly end the interrupted build.
 // TODO(philwo) - make sure that this actually *kills* the workers and not just politely waits
 // for them to finish.
 @Subscribe
 public void buildInterrupted(BuildInterruptedEvent event) {
   if (workers != null) {
     if (verbose) {
       env.getReporter().handle(Event.info("Build interrupted, shutting down worker pool..."));
     }
     workers.close();
     workers = null;
   }
 }
Exemple #3
0
 @Subscribe
 public void buildComplete(BuildCompleteEvent event) {
   if (workers != null && buildRequest.getOptions(WorkerOptions.class).workerQuitAfterBuild) {
     if (verbose) {
       env.getReporter().handle(Event.info("Build completed, shutting down worker pool..."));
     }
     workers.close();
     workers = null;
   }
 }
Exemple #4
0
  @Override
  public void beforeCommand(Command command, CommandEnvironment env) {
    this.env = env;
    env.getEventBus().register(this);

    if (workers == null) {
      Path logDir = env.getRuntime().getOutputBase().getRelative("worker-logs");
      try {
        logDir.createDirectory();
      } catch (IOException e) {
        env.getReporter()
            .handle(Event.error("Could not create directory for worker logs: " + logDir));
      }

      GenericKeyedObjectPoolConfig config = new GenericKeyedObjectPoolConfig();

      // It's better to re-use a worker as often as possible and keep it hot, in order to profit
      // from JIT optimizations as much as possible.
      config.setLifo(true);

      // Check for & deal with idle workers every 5 seconds.
      config.setTimeBetweenEvictionRunsMillis(5 * 1000);

      // Always test the liveliness of worker processes.
      config.setTestOnBorrow(true);
      config.setTestOnCreate(true);
      config.setTestOnReturn(true);
      config.setTestWhileIdle(true);

      // Don't limit the total number of worker processes, as otherwise the pool might be full of
      // e.g. Java workers and could never accommodate another request for a different kind of
      // worker.
      config.setMaxTotal(-1);

      workers = new WorkerPool(new WorkerFactory(), config);
      workers.setReporter(env.getReporter());
      workers.setLogDirectory(logDir);
    }
  }
 public Sender(
     String url, String secret, CommandEnvironment env, ExecutorService executorService)
     throws SenderException {
   this.reporter = env.getReporter();
   this.secret = readSecret(secret, reporter);
   try {
     this.url = new URL(url);
     if (!this.secret.isEmpty()) {
       if (!(this.url.getProtocol().equals("https")
           || this.url.getHost().equals("localhost")
           || this.url.getHost().matches("^127.0.0.[0-9]+$"))) {
         reporter.handle(
             Event.warn(
                 "Using authentication over unsecure channel, " + "consider using HTTPS."));
       }
     }
   } catch (MalformedURLException e) {
     throw new SenderException("Invalid server url " + url, e);
   }
   this.buildId = env.getCommandId().toString();
   this.executorService = executorService;
   sendMessage("test", null); // test connecting to the server.
   reporter.handle(Event.info("Results are being streamed to " + url + "/result/" + buildId));
 }
 @Override
 public void handleOptions(OptionsProvider optionsProvider) {
   DashOptions options = optionsProvider.getOptions(DashOptions.class);
   try {
     sender =
         (options == null || !options.useDash)
             ? NO_OP_SENDER
             : new Sender(options.url, options.secret, env, executorService);
   } catch (SenderException e) {
     env.getReporter().handle(e.toEvent());
     sender = NO_OP_SENDER;
   }
   if (optionsBuildData != null) {
     sender.send("options", optionsBuildData);
   }
   optionsBuildData = null;
 }
Exemple #7
0
  private ExitCode doTest(
      CommandEnvironment env, OptionsProvider options, AggregatingTestListener testListener) {
    BlazeRuntime runtime = env.getRuntime();
    // Run simultaneous build and test.
    List<String> targets = ProjectFileSupport.getTargets(runtime, options);
    BuildRequest request =
        BuildRequest.create(
            getClass().getAnnotation(Command.class).name(),
            options,
            runtime.getStartupOptionsProvider(),
            targets,
            env.getReporter().getOutErr(),
            env.getCommandId(),
            env.getCommandStartTime());
    request.setRunTests();

    BuildResult buildResult = new BuildTool(env).processRequest(request, null);

    Collection<ConfiguredTarget> testTargets = buildResult.getTestTargets();
    // TODO(bazel-team): don't handle isEmpty here or fix up a bunch of tests
    if (buildResult.getSuccessfulTargets() == null) {
      // This can happen if there were errors in the target parsing or loading phase
      // (original exitcode=BUILD_FAILURE) or if there weren't but --noanalyze was given
      // (original exitcode=SUCCESS).
      env.getReporter().handle(Event.error("Couldn't start the build. Unable to run tests"));
      return buildResult.getSuccess() ? ExitCode.PARSING_FAILURE : buildResult.getExitCondition();
    }
    // TODO(bazel-team): the check above shadows NO_TESTS_FOUND, but switching the conditions breaks
    // more tests
    if (testTargets.isEmpty()) {
      env.getReporter()
          .handle(Event.error(null, "No test targets were found, yet testing was requested"));
      return buildResult.getSuccess() ? ExitCode.NO_TESTS_FOUND : buildResult.getExitCondition();
    }

    boolean buildSuccess = buildResult.getSuccess();
    boolean testSuccess = analyzeTestResults(testTargets, testListener, options);

    if (testSuccess && !buildSuccess) {
      // If all tests run successfully, test summary should include warning if
      // there were build errors not associated with the test targets.
      printer.printLn(
          AnsiTerminalPrinter.Mode.ERROR
              + "One or more non-test targets failed to build.\n"
              + AnsiTerminalPrinter.Mode.DEFAULT);
    }

    return buildSuccess
        ? (testSuccess ? ExitCode.SUCCESS : ExitCode.TESTS_FAILED)
        : buildResult.getExitCondition();
  }
 private Log getLog(String logPath) {
   Log.Builder builder = Log.newBuilder().setPath(logPath);
   File log = new File(logPath);
   try {
     long fileSize = Files.size(log.toPath());
     if (fileSize > ONE_MB) {
       fileSize = ONE_MB;
       builder.setTruncated(true);
     }
     byte buffer[] = new byte[(int) fileSize];
     try (FileInputStream in = new FileInputStream(log)) {
       ByteStreams.readFully(in, buffer);
     }
     builder.setContents(ByteString.copyFrom(buffer));
   } catch (IOException e) {
     env.getReporter()
         .getOutErr()
         .printOutLn("Error reading log file " + logPath + ": " + e.getMessage());
     // TODO(kchodorow): add this info to the proto and send.
   }
   return builder.build();
 }
Exemple #9
0
  @Override
  public void editOptions(CommandEnvironment env, OptionsParser optionsParser)
      throws AbruptExitException {
    ProjectFileSupport.handleProjectFiles(env, optionsParser, commandName());

    TestOutputFormat testOutput = optionsParser.getOptions(ExecutionOptions.class).testOutput;

    try {
      if (testOutput == TestStrategy.TestOutputFormat.STREAMED) {
        env.getReporter()
            .handle(
                Event.warn(
                    "Streamed test output requested so all tests will be run locally, without sharding, "
                        + "one at a time"));
        optionsParser.parse(
            OptionPriority.SOFTWARE_REQUIREMENT,
            "streamed output requires locally run tests, without sharding",
            ImmutableList.of("--test_sharding_strategy=disabled", "--test_strategy=exclusive"));
      }
    } catch (OptionsParsingException e) {
      throw new IllegalStateException("Known options failed to parse", e);
    }
  }
Exemple #10
0
  @Override
  public ExitCode exec(CommandEnvironment env, OptionsProvider options) {
    TestResultAnalyzer resultAnalyzer =
        new TestResultAnalyzer(
            env.getDirectories().getExecRoot(),
            options.getOptions(TestSummaryOptions.class),
            options.getOptions(ExecutionOptions.class),
            env.getEventBus());

    printer =
        new AnsiTerminalPrinter(
            env.getReporter().getOutErr().getOutputStream(),
            options.getOptions(BlazeCommandEventHandler.Options.class).useColor());

    // Initialize test handler.
    AggregatingTestListener testListener =
        new AggregatingTestListener(resultAnalyzer, env.getEventBus(), env.getReporter());

    env.getEventBus().register(testListener);
    return doTest(env, options, testListener);
  }
Exemple #11
0
  @Override
  public ExitCode exec(CommandEnvironment env, OptionsProvider options)
      throws ShutdownBlazeServerException {
    BlazeRuntime runtime = env.getRuntime();
    Options cleanOptions = options.getOptions(Options.class);
    cleanOptions.expunge_async = cleanOptions.cleanStyle.equals("expunge_async");
    cleanOptions.expunge = cleanOptions.cleanStyle.equals("expunge");

    if (!cleanOptions.expunge
        && !cleanOptions.expunge_async
        && !cleanOptions.cleanStyle.isEmpty()) {
      env.getReporter()
          .handle(Event.error(null, "Invalid clean_style value '" + cleanOptions.cleanStyle + "'"));
      return ExitCode.COMMAND_LINE_ERROR;
    }

    String cleanBanner =
        cleanOptions.expunge_async
            ? "Starting clean."
            : "Starting clean (this may take a while). "
                + "Consider using --expunge_async if the clean takes more than several minutes.";

    env.getReporter().handle(Event.info(null /*location*/, cleanBanner));
    try {
      String symlinkPrefix =
          options.getOptions(BuildRequest.BuildRequestOptions.class).getSymlinkPrefix();
      actuallyClean(env, runtime.getOutputBase(), cleanOptions, symlinkPrefix);
      return ExitCode.SUCCESS;
    } catch (IOException e) {
      env.getReporter().handle(Event.error(e.getMessage()));
      return ExitCode.LOCAL_ENVIRONMENTAL_ERROR;
    } catch (CommandException | ExecException e) {
      env.getReporter().handle(Event.error(e.getMessage()));
      return ExitCode.RUN_FAILURE;
    } catch (InterruptedException e) {
      env.getReporter().handle(Event.error("clean interrupted"));
      return ExitCode.INTERRUPTED;
    }
  }
 @Override
 public void beforeCommand(Command command, CommandEnvironment env) {
   this.env = env;
   env.getEventBus().register(this);
 }
Exemple #13
0
  private void actuallyClean(
      CommandEnvironment env, Path outputBase, Options cleanOptions, String symlinkPrefix)
      throws IOException, ShutdownBlazeServerException, CommandException, ExecException,
          InterruptedException {
    BlazeRuntime runtime = env.getRuntime();
    if (env.getOutputService() != null) {
      env.getOutputService().clean();
    }
    if (cleanOptions.expunge) {
      LOG.info("Expunging...");
      // Delete the big subdirectories with the important content first--this
      // will take the most time. Then quickly delete the little locks, logs
      // and links right before we exit. Once the lock file is gone there will
      // be a small possibility of a server race if a client is waiting, but
      // all significant files will be gone by then.
      FileSystemUtils.deleteTreesBelow(outputBase);
      FileSystemUtils.deleteTree(outputBase);
    } else if (cleanOptions.expunge_async) {
      LOG.info("Expunging asynchronously...");
      String tempBaseName = outputBase.getBaseName() + "_tmp_" + ProcessUtils.getpid();

      // Keeping tempOutputBase in the same directory ensures it remains in the
      // same file system, and therefore the mv will be atomic and fast.
      Path tempOutputBase = outputBase.getParentDirectory().getChild(tempBaseName);
      outputBase.renameTo(tempOutputBase);
      env.getReporter()
          .handle(Event.info(null, "Output base moved to " + tempOutputBase + " for deletion"));

      // Daemonize the shell and use the double-fork idiom to ensure that the shell
      // exits even while the "rm -rf" command continues.
      String command =
          String.format(
              "exec >&- 2>&- <&- && (/usr/bin/setsid /bin/rm -rf %s &)&",
              ShellEscaper.escapeString(tempOutputBase.getPathString()));

      LOG.info("Executing shell commmand " + ShellEscaper.escapeString(command));

      // Doesn't throw iff command exited and was successful.
      new CommandBuilder()
          .addArg(command)
          .useShell(true)
          .setWorkingDir(tempOutputBase.getParentDirectory())
          .build()
          .execute();
    } else {
      LOG.info("Output cleaning...");
      runtime.clearCaches();
      // In order to be sure that we delete everything, delete the workspace directory both for
      // --deep_execroot and for --nodeep_execroot.
      for (String directory :
          new String[] {runtime.getWorkspaceName(), "execroot/" + runtime.getWorkspaceName()}) {
        Path child = outputBase.getRelative(directory);
        if (child.exists()) {
          LOG.finest("Cleaning " + child);
          FileSystemUtils.deleteTreesBelow(child);
        }
      }
    }
    // remove convenience links
    OutputDirectoryLinksUtils.removeOutputDirectoryLinks(
        runtime.getWorkspaceName(), runtime.getWorkspace(), env.getReporter(), symlinkPrefix);
    // shutdown on expunge cleans
    if (cleanOptions.expunge || cleanOptions.expunge_async) {
      throw new ShutdownBlazeServerException(0);
    }
  }