Beispiel #1
0
 private void reportTargets(AnalysisResult analysisResult) {
   Collection<ConfiguredTarget> targetsToBuild = analysisResult.getTargetsToBuild();
   Collection<ConfiguredTarget> targetsToTest = analysisResult.getTargetsToTest();
   if (targetsToTest != null) {
     int testCount = targetsToTest.size();
     int targetCount = targetsToBuild.size() - testCount;
     if (targetCount == 0) {
       getReporter()
           .handle(
               Event.info(
                   "Found "
                       + testCount
                       + (testCount == 1 ? " test target..." : " test targets...")));
     } else {
       getReporter()
           .handle(
               Event.info(
                   "Found "
                       + targetCount
                       + (targetCount == 1 ? " target and " : " targets and ")
                       + testCount
                       + (testCount == 1 ? " test target..." : " test targets...")));
     }
   } else {
     int targetCount = targetsToBuild.size();
     getReporter()
         .handle(
             Event.info(
                 "Found " + targetCount + (targetCount == 1 ? " target..." : " targets...")));
   }
 }
Beispiel #2
0
 Event toEvent() {
   if (getCause() != null) {
     return Event.error(getMessage() + ": " + getCause().getMessage());
   } else {
     return Event.error(getMessage());
   }
 }
Beispiel #3
0
  private ExitCode doTest(
      CommandEnvironment env, OptionsProvider options, AggregatingTestListener testListener) {
    BlazeRuntime runtime = env.getRuntime();
    // Run simultaneous build and test.
    List<String> targets = ProjectFileSupport.getTargets(runtime, options);
    BuildRequest request =
        BuildRequest.create(
            getClass().getAnnotation(Command.class).name(),
            options,
            runtime.getStartupOptionsProvider(),
            targets,
            env.getReporter().getOutErr(),
            env.getCommandId(),
            env.getCommandStartTime());
    request.setRunTests();

    BuildResult buildResult = new BuildTool(env).processRequest(request, null);

    Collection<ConfiguredTarget> testTargets = buildResult.getTestTargets();
    // TODO(bazel-team): don't handle isEmpty here or fix up a bunch of tests
    if (buildResult.getSuccessfulTargets() == null) {
      // This can happen if there were errors in the target parsing or loading phase
      // (original exitcode=BUILD_FAILURE) or if there weren't but --noanalyze was given
      // (original exitcode=SUCCESS).
      env.getReporter().handle(Event.error("Couldn't start the build. Unable to run tests"));
      return buildResult.getSuccess() ? ExitCode.PARSING_FAILURE : buildResult.getExitCondition();
    }
    // TODO(bazel-team): the check above shadows NO_TESTS_FOUND, but switching the conditions breaks
    // more tests
    if (testTargets.isEmpty()) {
      env.getReporter()
          .handle(Event.error(null, "No test targets were found, yet testing was requested"));
      return buildResult.getSuccess() ? ExitCode.NO_TESTS_FOUND : buildResult.getExitCondition();
    }

    boolean buildSuccess = buildResult.getSuccess();
    boolean testSuccess = analyzeTestResults(testTargets, testListener, options);

    if (testSuccess && !buildSuccess) {
      // If all tests run successfully, test summary should include warning if
      // there were build errors not associated with the test targets.
      printer.printLn(
          AnsiTerminalPrinter.Mode.ERROR
              + "One or more non-test targets failed to build.\n"
              + AnsiTerminalPrinter.Mode.DEFAULT);
    }

    return buildSuccess
        ? (testSuccess ? ExitCode.SUCCESS : ExitCode.TESTS_FAILED)
        : buildResult.getExitCondition();
  }
Beispiel #4
0
  /**
   * Performs the initial phases 0-2 of the build: Setup, Loading and Analysis.
   *
   * <p>Postcondition: On success, populates the BuildRequest's set of targets to build.
   *
   * @return null if loading / analysis phases were successful; a useful error message if loading or
   *     analysis phase errors were encountered and request.keepGoing.
   * @throws InterruptedException if the current thread was interrupted.
   * @throws ViewCreationFailedException if analysis failed for any reason.
   */
  private AnalysisResult runAnalysisPhase(
      BuildRequest request,
      LoadingResult loadingResult,
      BuildConfigurationCollection configurations)
      throws InterruptedException, ViewCreationFailedException {
    Stopwatch timer = Stopwatch.createStarted();
    if (!request.getBuildOptions().performAnalysisPhase) {
      getReporter().handle(Event.progress("Loading complete."));
      LOG.info("No analysis requested, so finished");
      return AnalysisResult.EMPTY;
    }

    getReporter().handle(Event.progress("Loading complete.  Analyzing..."));
    Profiler.instance().markPhase(ProfilePhase.ANALYZE);

    AnalysisResult analysisResult =
        getView()
            .update(
                loadingResult,
                configurations,
                request.getAspects(),
                request.getViewOptions(),
                request.getTopLevelArtifactContext(),
                getReporter(),
                getEventBus());

    // TODO(bazel-team): Merge these into one event.
    getEventBus()
        .post(
            new AnalysisPhaseCompleteEvent(
                analysisResult.getTargetsToBuild(),
                getView().getTargetsVisited(),
                timer.stop().elapsed(TimeUnit.MILLISECONDS)));
    getEventBus()
        .post(
            new TestFilteringCompleteEvent(
                analysisResult.getTargetsToBuild(), analysisResult.getTargetsToTest()));

    // Check licenses.
    // We check licenses if the first target configuration has license checking enabled. Right now,
    // it is not possible to have multiple target configurations with different settings for this
    // flag, which allows us to take this short cut.
    boolean checkLicenses = configurations.getTargetConfigurations().get(0).checkLicenses();
    if (checkLicenses) {
      Profiler.instance().markPhase(ProfilePhase.LICENSE);
      validateLicensingForTargets(
          analysisResult.getTargetsToBuild(), request.getViewOptions().keepGoing);
    }

    return analysisResult;
  }
Beispiel #5
0
  /**
   * Load Skylark aspect from an extension file. Is to be called from a SkyFunction.
   *
   * @return {@code null} if dependencies cannot be satisfied.
   */
  @Nullable
  public static SkylarkAspect loadSkylarkAspect(
      Environment env, Label extensionLabel, String skylarkValueName)
      throws AspectCreationException {
    SkyKey importFileKey = SkylarkImportLookupValue.key(extensionLabel, false);
    try {
      SkylarkImportLookupValue skylarkImportLookupValue =
          (SkylarkImportLookupValue)
              env.getValueOrThrow(importFileKey, SkylarkImportFailedException.class);
      if (skylarkImportLookupValue == null) {
        return null;
      }

      Object skylarkValue =
          skylarkImportLookupValue.getEnvironmentExtension().get(skylarkValueName);
      if (!(skylarkValue instanceof SkylarkAspect)) {
        throw new ConversionException(
            skylarkValueName + " from " + extensionLabel.toString() + " is not an aspect");
      }
      return (SkylarkAspect) skylarkValue;
    } catch (SkylarkImportFailedException | ConversionException e) {
      env.getListener().handle(Event.error(e.getMessage()));
      throw new AspectCreationException(e.getMessage());
    }
  }
 @ThreadSafety.ThreadSafe
 @Override
 public void missingEdge(Target target, Label label, NoSuchThingException e) {
   eventHandler.handle(
       Event.error(
           TargetUtils.getLocationMaybe(target), TargetUtils.formatMissingEdge(target, label, e)));
   super.missingEdge(target, label, e);
 }
Beispiel #7
0
 @Override
 public void reportInvalidOptions(EventHandler reporter, BuildOptions buildOptions) {
   if (generateDebugSymbols && !iosMultiCpus.isEmpty()) {
     reporter.handle(
         Event.error(
             "--objc_generate_debug_symbols is not supported when --ios_multi_cpus is set"));
   }
 }
Beispiel #8
0
 private static void deserializeEvent(StoredEventHandler eventHandler, Build.Event event) {
   String message = event.getMessage();
   switch (event.getKind()) {
     case ERROR:
       eventHandler.handle(Event.error(message));
       break;
     case WARNING:
       eventHandler.handle(Event.warn(message));
       break;
     case INFO:
       eventHandler.handle(Event.info(message));
       break;
     case PROGRESS:
       eventHandler.handle(Event.progress(message));
       break;
     default:
       break; // Ignore
   }
 }
Beispiel #9
0
 // Kill workers on Ctrl-C to quickly end the interrupted build.
 // TODO(philwo) - make sure that this actually *kills* the workers and not just politely waits
 // for them to finish.
 @Subscribe
 public void buildInterrupted(BuildInterruptedEvent event) {
   if (workers != null) {
     if (verbose) {
       env.getReporter().handle(Event.info("Build interrupted, shutting down worker pool..."));
     }
     workers.close();
     workers = null;
   }
 }
Beispiel #10
0
 @Subscribe
 public void buildComplete(BuildCompleteEvent event) {
   if (workers != null && buildRequest.getOptions(WorkerOptions.class).workerQuitAfterBuild) {
     if (verbose) {
       env.getReporter().handle(Event.info("Build completed, shutting down worker pool..."));
     }
     workers.close();
     workers = null;
   }
 }
 @Override
 public void reportInvalidOptions(EventHandler reporter, BuildOptions buildOptions) {
   if (!Collections.disjoint(translationFlags, J2OBJC_BLACKLISTED_TRANSLATION_FLAGS)) {
     String errorMsg =
         String.format(
             INVALID_TRANSLATION_FLAGS_MSG_TEMPLATE,
             Joiner.on(",").join(translationFlags),
             Joiner.on(",").join(J2OBJC_BLACKLISTED_TRANSLATION_FLAGS));
     reporter.handle(Event.error(errorMsg));
   }
 }
Beispiel #12
0
  @Override
  public ExitCode exec(CommandEnvironment env, OptionsProvider options)
      throws ShutdownBlazeServerException {
    BlazeRuntime runtime = env.getRuntime();
    Options cleanOptions = options.getOptions(Options.class);
    cleanOptions.expunge_async = cleanOptions.cleanStyle.equals("expunge_async");
    cleanOptions.expunge = cleanOptions.cleanStyle.equals("expunge");

    if (!cleanOptions.expunge
        && !cleanOptions.expunge_async
        && !cleanOptions.cleanStyle.isEmpty()) {
      env.getReporter()
          .handle(Event.error(null, "Invalid clean_style value '" + cleanOptions.cleanStyle + "'"));
      return ExitCode.COMMAND_LINE_ERROR;
    }

    String cleanBanner =
        cleanOptions.expunge_async
            ? "Starting clean."
            : "Starting clean (this may take a while). "
                + "Consider using --expunge_async if the clean takes more than several minutes.";

    env.getReporter().handle(Event.info(null /*location*/, cleanBanner));
    try {
      String symlinkPrefix =
          options.getOptions(BuildRequest.BuildRequestOptions.class).getSymlinkPrefix();
      actuallyClean(env, runtime.getOutputBase(), cleanOptions, symlinkPrefix);
      return ExitCode.SUCCESS;
    } catch (IOException e) {
      env.getReporter().handle(Event.error(e.getMessage()));
      return ExitCode.LOCAL_ENVIRONMENTAL_ERROR;
    } catch (CommandException | ExecException e) {
      env.getReporter().handle(Event.error(e.getMessage()));
      return ExitCode.RUN_FAILURE;
    } catch (InterruptedException e) {
      env.getReporter().handle(Event.error("clean interrupted"));
      return ExitCode.INTERRUPTED;
    }
  }
Beispiel #13
0
  /**
   * Ensure the runfiles tree exists and is consistent with the TestAction's manifest
   * ($0.runfiles_manifest), bringing it into consistency if not. The contents of the output file
   * $0.runfiles/MANIFEST, if it exists, are used a proxy for the set of existing symlinks, to avoid
   * the need for recursion.
   */
  private static void updateLocalRunfilesDirectory(
      TestRunnerAction testAction,
      Path runfilesDir,
      ActionExecutionContext actionExecutionContext,
      BinTools binTools,
      ImmutableMap<String, String> shellEnvironment,
      boolean enableRunfiles)
      throws ExecException, InterruptedException {
    Executor executor = actionExecutionContext.getExecutor();

    TestTargetExecutionSettings execSettings = testAction.getExecutionSettings();
    try {
      // Avoid rebuilding the runfiles directory if the manifest in it matches the input manifest,
      // implying the symlinks exist and are already up to date.
      if (Arrays.equals(
          runfilesDir.getRelative("MANIFEST").getMD5Digest(),
          execSettings.getInputManifest().getPath().getMD5Digest())) {
        return;
      }
    } catch (IOException e1) {
      // Ignore it - we will just try to create runfiles directory.
    }

    executor
        .getEventHandler()
        .handle(
            Event.progress(
                "Building runfiles directory for '"
                    + execSettings.getExecutable().prettyPrint()
                    + "'."));

    new SymlinkTreeHelper(execSettings.getInputManifest().getPath(), runfilesDir, false)
        .createSymlinks(
            testAction, actionExecutionContext, binTools, shellEnvironment, enableRunfiles);

    executor.getEventHandler().handle(Event.progress(testAction.getProgressMessage()));
  }
Beispiel #14
0
 public Sender(
     String url, String secret, CommandEnvironment env, ExecutorService executorService)
     throws SenderException {
   this.reporter = env.getReporter();
   this.secret = readSecret(secret, reporter);
   try {
     this.url = new URL(url);
     if (!this.secret.isEmpty()) {
       if (!(this.url.getProtocol().equals("https")
           || this.url.getHost().equals("localhost")
           || this.url.getHost().matches("^127.0.0.[0-9]+$"))) {
         reporter.handle(
             Event.warn(
                 "Using authentication over unsecure channel, " + "consider using HTTPS."));
       }
     }
   } catch (MalformedURLException e) {
     throw new SenderException("Invalid server url " + url, e);
   }
   this.buildId = env.getCommandId().toString();
   this.executorService = executorService;
   sendMessage("test", null); // test connecting to the server.
   reporter.handle(Event.info("Results are being streamed to " + url + "/result/" + buildId));
 }
Beispiel #15
0
  /**
   * The crux of the build system. Builds the targets specified in the request using the specified
   * Executor.
   *
   * <p>Performs loading, analysis and execution for the specified set of targets, honoring the
   * configuration options in the BuildRequest. Returns normally iff successful, throws an exception
   * otherwise.
   *
   * <p>The caller is responsible for setting up and syncing the package cache.
   *
   * <p>During this function's execution, the actualTargets and successfulTargets fields of the
   * request object are set.
   *
   * @param request the build request that this build tool is servicing, which specifies various
   *     options; during this method's execution, the actualTargets and successfulTargets fields of
   *     the request object are populated
   * @param validator target validator
   * @return the result as a {@link BuildResult} object
   */
  public BuildResult processRequest(BuildRequest request, TargetValidator validator) {
    BuildResult result = new BuildResult(request.getStartTime());
    runtime.getEventBus().register(result);
    Throwable catastrophe = null;
    ExitCode exitCode = ExitCode.BLAZE_INTERNAL_ERROR;
    try {
      buildTargets(request, result, validator);
      exitCode = ExitCode.SUCCESS;
    } catch (BuildFailedException e) {
      if (e.isErrorAlreadyShown()) {
        // The actual error has already been reported by the Builder.
      } else {
        reportExceptionError(e);
      }
      if (e.isCatastrophic()) {
        result.setCatastrophe();
      }
      exitCode = ExitCode.BUILD_FAILURE;
    } catch (InterruptedException e) {
      exitCode = ExitCode.INTERRUPTED;
      getReporter().handle(Event.error("build interrupted"));
      getEventBus().post(new BuildInterruptedEvent());
    } catch (TargetParsingException | LoadingFailedException | ViewCreationFailedException e) {
      exitCode = ExitCode.PARSING_FAILURE;
      reportExceptionError(e);
    } catch (TestExecException e) {
      // ExitCode.SUCCESS means that build was successful. Real return code of program
      // is going to be calculated in TestCommand.doTest().
      exitCode = ExitCode.SUCCESS;
      reportExceptionError(e);
    } catch (InvalidConfigurationException e) {
      exitCode = ExitCode.COMMAND_LINE_ERROR;
      reportExceptionError(e);
    } catch (AbruptExitException e) {
      exitCode = e.getExitCode();
      reportExceptionError(e);
      result.setCatastrophe();
    } catch (Throwable throwable) {
      catastrophe = throwable;
      Throwables.propagate(throwable);
    } finally {
      stopRequest(request, result, catastrophe, exitCode);
    }

    return result;
  }
Beispiel #16
0
 /**
  * If the action might read directories as inputs in a way that is unsound wrt dependency
  * checking, this method must be called.
  */
 protected void checkInputsForDirectories(
     EventHandler eventHandler, MetadataHandler metadataHandler) {
   // Report "directory dependency checking" warning only for non-generated directories (generated
   // ones will be reported earlier).
   for (Artifact input : getMandatoryInputs()) {
     // Assume that if the file did not exist, we would not have gotten here.
     if (input.isSourceArtifact() && !metadataHandler.isRegularFile(input)) {
       eventHandler.handle(
           Event.warn(
               getOwner().getLocation(),
               "input '"
                   + input.prettyPrint()
                   + "' to "
                   + getOwner().getLabel()
                   + " is a directory; dependency checking of directories is unsound"));
     }
   }
 }
Beispiel #17
0
  @Override
  public void buildTransitiveClosure(QueryExpression caller, Set<Target> targets, int maxDepth)
      throws QueryException {
    // Everything has already been loaded, so here we just check for errors so that we can
    // pre-emptively throw/report if needed.
    Iterable<SkyKey> transitiveTraversalKeys = makeTransitiveTraversalKeys(targets);
    ImmutableList.Builder<String> errorMessagesBuilder = ImmutableList.builder();

    // First, look for errors in the successfully evaluated TransitiveTraversalValues. They may
    // have encountered errors that they were able to recover from.
    Set<Entry<SkyKey, SkyValue>> successfulEntries =
        graph.getSuccessfulValues(transitiveTraversalKeys).entrySet();
    Builder<SkyKey> successfulKeysBuilder = ImmutableSet.builder();
    for (Entry<SkyKey, SkyValue> successfulEntry : successfulEntries) {
      successfulKeysBuilder.add(successfulEntry.getKey());
      TransitiveTraversalValue value = (TransitiveTraversalValue) successfulEntry.getValue();
      String firstErrorMessage = value.getFirstErrorMessage();
      if (firstErrorMessage != null) {
        errorMessagesBuilder.add(firstErrorMessage);
      }
    }
    ImmutableSet<SkyKey> successfulKeys = successfulKeysBuilder.build();

    // Next, look for errors from the unsuccessfully evaluated TransitiveTraversal skyfunctions.
    Iterable<SkyKey> unsuccessfulKeys =
        Iterables.filter(transitiveTraversalKeys, Predicates.not(Predicates.in(successfulKeys)));
    Set<Entry<SkyKey, Exception>> errorEntries =
        graph.getMissingAndExceptions(unsuccessfulKeys).entrySet();
    for (Map.Entry<SkyKey, Exception> entry : errorEntries) {
      if (entry.getValue() == null) {
        // Targets may be in the graph because they are not in the universe or depend on cycles.
        eventHandler.handle(Event.warn(entry.getKey().argument() + " does not exist in graph"));
      } else {
        errorMessagesBuilder.add(entry.getValue().getMessage());
      }
    }

    // Lastly, report all found errors.
    ImmutableList<String> errorMessages = errorMessagesBuilder.build();
    for (String errorMessage : errorMessages) {
      reportBuildFileError(caller, errorMessage);
    }
  }
Beispiel #18
0
  @Override
  public void beforeCommand(Command command, CommandEnvironment env) {
    this.env = env;
    env.getEventBus().register(this);

    if (workers == null) {
      Path logDir = env.getRuntime().getOutputBase().getRelative("worker-logs");
      try {
        logDir.createDirectory();
      } catch (IOException e) {
        env.getReporter()
            .handle(Event.error("Could not create directory for worker logs: " + logDir));
      }

      GenericKeyedObjectPoolConfig config = new GenericKeyedObjectPoolConfig();

      // It's better to re-use a worker as often as possible and keep it hot, in order to profit
      // from JIT optimizations as much as possible.
      config.setLifo(true);

      // Check for & deal with idle workers every 5 seconds.
      config.setTimeBetweenEvictionRunsMillis(5 * 1000);

      // Always test the liveliness of worker processes.
      config.setTestOnBorrow(true);
      config.setTestOnCreate(true);
      config.setTestOnReturn(true);
      config.setTestWhileIdle(true);

      // Don't limit the total number of worker processes, as otherwise the pool might be full of
      // e.g. Java workers and could never accommodate another request for a different kind of
      // worker.
      config.setMaxTotal(-1);

      workers = new WorkerPool(new WorkerFactory(), config);
      workers.setReporter(env.getReporter());
      workers.setLogDirectory(logDir);
    }
  }
Beispiel #19
0
 private static String readSecret(String secretFile, Reporter reporter) throws SenderException {
   if (secretFile.isEmpty()) {
     return "";
   }
   Path secretPath = new File(secretFile).toPath();
   if (!Files.isReadable(secretPath)) {
     throw new SenderException("Secret file " + secretFile + " doesn't exists or is unreadable");
   }
   try {
     if (Files.getPosixFilePermissions(secretPath).contains(PosixFilePermission.OTHERS_READ)
         || Files.getPosixFilePermissions(secretPath).contains(PosixFilePermission.GROUP_READ)) {
       reporter.handle(
           Event.warn(
               "Secret file "
                   + secretFile
                   + " is readable by non-owner. "
                   + "It is recommended to set its permission to 0600 (read-write only by the owner)."));
     }
     return new String(Files.readAllBytes(secretPath), StandardCharsets.UTF_8).trim();
   } catch (IOException e) {
     throw new SenderException("Invalid secret file " + secretFile, e);
   }
 }
Beispiel #20
0
  @Override
  public void editOptions(CommandEnvironment env, OptionsParser optionsParser)
      throws AbruptExitException {
    ProjectFileSupport.handleProjectFiles(env, optionsParser, commandName());

    TestOutputFormat testOutput = optionsParser.getOptions(ExecutionOptions.class).testOutput;

    try {
      if (testOutput == TestStrategy.TestOutputFormat.STREAMED) {
        env.getReporter()
            .handle(
                Event.warn(
                    "Streamed test output requested so all tests will be run locally, without sharding, "
                        + "one at a time"));
        optionsParser.parse(
            OptionPriority.SOFTWARE_REQUIREMENT,
            "streamed output requires locally run tests, without sharding",
            ImmutableList.of("--test_sharding_strategy=disabled", "--test_strategy=exclusive"));
      }
    } catch (OptionsParsingException e) {
      throw new IllegalStateException("Known options failed to parse", e);
    }
  }
  /** Creates the Extension to be imported. */
  private Extension createExtension(
      BuildFileAST ast, PathFragment file, Map<PathFragment, Extension> importMap, Environment env)
      throws InterruptedException, SkylarkImportLookupFunctionException {
    StoredEventHandler eventHandler = new StoredEventHandler();
    // TODO(bazel-team): this method overestimates the changes which can affect the
    // Skylark RuleClass. For example changes to comments or unused functions can modify the hash.
    // A more accurate - however much more complicated - way would be to calculate a hash based on
    // the transitive closure of the accessible AST nodes.
    try (Mutability mutability = Mutability.create("importing %s", file)) {
      com.google.devtools.build.lib.syntax.Environment extensionEnv =
          ruleClassProvider
              .createSkylarkRuleClassEnvironment(
                  mutability, eventHandler, ast.getContentHashCode(), importMap)
              .setupOverride("native", packageFactory.getNativeModule());
      ast.exec(extensionEnv, eventHandler);
      SkylarkRuleClassFunctions.exportRuleFunctions(extensionEnv, file);

      Event.replayEventsOn(env.getListener(), eventHandler.getEvents());
      if (eventHandler.hasErrors()) {
        throw new SkylarkImportLookupFunctionException(SkylarkImportFailedException.errors(file));
      }
      return new Extension(extensionEnv);
    }
  }
Beispiel #22
0
  private void actuallyClean(
      CommandEnvironment env, Path outputBase, Options cleanOptions, String symlinkPrefix)
      throws IOException, ShutdownBlazeServerException, CommandException, ExecException,
          InterruptedException {
    BlazeRuntime runtime = env.getRuntime();
    if (env.getOutputService() != null) {
      env.getOutputService().clean();
    }
    if (cleanOptions.expunge) {
      LOG.info("Expunging...");
      // Delete the big subdirectories with the important content first--this
      // will take the most time. Then quickly delete the little locks, logs
      // and links right before we exit. Once the lock file is gone there will
      // be a small possibility of a server race if a client is waiting, but
      // all significant files will be gone by then.
      FileSystemUtils.deleteTreesBelow(outputBase);
      FileSystemUtils.deleteTree(outputBase);
    } else if (cleanOptions.expunge_async) {
      LOG.info("Expunging asynchronously...");
      String tempBaseName = outputBase.getBaseName() + "_tmp_" + ProcessUtils.getpid();

      // Keeping tempOutputBase in the same directory ensures it remains in the
      // same file system, and therefore the mv will be atomic and fast.
      Path tempOutputBase = outputBase.getParentDirectory().getChild(tempBaseName);
      outputBase.renameTo(tempOutputBase);
      env.getReporter()
          .handle(Event.info(null, "Output base moved to " + tempOutputBase + " for deletion"));

      // Daemonize the shell and use the double-fork idiom to ensure that the shell
      // exits even while the "rm -rf" command continues.
      String command =
          String.format(
              "exec >&- 2>&- <&- && (/usr/bin/setsid /bin/rm -rf %s &)&",
              ShellEscaper.escapeString(tempOutputBase.getPathString()));

      LOG.info("Executing shell commmand " + ShellEscaper.escapeString(command));

      // Doesn't throw iff command exited and was successful.
      new CommandBuilder()
          .addArg(command)
          .useShell(true)
          .setWorkingDir(tempOutputBase.getParentDirectory())
          .build()
          .execute();
    } else {
      LOG.info("Output cleaning...");
      runtime.clearCaches();
      // In order to be sure that we delete everything, delete the workspace directory both for
      // --deep_execroot and for --nodeep_execroot.
      for (String directory :
          new String[] {runtime.getWorkspaceName(), "execroot/" + runtime.getWorkspaceName()}) {
        Path child = outputBase.getRelative(directory);
        if (child.exists()) {
          LOG.finest("Cleaning " + child);
          FileSystemUtils.deleteTreesBelow(child);
        }
      }
    }
    // remove convenience links
    OutputDirectoryLinksUtils.removeOutputDirectoryLinks(
        runtime.getWorkspaceName(), runtime.getWorkspace(), env.getReporter(), symlinkPrefix);
    // shutdown on expunge cleans
    if (cleanOptions.expunge || cleanOptions.expunge_async) {
      throw new ShutdownBlazeServerException(0);
    }
  }
  @Override
  @Nullable
  public BuildConfiguration createConfigurations(
      ConfigurationFactory configurationFactory,
      Cache<String, BuildConfiguration> cache,
      PackageProviderForConfigurations packageProvider,
      BuildOptions buildOptions,
      EventHandler eventHandler,
      boolean performSanityCheck)
      throws InvalidConfigurationException {
    // Target configuration
    BuildConfiguration targetConfiguration =
        configurationFactory.getConfiguration(packageProvider, buildOptions, false, cache);
    if (targetConfiguration == null) {
      return null;
    }

    BuildConfiguration dataConfiguration = targetConfiguration;

    // Host configuration
    // Note that this passes in the dataConfiguration, not the target
    // configuration. This is intentional.
    BuildConfiguration hostConfiguration =
        getHostConfigurationFromRequest(
            configurationFactory, packageProvider, dataConfiguration, buildOptions, cache);
    if (hostConfiguration == null) {
      return null;
    }

    ListMultimap<SplitTransition<?>, BuildConfiguration> splitTransitionsTable =
        ArrayListMultimap.create();
    for (SplitTransition<BuildOptions> transition : buildOptions.getPotentialSplitTransitions()) {
      List<BuildOptions> splitOptionsList = transition.split(buildOptions);

      // While it'd be clearer to condition the below on "if (!splitOptionsList.empty())",
      // IosExtension.ExtensionSplitArchTransition defaults to a single-value split. If we failed
      // that case then no builds would work, whether or not they're iOS builds (since iOS
      // configurations are unconditionally loaded). Once we have dynamic configuraiton support
      // for split transitions, this will all go away.
      if (splitOptionsList.size() > 1 && targetConfiguration.useDynamicConfigurations()) {
        throw new InvalidConfigurationException(
            "dynamic configurations don't yet support split transitions");
      }

      for (BuildOptions splitOptions : splitOptionsList) {
        BuildConfiguration splitConfig =
            configurationFactory.getConfiguration(packageProvider, splitOptions, false, cache);
        splitTransitionsTable.put(transition, splitConfig);
      }
    }
    if (packageProvider.valuesMissing()) {
      return null;
    }

    // Sanity check that the implicit labels are all in the transitive closure of explicit ones.
    // This also registers all targets in the cache entry and validates them on subsequent requests.
    Set<Label> reachableLabels = new HashSet<>();
    if (performSanityCheck) {
      // We allow the package provider to be null for testing.
      for (Map.Entry<String, Label> entry : buildOptions.getAllLabels().entries()) {
        Label label = entry.getValue();
        try {
          collectTransitiveClosure(packageProvider, reachableLabels, label);
        } catch (NoSuchThingException e) {
          eventHandler.handle(Event.error(e.getMessage()));
          throw new InvalidConfigurationException(
              String.format("Failed to load required %s target: '%s'", entry.getKey(), label));
        }
      }
      if (packageProvider.valuesMissing()) {
        return null;
      }
      sanityCheckImplicitLabels(reachableLabels, targetConfiguration);
      sanityCheckImplicitLabels(reachableLabels, hostConfiguration);
    }

    BuildConfiguration result =
        setupTransitions(
            targetConfiguration, dataConfiguration, hostConfiguration, splitTransitionsTable);
    result.reportInvalidOptions(eventHandler);
    return result;
  }
  @Override
  @ThreadCompatible
  public void execute(ActionExecutionContext actionExecutionContext)
      throws ActionExecutionException, InterruptedException {
    Executor executor = actionExecutionContext.getExecutor();

    // First, do a normal compilation, to generate the ".d" file. The generated object file is built
    // to a temporary location (tempOutputFile) and ignored afterwards.
    LOG.info("Generating " + getDotdFile());
    CppCompileActionContext context = executor.getContext(actionContext);
    CppCompileActionContext.Reply reply = null;
    try {
      // We delegate stdout/stderr to nowhere, i.e. same as redirecting to /dev/null.
      reply = context.execWithReply(this, actionExecutionContext.withFileOutErr(new FileOutErr()));
    } catch (ExecException e) {
      // We ignore failures here (other than capturing the Distributor reply).
      // The compilation may well fail (that's the whole point of negative compilation tests).
      // We execute it here just for the side effect of generating the ".d" file.
      reply = context.getReplyFromException(e, this);
      if (reply == null) {
        // This can only happen if the ExecException does not come from remote execution.
        throw e.toActionExecutionException(
            "Fake C++ Compilation of rule '" + getOwner().getLabel() + "'",
            executor.getVerboseFailures(),
            this);
      }
    }
    IncludeScanningContext scanningContext = executor.getContext(IncludeScanningContext.class);
    NestedSet<Artifact> discoveredInputs =
        discoverInputsFromDotdFiles(
            executor.getExecRoot(), scanningContext.getArtifactResolver(), reply);
    reply = null; // Clear in-memory .d files early.

    // Even cc_fake_binary rules need to properly declare their dependencies...
    // In fact, they need to declare their dependencies even more than cc_binary rules do.
    // CcCommonConfiguredTarget passes in an empty set of declaredIncludeDirs,
    // so this check below will only allow inclusion of header files that are explicitly
    // listed in the "srcs" of the cc_fake_binary or in the "srcs" of a cc_library that it
    // depends on.
    try {
      validateInclusions(
          discoveredInputs,
          actionExecutionContext.getArtifactExpander(),
          executor.getEventHandler());
    } catch (ActionExecutionException e) {
      // TODO(bazel-team): (2009) make this into an error, once most of the current warnings
      // are fixed.
      executor
          .getEventHandler()
          .handle(
              Event.warn(
                  getOwner().getLocation(),
                  e.getMessage() + ";\n  this warning may eventually become an error"));
    }

    updateActionInputs(discoveredInputs);

    // Generate a fake ".o" file containing the command line needed to generate
    // the real object file.
    LOG.info("Generating " + outputFile);

    // A cc_fake_binary rule generates fake .o files and a fake target file,
    // which merely contain instructions on building the real target. We need to
    // be careful to use a new set of output file names in the instructions, as
    // to not overwrite the fake output files when someone tries to follow the
    // instructions. As the real compilation is executed by the test from its
    // runfiles directory (where writing is forbidden), we patch the command
    // line to write to $TEST_TMPDIR instead.
    final String outputPrefix = "$TEST_TMPDIR/";
    String argv =
        Joiner.on(' ')
            .join(
                Iterables.transform(
                    getArgv(outputFile.getExecPath()),
                    new Function<String, String>() {
                      @Override
                      public String apply(String input) {
                        String result = ShellEscaper.escapeString(input);
                        // Once -c and -o options are added into action_config, the argument of
                        // getArgv(outputFile.getExecPath()) won't be used anymore. There will
                        // always be
                        // -c <tempOutputFile>, but here it has to be outputFile, so we replace it.
                        if (input.equals(tempOutputFile.getPathString())) {
                          result =
                              outputPrefix
                                  + ShellEscaper.escapeString(outputFile.getExecPathString());
                        }
                        if (input.equals(outputFile.getExecPathString())
                            || input.equals(getDotdFile().getSafeExecPath().getPathString())) {
                          result = outputPrefix + ShellEscaper.escapeString(input);
                        }
                        return result;
                      }
                    }));

    // Write the command needed to build the real .o file to the fake .o file.
    // Generate a command to ensure that the output directory exists; otherwise
    // the compilation would fail.
    try {
      // Ensure that the .d file and .o file are siblings, so that the "mkdir" below works for
      // both.
      Preconditions.checkState(
          outputFile
              .getExecPath()
              .getParentDirectory()
              .equals(getDotdFile().getSafeExecPath().getParentDirectory()));
      FileSystemUtils.writeContent(
          outputFile.getPath(),
          ISO_8859_1,
          outputFile.getPath().getBaseName()
              + ": "
              + "mkdir -p "
              + outputPrefix
              + "$(dirname "
              + outputFile.getExecPath()
              + ")"
              + " && "
              + argv
              + "\n");
    } catch (IOException e) {
      throw new ActionExecutionException(
          "failed to create fake compile command for rule '"
              + getOwner().getLabel()
              + ": "
              + e.getMessage(),
          this,
          false);
    }
  }
Beispiel #25
0
 /**
  * Validates the options for this BuildRequest.
  *
  * <p>Issues warnings for the use of deprecated options, and warnings or errors for any option
  * settings that conflict.
  */
 @VisibleForTesting
 public void validateOptions(BuildRequest request) throws InvalidConfigurationException {
   for (String issue : request.validateOptions()) {
     getReporter().handle(Event.warn(issue));
   }
 }
Beispiel #26
0
 @Override
 public void handle(Event event) {
   Preconditions.checkArgument(
       !EventKind.ERRORS_AND_WARNINGS.contains(event.getKind()), event);
 }
Beispiel #27
0
  /**
   * The crux of the build system. Builds the targets specified in the request using the specified
   * Executor.
   *
   * <p>Performs loading, analysis and execution for the specified set of targets, honoring the
   * configuration options in the BuildRequest. Returns normally iff successful, throws an exception
   * otherwise.
   *
   * <p>Callers must ensure that {@link #stopRequest} is called after this method, even if it
   * throws.
   *
   * <p>The caller is responsible for setting up and syncing the package cache.
   *
   * <p>During this function's execution, the actualTargets and successfulTargets fields of the
   * request object are set.
   *
   * @param request the build request that this build tool is servicing, which specifies various
   *     options; during this method's execution, the actualTargets and successfulTargets fields of
   *     the request object are populated
   * @param result the build result that is the mutable result of this build
   * @param validator target validator
   */
  public void buildTargets(BuildRequest request, BuildResult result, TargetValidator validator)
      throws BuildFailedException, LocalEnvironmentException, InterruptedException,
          ViewCreationFailedException, TargetParsingException, LoadingFailedException,
          ExecutorInitException, AbruptExitException, InvalidConfigurationException,
          TestExecException {
    validateOptions(request);
    BuildOptions buildOptions = runtime.createBuildOptions(request);
    // Sync the package manager before sending the BuildStartingEvent in runLoadingPhase()
    runtime.setupPackageCache(
        request.getPackageCacheOptions(), DefaultsPackage.getDefaultsPackageContent(buildOptions));

    ExecutionTool executionTool = null;
    LoadingResult loadingResult = null;
    BuildConfigurationCollection configurations = null;
    try {
      getEventBus().post(new BuildStartingEvent(runtime.getOutputFileSystem(), request));
      LOG.info("Build identifier: " + request.getId());
      executionTool = new ExecutionTool(runtime, request);
      if (needsExecutionPhase(request.getBuildOptions())) {
        // Initialize the execution tool early if we need it. This hides the latency of setting up
        // the execution backends.
        executionTool.init();
      }

      // Loading phase.
      loadingResult = runLoadingPhase(request, validator);

      // Create the build configurations.
      if (!request.getMultiCpus().isEmpty()) {
        getReporter()
            .handle(
                Event.warn(
                    "The --experimental_multi_cpu option is _very_ experimental and only intended for "
                        + "internal testing at this time. If you do not work on the build tool, then you "
                        + "should stop now!"));
        if (!"build".equals(request.getCommandName()) && !"test".equals(request.getCommandName())) {
          throw new InvalidConfigurationException(
              "The experimental setting to select multiple CPUs is only supported for 'build' and "
                  + "'test' right now!");
        }
      }
      configurations =
          getConfigurations(
              buildOptions, request.getMultiCpus(), request.getViewOptions().keepGoing);

      getEventBus().post(new ConfigurationsCreatedEvent(configurations));
      runtime.throwPendingException();
      if (configurations.getTargetConfigurations().size() == 1) {
        // TODO(bazel-team): This is not optimal - we retain backwards compatibility in the case
        // where there's only a single configuration, but we don't send an event in the multi-config
        // case. Can we do better? [multi-config]
        getEventBus()
            .post(
                new MakeEnvironmentEvent(
                    configurations.getTargetConfigurations().get(0).getMakeEnvironment()));
      }
      LOG.info("Configurations created");

      // Analysis phase.
      AnalysisResult analysisResult = runAnalysisPhase(request, loadingResult, configurations);
      result.setActualTargets(analysisResult.getTargetsToBuild());
      result.setTestTargets(analysisResult.getTargetsToTest());

      checkTargetEnvironmentRestrictions(
          analysisResult.getTargetsToBuild(), runtime.getPackageManager());
      reportTargets(analysisResult);

      // Execution phase.
      if (needsExecutionPhase(request.getBuildOptions())) {
        runtime.getSkyframeExecutor().injectTopLevelContext(request.getTopLevelArtifactContext());
        executionTool.executeBuild(
            request.getId(),
            analysisResult,
            result,
            runtime.getSkyframeExecutor(),
            configurations,
            transformPackageRoots(loadingResult.getPackageRoots()));
      }

      String delayedErrorMsg = analysisResult.getError();
      if (delayedErrorMsg != null) {
        throw new BuildFailedException(delayedErrorMsg);
      }
    } catch (RuntimeException e) {
      // Print an error message for unchecked runtime exceptions. This does not concern Error
      // subclasses such as OutOfMemoryError.
      request
          .getOutErr()
          .printErrLn("Unhandled exception thrown during build; message: " + e.getMessage());
      throw e;
    } finally {
      // Delete dirty nodes to ensure that they do not accumulate indefinitely.
      long versionWindow = request.getViewOptions().versionWindowForDirtyNodeGc;
      if (versionWindow != -1) {
        runtime.getSkyframeExecutor().deleteOldNodes(versionWindow);
      }

      if (executionTool != null) {
        executionTool.shutdown();
      }
      // The workspace status actions will not run with certain flags, or if an error
      // occurs early in the build. Tell a lie so that the event is not missing.
      // If multiple build_info events are sent, only the first is kept, so this does not harm
      // successful runs (which use the workspace status action).
      getEventBus()
          .post(
              new BuildInfoEvent(
                  runtime.getworkspaceStatusActionFactory().createDummyWorkspaceStatus()));
    }

    if (loadingResult != null && loadingResult.hasTargetPatternError()) {
      throw new BuildFailedException(
          "execution phase successful, but there were errors " + "parsing the target pattern");
    }
  }
Beispiel #28
0
 private void reportExceptionError(Exception e) {
   if (e.getMessage() != null) {
     getReporter().handle(Event.error(e.getMessage()));
   }
 }
Beispiel #29
0
  /**
   * Analyzes the specified targets using Skyframe as the driving framework.
   *
   * @return the configured targets that should be built along with a WalkableGraph of the analysis.
   */
  public SkyframeAnalysisResult configureTargets(
      EventHandler eventHandler,
      List<ConfiguredTargetKey> values,
      List<AspectValueKey> aspectKeys,
      EventBus eventBus,
      boolean keepGoing)
      throws InterruptedException, ViewCreationFailedException {
    enableAnalysis(true);
    EvaluationResult<ActionLookupValue> result;
    try {
      result = skyframeExecutor.configureTargets(eventHandler, values, aspectKeys, keepGoing);
    } finally {
      enableAnalysis(false);
    }
    ImmutableMap<Action, ConflictException> badActions = skyframeExecutor.findArtifactConflicts();

    Collection<AspectValue> goodAspects = Lists.newArrayListWithCapacity(values.size());
    NestedSetBuilder<Package> packages = NestedSetBuilder.stableOrder();
    for (AspectValueKey aspectKey : aspectKeys) {
      AspectValue value = (AspectValue) result.get(AspectValue.key(aspectKey));
      if (value == null) {
        // Skip aspects that couldn't be applied to targets.
        continue;
      }
      goodAspects.add(value);
      packages.addTransitive(value.getTransitivePackages());
    }

    // Filter out all CTs that have a bad action and convert to a list of configured targets. This
    // code ensures that the resulting list of configured targets has the same order as the incoming
    // list of values, i.e., that the order is deterministic.
    Collection<ConfiguredTarget> goodCts = Lists.newArrayListWithCapacity(values.size());
    for (ConfiguredTargetKey value : values) {
      ConfiguredTargetValue ctValue =
          (ConfiguredTargetValue) result.get(ConfiguredTargetValue.key(value));
      if (ctValue == null) {
        continue;
      }
      goodCts.add(ctValue.getConfiguredTarget());
      packages.addTransitive(ctValue.getTransitivePackages());
    }

    if (!result.hasError() && badActions.isEmpty()) {
      setDeserializedArtifactOwners();
      return new SkyframeAnalysisResult(
          /*hasLoadingError=*/ false,
          /*hasAnalysisError=*/ false,
          ImmutableList.copyOf(goodCts),
          result.getWalkableGraph(),
          ImmutableList.copyOf(goodAspects),
          LoadingPhaseRunner.collectPackageRoots(packages.build().toCollection()));
    }

    // --nokeep_going so we fail with an exception for the first error.
    // TODO(bazel-team): We might want to report the other errors through the event bus but
    // for keeping this code in parity with legacy we just report the first error for now.
    if (!keepGoing) {
      for (Map.Entry<Action, ConflictException> bad : badActions.entrySet()) {
        ConflictException ex = bad.getValue();
        try {
          ex.rethrowTyped();
        } catch (MutableActionGraph.ActionConflictException ace) {
          ace.reportTo(eventHandler);
          String errorMsg =
              "Analysis of target '"
                  + bad.getKey().getOwner().getLabel()
                  + "' failed; build aborted";
          throw new ViewCreationFailedException(errorMsg);
        } catch (ArtifactPrefixConflictException apce) {
          eventHandler.handle(Event.error(apce.getMessage()));
        }
        throw new ViewCreationFailedException(ex.getMessage());
      }

      Map.Entry<SkyKey, ErrorInfo> error = result.errorMap().entrySet().iterator().next();
      SkyKey topLevel = error.getKey();
      ErrorInfo errorInfo = error.getValue();
      assertSaneAnalysisError(errorInfo, topLevel);
      skyframeExecutor
          .getCyclesReporter()
          .reportCycles(errorInfo.getCycleInfo(), topLevel, eventHandler);
      Throwable cause = errorInfo.getException();
      Preconditions.checkState(
          cause != null || !Iterables.isEmpty(errorInfo.getCycleInfo()), errorInfo);
      String errorMsg = null;
      if (topLevel.argument() instanceof ConfiguredTargetKey) {
        errorMsg =
            "Analysis of target '"
                + ConfiguredTargetValue.extractLabel(topLevel)
                + "' failed; build aborted";
      } else if (topLevel.argument() instanceof AspectValueKey) {
        AspectValueKey aspectKey = (AspectValueKey) topLevel.argument();
        errorMsg = "Analysis of aspect '" + aspectKey.getDescription() + "' failed; build aborted";
      } else {
        assert false;
      }
      if (cause instanceof ActionConflictException) {
        ((ActionConflictException) cause).reportTo(eventHandler);
      }
      throw new ViewCreationFailedException(errorMsg);
    }

    boolean hasLoadingError = false;
    // --keep_going : We notify the error and return a ConfiguredTargetValue
    for (Map.Entry<SkyKey, ErrorInfo> errorEntry : result.errorMap().entrySet()) {
      // Only handle errors of configured targets, not errors of top-level aspects.
      // TODO(ulfjack): this is quadratic - if there are a lot of CTs, this could be rather slow.
      if (!values.contains(errorEntry.getKey().argument())) {
        continue;
      }
      SkyKey errorKey = errorEntry.getKey();
      ConfiguredTargetKey label = (ConfiguredTargetKey) errorKey.argument();
      Label topLevelLabel = label.getLabel();
      ErrorInfo errorInfo = errorEntry.getValue();
      assertSaneAnalysisError(errorInfo, errorKey);

      skyframeExecutor
          .getCyclesReporter()
          .reportCycles(errorInfo.getCycleInfo(), errorKey, eventHandler);
      Exception cause = errorInfo.getException();
      Label analysisRootCause = null;
      if (cause instanceof ConfiguredValueCreationException) {
        ConfiguredValueCreationException ctCause = (ConfiguredValueCreationException) cause;
        for (Label rootCause : ctCause.getRootCauses()) {
          hasLoadingError = true;
          eventBus.post(new LoadingFailureEvent(topLevelLabel, rootCause));
        }
        analysisRootCause = ctCause.getAnalysisRootCause();
      } else if (!Iterables.isEmpty(errorInfo.getCycleInfo())) {
        analysisRootCause =
            maybeGetConfiguredTargetCycleCulprit(topLevelLabel, errorInfo.getCycleInfo());
      } else if (cause instanceof ActionConflictException) {
        ((ActionConflictException) cause).reportTo(eventHandler);
      }
      eventHandler.handle(
          Event.warn(
              "errors encountered while analyzing target '"
                  + topLevelLabel
                  + "': it will not be built"));
      if (analysisRootCause != null) {
        eventBus.post(
            new AnalysisFailureEvent(
                LabelAndConfiguration.of(topLevelLabel, label.getConfiguration()),
                analysisRootCause));
      }
    }

    Collection<Exception> reportedExceptions = Sets.newHashSet();
    for (Map.Entry<Action, ConflictException> bad : badActions.entrySet()) {
      ConflictException ex = bad.getValue();
      try {
        ex.rethrowTyped();
      } catch (MutableActionGraph.ActionConflictException ace) {
        ace.reportTo(eventHandler);
        eventHandler.handle(
            Event.warn(
                "errors encountered while analyzing target '"
                    + bad.getKey().getOwner().getLabel()
                    + "': it will not be built"));
      } catch (ArtifactPrefixConflictException apce) {
        if (reportedExceptions.add(apce)) {
          eventHandler.handle(Event.error(apce.getMessage()));
        }
      }
    }

    if (!badActions.isEmpty()) {
      // In order to determine the set of configured targets transitively error free from action
      // conflict issues, we run a post-processing update() that uses the bad action map.
      EvaluationResult<PostConfiguredTargetValue> actionConflictResult =
          skyframeExecutor.postConfigureTargets(eventHandler, values, keepGoing, badActions);

      goodCts = Lists.newArrayListWithCapacity(values.size());
      for (ConfiguredTargetKey value : values) {
        PostConfiguredTargetValue postCt =
            actionConflictResult.get(PostConfiguredTargetValue.key(value));
        if (postCt != null) {
          goodCts.add(postCt.getCt());
        }
      }
    }
    setDeserializedArtifactOwners();
    return new SkyframeAnalysisResult(
        hasLoadingError,
        result.hasError() || !badActions.isEmpty(),
        ImmutableList.copyOf(goodCts),
        result.getWalkableGraph(),
        ImmutableList.copyOf(goodAspects),
        LoadingPhaseRunner.collectPackageRoots(packages.build().toCollection()));
  }
Beispiel #30
0
 private void error(String message, int start, int end) {
   this.containsErrors = true;
   eventHandler.handle(Event.error(createLocation(start, end), message));
 }