Beispiel #1
0
  @Test
  public void whenSeparateTestFailsThenBuildFails() throws Exception {
    CommandRunnerParams commandRunnerParams = CommandRunnerParamsForTesting.builder().build();
    final TestResults failingTestResults =
        new TestResults(
            ImmutableList.of(
                new TestCaseSummary(
                    "TestCase",
                    ImmutableList.of(
                        new TestResultSummary(
                            "TestCaseResult",
                            "failTest",
                            ResultType.FAILURE,
                            5000,
                            null,
                            null,
                            null,
                            null)))));
    BuildTarget failingTestTarget = BuildTargetFactory.newInstance("//:failingtest");
    FakeTestRule failingTest =
        new FakeTestRule(
            BuildRuleParamsFactory.createTrivialBuildRuleParams(failingTestTarget),
            new SourcePathResolver(new BuildRuleResolver()),
            ImmutableSet.<Label>of(),
            Optional.of(Paths.get("failingTestStep1OutputDir")),
            true, // runTestSeparately
            ImmutableList.<Step>of(),
            new Callable<TestResults>() {
              @Override
              public TestResults call() {
                return failingTestResults;
              }
            });

    ListeningExecutorService service =
        MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(3));
    FakeBuildEngine fakeBuildEngine =
        new FakeBuildEngine(
            ImmutableMap.of(
                failingTestTarget, new BuildResult(failingTest, BUILT_LOCALLY, CacheResult.skip())),
            ImmutableMap.of(failingTestTarget, new RuleKey("00")));
    ExecutionContext fakeExecutionContext =
        TestExecutionContext.newBuilder().setProjectFilesystem(new FakeProjectFilesystem()).build();
    DefaultStepRunner stepRunner = new DefaultStepRunner(fakeExecutionContext);
    int ret =
        TestRunning.runTests(
            commandRunnerParams,
            ImmutableList.<TestRule>of(failingTest),
            FakeBuildContext.NOOP_CONTEXT,
            fakeExecutionContext,
            DEFAULT_OPTIONS,
            service,
            fakeBuildEngine,
            stepRunner);

    assertThat(ret, equalTo(TestRunning.TEST_FAILURES_EXIT_CODE));
  }
Beispiel #2
0
  private int runTestsInternal(
      CommandRunnerParams params,
      BuildEngine buildEngine,
      Build build,
      Iterable<TestRule> testRules)
      throws InterruptedException, IOException {

    if (!withDashArguments.isEmpty()) {
      params
          .getBuckEventBus()
          .post(
              ConsoleEvent.severe("Unexpected arguments after \"--\" when using internal runner"));
      return 1;
    }

    ConcurrencyLimit concurrencyLimit =
        new ConcurrencyLimit(
            getNumTestThreads(params.getBuckConfig()), params.getBuckConfig().getLoadLimit());
    try (CommandThreadManager testPool =
        new CommandThreadManager(
            "Test-Run", params.getBuckConfig().getWorkQueueExecutionOrder(), concurrencyLimit)) {
      return TestRunning.runTests(
          params,
          testRules,
          Preconditions.checkNotNull(build.getBuildContext()),
          build.getExecutionContext(),
          getTestRunningOptions(params),
          testPool.getExecutor(),
          buildEngine,
          new DefaultStepRunner(build.getExecutionContext()));
    } catch (ExecutionException e) {
      params
          .getBuckEventBus()
          .post(ConsoleEvent.severe(MoreExceptions.getHumanReadableOrLocalizedMessage(e)));
      return 1;
    }
  }
Beispiel #3
0
  @Test
  public void whenSomeTestsAreSeparateThenSeparateTestsRunAtEnd() throws Exception {
    CommandRunnerParams commandRunnerParams = CommandRunnerParamsForTesting.builder().build();

    AtomicInteger atomicExecutionOrder = new AtomicInteger(0);
    ExecutionOrderAwareFakeStep separateTestStep1 =
        new ExecutionOrderAwareFakeStep("teststep1", "teststep1", 0, atomicExecutionOrder);
    final TestResults fakeTestResults =
        new TestResults(
            ImmutableList.of(
                new TestCaseSummary(
                    "TestCase",
                    ImmutableList.of(
                        new TestResultSummary(
                            "TestCaseResult",
                            "passTest",
                            ResultType.SUCCESS,
                            5000,
                            null,
                            null,
                            null,
                            null)))));

    BuildTarget separateTest1Target = BuildTargetFactory.newInstance("//:test1");
    FakeTestRule separateTest1 =
        new FakeTestRule(
            BuildRuleParamsFactory.createTrivialBuildRuleParams(separateTest1Target),
            new SourcePathResolver(new BuildRuleResolver()),
            ImmutableSet.<Label>of(),
            Optional.of(Paths.get("separateTestStep1OutputDir")),
            true, // runTestSeparately
            ImmutableList.<Step>of(separateTestStep1),
            new Callable<TestResults>() {
              @Override
              public TestResults call() {
                return fakeTestResults;
              }
            });

    ExecutionOrderAwareFakeStep separateTestStep2 =
        new ExecutionOrderAwareFakeStep("teststep2", "teststep2", 0, atomicExecutionOrder);
    BuildTarget separateTest2Target = BuildTargetFactory.newInstance("//:test2");
    FakeTestRule separateTest2 =
        new FakeTestRule(
            BuildRuleParamsFactory.createTrivialBuildRuleParams(separateTest2Target),
            new SourcePathResolver(new BuildRuleResolver()),
            ImmutableSet.<Label>of(),
            Optional.of(Paths.get("separateTestStep2OutputDir")),
            true, // runTestSeparately
            ImmutableList.<Step>of(separateTestStep2),
            new Callable<TestResults>() {
              @Override
              public TestResults call() {
                return fakeTestResults;
              }
            });

    ExecutionOrderAwareFakeStep separateTestStep3 =
        new ExecutionOrderAwareFakeStep("teststep3", "teststep3", 0, atomicExecutionOrder);
    BuildTarget separateTest3Target = BuildTargetFactory.newInstance("//:test3");
    FakeTestRule separateTest3 =
        new FakeTestRule(
            BuildRuleParamsFactory.createTrivialBuildRuleParams(separateTest3Target),
            new SourcePathResolver(new BuildRuleResolver()),
            ImmutableSet.<Label>of(),
            Optional.of(Paths.get("separateTestStep3OutputDir")),
            true, // runTestSeparately
            ImmutableList.<Step>of(separateTestStep3),
            new Callable<TestResults>() {
              @Override
              public TestResults call() {
                return fakeTestResults;
              }
            });

    ExecutionOrderAwareFakeStep parallelTestStep1 =
        new ExecutionOrderAwareFakeStep(
            "parallelteststep1", "parallelteststep1", 0, atomicExecutionOrder);
    BuildTarget parallelTest1Target = BuildTargetFactory.newInstance("//:paralleltest1");
    FakeTestRule parallelTest1 =
        new FakeTestRule(
            BuildRuleParamsFactory.createTrivialBuildRuleParams(parallelTest1Target),
            new SourcePathResolver(new BuildRuleResolver()),
            ImmutableSet.<Label>of(),
            Optional.of(Paths.get("parallelTestStep1OutputDir")),
            false, // runTestSeparately
            ImmutableList.<Step>of(parallelTestStep1),
            new Callable<TestResults>() {
              @Override
              public TestResults call() {
                return fakeTestResults;
              }
            });

    ExecutionOrderAwareFakeStep parallelTestStep2 =
        new ExecutionOrderAwareFakeStep(
            "parallelteststep2", "parallelteststep2", 0, atomicExecutionOrder);
    BuildTarget parallelTest2Target = BuildTargetFactory.newInstance("//:paralleltest2");
    FakeTestRule parallelTest2 =
        new FakeTestRule(
            BuildRuleParamsFactory.createTrivialBuildRuleParams(parallelTest2Target),
            new SourcePathResolver(new BuildRuleResolver()),
            ImmutableSet.<Label>of(),
            Optional.of(Paths.get("parallelTestStep2OutputDir")),
            false, // runTestSeparately
            ImmutableList.<Step>of(parallelTestStep2),
            new Callable<TestResults>() {
              @Override
              public TestResults call() {
                return fakeTestResults;
              }
            });

    ExecutionOrderAwareFakeStep parallelTestStep3 =
        new ExecutionOrderAwareFakeStep(
            "parallelteststep3", "parallelteststep3", 0, atomicExecutionOrder);
    BuildTarget parallelTest3Target = BuildTargetFactory.newInstance("//:paralleltest3");
    FakeTestRule parallelTest3 =
        new FakeTestRule(
            BuildRuleParamsFactory.createTrivialBuildRuleParams(parallelTest3Target),
            new SourcePathResolver(new BuildRuleResolver()),
            ImmutableSet.<Label>of(),
            Optional.of(Paths.get("parallelTestStep3OutputDir")),
            false, // runTestSeparately
            ImmutableList.<Step>of(parallelTestStep3),
            new Callable<TestResults>() {
              @Override
              public TestResults call() {
                return fakeTestResults;
              }
            });

    // We explicitly use an actual thread pool here; the logic should ensure the
    // separate tests are run in the correct order.
    ListeningExecutorService service =
        MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(3));
    FakeBuildEngine fakeBuildEngine =
        new FakeBuildEngine(
            ImmutableMap.<BuildTarget, BuildResult>builder()
                .put(
                    separateTest1Target,
                    new BuildResult(separateTest1, BUILT_LOCALLY, CacheResult.skip()))
                .put(
                    separateTest2Target,
                    new BuildResult(separateTest2, BUILT_LOCALLY, CacheResult.skip()))
                .put(
                    separateTest3Target,
                    new BuildResult(separateTest3, BUILT_LOCALLY, CacheResult.skip()))
                .put(
                    parallelTest1Target,
                    new BuildResult(parallelTest1, BUILT_LOCALLY, CacheResult.skip()))
                .put(
                    parallelTest2Target,
                    new BuildResult(parallelTest2, BUILT_LOCALLY, CacheResult.skip()))
                .put(
                    parallelTest3Target,
                    new BuildResult(parallelTest3, BUILT_LOCALLY, CacheResult.skip()))
                .build(),
            ImmutableMap.<BuildTarget, RuleKey>builder()
                .put(separateTest1Target, new RuleKey("00"))
                .put(separateTest2Target, new RuleKey("00"))
                .put(separateTest3Target, new RuleKey("00"))
                .put(parallelTest1Target, new RuleKey("00"))
                .put(parallelTest2Target, new RuleKey("00"))
                .put(parallelTest3Target, new RuleKey("00"))
                .build());
    ExecutionContext fakeExecutionContext =
        TestExecutionContext.newBuilder().setProjectFilesystem(new FakeProjectFilesystem()).build();
    DefaultStepRunner stepRunner = new DefaultStepRunner(fakeExecutionContext);
    int ret =
        TestRunning.runTests(
            commandRunnerParams,
            ImmutableList.<TestRule>of(
                separateTest1,
                parallelTest1,
                separateTest2,
                parallelTest2,
                separateTest3,
                parallelTest3),
            FakeBuildContext.NOOP_CONTEXT,
            fakeExecutionContext,
            DEFAULT_OPTIONS,
            service,
            fakeBuildEngine,
            stepRunner);

    assertThat(ret, equalTo(0));

    // The tests not marked as separate could run in any order -- but they must run
    // before the separate test steps.
    ImmutableSet<Optional<Integer>> expectedParallelStepExecutionOrderSet =
        ImmutableSet.<Optional<Integer>>builder()
            .add(Optional.of(0))
            .add(Optional.of(1))
            .add(Optional.of(2))
            .add(Optional.of(3))
            .add(Optional.of(4))
            .add(Optional.of(5))
            .build();

    ImmutableSet<Optional<Integer>> actualParallelStepExecutionOrderSet =
        ImmutableSet.<Optional<Integer>>builder()
            .add(parallelTestStep1.getExecutionBeginOrder())
            .add(parallelTestStep1.getExecutionEndOrder())
            .add(parallelTestStep2.getExecutionBeginOrder())
            .add(parallelTestStep2.getExecutionEndOrder())
            .add(parallelTestStep3.getExecutionBeginOrder())
            .add(parallelTestStep3.getExecutionEndOrder())
            .build();

    LOG.debug(
        "Expected parallel execution order: %s Actual parallel execution order: %s",
        expectedParallelStepExecutionOrderSet, actualParallelStepExecutionOrderSet);

    // We allow the parallel steps to begin and end in any order (note the thread
    // pool of size 3 above), so we use a set.
    assertThat(actualParallelStepExecutionOrderSet, equalTo(expectedParallelStepExecutionOrderSet));

    // The separate test steps must begin and end in a specific order, so we use a list.
    ImmutableList<Optional<Integer>> expectedSeparateStepExecutionOrderList =
        ImmutableList.<Optional<Integer>>builder()
            .add(Optional.of(6))
            .add(Optional.of(7))
            .add(Optional.of(8))
            .add(Optional.of(9))
            .add(Optional.of(10))
            .add(Optional.of(11))
            .build();

    ImmutableList<Optional<Integer>> actualSeparateStepExecutionOrderList =
        ImmutableList.<Optional<Integer>>builder()
            .add(separateTestStep1.getExecutionBeginOrder())
            .add(separateTestStep1.getExecutionEndOrder())
            .add(separateTestStep2.getExecutionBeginOrder())
            .add(separateTestStep2.getExecutionEndOrder())
            .add(separateTestStep3.getExecutionBeginOrder())
            .add(separateTestStep3.getExecutionEndOrder())
            .build();

    LOG.debug(
        "Expected separate execution order: %s Actual separate execution order: %s",
        expectedSeparateStepExecutionOrderList, actualSeparateStepExecutionOrderList);

    assertThat(
        actualSeparateStepExecutionOrderList, equalTo(expectedSeparateStepExecutionOrderList));
  }
Beispiel #4
0
  @Test
  public void whenAllTestsAreSeparateTestsRunInOrder() throws Exception {
    CommandRunnerParams commandRunnerParams = CommandRunnerParamsForTesting.builder().build();

    AtomicInteger atomicExecutionOrder = new AtomicInteger(0);
    ExecutionOrderAwareFakeStep separateTestStep1 =
        new ExecutionOrderAwareFakeStep("teststep1", "teststep1", 0, atomicExecutionOrder);
    final TestResults fakeTestResults =
        new TestResults(
            ImmutableList.of(
                new TestCaseSummary(
                    "TestCase",
                    ImmutableList.of(
                        new TestResultSummary(
                            "TestCaseResult",
                            "passTest",
                            ResultType.SUCCESS,
                            5000,
                            null,
                            null,
                            null,
                            null)))));
    BuildTarget separateTest1Target = BuildTargetFactory.newInstance("//:test1");
    FakeTestRule separateTest1 =
        new FakeTestRule(
            BuildRuleParamsFactory.createTrivialBuildRuleParams(separateTest1Target),
            new SourcePathResolver(new BuildRuleResolver()),
            ImmutableSet.<Label>of(),
            Optional.of(Paths.get("separateTestStep1OutputDir")),
            true, // runTestSeparately
            ImmutableList.<Step>of(separateTestStep1),
            new Callable<TestResults>() {
              @Override
              public TestResults call() {
                return fakeTestResults;
              }
            });

    ExecutionOrderAwareFakeStep separateTestStep2 =
        new ExecutionOrderAwareFakeStep("teststep2", "teststep2", 0, atomicExecutionOrder);
    BuildTarget separateTest2Target = BuildTargetFactory.newInstance("//:test2");
    FakeTestRule separateTest2 =
        new FakeTestRule(
            BuildRuleParamsFactory.createTrivialBuildRuleParams(separateTest2Target),
            new SourcePathResolver(new BuildRuleResolver()),
            ImmutableSet.<Label>of(),
            Optional.of(Paths.get("separateTestStep2OutputDir")),
            true, // runTestSeparately
            ImmutableList.<Step>of(separateTestStep2),
            new Callable<TestResults>() {
              @Override
              public TestResults call() {
                return fakeTestResults;
              }
            });

    ExecutionOrderAwareFakeStep separateTestStep3 =
        new ExecutionOrderAwareFakeStep("teststep3", "teststep3", 0, atomicExecutionOrder);
    BuildTarget separateTest3Target = BuildTargetFactory.newInstance("//:test3");
    FakeTestRule separateTest3 =
        new FakeTestRule(
            BuildRuleParamsFactory.createTrivialBuildRuleParams(separateTest3Target),
            new SourcePathResolver(new BuildRuleResolver()),
            ImmutableSet.<Label>of(),
            Optional.of(Paths.get("separateTestStep3OutputDir")),
            true, // runTestSeparately
            ImmutableList.<Step>of(separateTestStep3),
            new Callable<TestResults>() {
              @Override
              public TestResults call() {
                return fakeTestResults;
              }
            });

    // We explicitly use an actual thread pool here; the logic should ensure the
    // separate tests are run in the correct order.
    ListeningExecutorService service =
        MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(3));
    FakeBuildEngine fakeBuildEngine =
        new FakeBuildEngine(
            ImmutableMap.of(
                separateTest1Target,
                    new BuildResult(separateTest1, BUILT_LOCALLY, CacheResult.skip()),
                separateTest2Target,
                    new BuildResult(separateTest2, BUILT_LOCALLY, CacheResult.skip()),
                separateTest3Target,
                    new BuildResult(separateTest3, BUILT_LOCALLY, CacheResult.skip())),
            ImmutableMap.of(
                separateTest1Target, new RuleKey("00"),
                separateTest2Target, new RuleKey("00"),
                separateTest3Target, new RuleKey("00")));
    ExecutionContext fakeExecutionContext =
        TestExecutionContext.newBuilder().setProjectFilesystem(new FakeProjectFilesystem()).build();
    DefaultStepRunner stepRunner = new DefaultStepRunner(fakeExecutionContext);
    int ret =
        TestRunning.runTests(
            commandRunnerParams,
            ImmutableList.<TestRule>of(separateTest1, separateTest2, separateTest3),
            FakeBuildContext.NOOP_CONTEXT,
            fakeExecutionContext,
            DEFAULT_OPTIONS,
            service,
            fakeBuildEngine,
            stepRunner);

    assertThat(ret, equalTo(0));
    assertThat(separateTestStep1.getExecutionBeginOrder(), equalTo(Optional.of(0)));
    assertThat(separateTestStep1.getExecutionEndOrder(), equalTo(Optional.of(1)));
    assertThat(separateTestStep2.getExecutionBeginOrder(), equalTo(Optional.of(2)));
    assertThat(separateTestStep2.getExecutionEndOrder(), equalTo(Optional.of(3)));
    assertThat(separateTestStep3.getExecutionBeginOrder(), equalTo(Optional.of(4)));
    assertThat(separateTestStep3.getExecutionEndOrder(), equalTo(Optional.of(5)));
  }