@Test
public void whenSeparateTestFailsThenBuildFails() throws Exception {
CommandRunnerParams commandRunnerParams = CommandRunnerParamsForTesting.builder().build();
final TestResults failingTestResults =
new TestResults(
ImmutableList.of(
new TestCaseSummary(
"TestCase",
ImmutableList.of(
new TestResultSummary(
"TestCaseResult",
"failTest",
ResultType.FAILURE,
5000,
null,
null,
null,
null)))));
BuildTarget failingTestTarget = BuildTargetFactory.newInstance("//:failingtest");
FakeTestRule failingTest =
new FakeTestRule(
BuildRuleParamsFactory.createTrivialBuildRuleParams(failingTestTarget),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSet.<Label>of(),
Optional.of(Paths.get("failingTestStep1OutputDir")),
true, // runTestSeparately
ImmutableList.<Step>of(),
new Callable<TestResults>() {
@Override
public TestResults call() {
return failingTestResults;
}
});
ListeningExecutorService service =
MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(3));
FakeBuildEngine fakeBuildEngine =
new FakeBuildEngine(
ImmutableMap.of(
failingTestTarget, new BuildResult(failingTest, BUILT_LOCALLY, CacheResult.skip())),
ImmutableMap.of(failingTestTarget, new RuleKey("00")));
ExecutionContext fakeExecutionContext =
TestExecutionContext.newBuilder().setProjectFilesystem(new FakeProjectFilesystem()).build();
DefaultStepRunner stepRunner = new DefaultStepRunner(fakeExecutionContext);
int ret =
TestRunning.runTests(
commandRunnerParams,
ImmutableList.<TestRule>of(failingTest),
FakeBuildContext.NOOP_CONTEXT,
fakeExecutionContext,
DEFAULT_OPTIONS,
service,
fakeBuildEngine,
stepRunner);
assertThat(ret, equalTo(TestRunning.TEST_FAILURES_EXIT_CODE));
}
private int runTestsInternal(
CommandRunnerParams params,
BuildEngine buildEngine,
Build build,
Iterable<TestRule> testRules)
throws InterruptedException, IOException {
if (!withDashArguments.isEmpty()) {
params
.getBuckEventBus()
.post(
ConsoleEvent.severe("Unexpected arguments after \"--\" when using internal runner"));
return 1;
}
ConcurrencyLimit concurrencyLimit =
new ConcurrencyLimit(
getNumTestThreads(params.getBuckConfig()), params.getBuckConfig().getLoadLimit());
try (CommandThreadManager testPool =
new CommandThreadManager(
"Test-Run", params.getBuckConfig().getWorkQueueExecutionOrder(), concurrencyLimit)) {
return TestRunning.runTests(
params,
testRules,
Preconditions.checkNotNull(build.getBuildContext()),
build.getExecutionContext(),
getTestRunningOptions(params),
testPool.getExecutor(),
buildEngine,
new DefaultStepRunner(build.getExecutionContext()));
} catch (ExecutionException e) {
params
.getBuckEventBus()
.post(ConsoleEvent.severe(MoreExceptions.getHumanReadableOrLocalizedMessage(e)));
return 1;
}
}
@Test
public void whenSomeTestsAreSeparateThenSeparateTestsRunAtEnd() throws Exception {
CommandRunnerParams commandRunnerParams = CommandRunnerParamsForTesting.builder().build();
AtomicInteger atomicExecutionOrder = new AtomicInteger(0);
ExecutionOrderAwareFakeStep separateTestStep1 =
new ExecutionOrderAwareFakeStep("teststep1", "teststep1", 0, atomicExecutionOrder);
final TestResults fakeTestResults =
new TestResults(
ImmutableList.of(
new TestCaseSummary(
"TestCase",
ImmutableList.of(
new TestResultSummary(
"TestCaseResult",
"passTest",
ResultType.SUCCESS,
5000,
null,
null,
null,
null)))));
BuildTarget separateTest1Target = BuildTargetFactory.newInstance("//:test1");
FakeTestRule separateTest1 =
new FakeTestRule(
BuildRuleParamsFactory.createTrivialBuildRuleParams(separateTest1Target),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSet.<Label>of(),
Optional.of(Paths.get("separateTestStep1OutputDir")),
true, // runTestSeparately
ImmutableList.<Step>of(separateTestStep1),
new Callable<TestResults>() {
@Override
public TestResults call() {
return fakeTestResults;
}
});
ExecutionOrderAwareFakeStep separateTestStep2 =
new ExecutionOrderAwareFakeStep("teststep2", "teststep2", 0, atomicExecutionOrder);
BuildTarget separateTest2Target = BuildTargetFactory.newInstance("//:test2");
FakeTestRule separateTest2 =
new FakeTestRule(
BuildRuleParamsFactory.createTrivialBuildRuleParams(separateTest2Target),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSet.<Label>of(),
Optional.of(Paths.get("separateTestStep2OutputDir")),
true, // runTestSeparately
ImmutableList.<Step>of(separateTestStep2),
new Callable<TestResults>() {
@Override
public TestResults call() {
return fakeTestResults;
}
});
ExecutionOrderAwareFakeStep separateTestStep3 =
new ExecutionOrderAwareFakeStep("teststep3", "teststep3", 0, atomicExecutionOrder);
BuildTarget separateTest3Target = BuildTargetFactory.newInstance("//:test3");
FakeTestRule separateTest3 =
new FakeTestRule(
BuildRuleParamsFactory.createTrivialBuildRuleParams(separateTest3Target),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSet.<Label>of(),
Optional.of(Paths.get("separateTestStep3OutputDir")),
true, // runTestSeparately
ImmutableList.<Step>of(separateTestStep3),
new Callable<TestResults>() {
@Override
public TestResults call() {
return fakeTestResults;
}
});
ExecutionOrderAwareFakeStep parallelTestStep1 =
new ExecutionOrderAwareFakeStep(
"parallelteststep1", "parallelteststep1", 0, atomicExecutionOrder);
BuildTarget parallelTest1Target = BuildTargetFactory.newInstance("//:paralleltest1");
FakeTestRule parallelTest1 =
new FakeTestRule(
BuildRuleParamsFactory.createTrivialBuildRuleParams(parallelTest1Target),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSet.<Label>of(),
Optional.of(Paths.get("parallelTestStep1OutputDir")),
false, // runTestSeparately
ImmutableList.<Step>of(parallelTestStep1),
new Callable<TestResults>() {
@Override
public TestResults call() {
return fakeTestResults;
}
});
ExecutionOrderAwareFakeStep parallelTestStep2 =
new ExecutionOrderAwareFakeStep(
"parallelteststep2", "parallelteststep2", 0, atomicExecutionOrder);
BuildTarget parallelTest2Target = BuildTargetFactory.newInstance("//:paralleltest2");
FakeTestRule parallelTest2 =
new FakeTestRule(
BuildRuleParamsFactory.createTrivialBuildRuleParams(parallelTest2Target),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSet.<Label>of(),
Optional.of(Paths.get("parallelTestStep2OutputDir")),
false, // runTestSeparately
ImmutableList.<Step>of(parallelTestStep2),
new Callable<TestResults>() {
@Override
public TestResults call() {
return fakeTestResults;
}
});
ExecutionOrderAwareFakeStep parallelTestStep3 =
new ExecutionOrderAwareFakeStep(
"parallelteststep3", "parallelteststep3", 0, atomicExecutionOrder);
BuildTarget parallelTest3Target = BuildTargetFactory.newInstance("//:paralleltest3");
FakeTestRule parallelTest3 =
new FakeTestRule(
BuildRuleParamsFactory.createTrivialBuildRuleParams(parallelTest3Target),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSet.<Label>of(),
Optional.of(Paths.get("parallelTestStep3OutputDir")),
false, // runTestSeparately
ImmutableList.<Step>of(parallelTestStep3),
new Callable<TestResults>() {
@Override
public TestResults call() {
return fakeTestResults;
}
});
// We explicitly use an actual thread pool here; the logic should ensure the
// separate tests are run in the correct order.
ListeningExecutorService service =
MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(3));
FakeBuildEngine fakeBuildEngine =
new FakeBuildEngine(
ImmutableMap.<BuildTarget, BuildResult>builder()
.put(
separateTest1Target,
new BuildResult(separateTest1, BUILT_LOCALLY, CacheResult.skip()))
.put(
separateTest2Target,
new BuildResult(separateTest2, BUILT_LOCALLY, CacheResult.skip()))
.put(
separateTest3Target,
new BuildResult(separateTest3, BUILT_LOCALLY, CacheResult.skip()))
.put(
parallelTest1Target,
new BuildResult(parallelTest1, BUILT_LOCALLY, CacheResult.skip()))
.put(
parallelTest2Target,
new BuildResult(parallelTest2, BUILT_LOCALLY, CacheResult.skip()))
.put(
parallelTest3Target,
new BuildResult(parallelTest3, BUILT_LOCALLY, CacheResult.skip()))
.build(),
ImmutableMap.<BuildTarget, RuleKey>builder()
.put(separateTest1Target, new RuleKey("00"))
.put(separateTest2Target, new RuleKey("00"))
.put(separateTest3Target, new RuleKey("00"))
.put(parallelTest1Target, new RuleKey("00"))
.put(parallelTest2Target, new RuleKey("00"))
.put(parallelTest3Target, new RuleKey("00"))
.build());
ExecutionContext fakeExecutionContext =
TestExecutionContext.newBuilder().setProjectFilesystem(new FakeProjectFilesystem()).build();
DefaultStepRunner stepRunner = new DefaultStepRunner(fakeExecutionContext);
int ret =
TestRunning.runTests(
commandRunnerParams,
ImmutableList.<TestRule>of(
separateTest1,
parallelTest1,
separateTest2,
parallelTest2,
separateTest3,
parallelTest3),
FakeBuildContext.NOOP_CONTEXT,
fakeExecutionContext,
DEFAULT_OPTIONS,
service,
fakeBuildEngine,
stepRunner);
assertThat(ret, equalTo(0));
// The tests not marked as separate could run in any order -- but they must run
// before the separate test steps.
ImmutableSet<Optional<Integer>> expectedParallelStepExecutionOrderSet =
ImmutableSet.<Optional<Integer>>builder()
.add(Optional.of(0))
.add(Optional.of(1))
.add(Optional.of(2))
.add(Optional.of(3))
.add(Optional.of(4))
.add(Optional.of(5))
.build();
ImmutableSet<Optional<Integer>> actualParallelStepExecutionOrderSet =
ImmutableSet.<Optional<Integer>>builder()
.add(parallelTestStep1.getExecutionBeginOrder())
.add(parallelTestStep1.getExecutionEndOrder())
.add(parallelTestStep2.getExecutionBeginOrder())
.add(parallelTestStep2.getExecutionEndOrder())
.add(parallelTestStep3.getExecutionBeginOrder())
.add(parallelTestStep3.getExecutionEndOrder())
.build();
LOG.debug(
"Expected parallel execution order: %s Actual parallel execution order: %s",
expectedParallelStepExecutionOrderSet, actualParallelStepExecutionOrderSet);
// We allow the parallel steps to begin and end in any order (note the thread
// pool of size 3 above), so we use a set.
assertThat(actualParallelStepExecutionOrderSet, equalTo(expectedParallelStepExecutionOrderSet));
// The separate test steps must begin and end in a specific order, so we use a list.
ImmutableList<Optional<Integer>> expectedSeparateStepExecutionOrderList =
ImmutableList.<Optional<Integer>>builder()
.add(Optional.of(6))
.add(Optional.of(7))
.add(Optional.of(8))
.add(Optional.of(9))
.add(Optional.of(10))
.add(Optional.of(11))
.build();
ImmutableList<Optional<Integer>> actualSeparateStepExecutionOrderList =
ImmutableList.<Optional<Integer>>builder()
.add(separateTestStep1.getExecutionBeginOrder())
.add(separateTestStep1.getExecutionEndOrder())
.add(separateTestStep2.getExecutionBeginOrder())
.add(separateTestStep2.getExecutionEndOrder())
.add(separateTestStep3.getExecutionBeginOrder())
.add(separateTestStep3.getExecutionEndOrder())
.build();
LOG.debug(
"Expected separate execution order: %s Actual separate execution order: %s",
expectedSeparateStepExecutionOrderList, actualSeparateStepExecutionOrderList);
assertThat(
actualSeparateStepExecutionOrderList, equalTo(expectedSeparateStepExecutionOrderList));
}
@Test
public void whenAllTestsAreSeparateTestsRunInOrder() throws Exception {
CommandRunnerParams commandRunnerParams = CommandRunnerParamsForTesting.builder().build();
AtomicInteger atomicExecutionOrder = new AtomicInteger(0);
ExecutionOrderAwareFakeStep separateTestStep1 =
new ExecutionOrderAwareFakeStep("teststep1", "teststep1", 0, atomicExecutionOrder);
final TestResults fakeTestResults =
new TestResults(
ImmutableList.of(
new TestCaseSummary(
"TestCase",
ImmutableList.of(
new TestResultSummary(
"TestCaseResult",
"passTest",
ResultType.SUCCESS,
5000,
null,
null,
null,
null)))));
BuildTarget separateTest1Target = BuildTargetFactory.newInstance("//:test1");
FakeTestRule separateTest1 =
new FakeTestRule(
BuildRuleParamsFactory.createTrivialBuildRuleParams(separateTest1Target),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSet.<Label>of(),
Optional.of(Paths.get("separateTestStep1OutputDir")),
true, // runTestSeparately
ImmutableList.<Step>of(separateTestStep1),
new Callable<TestResults>() {
@Override
public TestResults call() {
return fakeTestResults;
}
});
ExecutionOrderAwareFakeStep separateTestStep2 =
new ExecutionOrderAwareFakeStep("teststep2", "teststep2", 0, atomicExecutionOrder);
BuildTarget separateTest2Target = BuildTargetFactory.newInstance("//:test2");
FakeTestRule separateTest2 =
new FakeTestRule(
BuildRuleParamsFactory.createTrivialBuildRuleParams(separateTest2Target),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSet.<Label>of(),
Optional.of(Paths.get("separateTestStep2OutputDir")),
true, // runTestSeparately
ImmutableList.<Step>of(separateTestStep2),
new Callable<TestResults>() {
@Override
public TestResults call() {
return fakeTestResults;
}
});
ExecutionOrderAwareFakeStep separateTestStep3 =
new ExecutionOrderAwareFakeStep("teststep3", "teststep3", 0, atomicExecutionOrder);
BuildTarget separateTest3Target = BuildTargetFactory.newInstance("//:test3");
FakeTestRule separateTest3 =
new FakeTestRule(
BuildRuleParamsFactory.createTrivialBuildRuleParams(separateTest3Target),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSet.<Label>of(),
Optional.of(Paths.get("separateTestStep3OutputDir")),
true, // runTestSeparately
ImmutableList.<Step>of(separateTestStep3),
new Callable<TestResults>() {
@Override
public TestResults call() {
return fakeTestResults;
}
});
// We explicitly use an actual thread pool here; the logic should ensure the
// separate tests are run in the correct order.
ListeningExecutorService service =
MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(3));
FakeBuildEngine fakeBuildEngine =
new FakeBuildEngine(
ImmutableMap.of(
separateTest1Target,
new BuildResult(separateTest1, BUILT_LOCALLY, CacheResult.skip()),
separateTest2Target,
new BuildResult(separateTest2, BUILT_LOCALLY, CacheResult.skip()),
separateTest3Target,
new BuildResult(separateTest3, BUILT_LOCALLY, CacheResult.skip())),
ImmutableMap.of(
separateTest1Target, new RuleKey("00"),
separateTest2Target, new RuleKey("00"),
separateTest3Target, new RuleKey("00")));
ExecutionContext fakeExecutionContext =
TestExecutionContext.newBuilder().setProjectFilesystem(new FakeProjectFilesystem()).build();
DefaultStepRunner stepRunner = new DefaultStepRunner(fakeExecutionContext);
int ret =
TestRunning.runTests(
commandRunnerParams,
ImmutableList.<TestRule>of(separateTest1, separateTest2, separateTest3),
FakeBuildContext.NOOP_CONTEXT,
fakeExecutionContext,
DEFAULT_OPTIONS,
service,
fakeBuildEngine,
stepRunner);
assertThat(ret, equalTo(0));
assertThat(separateTestStep1.getExecutionBeginOrder(), equalTo(Optional.of(0)));
assertThat(separateTestStep1.getExecutionEndOrder(), equalTo(Optional.of(1)));
assertThat(separateTestStep2.getExecutionBeginOrder(), equalTo(Optional.of(2)));
assertThat(separateTestStep2.getExecutionEndOrder(), equalTo(Optional.of(3)));
assertThat(separateTestStep3.getExecutionBeginOrder(), equalTo(Optional.of(4)));
assertThat(separateTestStep3.getExecutionEndOrder(), equalTo(Optional.of(5)));
}