@Test
public void testBrokenOperatorProcessWhileClosing() throws Exception {
BrokenOperator brokenOperator =
new BrokenOperator(
driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"), true);
final Driver driver =
new Driver(driverContext, brokenOperator, createSinkOperator(brokenOperator));
assertSame(driver.getDriverContext(), driverContext);
// block thread in operator close
Future<Boolean> driverClose =
executor.submit(
new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
driver.close();
return true;
}
});
brokenOperator.waitForLocked();
assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
assertTrue(driver.isFinished());
brokenOperator.unlock();
assertTrue(driverClose.get());
}
@Override
public synchronized ListenableFuture<?> processFor(Duration duration) {
if (driver == null) {
driver = driverSplitRunnerFactory.createDriver(partitionedSplit);
}
return driver.processFor(duration);
}
@Test
public void testAddSourceFinish() {
PlanNodeId sourceId = new PlanNodeId("source");
TableScanOperator source =
new TableScanOperator(
driverContext.addOperatorContext(99, "values"),
sourceId,
new DataStreamProvider() {
@Override
public Operator createNewDataStream(
OperatorContext operatorContext, Split split, List<ColumnHandle> columns) {
return new ValuesOperator(
driverContext.addOperatorContext(0, "values"),
rowPagesBuilder(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG)
.addSequencePage(10, 20, 30, 40)
.build());
}
},
ImmutableList.of(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG),
ImmutableList.<ColumnHandle>of());
MaterializingOperator sink = createSinkOperator(source);
Driver driver = new Driver(driverContext, source, sink);
assertSame(driver.getDriverContext(), driverContext);
assertFalse(driver.isFinished());
// todo TableScanOperator should be blocked until split is set
assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
assertFalse(driver.isFinished());
driver.updateSource(
new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, new MockSplit())), true));
assertFalse(driver.isFinished());
assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone());
assertTrue(driver.isFinished());
assertTrue(sink.isFinished());
assertTrue(source.isFinished());
}
@Test
public void testAddSourceFinish() {
PlanNodeId sourceId = new PlanNodeId("source");
final List<Type> types = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT);
TableScanOperator source =
new TableScanOperator(
driverContext.addOperatorContext(99, new PlanNodeId("test"), "values"),
sourceId,
new PageSourceProvider() {
@Override
public ConnectorPageSource createPageSource(
Session session, Split split, List<ColumnHandle> columns) {
return new FixedPageSource(
rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build());
}
},
types,
ImmutableList.<ColumnHandle>of());
PageConsumerOperator sink = createSinkOperator(source);
Driver driver = new Driver(driverContext, source, sink);
assertSame(driver.getDriverContext(), driverContext);
assertFalse(driver.isFinished());
assertFalse(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
assertFalse(driver.isFinished());
driver.updateSource(
new TaskSource(
sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, newMockSplit())), true));
assertFalse(driver.isFinished());
assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone());
assertTrue(driver.isFinished());
assertTrue(sink.isFinished());
assertTrue(source.isFinished());
}
@Override
public ListenableFuture<?> processFor(Duration duration) {
Driver driver;
synchronized (this) {
// if close() was called before we get here, there's not point in even creating the driver
if (closed) {
return Futures.immediateFuture(null);
}
if (this.driver == null) {
this.driver = driverSplitRunnerFactory.createDriver(driverContext, partitionedSplit);
}
driver = this.driver;
}
return driver.processFor(duration);
}
@Test
public void testNormalFinish() {
List<Type> types = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT);
ValuesOperator source =
new ValuesOperator(
driverContext.addOperatorContext(0, new PlanNodeId("test"), "values"),
types,
rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build());
Operator sink = createSinkOperator(source);
Driver driver = new Driver(driverContext, source, sink);
assertSame(driver.getDriverContext(), driverContext);
assertFalse(driver.isFinished());
ListenableFuture<?> blocked = driver.processFor(new Duration(1, TimeUnit.SECONDS));
assertTrue(blocked.isDone());
assertTrue(driver.isFinished());
assertTrue(sink.isFinished());
assertTrue(source.isFinished());
}
@Test
public void testNormalFinish() {
ValuesOperator source =
new ValuesOperator(
driverContext.addOperatorContext(0, "values"),
rowPagesBuilder(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG)
.addSequencePage(10, 20, 30, 40)
.build());
MaterializingOperator sink = createSinkOperator(source);
Driver driver = new Driver(driverContext, source, sink);
assertSame(driver.getDriverContext(), driverContext);
assertFalse(driver.isFinished());
ListenableFuture<?> blocked = driver.processFor(new Duration(1, TimeUnit.SECONDS));
assertTrue(blocked.isDone());
assertTrue(driver.isFinished());
assertTrue(sink.isFinished());
assertTrue(source.isFinished());
}
@Test
public void testBrokenOperatorAddSource() throws Exception {
PlanNodeId sourceId = new PlanNodeId("source");
final List<Type> types = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT);
// create a table scan operator that does not block, which will cause the driver loop to busy
// wait
TableScanOperator source =
new NotBlockedTableScanOperator(
driverContext.addOperatorContext(99, new PlanNodeId("test"), "values"),
sourceId,
new PageSourceProvider() {
@Override
public ConnectorPageSource createPageSource(
Session session, Split split, List<ColumnHandle> columns) {
return new FixedPageSource(
rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build());
}
},
types,
ImmutableList.<ColumnHandle>of());
BrokenOperator brokenOperator =
new BrokenOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"));
final Driver driver = new Driver(driverContext, source, brokenOperator);
// block thread in operator processing
Future<Boolean> driverProcessFor =
executor.submit(
new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone();
}
});
brokenOperator.waitForLocked();
assertSame(driver.getDriverContext(), driverContext);
assertFalse(driver.isFinished());
// processFor always returns NOT_BLOCKED, because DriveLockResult was not acquired
assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
assertFalse(driver.isFinished());
driver.updateSource(
new TaskSource(
sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, newMockSplit())), true));
assertFalse(driver.isFinished());
// processFor always returns NOT_BLOCKED, because DriveLockResult was not acquired
assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone());
assertFalse(driver.isFinished());
driver.close();
assertTrue(driver.isFinished());
try {
driverProcessFor.get(1, TimeUnit.SECONDS);
fail("Expected InterruptedException");
} catch (ExecutionException e) {
checkArgument(
getRootCause(e) instanceof InterruptedException,
"Expected root cause exception to be an instance of InterruptedException");
}
}
@Test
public void testBrokenOperatorAddSource() throws Exception {
PlanNodeId sourceId = new PlanNodeId("source");
TableScanOperator source =
new TableScanOperator(
driverContext.addOperatorContext(99, "values"),
sourceId,
new DataStreamProvider() {
@Override
public Operator createNewDataStream(
OperatorContext operatorContext, Split split, List<ColumnHandle> columns) {
return new ValuesOperator(
driverContext.addOperatorContext(0, "values"),
rowPagesBuilder(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG)
.addSequencePage(10, 20, 30, 40)
.build());
}
},
ImmutableList.of(SINGLE_VARBINARY, SINGLE_LONG, SINGLE_LONG),
ImmutableList.<ColumnHandle>of());
BrokenOperator brokenOperator =
new BrokenOperator(driverContext.addOperatorContext(0, "source"));
final Driver driver = new Driver(driverContext, source, brokenOperator);
// block thread in operator processing
Future<Boolean> driverProcessFor =
executor.submit(
new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone();
}
});
brokenOperator.waitForLocked();
assertSame(driver.getDriverContext(), driverContext);
assertFalse(driver.isFinished());
// todo TableScanOperator should be blocked until split is set
assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
assertFalse(driver.isFinished());
driver.updateSource(
new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, new MockSplit())), true));
assertFalse(driver.isFinished());
assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone());
assertFalse(driver.isFinished());
driver.close();
assertTrue(driver.isFinished());
try {
driverProcessFor.get(1, TimeUnit.SECONDS);
fail("Expected InterruptedException");
} catch (ExecutionException e) {
checkArgument(
getRootCause(e) instanceof InterruptedException,
"Expected root cause exception to be an instance of InterruptedException");
}
}