@Test
public void testExecuteConcurrentModification() throws Exception {
String sql = "SELECT * FROM " + DATA_SERVICE_NAME;
DataServiceExecutor executor =
new DataServiceExecutor.Builder(new SQL(sql), dataService, context)
.prepareExecution(false)
.sqlTransGenerator(sqlTransGenerator)
.serviceTrans(serviceTrans)
.genTrans(genTrans)
.build();
final DataServiceExecutor.ExecutionPoint stage = DataServiceExecutor.ExecutionPoint.OPTIMIZE;
final ListMultimap<DataServiceExecutor.ExecutionPoint, Runnable> listenerMap =
executor.getListenerMap();
final Runnable task =
new Runnable() {
@Override
public void run() {
// Remove itself on run
assertTrue(listenerMap.remove(stage, this));
}
};
listenerMap.put(stage, task);
executor.executeQuery();
// Note the error reported to logs
verify(genTrans.getLogChannel())
.logError(anyString(), eq(stage), eq(ImmutableList.of(task)), eq(ImmutableList.of()));
}
@Test
public void testQueryWithParams() throws Exception {
String sql =
"SELECT * FROM "
+ DATA_SERVICE_NAME
+ " WHERE PARAMETER('foo') = 'bar' AND PARAMETER('baz') = 'bop'";
final SQL theSql = new SQL(sql);
DataServiceExecutor executor =
new DataServiceExecutor.Builder(theSql, dataService, context)
.serviceTrans(serviceTrans)
.sqlTransGenerator(sqlTransGenerator)
.genTrans(genTrans)
.parameters(ImmutableMap.of("BUILD_PARAM", "TRUE"))
.build();
List<Condition> conditions = theSql.getWhereCondition().getCondition().getChildren();
assertEquals(2, conditions.size());
for (Condition condition : conditions) {
// verifies that each of the parameter conditions have their left and right valuename
// set to null after executor initialization. This prevents failure due to non-existent
// fieldnames being present.
assertNull(condition.getLeftValuename());
assertNull(condition.getRightValuename());
}
assertThat(
executor.getParameters(),
equalTo(
(Map<String, String>)
ImmutableMap.of(
"baz", "bop",
"foo", "bar",
"BUILD_PARAM", "TRUE")));
// Late parameter modification is okay
executor.getParameters().put("AFTER_BUILD", "TRUE");
// Parameters should not be set on the trans until execute
verify(serviceTrans, never()).setParameterValue(anyString(), anyString());
executor.executeQuery();
// verify that the parameter values were correctly extracted from the WHERE and applied
verify(serviceTrans).setParameterValue("foo", "bar");
verify(serviceTrans).setParameterValue("baz", "bop");
verify(serviceTrans).setParameterValue("BUILD_PARAM", "TRUE");
verify(serviceTrans).setParameterValue("AFTER_BUILD", "TRUE");
}
@Test
public void testExecuteQuery() throws Exception {
SQL sql = new SQL("SELECT * FROM " + DATA_SERVICE_NAME);
StepInterface serviceStep = serviceTrans.findRunThread(DATA_SERVICE_STEP);
StepInterface resultStep = genTrans.findRunThread(RESULT_STEP_NAME);
when(serviceTrans.getTransMeta().listParameters()).thenReturn(new String[0]);
PushDownOptimizationMeta optimization = mock(PushDownOptimizationMeta.class);
when(optimization.isEnabled()).thenReturn(true);
dataService.getPushDownOptimizationMeta().add(optimization);
IMetaStore metastore = mock(IMetaStore.class);
DataServiceExecutor executor =
new DataServiceExecutor.Builder(sql, dataService, context)
.serviceTrans(serviceTrans)
.sqlTransGenerator(sqlTransGenerator)
.genTrans(genTrans)
.metastore(metastore)
.build();
ArgumentCaptor<String> objectIds = ArgumentCaptor.forClass(String.class);
verify(serviceTrans).setContainerObjectId(objectIds.capture());
when(serviceTrans.getContainerObjectId()).thenReturn(objectIds.getValue());
verify(genTrans).setContainerObjectId(objectIds.capture());
when(genTrans.getContainerObjectId()).thenReturn(objectIds.getValue());
verify(serviceTrans).setMetaStore(metastore);
verify(genTrans).setMetaStore(metastore);
RowProducer sqlTransRowProducer = mock(RowProducer.class);
when(genTrans.addRowProducer(INJECTOR_STEP_NAME, 0)).thenReturn(sqlTransRowProducer);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
// Start Execution
executor.executeQuery(new DataOutputStream(outputStream));
// Check header was written
assertThat(outputStream.size(), greaterThan(0));
outputStream.reset();
InOrder genTransStartup = inOrder(genTrans, resultStep);
InOrder serviceTransStartup = inOrder(optimization, serviceTrans, serviceStep);
ArgumentCaptor<RowListener> listenerArgumentCaptor = ArgumentCaptor.forClass(RowListener.class);
ArgumentCaptor<StepListener> resultStepListener = ArgumentCaptor.forClass(StepListener.class);
ArgumentCaptor<TransListener> transListenerCaptor =
ArgumentCaptor.forClass(TransListener.class);
genTransStartup.verify(genTrans).addTransListener(transListenerCaptor.capture());
genTransStartup.verify(genTrans).addRowProducer(INJECTOR_STEP_NAME, 0);
genTransStartup.verify(resultStep).addStepListener(resultStepListener.capture());
genTransStartup.verify(resultStep).addRowListener(listenerArgumentCaptor.capture());
RowListener clientRowListener = listenerArgumentCaptor.getValue();
genTransStartup.verify(genTrans).startThreads();
serviceTransStartup.verify(optimization).activate(executor);
serviceTransStartup.verify(serviceStep).addRowListener(listenerArgumentCaptor.capture());
serviceTransStartup.verify(serviceTrans).startThreads();
// Verify linkage
RowListener serviceRowListener = listenerArgumentCaptor.getValue();
assertNotNull(serviceRowListener);
// Push row from service to sql Trans
RowMetaInterface rowMeta = genTrans.getTransMeta().getStepFields(RESULT_STEP_NAME);
Object[] data;
for (int i = 0; i < 50; i++) {
data = new Object[] {i};
Object[] dataClone = {i};
when(rowMeta.cloneRow(data)).thenReturn(dataClone);
serviceRowListener.rowWrittenEvent(rowMeta, data);
verify(sqlTransRowProducer)
.putRowWait(
same(rowMeta),
and(eq(dataClone), not(same(data))),
any(Long.class),
any(TimeUnit.class));
verify(rowMeta).cloneRow(data);
}
doReturn(true).when(serviceTrans).isRunning();
resultStepListener.getValue().stepFinished(genTrans, resultStep.getStepMeta(), resultStep);
verify(serviceTrans).stopAll();
// Verify Service Trans finished
ArgumentCaptor<StepListener> serviceStepListener = ArgumentCaptor.forClass(StepListener.class);
verify(serviceStep).addStepListener(serviceStepListener.capture());
serviceStepListener
.getValue()
.stepFinished(serviceTrans, serviceStep.getStepMeta(), serviceStep);
verify(sqlTransRowProducer).finished();
// Push row from service to sql Trans
for (int i = 0; i < 50; i++) {
Object[] row = {i};
clientRowListener.rowWrittenEvent(rowMeta, row);
}
transListenerCaptor.getValue().transFinished(genTrans);
InOrder writeRows = inOrder(rowMeta);
ArgumentCaptor<DataOutputStream> streamCaptor = ArgumentCaptor.forClass(DataOutputStream.class);
writeRows.verify(rowMeta).writeMeta(streamCaptor.capture());
DataOutputStream dataOutputStream = streamCaptor.getValue();
writeRows
.verify(rowMeta, times(50))
.writeData(same(dataOutputStream), argThat(arrayWithSize(1)));
writeRows.verifyNoMoreInteractions();
executor.waitUntilFinished();
verify(serviceTrans).waitUntilFinished();
verify(genTrans).waitUntilFinished();
}
