@Test
public void testDynamicLimitDefault() throws Exception {
final String limitProp = "det.dataservice.dynamic.limit";
final int defaultLimit = 50000;
boolean userDef = dataService.isUserDefined();
try {
System.getProperties().remove(limitProp);
dataService.setUserDefined(false);
DataServiceExecutor executor =
new DataServiceExecutor.Builder(
new SQL("SELECT * FROM " + DATA_SERVICE_NAME), dataService, context)
.serviceTrans(serviceTrans)
.genTrans(genTrans)
.build();
assertEquals(defaultLimit, executor.getServiceRowLimit());
System.setProperty(limitProp, "baah");
executor =
new DataServiceExecutor.Builder(
new SQL("SELECT * FROM " + DATA_SERVICE_NAME), dataService, context)
.serviceTrans(serviceTrans)
.genTrans(genTrans)
.build();
assertEquals(defaultLimit, executor.getServiceRowLimit());
verify(logChannel).logError(anyString());
} finally {
dataService.setUserDefined(userDef);
System.getProperties().remove(limitProp);
}
}
@Test
public void testExecuteConcurrentModification() throws Exception {
String sql = "SELECT * FROM " + DATA_SERVICE_NAME;
DataServiceExecutor executor =
new DataServiceExecutor.Builder(new SQL(sql), dataService, context)
.prepareExecution(false)
.sqlTransGenerator(sqlTransGenerator)
.serviceTrans(serviceTrans)
.genTrans(genTrans)
.build();
final DataServiceExecutor.ExecutionPoint stage = DataServiceExecutor.ExecutionPoint.OPTIMIZE;
final ListMultimap<DataServiceExecutor.ExecutionPoint, Runnable> listenerMap =
executor.getListenerMap();
final Runnable task =
new Runnable() {
@Override
public void run() {
// Remove itself on run
assertTrue(listenerMap.remove(stage, this));
}
};
listenerMap.put(stage, task);
executor.executeQuery();
// Note the error reported to logs
verify(genTrans.getLogChannel())
.logError(anyString(), eq(stage), eq(ImmutableList.of(task)), eq(ImmutableList.of()));
}
@Test
public void testDynamicLimit() throws Exception {
final String limitProp = "det.dataservice.dynamic.limit";
boolean userDef = dataService.isUserDefined();
try {
System.setProperty(limitProp, "2");
dataService.setUserDefined(false);
DataServiceExecutor executor =
new DataServiceExecutor.Builder(
new SQL("SELECT * FROM " + DATA_SERVICE_NAME), dataService, context)
.serviceTrans(serviceTrans)
.genTrans(genTrans)
.build();
assertEquals(2, executor.getServiceRowLimit());
dataService.setUserDefined(true);
System.setProperty(limitProp, "2");
executor =
new DataServiceExecutor.Builder(
new SQL("SELECT * FROM " + DATA_SERVICE_NAME), dataService, context)
.serviceTrans(serviceTrans)
.genTrans(genTrans)
.build();
assertEquals(0, executor.getServiceRowLimit());
} finally {
dataService.setUserDefined(userDef);
System.getProperties().remove(limitProp);
}
}
@Test
public void testNullNotNullKeywords() throws Exception {
String sql =
"SELECT * FROM " + DATA_SERVICE_NAME + " WHERE column1 IS NOT NULL AND column2 IS NULL";
DataServiceExecutor executor =
new DataServiceExecutor.Builder(new SQL(sql), dataService, context)
.serviceTrans(serviceTrans)
.sqlTransGenerator(sqlTransGenerator)
.genTrans(genTrans)
.build();
Condition condition = executor.getSql().getWhereCondition().getCondition();
Condition condition1 = condition.getCondition(0);
Condition condition2 = condition.getCondition(1);
assertEquals("column1", condition1.getLeftValuename());
assertNull(condition1.getRightExact());
assertEquals(Condition.FUNC_NOT_NULL, condition1.getFunction());
assertEquals("column2", condition2.getLeftValuename());
assertNull(condition2.getRightExact());
assertEquals(Condition.FUNC_NULL, condition2.getFunction());
}
@Test
public void testConditionResolution() throws Exception {
RowMeta rowMeta = new RowMeta();
rowMeta.addValueMeta(new ValueMetaString("aString"));
rowMeta.addValueMeta(new ValueMetaInteger("anInt"));
rowMeta.addValueMeta(new ValueMetaDate("aDate"));
String query =
"SELECT * FROM "
+ DATA_SERVICE_NAME
+ " WHERE anInt = 2 AND aDate IN ('2014-12-05','2008-01-01')";
when(transMeta.getStepFields(DATA_SERVICE_STEP)).thenReturn(rowMeta);
DataServiceExecutor executor =
new DataServiceExecutor.Builder(new SQL(query), dataService, context)
.serviceTrans(transMeta)
.prepareExecution(false)
.build();
Condition condition = executor.getSql().getWhereCondition().getCondition();
Calendar calendar = Calendar.getInstance();
calendar.clear();
calendar.set(2014, Calendar.DECEMBER, 5);
assertThat(
condition.evaluate(rowMeta, new Object[] {"value", 2L, calendar.getTime()}), is(true));
assertThat(condition.evaluate(rowMeta, new Object[] {"value", 2L, new Date()}), is(false));
}
@Test
public void testQueryWithParams() throws Exception {
String sql =
"SELECT * FROM "
+ DATA_SERVICE_NAME
+ " WHERE PARAMETER('foo') = 'bar' AND PARAMETER('baz') = 'bop'";
final SQL theSql = new SQL(sql);
DataServiceExecutor executor =
new DataServiceExecutor.Builder(theSql, dataService, context)
.serviceTrans(serviceTrans)
.sqlTransGenerator(sqlTransGenerator)
.genTrans(genTrans)
.parameters(ImmutableMap.of("BUILD_PARAM", "TRUE"))
.build();
List<Condition> conditions = theSql.getWhereCondition().getCondition().getChildren();
assertEquals(2, conditions.size());
for (Condition condition : conditions) {
// verifies that each of the parameter conditions have their left and right valuename
// set to null after executor initialization. This prevents failure due to non-existent
// fieldnames being present.
assertNull(condition.getLeftValuename());
assertNull(condition.getRightValuename());
}
assertThat(
executor.getParameters(),
equalTo(
(Map<String, String>)
ImmutableMap.of(
"baz", "bop",
"foo", "bar",
"BUILD_PARAM", "TRUE")));
// Late parameter modification is okay
executor.getParameters().put("AFTER_BUILD", "TRUE");
// Parameters should not be set on the trans until execute
verify(serviceTrans, never()).setParameterValue(anyString(), anyString());
executor.executeQuery();
// verify that the parameter values were correctly extracted from the WHERE and applied
verify(serviceTrans).setParameterValue("foo", "bar");
verify(serviceTrans).setParameterValue("baz", "bop");
verify(serviceTrans).setParameterValue("BUILD_PARAM", "TRUE");
verify(serviceTrans).setParameterValue("AFTER_BUILD", "TRUE");
}
@Test
public void testIsComplete() throws KettleException {
String sql = "SELECT * FROM " + DATA_SERVICE_NAME;
when(genTrans.isStopped()).thenReturn(true);
DataServiceExecutor executor =
new DataServiceExecutor.Builder(new SQL(sql), dataService, context)
.serviceTrans(serviceTrans)
.sqlTransGenerator(sqlTransGenerator)
.genTrans(genTrans)
.build();
assertTrue(executor.isStopped());
}
@Test
public void testStop() throws KettleException {
String sql = "SELECT * FROM " + DATA_SERVICE_NAME;
when(serviceTrans.isRunning()).thenReturn(true);
when(genTrans.isRunning()).thenReturn(true);
DataServiceExecutor executor =
new DataServiceExecutor.Builder(new SQL(sql), dataService, context)
.serviceTrans(serviceTrans)
.sqlTransGenerator(sqlTransGenerator)
.genTrans(genTrans)
.build();
executor.stop();
verify(serviceTrans).stopAll();
verify(genTrans).stopAll();
}
@Test
public void testWithLazyConversion() throws Exception {
RowMeta rowMeta = new RowMeta();
ValueMetaInterface vm = new ValueMetaString("aBinaryStoredString");
vm.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING);
vm.setStorageMetadata(new ValueMetaString());
rowMeta.addValueMeta(vm);
String query = "SELECT * FROM " + DATA_SERVICE_NAME + " WHERE aBinaryStoredString = 'value'";
when(transMeta.getStepFields(DATA_SERVICE_STEP)).thenReturn(rowMeta);
DataServiceExecutor executor =
new DataServiceExecutor.Builder(new SQL(query), dataService, context)
.serviceTrans(new Trans(transMeta))
.prepareExecution(false)
.build();
executor
.getSql()
.getWhereCondition()
.getCondition()
.evaluate(rowMeta, new Object[] {"value".getBytes()});
}
@Test
public void testExecuteQuery() throws Exception {
SQL sql = new SQL("SELECT * FROM " + DATA_SERVICE_NAME);
StepInterface serviceStep = serviceTrans.findRunThread(DATA_SERVICE_STEP);
StepInterface resultStep = genTrans.findRunThread(RESULT_STEP_NAME);
when(serviceTrans.getTransMeta().listParameters()).thenReturn(new String[0]);
PushDownOptimizationMeta optimization = mock(PushDownOptimizationMeta.class);
when(optimization.isEnabled()).thenReturn(true);
dataService.getPushDownOptimizationMeta().add(optimization);
IMetaStore metastore = mock(IMetaStore.class);
DataServiceExecutor executor =
new DataServiceExecutor.Builder(sql, dataService, context)
.serviceTrans(serviceTrans)
.sqlTransGenerator(sqlTransGenerator)
.genTrans(genTrans)
.metastore(metastore)
.build();
ArgumentCaptor<String> objectIds = ArgumentCaptor.forClass(String.class);
verify(serviceTrans).setContainerObjectId(objectIds.capture());
when(serviceTrans.getContainerObjectId()).thenReturn(objectIds.getValue());
verify(genTrans).setContainerObjectId(objectIds.capture());
when(genTrans.getContainerObjectId()).thenReturn(objectIds.getValue());
verify(serviceTrans).setMetaStore(metastore);
verify(genTrans).setMetaStore(metastore);
RowProducer sqlTransRowProducer = mock(RowProducer.class);
when(genTrans.addRowProducer(INJECTOR_STEP_NAME, 0)).thenReturn(sqlTransRowProducer);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
// Start Execution
executor.executeQuery(new DataOutputStream(outputStream));
// Check header was written
assertThat(outputStream.size(), greaterThan(0));
outputStream.reset();
InOrder genTransStartup = inOrder(genTrans, resultStep);
InOrder serviceTransStartup = inOrder(optimization, serviceTrans, serviceStep);
ArgumentCaptor<RowListener> listenerArgumentCaptor = ArgumentCaptor.forClass(RowListener.class);
ArgumentCaptor<StepListener> resultStepListener = ArgumentCaptor.forClass(StepListener.class);
ArgumentCaptor<TransListener> transListenerCaptor =
ArgumentCaptor.forClass(TransListener.class);
genTransStartup.verify(genTrans).addTransListener(transListenerCaptor.capture());
genTransStartup.verify(genTrans).addRowProducer(INJECTOR_STEP_NAME, 0);
genTransStartup.verify(resultStep).addStepListener(resultStepListener.capture());
genTransStartup.verify(resultStep).addRowListener(listenerArgumentCaptor.capture());
RowListener clientRowListener = listenerArgumentCaptor.getValue();
genTransStartup.verify(genTrans).startThreads();
serviceTransStartup.verify(optimization).activate(executor);
serviceTransStartup.verify(serviceStep).addRowListener(listenerArgumentCaptor.capture());
serviceTransStartup.verify(serviceTrans).startThreads();
// Verify linkage
RowListener serviceRowListener = listenerArgumentCaptor.getValue();
assertNotNull(serviceRowListener);
// Push row from service to sql Trans
RowMetaInterface rowMeta = genTrans.getTransMeta().getStepFields(RESULT_STEP_NAME);
Object[] data;
for (int i = 0; i < 50; i++) {
data = new Object[] {i};
Object[] dataClone = {i};
when(rowMeta.cloneRow(data)).thenReturn(dataClone);
serviceRowListener.rowWrittenEvent(rowMeta, data);
verify(sqlTransRowProducer)
.putRowWait(
same(rowMeta),
and(eq(dataClone), not(same(data))),
any(Long.class),
any(TimeUnit.class));
verify(rowMeta).cloneRow(data);
}
doReturn(true).when(serviceTrans).isRunning();
resultStepListener.getValue().stepFinished(genTrans, resultStep.getStepMeta(), resultStep);
verify(serviceTrans).stopAll();
// Verify Service Trans finished
ArgumentCaptor<StepListener> serviceStepListener = ArgumentCaptor.forClass(StepListener.class);
verify(serviceStep).addStepListener(serviceStepListener.capture());
serviceStepListener
.getValue()
.stepFinished(serviceTrans, serviceStep.getStepMeta(), serviceStep);
verify(sqlTransRowProducer).finished();
// Push row from service to sql Trans
for (int i = 0; i < 50; i++) {
Object[] row = {i};
clientRowListener.rowWrittenEvent(rowMeta, row);
}
transListenerCaptor.getValue().transFinished(genTrans);
InOrder writeRows = inOrder(rowMeta);
ArgumentCaptor<DataOutputStream> streamCaptor = ArgumentCaptor.forClass(DataOutputStream.class);
writeRows.verify(rowMeta).writeMeta(streamCaptor.capture());
DataOutputStream dataOutputStream = streamCaptor.getValue();
writeRows
.verify(rowMeta, times(50))
.writeData(same(dataOutputStream), argThat(arrayWithSize(1)));
writeRows.verifyNoMoreInteractions();
executor.waitUntilFinished();
verify(serviceTrans).waitUntilFinished();
verify(genTrans).waitUntilFinished();
}