public void getFields(
RowMetaInterface inputRowMeta,
String name,
RowMetaInterface[] info,
StepMeta nextStep,
VariableSpace space,
Repository repository,
IMetaStore metaStore)
throws KettleStepException {
// Set the sorted properties: ascending/descending
for (int i = 0; i < fieldName.length; i++) {
int idx = inputRowMeta.indexOfValue(fieldName[i]);
if (idx >= 0) {
ValueMetaInterface valueMeta = inputRowMeta.getValueMeta(idx);
valueMeta.setSortedDescending(!ascending[i]);
valueMeta.setCaseInsensitive(!caseSensitive[i]);
valueMeta.setCollatorDisabled(!collatorEnabled[i]);
valueMeta.setCollatorStrength(collatorStrength[i]);
// Also see if lazy conversion is active on these key fields.
// If so we want to automatically convert them to the normal storage type.
// This will improve performance, see also: PDI-346
//
valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
valueMeta.setStorageMetadata(null);
}
}
}
public static ValueMetaInterface cloneValueMeta(ValueMetaInterface source, int targetType)
throws KettlePluginException {
ValueMetaInterface target = null;
// If we're Cloneable and not changing types, call clone()
if (source instanceof Cloneable && source.getType() == targetType) {
target = source.clone();
} else {
target =
createValueMeta(source.getName(), targetType, source.getLength(), source.getPrecision());
}
target.setConversionMask(source.getConversionMask());
target.setDecimalSymbol(source.getDecimalSymbol());
target.setGroupingSymbol(source.getGroupingSymbol());
target.setStorageType(source.getStorageType());
if (source.getStorageMetadata() != null) {
target.setStorageMetadata(
cloneValueMeta(source.getStorageMetadata(), source.getStorageMetadata().getType()));
}
target.setStringEncoding(source.getStringEncoding());
target.setTrimType(source.getTrimType());
target.setDateFormatLenient(source.isDateFormatLenient());
target.setDateFormatLocale(source.getDateFormatLocale());
target.setDateFormatTimeZone(source.getDateFormatTimeZone());
target.setLenientStringToNumber(source.isLenientStringToNumber());
target.setLargeTextField(source.isLargeTextField());
target.setComments(source.getComments());
target.setCaseInsensitive(source.isCaseInsensitive());
target.setIndex(source.getIndex());
target.setOrigin(source.getOrigin());
target.setOriginalAutoIncrement(source.isOriginalAutoIncrement());
target.setOriginalColumnType(source.getOriginalColumnType());
target.setOriginalColumnTypeName(source.getOriginalColumnTypeName());
target.setOriginalNullable(source.isOriginalNullable());
target.setOriginalPrecision(source.getOriginalPrecision());
target.setOriginalScale(source.getOriginalScale());
target.setOriginalSigned(source.isOriginalSigned());
return target;
}
@Test
public void testWithLazyConversion() throws Exception {
RowMeta rowMeta = new RowMeta();
ValueMetaInterface vm = new ValueMetaString("aBinaryStoredString");
vm.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING);
vm.setStorageMetadata(new ValueMetaString());
rowMeta.addValueMeta(vm);
String query = "SELECT * FROM " + DATA_SERVICE_NAME + " WHERE aBinaryStoredString = 'value'";
when(transMeta.getStepFields(DATA_SERVICE_STEP)).thenReturn(rowMeta);
DataServiceExecutor executor =
new DataServiceExecutor.Builder(new SQL(query), dataService, context)
.serviceTrans(new Trans(transMeta))
.prepareExecution(false)
.build();
executor
.getSql()
.getWhereCondition()
.getCondition()
.evaluate(rowMeta, new Object[] {"value".getBytes()});
}
public void getFields(
RowMetaInterface rowMeta,
String origin,
RowMetaInterface[] info,
StepMeta nextStep,
VariableSpace space)
throws KettleStepException {
rowMeta.clear(); // Start with a clean slate, eats the input
for (int i = 0; i < inputFields.length; i++) {
TextFileInputField field = inputFields[i];
ValueMetaInterface valueMeta = new ValueMeta(field.getName(), field.getType());
valueMeta.setConversionMask(field.getFormat());
valueMeta.setLength(field.getLength());
valueMeta.setPrecision(field.getPrecision());
valueMeta.setConversionMask(field.getFormat());
valueMeta.setDecimalSymbol(field.getDecimalSymbol());
valueMeta.setGroupingSymbol(field.getGroupSymbol());
valueMeta.setCurrencySymbol(field.getCurrencySymbol());
valueMeta.setTrimType(field.getTrimType());
if (lazyConversionActive)
valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING);
valueMeta.setStringEncoding(space.environmentSubstitute(encoding));
// In case we want to convert Strings...
// Using a copy of the valueMeta object means that the inner and outer representation format
// is the same.
// Preview will show the data the same way as we read it.
// This layout is then taken further down the road by the metadata through the transformation.
//
ValueMetaInterface storageMetadata = valueMeta.clone();
storageMetadata.setType(ValueMetaInterface.TYPE_STRING);
storageMetadata.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
storageMetadata.setLength(
-1, -1); // we don't really know the lengths of the strings read in advance.
valueMeta.setStorageMetadata(storageMetadata);
valueMeta.setOrigin(origin);
rowMeta.addValueMeta(valueMeta);
}
if (!Const.isEmpty(filenameField) && includingFilename) {
ValueMetaInterface filenameMeta =
new ValueMeta(filenameField, ValueMetaInterface.TYPE_STRING);
filenameMeta.setOrigin(origin);
if (lazyConversionActive) {
filenameMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING);
filenameMeta.setStorageMetadata(
new ValueMeta(filenameField, ValueMetaInterface.TYPE_STRING));
}
rowMeta.addValueMeta(filenameMeta);
}
if (!Const.isEmpty(rowNumField)) {
ValueMetaInterface rowNumMeta = new ValueMeta(rowNumField, ValueMetaInterface.TYPE_INTEGER);
rowNumMeta.setLength(10);
rowNumMeta.setOrigin(origin);
rowMeta.addValueMeta(rowNumMeta);
}
}
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
meta = (StreamLookupMeta) smi;
data = (StreamLookupData) sdi;
if (data.readLookupValues) {
data.readLookupValues = false;
if (!readLookupValues()) // Read values in lookup table (look)
{
logError(
BaseMessages.getString(
PKG, "StreamLookup.Log.UnableToReadDataFromLookupStream")); // $NON-NLS-1$
setErrors(1);
stopAll();
return false;
}
// At this point, all the values in the cache are of normal storage data type...
// We should reflect this in the metadata...
//
if (data.keyMeta != null) { // null when no rows coming from lookup stream
for (ValueMetaInterface valueMeta : data.keyMeta.getValueMetaList()) {
valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
}
}
if (data.valueMeta != null) { // null when no rows coming from lookup stream
for (ValueMetaInterface valueMeta : data.valueMeta.getValueMetaList()) {
valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
}
}
return true;
}
Object[] r = getRow(); // Get row from input rowset & set row busy!
if (r == null) // no more input to be expected...
{
if (log.isDetailed())
logDetailed(
BaseMessages.getString(
PKG,
"StreamLookup.Log.StoppedProcessingWithEmpty",
getLinesRead() + "")); // $NON-NLS-1$ //$NON-NLS-2$
setOutputDone();
return false;
}
if (first) {
first = false;
// read the lookup values!
data.keynrs = new int[meta.getKeystream().length];
data.lookupMeta = new RowMeta();
data.convertKeysToNative = new boolean[meta.getKeystream().length];
for (int i = 0; i < meta.getKeystream().length; i++) {
// Find the keynr in the row (only once)
data.keynrs[i] = getInputRowMeta().indexOfValue(meta.getKeystream()[i]);
if (data.keynrs[i] < 0) {
throw new KettleStepException(
BaseMessages.getString(
PKG,
"StreamLookup.Log.FieldNotFound",
meta.getKeystream()[i],
"" + getInputRowMeta().getString(r))); // $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
} else {
if (log.isDetailed())
logDetailed(
BaseMessages.getString(
PKG,
"StreamLookup.Log.FieldInfo",
meta.getKeystream()[i],
"" + data.keynrs[i])); // $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
}
data.lookupMeta.addValueMeta(getInputRowMeta().getValueMeta(data.keynrs[i]).clone());
// If we have binary storage data coming in, we convert it to normal data storage.
// The storage in the lookup data store is also normal data storage. TODO: enforce normal
// data storage??
//
data.convertKeysToNative[i] =
getInputRowMeta().getValueMeta(data.keynrs[i]).isStorageBinaryString();
}
data.outputRowMeta = getInputRowMeta().clone();
meta.getFields(
data.outputRowMeta, getStepname(), new RowMetaInterface[] {data.infoMeta}, null, this);
// Handle the NULL values (not found...)
handleNullIf();
}
Object[] outputRow =
lookupValues(getInputRowMeta(), r); // Do the actual lookup in the hastable.
if (outputRow == null) {
setOutputDone(); // signal end to receiver(s)
return false;
}
putRow(data.outputRowMeta, outputRow); // copy row to output rowset(s);
if (checkFeedback(getLinesRead())) {
if (log.isBasic())
logBasic(
BaseMessages.getString(PKG, "StreamLookup.Log.LineNumber")
+ getLinesRead()); //$NON-NLS-1$
}
return true;
}
private boolean readLookupValues() throws KettleException {
data.infoStream = meta.getStepIOMeta().getInfoStreams().get(0);
if (data.infoStream.getStepMeta() == null) {
logError(
BaseMessages.getString(PKG, "StreamLookup.Log.NoLookupStepSpecified")); // $NON-NLS-1$
return false;
}
if (log.isDetailed())
logDetailed(
BaseMessages.getString(PKG, "StreamLookup.Log.ReadingFromStream")
+ data.infoStream.getStepname()
+ "]"); //$NON-NLS-1$ //$NON-NLS-2$
int[] keyNrs = new int[meta.getKeylookup().length];
int[] valueNrs = new int[meta.getValue().length];
boolean firstRun = true;
// Which row set do we read from?
//
RowSet rowSet = findInputRowSet(data.infoStream.getStepname());
Object[] rowData = getRowFrom(rowSet); // rows are originating from "lookup_from"
while (rowData != null) {
if (log.isRowLevel())
logRowlevel(
BaseMessages.getString(PKG, "StreamLookup.Log.ReadLookupRow")
+ rowSet.getRowMeta().getString(rowData)); // $NON-NLS-1$
if (firstRun) {
firstRun = false;
data.hasLookupRows = true;
data.infoMeta = rowSet.getRowMeta().clone();
data.keyMeta = new RowMeta();
data.valueMeta = new RowMeta();
// Look up the keys in the source rows
for (int i = 0; i < meta.getKeylookup().length; i++) {
keyNrs[i] = rowSet.getRowMeta().indexOfValue(meta.getKeylookup()[i]);
if (keyNrs[i] < 0) {
throw new KettleStepException(
BaseMessages.getString(
PKG,
"StreamLookup.Exception.UnableToFindField",
meta.getKeylookup()[i])); // $NON-NLS-1$ //$NON-NLS-2$
}
data.keyMeta.addValueMeta(rowSet.getRowMeta().getValueMeta(keyNrs[i]));
}
// Save the data types of the keys to optionally convert input rows later on...
if (data.keyTypes == null) {
data.keyTypes = data.keyMeta.clone();
}
// set the meta data for the keys also to STORAGE_TYPE_NORMAL, otherwise it will conflict
// later on
// for the data is is already set to STORAGE_TYPE_NORMAL in StreamLookupMeta.getFields()
// all values in the cache are of this storage type (see convertToNormalStorageType below)
// position here after keyTypes are stored (needed below for correct
// convertToNormalStorageType)
for (int i = 0; i < keyNrs.length; i++) {
data.keyMeta.getValueMeta(i).setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
}
for (int v = 0; v < meta.getValue().length; v++) {
valueNrs[v] = rowSet.getRowMeta().indexOfValue(meta.getValue()[v]);
if (valueNrs[v] < 0) {
throw new KettleStepException(
BaseMessages.getString(
PKG,
"StreamLookup.Exception.UnableToFindField",
meta.getValue()[v])); // $NON-NLS-1$ //$NON-NLS-2$
}
data.valueMeta.addValueMeta(rowSet.getRowMeta().getValueMeta(valueNrs[v]));
}
}
Object[] keyData = new Object[keyNrs.length];
for (int i = 0; i < keyNrs.length; i++) {
ValueMetaInterface keyMeta = data.keyTypes.getValueMeta(i);
keyData[i] =
keyMeta.convertToNormalStorageType(
rowData[keyNrs[i]]); // Make sure only normal storage goes in
keyMeta.setStorageType(
ValueMetaInterface
.STORAGE_TYPE_NORMAL); // now we need to change keyMeta/keyTypes also to normal
}
Object[] valueData = new Object[valueNrs.length];
for (int i = 0; i < valueNrs.length; i++) {
ValueMetaInterface valueMeta = data.valueMeta.getValueMeta(i);
valueData[i] =
valueMeta.convertToNormalStorageType(
rowData[valueNrs[i]]); // make sure only normal storage goes in
}
addToCache(data.keyMeta, keyData, data.valueMeta, valueData);
rowData = getRowFrom(rowSet);
}
return true;
}
