public void getFields( RowMetaInterface inputRowMeta, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore) throws KettleStepException { // Set the sorted properties: ascending/descending for (int i = 0; i < fieldName.length; i++) { int idx = inputRowMeta.indexOfValue(fieldName[i]); if (idx >= 0) { ValueMetaInterface valueMeta = inputRowMeta.getValueMeta(idx); valueMeta.setSortedDescending(!ascending[i]); valueMeta.setCaseInsensitive(!caseSensitive[i]); valueMeta.setCollatorDisabled(!collatorEnabled[i]); valueMeta.setCollatorStrength(collatorStrength[i]); // Also see if lazy conversion is active on these key fields. // If so we want to automatically convert them to the normal storage type. // This will improve performance, see also: PDI-346 // valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL); valueMeta.setStorageMetadata(null); } } }
public static ValueMetaInterface cloneValueMeta(ValueMetaInterface source, int targetType) throws KettlePluginException { ValueMetaInterface target = null; // If we're Cloneable and not changing types, call clone() if (source instanceof Cloneable && source.getType() == targetType) { target = source.clone(); } else { target = createValueMeta(source.getName(), targetType, source.getLength(), source.getPrecision()); } target.setConversionMask(source.getConversionMask()); target.setDecimalSymbol(source.getDecimalSymbol()); target.setGroupingSymbol(source.getGroupingSymbol()); target.setStorageType(source.getStorageType()); if (source.getStorageMetadata() != null) { target.setStorageMetadata( cloneValueMeta(source.getStorageMetadata(), source.getStorageMetadata().getType())); } target.setStringEncoding(source.getStringEncoding()); target.setTrimType(source.getTrimType()); target.setDateFormatLenient(source.isDateFormatLenient()); target.setDateFormatLocale(source.getDateFormatLocale()); target.setDateFormatTimeZone(source.getDateFormatTimeZone()); target.setLenientStringToNumber(source.isLenientStringToNumber()); target.setLargeTextField(source.isLargeTextField()); target.setComments(source.getComments()); target.setCaseInsensitive(source.isCaseInsensitive()); target.setIndex(source.getIndex()); target.setOrigin(source.getOrigin()); target.setOriginalAutoIncrement(source.isOriginalAutoIncrement()); target.setOriginalColumnType(source.getOriginalColumnType()); target.setOriginalColumnTypeName(source.getOriginalColumnTypeName()); target.setOriginalNullable(source.isOriginalNullable()); target.setOriginalPrecision(source.getOriginalPrecision()); target.setOriginalScale(source.getOriginalScale()); target.setOriginalSigned(source.isOriginalSigned()); return target; }
@Test public void testWithLazyConversion() throws Exception { RowMeta rowMeta = new RowMeta(); ValueMetaInterface vm = new ValueMetaString("aBinaryStoredString"); vm.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING); vm.setStorageMetadata(new ValueMetaString()); rowMeta.addValueMeta(vm); String query = "SELECT * FROM " + DATA_SERVICE_NAME + " WHERE aBinaryStoredString = 'value'"; when(transMeta.getStepFields(DATA_SERVICE_STEP)).thenReturn(rowMeta); DataServiceExecutor executor = new DataServiceExecutor.Builder(new SQL(query), dataService, context) .serviceTrans(new Trans(transMeta)) .prepareExecution(false) .build(); executor .getSql() .getWhereCondition() .getCondition() .evaluate(rowMeta, new Object[] {"value".getBytes()}); }
public void getFields( RowMetaInterface rowMeta, String origin, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space) throws KettleStepException { rowMeta.clear(); // Start with a clean slate, eats the input for (int i = 0; i < inputFields.length; i++) { TextFileInputField field = inputFields[i]; ValueMetaInterface valueMeta = new ValueMeta(field.getName(), field.getType()); valueMeta.setConversionMask(field.getFormat()); valueMeta.setLength(field.getLength()); valueMeta.setPrecision(field.getPrecision()); valueMeta.setConversionMask(field.getFormat()); valueMeta.setDecimalSymbol(field.getDecimalSymbol()); valueMeta.setGroupingSymbol(field.getGroupSymbol()); valueMeta.setCurrencySymbol(field.getCurrencySymbol()); valueMeta.setTrimType(field.getTrimType()); if (lazyConversionActive) valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING); valueMeta.setStringEncoding(space.environmentSubstitute(encoding)); // In case we want to convert Strings... // Using a copy of the valueMeta object means that the inner and outer representation format // is the same. // Preview will show the data the same way as we read it. // This layout is then taken further down the road by the metadata through the transformation. // ValueMetaInterface storageMetadata = valueMeta.clone(); storageMetadata.setType(ValueMetaInterface.TYPE_STRING); storageMetadata.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL); storageMetadata.setLength( -1, -1); // we don't really know the lengths of the strings read in advance. valueMeta.setStorageMetadata(storageMetadata); valueMeta.setOrigin(origin); rowMeta.addValueMeta(valueMeta); } if (!Const.isEmpty(filenameField) && includingFilename) { ValueMetaInterface filenameMeta = new ValueMeta(filenameField, ValueMetaInterface.TYPE_STRING); filenameMeta.setOrigin(origin); if (lazyConversionActive) { filenameMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING); filenameMeta.setStorageMetadata( new ValueMeta(filenameField, ValueMetaInterface.TYPE_STRING)); } rowMeta.addValueMeta(filenameMeta); } if (!Const.isEmpty(rowNumField)) { ValueMetaInterface rowNumMeta = new ValueMeta(rowNumField, ValueMetaInterface.TYPE_INTEGER); rowNumMeta.setLength(10); rowNumMeta.setOrigin(origin); rowMeta.addValueMeta(rowNumMeta); } }
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException { meta = (StreamLookupMeta) smi; data = (StreamLookupData) sdi; if (data.readLookupValues) { data.readLookupValues = false; if (!readLookupValues()) // Read values in lookup table (look) { logError( BaseMessages.getString( PKG, "StreamLookup.Log.UnableToReadDataFromLookupStream")); // $NON-NLS-1$ setErrors(1); stopAll(); return false; } // At this point, all the values in the cache are of normal storage data type... // We should reflect this in the metadata... // if (data.keyMeta != null) { // null when no rows coming from lookup stream for (ValueMetaInterface valueMeta : data.keyMeta.getValueMetaList()) { valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL); } } if (data.valueMeta != null) { // null when no rows coming from lookup stream for (ValueMetaInterface valueMeta : data.valueMeta.getValueMetaList()) { valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL); } } return true; } Object[] r = getRow(); // Get row from input rowset & set row busy! if (r == null) // no more input to be expected... { if (log.isDetailed()) logDetailed( BaseMessages.getString( PKG, "StreamLookup.Log.StoppedProcessingWithEmpty", getLinesRead() + "")); // $NON-NLS-1$ //$NON-NLS-2$ setOutputDone(); return false; } if (first) { first = false; // read the lookup values! data.keynrs = new int[meta.getKeystream().length]; data.lookupMeta = new RowMeta(); data.convertKeysToNative = new boolean[meta.getKeystream().length]; for (int i = 0; i < meta.getKeystream().length; i++) { // Find the keynr in the row (only once) data.keynrs[i] = getInputRowMeta().indexOfValue(meta.getKeystream()[i]); if (data.keynrs[i] < 0) { throw new KettleStepException( BaseMessages.getString( PKG, "StreamLookup.Log.FieldNotFound", meta.getKeystream()[i], "" + getInputRowMeta().getString(r))); // $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ } else { if (log.isDetailed()) logDetailed( BaseMessages.getString( PKG, "StreamLookup.Log.FieldInfo", meta.getKeystream()[i], "" + data.keynrs[i])); // $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ } data.lookupMeta.addValueMeta(getInputRowMeta().getValueMeta(data.keynrs[i]).clone()); // If we have binary storage data coming in, we convert it to normal data storage. // The storage in the lookup data store is also normal data storage. TODO: enforce normal // data storage?? // data.convertKeysToNative[i] = getInputRowMeta().getValueMeta(data.keynrs[i]).isStorageBinaryString(); } data.outputRowMeta = getInputRowMeta().clone(); meta.getFields( data.outputRowMeta, getStepname(), new RowMetaInterface[] {data.infoMeta}, null, this); // Handle the NULL values (not found...) handleNullIf(); } Object[] outputRow = lookupValues(getInputRowMeta(), r); // Do the actual lookup in the hastable. if (outputRow == null) { setOutputDone(); // signal end to receiver(s) return false; } putRow(data.outputRowMeta, outputRow); // copy row to output rowset(s); if (checkFeedback(getLinesRead())) { if (log.isBasic()) logBasic( BaseMessages.getString(PKG, "StreamLookup.Log.LineNumber") + getLinesRead()); //$NON-NLS-1$ } return true; }
private boolean readLookupValues() throws KettleException { data.infoStream = meta.getStepIOMeta().getInfoStreams().get(0); if (data.infoStream.getStepMeta() == null) { logError( BaseMessages.getString(PKG, "StreamLookup.Log.NoLookupStepSpecified")); // $NON-NLS-1$ return false; } if (log.isDetailed()) logDetailed( BaseMessages.getString(PKG, "StreamLookup.Log.ReadingFromStream") + data.infoStream.getStepname() + "]"); //$NON-NLS-1$ //$NON-NLS-2$ int[] keyNrs = new int[meta.getKeylookup().length]; int[] valueNrs = new int[meta.getValue().length]; boolean firstRun = true; // Which row set do we read from? // RowSet rowSet = findInputRowSet(data.infoStream.getStepname()); Object[] rowData = getRowFrom(rowSet); // rows are originating from "lookup_from" while (rowData != null) { if (log.isRowLevel()) logRowlevel( BaseMessages.getString(PKG, "StreamLookup.Log.ReadLookupRow") + rowSet.getRowMeta().getString(rowData)); // $NON-NLS-1$ if (firstRun) { firstRun = false; data.hasLookupRows = true; data.infoMeta = rowSet.getRowMeta().clone(); data.keyMeta = new RowMeta(); data.valueMeta = new RowMeta(); // Look up the keys in the source rows for (int i = 0; i < meta.getKeylookup().length; i++) { keyNrs[i] = rowSet.getRowMeta().indexOfValue(meta.getKeylookup()[i]); if (keyNrs[i] < 0) { throw new KettleStepException( BaseMessages.getString( PKG, "StreamLookup.Exception.UnableToFindField", meta.getKeylookup()[i])); // $NON-NLS-1$ //$NON-NLS-2$ } data.keyMeta.addValueMeta(rowSet.getRowMeta().getValueMeta(keyNrs[i])); } // Save the data types of the keys to optionally convert input rows later on... if (data.keyTypes == null) { data.keyTypes = data.keyMeta.clone(); } // set the meta data for the keys also to STORAGE_TYPE_NORMAL, otherwise it will conflict // later on // for the data is is already set to STORAGE_TYPE_NORMAL in StreamLookupMeta.getFields() // all values in the cache are of this storage type (see convertToNormalStorageType below) // position here after keyTypes are stored (needed below for correct // convertToNormalStorageType) for (int i = 0; i < keyNrs.length; i++) { data.keyMeta.getValueMeta(i).setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL); } for (int v = 0; v < meta.getValue().length; v++) { valueNrs[v] = rowSet.getRowMeta().indexOfValue(meta.getValue()[v]); if (valueNrs[v] < 0) { throw new KettleStepException( BaseMessages.getString( PKG, "StreamLookup.Exception.UnableToFindField", meta.getValue()[v])); // $NON-NLS-1$ //$NON-NLS-2$ } data.valueMeta.addValueMeta(rowSet.getRowMeta().getValueMeta(valueNrs[v])); } } Object[] keyData = new Object[keyNrs.length]; for (int i = 0; i < keyNrs.length; i++) { ValueMetaInterface keyMeta = data.keyTypes.getValueMeta(i); keyData[i] = keyMeta.convertToNormalStorageType( rowData[keyNrs[i]]); // Make sure only normal storage goes in keyMeta.setStorageType( ValueMetaInterface .STORAGE_TYPE_NORMAL); // now we need to change keyMeta/keyTypes also to normal } Object[] valueData = new Object[valueNrs.length]; for (int i = 0; i < valueNrs.length; i++) { ValueMetaInterface valueMeta = data.valueMeta.getValueMeta(i); valueData[i] = valueMeta.convertToNormalStorageType( rowData[valueNrs[i]]); // make sure only normal storage goes in } addToCache(data.keyMeta, keyData, data.valueMeta, valueData); rowData = getRowFrom(rowSet); } return true; }