public void getFields( RowMetaInterface row, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore) throws KettleStepException { // No values are added to the row in this type of step // However, in case of Fixed length records, // the field precisions and lengths are altered! for (int i = 0; i < outputFields.length; i++) { TextFileField field = outputFields[i]; ValueMetaInterface v = row.searchValueMeta(field.getName()); if (v != null) { v.setLength(field.getLength()); v.setPrecision(field.getPrecision()); v.setConversionMask(field.getFormat()); v.setDecimalSymbol(field.getDecimalSymbol()); v.setGroupingSymbol(field.getGroupingSymbol()); v.setCurrencySymbol(field.getCurrencySymbol()); v.setOutputPaddingEnabled(isPadded()); v.setTrimType(field.getTrimType()); if (!Const.isEmpty(getEncoding())) { v.setStringEncoding(getEncoding()); } // enable output padding by default to be compatible with v2.5.x // v.setOutputPaddingEnabled(true); } } }
@Test public void testGetFieldDefinitionBigNumber() { String bigNumberName = "bigNumberName"; ValueMetaInterface valueMetaInterface = new ValueMetaBigNumber(); valueMetaInterface.setName(bigNumberName); valueMetaInterface.setPrecision(0); valueMetaInterface.setLength(9); assertGetFieldDefinition(valueMetaInterface, "INT"); valueMetaInterface.setLength(18); assertGetFieldDefinition(valueMetaInterface, "BIGINT"); valueMetaInterface.setLength(19); assertGetFieldDefinition(valueMetaInterface, "FLOAT"); valueMetaInterface.setPrecision(10); valueMetaInterface.setLength(16); assertGetFieldDefinition(valueMetaInterface, "FLOAT"); valueMetaInterface.setLength(15); assertGetFieldDefinition(valueMetaInterface, "DOUBLE"); }
@Test public void testGetFieldDefinitionInteger() { String integerName = "integerName"; ValueMetaInterface valueMetaInterface = new ValueMetaInteger(); valueMetaInterface.setName(integerName); valueMetaInterface.setPrecision(0); valueMetaInterface.setLength(9); assertGetFieldDefinition(valueMetaInterface, "INT"); valueMetaInterface.setLength(18); assertGetFieldDefinition(valueMetaInterface, "BIGINT"); valueMetaInterface.setLength(19); assertGetFieldDefinition(valueMetaInterface, "FLOAT"); }
/** * Generates row meta structure from a fields array. * * @param fields the fields array * @param origin the data origin * @param rowMeta the row meta to generate */ public static void fieldsToRowMeta( final CobFileInputField[] fields, final String origin, final RowMetaInterface rowMeta) { rowMeta.clear(); // Start with a clean slate, eats the input for (int i = 0; i < fields.length; i++) { CobFileInputField field = fields[i]; ValueMetaInterface valueMeta = new ValueMeta(field.getName(), field.getType()); valueMeta.setConversionMask(field.getFormat()); valueMeta.setLength(field.getLength()); valueMeta.setPrecision(field.getPrecision()); valueMeta.setConversionMask(field.getFormat()); valueMeta.setDecimalSymbol(field.getDecimalSymbol()); valueMeta.setGroupingSymbol(field.getGroupSymbol()); valueMeta.setCurrencySymbol(field.getCurrencySymbol()); valueMeta.setTrimType(field.getTrimType()); valueMeta.setOrigin(origin); rowMeta.addValueMeta(valueMeta); } }
public void getFields( RowMetaInterface row, String origin, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space) throws KettleStepException { if (!Const.isEmpty(technicalKeyField)) { ValueMetaInterface v = new ValueMeta(technicalKeyField, ValueMetaInterface.TYPE_INTEGER); v.setLength(10); v.setPrecision(0); v.setOrigin(origin); row.addValueMeta(v); } if (replaceFields) { for (int i = 0; i < keyField.length; i++) { int idx = row.indexOfValue(keyField[i]); if (idx >= 0) { row.removeValueMeta(idx); } } } }
public static final RowMetaAndData buildRow( RowGeneratorMeta meta, List<CheckResultInterface> remarks, String origin) { RowMetaInterface rowMeta = new RowMeta(); Object[] rowData = RowDataUtil.allocateRowData(meta.getFieldName().length); for (int i = 0; i < meta.getFieldName().length; i++) { int valtype = ValueMeta.getType(meta.getFieldType()[i]); if (meta.getFieldName()[i] != null) { ValueMetaInterface valueMeta = new ValueMeta(meta.getFieldName()[i], valtype); // build a value! valueMeta.setLength(meta.getFieldLength()[i]); valueMeta.setPrecision(meta.getFieldPrecision()[i]); valueMeta.setConversionMask(meta.getFieldFormat()[i]); valueMeta.setGroupingSymbol(meta.getGroup()[i]); valueMeta.setDecimalSymbol(meta.getDecimal()[i]); valueMeta.setOrigin(origin); ValueMetaInterface stringMeta = valueMeta.clone(); stringMeta.setType(ValueMetaInterface.TYPE_STRING); String stringValue = meta.getValue()[i]; // If the value is empty: consider it to be NULL. if (Const.isEmpty(stringValue)) { rowData[i] = null; if (valueMeta.getType() == ValueMetaInterface.TYPE_NONE) { String message = BaseMessages.getString( PKG, "RowGenerator.CheckResult.SpecifyTypeError", valueMeta.getName(), stringValue); remarks.add(new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null)); } } else { // Convert the data from String to the specified type ... // try { rowData[i] = valueMeta.convertData(stringMeta, stringValue); } catch (KettleValueException e) { switch (valueMeta.getType()) { case ValueMetaInterface.TYPE_NUMBER: { String message = BaseMessages.getString( PKG, "RowGenerator.BuildRow.Error.Parsing.Number", valueMeta.getName(), stringValue, e.toString()); remarks.add( new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null)); } break; case ValueMetaInterface.TYPE_DATE: { String message = BaseMessages.getString( PKG, "RowGenerator.BuildRow.Error.Parsing.Date", valueMeta.getName(), stringValue, e.toString()); remarks.add( new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null)); } break; case ValueMetaInterface.TYPE_INTEGER: { String message = BaseMessages.getString( PKG, "RowGenerator.BuildRow.Error.Parsing.Integer", valueMeta.getName(), stringValue, e.toString()); remarks.add( new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null)); } break; case ValueMetaInterface.TYPE_BIGNUMBER: { String message = BaseMessages.getString( PKG, "RowGenerator.BuildRow.Error.Parsing.BigNumber", valueMeta.getName(), stringValue, e.toString()); remarks.add( new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null)); } break; default: // Boolean and binary don't throw errors normally, so it's probably an unspecified // error problem... { String message = BaseMessages.getString( PKG, "RowGenerator.CheckResult.SpecifyTypeError", valueMeta.getName(), stringValue); remarks.add( new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null)); } break; } } } // Now add value to the row! // This is in fact a copy from the fields row, but now with data. rowMeta.addValueMeta(valueMeta); } } return new RowMetaAndData(rowMeta, rowData); }
public void getFields( RowMetaInterface rowMeta, String origin, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space) throws KettleStepException { rowMeta.clear(); // Start with a clean slate, eats the input for (int i = 0; i < inputFields.length; i++) { TextFileInputField field = inputFields[i]; ValueMetaInterface valueMeta = new ValueMeta(field.getName(), field.getType()); valueMeta.setConversionMask(field.getFormat()); valueMeta.setLength(field.getLength()); valueMeta.setPrecision(field.getPrecision()); valueMeta.setConversionMask(field.getFormat()); valueMeta.setDecimalSymbol(field.getDecimalSymbol()); valueMeta.setGroupingSymbol(field.getGroupSymbol()); valueMeta.setCurrencySymbol(field.getCurrencySymbol()); valueMeta.setTrimType(field.getTrimType()); if (lazyConversionActive) valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING); valueMeta.setStringEncoding(space.environmentSubstitute(encoding)); // In case we want to convert Strings... // Using a copy of the valueMeta object means that the inner and outer representation format // is the same. // Preview will show the data the same way as we read it. // This layout is then taken further down the road by the metadata through the transformation. // ValueMetaInterface storageMetadata = valueMeta.clone(); storageMetadata.setType(ValueMetaInterface.TYPE_STRING); storageMetadata.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL); storageMetadata.setLength( -1, -1); // we don't really know the lengths of the strings read in advance. valueMeta.setStorageMetadata(storageMetadata); valueMeta.setOrigin(origin); rowMeta.addValueMeta(valueMeta); } if (!Const.isEmpty(filenameField) && includingFilename) { ValueMetaInterface filenameMeta = new ValueMeta(filenameField, ValueMetaInterface.TYPE_STRING); filenameMeta.setOrigin(origin); if (lazyConversionActive) { filenameMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING); filenameMeta.setStorageMetadata( new ValueMeta(filenameField, ValueMetaInterface.TYPE_STRING)); } rowMeta.addValueMeta(filenameMeta); } if (!Const.isEmpty(rowNumField)) { ValueMetaInterface rowNumMeta = new ValueMeta(rowNumField, ValueMetaInterface.TYPE_INTEGER); rowNumMeta.setLength(10); rowNumMeta.setOrigin(origin); rowMeta.addValueMeta(rowNumMeta); } }
public void getFields( RowMetaInterface row, String origin, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space) throws KettleStepException { // Row should normally be empty when we get here. // That is because there is no previous step to this mapping input step from the viewpoint of // this single sub-transformation. // From the viewpoint of the transformation that executes the mapping, it's important to know // what comes out at the exit points. // For that reason we need to re-order etc, based on the input specification... // if (inputRowMeta != null && !inputRowMeta.isEmpty()) { // this gets set only in the parent transformation... // It includes all the renames that needed to be done // if (selectingAndSortingUnspecifiedFields) { // First rename any fields... if (valueRenames != null) { for (MappingValueRename valueRename : valueRenames) { ValueMetaInterface valueMeta = inputRowMeta.searchValueMeta(valueRename.getSourceValueName()); if (valueMeta == null) { throw new KettleStepException( BaseMessages.getString( PKG, "MappingInput.Exception.UnableToFindMappedValue", valueRename.getSourceValueName())); } valueMeta.setName(valueRename.getTargetValueName()); } } // Select the specified fields from the input, re-order everything and put the other fields // at the back, sorted... // RowMetaInterface newRow = new RowMeta(); for (int i = 0; i < fieldName.length; i++) { int index = inputRowMeta.indexOfValue(fieldName[i]); if (index < 0) { throw new KettleStepException( BaseMessages.getString( PKG, "MappingInputMeta.Exception.UnknownField", fieldName[i])); } newRow.addValueMeta(inputRowMeta.getValueMeta(index)); } // Now get the unspecified fields. // Sort the fields // Add them after the specified fields... // List<String> extra = new ArrayList<String>(); for (int i = 0; i < inputRowMeta.size(); i++) { String fieldName = inputRowMeta.getValueMeta(i).getName(); if (newRow.indexOfValue(fieldName) < 0) { extra.add(fieldName); } } Collections.sort(extra); for (String fieldName : extra) { ValueMetaInterface extraValue = inputRowMeta.searchValueMeta(fieldName); newRow.addValueMeta(extraValue); } // now merge the new row... // This is basically the input row meta data with the fields re-ordered. // row.mergeRowMeta(newRow); } else { row.mergeRowMeta(inputRowMeta); // Validate the existence of all the specified fields... // if (!row.isEmpty()) { for (int i = 0; i < fieldName.length; i++) { if (row.indexOfValue(fieldName[i]) < 0) { throw new KettleStepException( BaseMessages.getString( PKG, "MappingInputMeta.Exception.UnknownField", fieldName[i])); } } } } } else { // We'll have to work with the statically provided information for (int i = 0; i < fieldName.length; i++) { if (!Const.isEmpty(fieldName[i])) { ValueMetaInterface v = new ValueMeta(fieldName[i], fieldType[i]); if (v.getType() == ValueMetaInterface.TYPE_NONE) v.setType(ValueMetaInterface.TYPE_STRING); v.setLength(fieldLength[i]); v.setPrecision(fieldPrecision[i]); v.setOrigin(origin); row.addValueMeta(v); } } } }
private ValueMetaInterface getValueMeta(CalculatorMetaFunction fn, String origin) { ValueMetaInterface v = new ValueMeta(fn.getFieldName(), fn.getValueType()); v.setLength(fn.getValueLength()); v.setPrecision(fn.getValuePrecision()); v.setOrigin(origin); v.setComments(fn.getCalcTypeDesc()); v.setConversionMask(fn.getConversionMask()); v.setDecimalSymbol(fn.getDecimalSymbol()); v.setGroupingSymbol(fn.getGroupingSymbol()); v.setCurrencySymbol(fn.getCurrencySymbol()); // What if the user didn't specify a data type? // In that case we look for the default data type // if (fn.getValueType() == ValueMetaInterface.TYPE_NONE) { int defaultResultType = ValueMetaInterface.TYPE_NONE; switch (fn.getCalcType()) { case CalculatorMetaFunction.CALC_NONE: break; case CalculatorMetaFunction.CALC_ADD: // A + B defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_SUBTRACT: // A - B defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_MULTIPLY: // A * B defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_DIVIDE: // A / B defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_SQUARE: // A * A defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_SQUARE_ROOT: // SQRT( A ) defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_PERCENT_1: // 100 * A / B defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_PERCENT_2: // A - ( A * B / 100 ) defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_PERCENT_3: // A + ( A * B / 100 ) defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_COMBINATION_1: // A + B * C defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_COMBINATION_2: // SQRT( A*A + B*B ) defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_ROUND_1: // ROUND( A ) defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_ROUND_2: // ROUND( A , B ) defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_CONSTANT: // Set field to constant value... defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_NVL: // Replace null values with another value break; case CalculatorMetaFunction.CALC_ADD_DAYS: // Add B days to date field A defaultResultType = ValueMetaInterface.TYPE_DATE; break; case CalculatorMetaFunction.CALC_YEAR_OF_DATE: // What is the year (Integer) of a date? defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_MONTH_OF_DATE: // What is the month (Integer) of a date? defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction .CALC_DAY_OF_YEAR: // What is the day of year (Integer) of a date? defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction .CALC_DAY_OF_MONTH: // What is the day of month (Integer) of a date? defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction .CALC_DAY_OF_WEEK: // What is the day of week (Integer) of a date? defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction .CALC_WEEK_OF_YEAR: // What is the week of year (Integer) of a date? defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction .CALC_WEEK_OF_YEAR_ISO8601: // What is the week of year (Integer) of a date ISO8601 // style? defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction .CALC_YEAR_OF_DATE_ISO8601: // What is the year (Integer) of a date ISO8601 style? defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_BYTE_TO_HEX_ENCODE: // Byte to Hex encode string field A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_HEX_TO_BYTE_DECODE: // Hex to Byte decode string field A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_CHAR_TO_HEX_ENCODE: // Char to Hex encode string field A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_HEX_TO_CHAR_DECODE: // Hex to Char decode string field A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_CRC32: // CRC32 of a file A defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_ADLER32: // ADLER32 of a file A defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_MD5: // MD5 of a file A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_SHA1: // SHA1 of a file Al defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction .CALC_LEVENSHTEIN_DISTANCE: // LEVENSHTEIN_DISTANCE of string A and string B defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_METAPHONE: // METAPHONE of string A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_DOUBLE_METAPHONE: // Double METAPHONE of string A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_ABS: // ABS( A ) defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_REMOVE_TIME_FROM_DATE: // Remove time from field A defaultResultType = ValueMetaInterface.TYPE_DATE; break; case CalculatorMetaFunction.CALC_DATE_DIFF: // DateA - DateB defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_ADD3: // A + B +C defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_INITCAP: // InitCap(A) defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_UPPER_CASE: // UpperCase(A) defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_LOWER_CASE: // LowerCase(A) defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_MASK_XML: // MaskXML(A) defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_USE_CDATA: // CDATA(A) defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_REMOVE_CR: // REMOVE CR FROM string A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_REMOVE_LF: // REMOVE LF FROM string A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_REMOVE_CRLF: // REMOVE CRLF FROM string A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_REMOVE_TAB: // REMOVE TAB FROM string A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_GET_ONLY_DIGITS: // GET ONLY DIGITS FROM string A defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_REMOVE_DIGITS: // REMOVE DIGITS FROM string A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_STRING_LEN: // LENGTH OF string A defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_LOAD_FILE_CONTENT_BINARY: // LOAD FILE CONTENT IN BLOB defaultResultType = ValueMetaInterface.TYPE_BINARY; break; case CalculatorMetaFunction.CALC_ADD_TIME_TO_DATE: // ADD TIME TO A DATE defaultResultType = ValueMetaInterface.TYPE_DATE; break; case CalculatorMetaFunction.CALC_GEOM_UNION: // Calculate geometry union defaultResultType = ValueMetaInterface.TYPE_GEOMETRY; break; case CalculatorMetaFunction.CALC_GEOM_INTERSECTION: // Calculate geometry intersection defaultResultType = ValueMetaInterface.TYPE_GEOMETRY; break; case CalculatorMetaFunction.CALC_GEOM_DIFFERENCE: // Calculate geometry difference defaultResultType = ValueMetaInterface.TYPE_GEOMETRY; break; case CalculatorMetaFunction .CALC_GEOM_SYMETRIC_DIFFERENCE: // Calculate geometry symetric difference defaultResultType = ValueMetaInterface.TYPE_GEOMETRY; break; case CalculatorMetaFunction.CALC_GEOM_AREA: // Calculate area defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_GEOM_LENGTH: // Calculate length defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_GEOM_CENTROID: // Calculate centroid defaultResultType = ValueMetaInterface.TYPE_GEOMETRY; break; case CalculatorMetaFunction.CALC_GEOM_POINT_ON_SURFACE: // Calculate random point on surface defaultResultType = ValueMetaInterface.TYPE_GEOMETRY; break; case CalculatorMetaFunction.CALC_GEOM_REVERSE: // Reverse geometry defaultResultType = ValueMetaInterface.TYPE_GEOMETRY; break; case CalculatorMetaFunction.CALC_GEOM_BOUNDARY: // Calculate geometry boundary defaultResultType = ValueMetaInterface.TYPE_GEOMETRY; break; case CalculatorMetaFunction.CALC_GEOM_ENVELOPE: // Calculate geometry envelope defaultResultType = ValueMetaInterface.TYPE_GEOMETRY; break; case CalculatorMetaFunction.CALC_GEOM_CONVEX_HULL: // Calculate geometry convex hull defaultResultType = ValueMetaInterface.TYPE_GEOMETRY; break; case CalculatorMetaFunction.CALC_GEOM_BUFFER: // Calculate geometry buffer defaultResultType = ValueMetaInterface.TYPE_GEOMETRY; break; case CalculatorMetaFunction.CALC_GEOM_DISTANCE: // Calculate distance between geometries defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; default: break; } v.setType(defaultResultType); } return v; }
public SQLStatement getSQLStatements( TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev) { SQLStatement retval = new SQLStatement(stepMeta.getName(), databaseWriteMeta, null); // default: nothing to do! int i; if (databaseWriteMeta != null) { if (prev != null && prev.size() > 0) { if (!Const.isEmpty(tablename)) { String schemaTable = databaseWriteMeta.getQuotedSchemaTableCombination(schemaName, tablename); Database db = new Database(databaseWriteMeta); try { boolean doHash = false; String cr_table = null; db.connect(); // OK, what do we put in the new table?? RowMetaInterface fields = new RowMeta(); ValueMetaInterface vkeyfield = null; if (!Const.isEmpty(technicalKeyField)) { // First, the new technical key... vkeyfield = new ValueMeta(technicalKeyField, ValueMetaInterface.TYPE_INTEGER); vkeyfield.setLength(10); vkeyfield.setPrecision(0); } // Then the hashcode (optional) ValueMetaInterface vhashfield = null; if (useHash && !Const.isEmpty(hashField)) { vhashfield = new ValueMeta(hashField, ValueMetaInterface.TYPE_INTEGER); vhashfield.setLength(15); vhashfield.setPrecision(0); doHash = true; } // Then the last update field (optional) ValueMetaInterface vLastUpdateField = null; if (!Const.isEmpty(lastUpdateField)) { vLastUpdateField = new ValueMeta(lastUpdateField, ValueMetaInterface.TYPE_DATE); } if (!db.checkTableExists(schemaTable)) { if (vkeyfield != null) { // Add technical key field. fields.addValueMeta(vkeyfield); } // Add the keys only to the table if (keyField != null && keyLookup != null) { int cnt = keyField.length; for (i = 0; i < cnt; i++) { String error_field = ""; // $NON-NLS-1$ // Find the value in the stream ValueMetaInterface v = prev.searchValueMeta(keyField[i]); if (v != null) { String name = keyLookup[i]; ValueMetaInterface newValue = v.clone(); newValue.setName(name); if (vkeyfield != null) { if (name.equals(vkeyfield.getName()) || (doHash == true && name.equals(vhashfield.getName()))) { error_field += name; } } if (error_field.length() > 0) { retval.setError( Messages.getString( "ConcurrentCombinationLookupMeta.ReturnValue.NameCollision", error_field)); //$NON-NLS-1$ } else { fields.addValueMeta(newValue); } } } } if (doHash == true) { fields.addValueMeta(vhashfield); } if (vLastUpdateField != null) { fields.addValueMeta(vLastUpdateField); } } else { // Table already exists // Get the fields that are in the table now: RowMetaInterface tabFields = db.getTableFields(schemaTable); // Don't forget to quote these as well... databaseWriteMeta.quoteReservedWords(tabFields); if (vkeyfield != null && tabFields.searchValueMeta(vkeyfield.getName()) == null) { // Add technical key field if it didn't exist yet fields.addValueMeta(vkeyfield); } // Add the already existing fields int cnt = tabFields.size(); for (i = 0; i < cnt; i++) { ValueMetaInterface v = tabFields.getValueMeta(i); fields.addValueMeta(v); } // Find the missing fields in the real table String keyLookup[] = getKeyLookup(); String keyField[] = getKeyField(); if (keyField != null && keyLookup != null) { cnt = keyField.length; for (i = 0; i < cnt; i++) { // Find the value in the stream ValueMetaInterface v = prev.searchValueMeta(keyField[i]); if (v != null) { ValueMetaInterface newValue = v.clone(); newValue.setName(keyLookup[i]); // Does the corresponding name exist in the table if (tabFields.searchValueMeta(newValue.getName()) == null) { fields.addValueMeta(newValue); // nope --> add } } } } if (doHash == true && tabFields.searchValueMeta(vhashfield.getName()) == null) { // Add hash field fields.addValueMeta(vhashfield); } if (vLastUpdateField != null && tabFields.searchValueMeta(vLastUpdateField.getName()) == null) { fields.addValueMeta(vLastUpdateField); } } cr_table = db.getDDL( schemaTable, fields, (CREATION_METHOD_SEQUENCE.equals(getTechKeyCreation()) && sequenceFrom != null && sequenceFrom.length() != 0) ? null : technicalKeyField, CREATION_METHOD_AUTOINC.equals(getTechKeyCreation()), null, true); // // OK, now let's build the index // // What fields do we put int the index? // Only the hashcode or all fields? String cr_index = ""; // $NON-NLS-1$ String cr_uniq_index = ""; // $NON-NLS-1$ String idx_fields[] = null; if (useHash) { if (hashField != null && hashField.length() > 0) { idx_fields = new String[] {hashField}; } else { retval.setError( Messages.getString( "ConcurrentCombinationLookupMeta.ReturnValue.NotHashFieldSpecified")); //$NON-NLS-1$ } } else // index on all key fields... { if (!Const.isEmpty(keyLookup)) { int nrfields = keyLookup.length; if (nrfields > 32 && databaseWriteMeta.getDatabaseType() == DatabaseMeta.TYPE_DATABASE_ORACLE) { nrfields = 32; // Oracle indexes are limited to 32 fields... } idx_fields = new String[nrfields]; for (i = 0; i < nrfields; i++) idx_fields[i] = keyLookup[i]; } else { retval.setError( Messages.getString( "ConcurrentCombinationLookupMeta.ReturnValue.NotFieldsSpecified")); //$NON-NLS-1$ } } // OK, now get the create index statement... if (!Const.isEmpty(technicalKeyField)) { String techKeyArr[] = new String[] {technicalKeyField}; if (!db.checkIndexExists(schemaName, tablename, techKeyArr)) { String indexname = "idx_" + tablename + "_pk"; // $NON-NLS-1$ //$NON-NLS-2$ cr_uniq_index = db.getCreateIndexStatement( schemaName, tablename, indexname, techKeyArr, true, true, false, true); cr_uniq_index += Const.CR; } } // OK, now get the create lookup index statement... if (!Const.isEmpty(idx_fields) && !db.checkIndexExists(schemaName, tablename, idx_fields)) { String indexname = "idx_" + tablename + "_lookup"; // $NON-NLS-1$ //$NON-NLS-2$ cr_index = db.getCreateIndexStatement( schemaName, tablename, indexname, idx_fields, false, false, false, true); cr_index += Const.CR; } // // Don't forget the sequence (optional) // String cr_seq = ""; // $NON-NLS-1$ if (databaseWriteMeta.supportsSequences() && !Const.isEmpty(sequenceFrom)) { if (!db.checkSequenceExists(schemaName, sequenceFrom)) { cr_seq += db.getCreateSequenceStatement(schemaName, sequenceFrom, 1L, 1L, -1L, true); cr_seq += Const.CR; } } retval.setSQL(cr_table + cr_uniq_index + cr_index + cr_seq); } catch (KettleException e) { retval.setError( Messages.getString("ConcurrentCombinationLookupMeta.ReturnValue.ErrorOccurred") + Const.CR + e.getMessage()); // $NON-NLS-1$ } } else { retval.setError( Messages.getString( "ConcurrentCombinationLookupMeta.ReturnValue.NotTableDefined")); //$NON-NLS-1$ } } else { retval.setError( Messages.getString( "ConcurrentCombinationLookupMeta.ReturnValue.NotReceivingField")); //$NON-NLS-1$ } } else { retval.setError( Messages.getString( "ConcurrentCombinationLookupMeta.ReturnValue.NotConnectionDefined")); //$NON-NLS-1$ } return retval; }
private ValueMetaInterface getValueMeta(CalculatorMetaFunction fn, String origin) { ValueMetaInterface v = new ValueMeta(fn.getFieldName(), fn.getValueType()); // What if the user didn't specify a data type? // In that case we look for the default data type // if (fn.getValueType() == ValueMetaInterface.TYPE_NONE) { int defaultResultType = ValueMetaInterface.TYPE_NONE; switch (fn.getCalcType()) { case CalculatorMetaFunction.CALC_NONE: break; case CalculatorMetaFunction.CALC_ADD: // A + B defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_SUBTRACT: // A - B defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_MULTIPLY: // A * B defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_DIVIDE: // A / B defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_SQUARE: // A * A defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_SQUARE_ROOT: // SQRT( A ) defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_PERCENT_1: // 100 * A / B defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_PERCENT_2: // A - ( A * B / 100 ) defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_PERCENT_3: // A + ( A * B / 100 ) defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_COMBINATION_1: // A + B * C defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_COMBINATION_2: // SQRT( A*A + B*B ) defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_ROUND_1: // ROUND( A ) defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_ROUND_2: // ROUND( A , B ) defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_ROUND_STD_1: // STDROUND( A ) defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_ROUND_STD_2: // STDROUND( A , B ) defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_CONSTANT: // Set field to constant value... defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_NVL: // Replace null values with another value break; case CalculatorMetaFunction.CALC_ADD_DAYS: // Add B days to date field A defaultResultType = ValueMetaInterface.TYPE_DATE; break; case CalculatorMetaFunction.CALC_YEAR_OF_DATE: // What is the year (Integer) of a date? defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_MONTH_OF_DATE: // What is the month (Integer) of a date? defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction .CALC_DAY_OF_YEAR: // What is the day of year (Integer) of a date? defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction .CALC_DAY_OF_MONTH: // What is the day of month (Integer) of a date? defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction .CALC_DAY_OF_WEEK: // What is the day of week (Integer) of a date? defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction .CALC_WEEK_OF_YEAR: // What is the week of year (Integer) of a date? defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction .CALC_WEEK_OF_YEAR_ISO8601: // What is the week of year (Integer) of a date ISO8601 // style? defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction .CALC_YEAR_OF_DATE_ISO8601: // What is the year (Integer) of a date ISO8601 style? defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_BYTE_TO_HEX_ENCODE: // Byte to Hex encode string field A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_HEX_TO_BYTE_DECODE: // Hex to Byte decode string field A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_CHAR_TO_HEX_ENCODE: // Char to Hex encode string field A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_HEX_TO_CHAR_DECODE: // Hex to Char decode string field A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_CRC32: // CRC32 of a file A defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_ADLER32: // ADLER32 of a file A defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_MD5: // MD5 of a file A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_SHA1: // SHA1 of a file Al defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction .CALC_LEVENSHTEIN_DISTANCE: // LEVENSHTEIN_DISTANCE of string A and string B defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_METAPHONE: // METAPHONE of string A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_DOUBLE_METAPHONE: // Double METAPHONE of string A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_ABS: // ABS( A ) defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_REMOVE_TIME_FROM_DATE: // Remove time from field A defaultResultType = ValueMetaInterface.TYPE_DATE; break; case CalculatorMetaFunction.CALC_DATE_DIFF: // DateA - DateB defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_ADD3: // A + B +C defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_INITCAP: // InitCap(A) defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_UPPER_CASE: // UpperCase(A) defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_LOWER_CASE: // LowerCase(A) defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_MASK_XML: // MaskXML(A) defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_USE_CDATA: // CDATA(A) defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_REMOVE_CR: // REMOVE CR FROM string A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_REMOVE_LF: // REMOVE LF FROM string A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_REMOVE_CRLF: // REMOVE CRLF FROM string A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_REMOVE_TAB: // REMOVE TAB FROM string A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_GET_ONLY_DIGITS: // GET ONLY DIGITS FROM string A defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_REMOVE_DIGITS: // REMOVE DIGITS FROM string A defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_STRING_LEN: // LENGTH OF string A defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_LOAD_FILE_CONTENT_BINARY: // LOAD FILE CONTENT IN BLOB defaultResultType = ValueMetaInterface.TYPE_BINARY; break; case CalculatorMetaFunction.CALC_ADD_TIME_TO_DATE: // ADD TIME TO A DATE defaultResultType = ValueMetaInterface.TYPE_DATE; break; case CalculatorMetaFunction .CALC_QUARTER_OF_DATE: // What is the quarter (Integer) of a date? defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_SUBSTITUTE_VARIABLE: // variable substitution in string defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_ESCAPE_HTML: // escape HTML defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_ESCAPE_SQL: // escape SQL defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_UNESCAPE_HTML: // unEscape HTML defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_UNESCAPE_XML: // unEscape XML defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_DATE_WORKING_DIFF: // Date A - Date B defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_ADD_MONTHS: // Date A - B Months defaultResultType = ValueMetaInterface.TYPE_DATE; break; case CalculatorMetaFunction.CALC_CHECK_XML_FILE_WELL_FORMED: // XML file A well formed defaultResultType = ValueMetaInterface.TYPE_BOOLEAN; break; case CalculatorMetaFunction.CALC_CHECK_XML_WELL_FORMED: // XML string A well formed defaultResultType = ValueMetaInterface.TYPE_BOOLEAN; break; case CalculatorMetaFunction.CALC_GET_FILE_ENCODING: // get file encoding defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_DAMERAU_LEVENSHTEIN: defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_NEEDLEMAN_WUNSH: defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_JARO: defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_JARO_WINKLER: defaultResultType = ValueMetaInterface.TYPE_NUMBER; break; case CalculatorMetaFunction.CALC_SOUNDEX: defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_REFINED_SOUNDEX: defaultResultType = ValueMetaInterface.TYPE_STRING; break; case CalculatorMetaFunction.CALC_CEIL: // CEIL( A ) defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_FLOOR: // FLOOR( A ) defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_HOUR_OF_DAY: defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_MINUTE_OF_HOUR: defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; case CalculatorMetaFunction.CALC_SECOND_OF_MINUTE: defaultResultType = ValueMetaInterface.TYPE_INTEGER; break; default: break; } v = new ValueMeta(fn.getFieldName(), defaultResultType); } v.setLength(fn.getValueLength()); v.setPrecision(fn.getValuePrecision()); v.setOrigin(origin); v.setComments(fn.getCalcTypeDesc()); v.setConversionMask(fn.getConversionMask()); v.setDecimalSymbol(fn.getDecimalSymbol()); v.setGroupingSymbol(fn.getGroupingSymbol()); v.setCurrencySymbol(fn.getCurrencySymbol()); return v; }
public void getFields( RowMetaInterface r, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore) throws KettleStepException { int i; for (i = 0; i < inputFields.length; i++) { PropertyInputField field = inputFields[i]; int type = field.getType(); if (type == ValueMetaInterface.TYPE_NONE) { type = ValueMetaInterface.TYPE_STRING; } try { ValueMetaInterface v = ValueMetaFactory.createValueMeta(space.environmentSubstitute(field.getName()), type); v.setLength(field.getLength()); v.setPrecision(field.getPrecision()); v.setOrigin(name); v.setConversionMask(field.getFormat()); v.setDecimalSymbol(field.getDecimalSymbol()); v.setGroupingSymbol(field.getGroupSymbol()); v.setCurrencySymbol(field.getCurrencySymbol()); r.addValueMeta(v); } catch (Exception e) { throw new KettleStepException(e); } } String realFilenameField = space.environmentSubstitute(filenameField); if (includeFilename && !Utils.isEmpty(realFilenameField)) { ValueMetaInterface v = new ValueMetaString(realFilenameField); v.setLength(500); v.setPrecision(-1); v.setOrigin(name); r.addValueMeta(v); } String realRowNumberField = space.environmentSubstitute(rowNumberField); if (includeRowNumber && !Utils.isEmpty(realRowNumberField)) { ValueMetaInterface v = new ValueMetaInteger(realRowNumberField); v.setLength(ValueMetaInterface.DEFAULT_INTEGER_LENGTH, 0); v.setOrigin(name); r.addValueMeta(v); } String realSectionField = space.environmentSubstitute(iniSectionField); if (includeIniSection && !Utils.isEmpty(realSectionField)) { ValueMetaInterface v = new ValueMetaString(realSectionField); v.setLength(500); v.setPrecision(-1); v.setOrigin(name); r.addValueMeta(v); } // Add additional fields if (getShortFileNameField() != null && getShortFileNameField().length() > 0) { ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(getShortFileNameField())); v.setLength(100, -1); v.setOrigin(name); r.addValueMeta(v); } if (getExtensionField() != null && getExtensionField().length() > 0) { ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(getExtensionField())); v.setLength(100, -1); v.setOrigin(name); r.addValueMeta(v); } if (getPathField() != null && getPathField().length() > 0) { ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(getPathField())); v.setLength(100, -1); v.setOrigin(name); r.addValueMeta(v); } if (getSizeField() != null && getSizeField().length() > 0) { ValueMetaInterface v = new ValueMetaInteger(space.environmentSubstitute(getSizeField())); v.setOrigin(name); v.setLength(9); r.addValueMeta(v); } if (isHiddenField() != null && isHiddenField().length() > 0) { ValueMetaInterface v = new ValueMetaBoolean(space.environmentSubstitute(isHiddenField())); v.setOrigin(name); r.addValueMeta(v); } if (getLastModificationDateField() != null && getLastModificationDateField().length() > 0) { ValueMetaInterface v = new ValueMetaDate(space.environmentSubstitute(getLastModificationDateField())); v.setOrigin(name); r.addValueMeta(v); } if (getUriField() != null && getUriField().length() > 0) { ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(getUriField())); v.setLength(100, -1); v.setOrigin(name); r.addValueMeta(v); } if (getRootUriField() != null && getRootUriField().length() > 0) { ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(getRootUriField())); v.setLength(100, -1); v.setOrigin(name); r.addValueMeta(v); } }