@Test
public void testGetFieldDefinitionInteger() {
String integerName = "integerName";
ValueMetaInterface valueMetaInterface = new ValueMetaInteger();
valueMetaInterface.setName(integerName);
valueMetaInterface.setPrecision(0);
valueMetaInterface.setLength(9);
assertGetFieldDefinition(valueMetaInterface, "INT");
valueMetaInterface.setLength(18);
assertGetFieldDefinition(valueMetaInterface, "BIGINT");
valueMetaInterface.setLength(19);
assertGetFieldDefinition(valueMetaInterface, "FLOAT");
}
public void getFields(
RowMetaInterface row,
String name,
RowMetaInterface[] info,
StepMeta nextStep,
VariableSpace space,
Repository repository,
IMetaStore metaStore)
throws KettleStepException {
// No values are added to the row in this type of step
// However, in case of Fixed length records,
// the field precisions and lengths are altered!
for (int i = 0; i < outputFields.length; i++) {
TextFileField field = outputFields[i];
ValueMetaInterface v = row.searchValueMeta(field.getName());
if (v != null) {
v.setLength(field.getLength());
v.setPrecision(field.getPrecision());
v.setConversionMask(field.getFormat());
v.setDecimalSymbol(field.getDecimalSymbol());
v.setGroupingSymbol(field.getGroupingSymbol());
v.setCurrencySymbol(field.getCurrencySymbol());
v.setOutputPaddingEnabled(isPadded());
v.setTrimType(field.getTrimType());
if (!Const.isEmpty(getEncoding())) {
v.setStringEncoding(getEncoding());
}
// enable output padding by default to be compatible with v2.5.x
//
v.setOutputPaddingEnabled(true);
}
}
}
@Override
public void getFields(
RowMetaInterface row,
String name,
RowMetaInterface[] info,
StepMeta nextStep,
VariableSpace space,
Repository repository,
IMetaStore metaStore)
throws KettleStepException {
for (SasInputField field : outputFields) {
try {
ValueMetaInterface valueMeta =
ValueMetaFactory.createValueMeta(field.getRename(), field.getType());
valueMeta.setLength(field.getLength(), field.getPrecision());
valueMeta.setDecimalSymbol(field.getDecimalSymbol());
valueMeta.setGroupingSymbol(field.getGroupingSymbol());
valueMeta.setConversionMask(field.getConversionMask());
valueMeta.setTrimType(field.getTrimType());
valueMeta.setOrigin(name);
row.addValueMeta(valueMeta);
} catch (Exception e) {
throw new KettleStepException(e);
}
}
}
public void getFields(
RowMetaInterface r,
String name,
RowMetaInterface info[],
StepMeta nextStep,
VariableSpace space,
Repository repository,
IMetaStore metaStore)
throws KettleStepException {
int i;
for (i = 0; i < inputFields.length; i++) {
RssInputField field = inputFields[i];
int type = field.getType();
if (type == ValueMeta.TYPE_NONE) type = ValueMeta.TYPE_STRING;
try {
ValueMetaInterface v =
ValueMetaFactory.createValueMeta(space.environmentSubstitute(field.getName()), type);
v.setLength(field.getLength(), field.getPrecision());
v.setOrigin(name);
r.addValueMeta(v);
} catch (Exception e) {
throw new KettleStepException(e);
}
}
if (includeUrl) {
ValueMetaInterface v =
new ValueMeta(space.environmentSubstitute(urlField), ValueMeta.TYPE_STRING);
v.setLength(100, -1);
v.setOrigin(name);
r.addValueMeta(v);
}
if (includeRowNumber) {
ValueMetaInterface v =
new ValueMeta(space.environmentSubstitute(rowNumberField), ValueMeta.TYPE_INTEGER);
v.setLength(ValueMetaInterface.DEFAULT_INTEGER_LENGTH, 0);
v.setOrigin(name);
r.addValueMeta(v);
}
}
public static ValueMetaInterface createValueMeta(String name, int type, int length, int precision)
throws KettlePluginException {
PluginInterface stringPlugin =
pluginRegistry.getPlugin(ValueMetaPluginType.class, String.valueOf(type));
if (stringPlugin == null) {
throw new KettlePluginException("Unable to locate value meta plugin of type (id) " + type);
}
ValueMetaInterface valueMeta = pluginRegistry.loadClass(stringPlugin, ValueMetaInterface.class);
valueMeta.setName(name);
valueMeta.setLength(length, precision);
return valueMeta;
}
@Override
public void getFields(
RowMetaInterface row,
String name,
RowMetaInterface[] info,
StepMeta nextStep,
VariableSpace space)
throws KettleStepException {
// Remove the key value (there will be different entries for each output row)
//
if (keyField != null && keyField.length() > 0) {
int idx = row.indexOfValue(keyField);
if (idx < 0) {
throw new KettleStepException(
BaseMessages.getString(
PKG, "DenormaliserMeta.Exception.UnableToLocateKeyField", keyField));
} //$NON-NLS-1$ //$NON-NLS-2$
row.removeValueMeta(idx);
} else {
throw new KettleStepException(
BaseMessages.getString(
PKG, "DenormaliserMeta.Exception.RequiredKeyField")); // $NON-NLS-1$
}
// Remove all field value(s) (there will be different entries for each output row)
//
for (int i = 0; i < denormaliserTargetField.length; i++) {
String fieldname = denormaliserTargetField[i].getFieldName();
if (fieldname != null && fieldname.length() > 0) {
int idx = row.indexOfValue(fieldname);
if (idx >= 0) {
row.removeValueMeta(idx);
}
} else {
throw new KettleStepException(
BaseMessages.getString(
PKG,
"DenormaliserMeta.Exception.RequiredTargetFieldName",
(i + 1) + "")); // $NON-NLS-1$ //$NON-NLS-2$
}
}
// Re-add the target fields
for (int i = 0; i < denormaliserTargetField.length; i++) {
DenormaliserTargetField field = denormaliserTargetField[i];
ValueMetaInterface target = new ValueMeta(field.getTargetName(), field.getTargetType());
target.setLength(field.getTargetLength(), field.getTargetPrecision());
target.setOrigin(name);
row.addValueMeta(target);
}
}
@Test
public void testGetFieldDefinitionBigNumber() {
String bigNumberName = "bigNumberName";
ValueMetaInterface valueMetaInterface = new ValueMetaBigNumber();
valueMetaInterface.setName(bigNumberName);
valueMetaInterface.setPrecision(0);
valueMetaInterface.setLength(9);
assertGetFieldDefinition(valueMetaInterface, "INT");
valueMetaInterface.setLength(18);
assertGetFieldDefinition(valueMetaInterface, "BIGINT");
valueMetaInterface.setLength(19);
assertGetFieldDefinition(valueMetaInterface, "FLOAT");
valueMetaInterface.setPrecision(10);
valueMetaInterface.setLength(16);
assertGetFieldDefinition(valueMetaInterface, "FLOAT");
valueMetaInterface.setLength(15);
assertGetFieldDefinition(valueMetaInterface, "DOUBLE");
}
/* This function adds meta data to the rows being pushed out */
public void getFields(
RowMetaInterface r,
String name,
RowMetaInterface[] info,
StepMeta nextStep,
VariableSpace space,
Repository repository,
IMetaStore metaStore)
throws KettleStepException {
String realfieldname = space.environmentSubstitute(getSalesforceIDFieldName());
if (!Const.isEmpty(realfieldname)) {
ValueMetaInterface v = new ValueMetaString(realfieldname);
v.setLength(18);
v.setOrigin(name);
r.addValueMeta(v);
}
}
public void getFields(
RowMetaInterface inputRowMeta,
String name,
RowMetaInterface info[],
StepMeta nextStep,
VariableSpace space)
throws KettleStepException {
for (int i = 0; i < fieldOutStream.length; i++) {
if (!Const.isEmpty(fieldOutStream[i])) {
ValueMetaInterface v =
new ValueMeta(space.environmentSubstitute(fieldOutStream[i]), ValueMeta.TYPE_STRING);
v.setLength(100, -1);
v.setOrigin(name);
inputRowMeta.addValueMeta(v);
}
}
}
public void getFields(
RowMetaInterface r,
String name,
RowMetaInterface[] info,
StepMeta nextStep,
VariableSpace space) {
// Remove the field to split
int idx = r.indexOfValue(splitField);
if (idx < 0) // not found
{
throw new RuntimeException(
BaseMessages.getString(
PKG,
"FieldSplitter.Log.CouldNotFindFieldToSplit",
splitField)); //$NON-NLS-1$ //$NON-NLS-2$
}
// Add the new fields at the place of the index --> replace!
for (int i = 0; i < fieldName.length; i++) {
final ValueMetaInterface v = new ValueMeta(fieldName[i], fieldType[i]);
v.setLength(fieldLength[i], fieldPrecision[i]);
v.setOrigin(name);
v.setConversionMask(fieldFormat[i]);
v.setDecimalSymbol(fieldDecimal[i]);
v.setGroupingSymbol(fieldGroup[i]);
v.setCurrencySymbol(fieldCurrency[i]);
v.setTrimType(fieldTrimType[i]);
// TODO when implemented in UI
// v.setDateFormatLenient(dateFormatLenient);
// TODO when implemented in UI
// v.setDateFormatLocale(dateFormatLocale);
if (i == 0 && idx >= 0) {
// the first valueMeta (splitField) will be replaced
r.setValueMeta(idx, v);
} else {
// other valueMeta will be added
if (idx >= r.size()) r.addValueMeta(v);
r.addValueMeta(idx + i, v);
}
}
}
/**
* Generates row meta structure from a fields array.
*
* @param fields the fields array
* @param origin the data origin
* @param rowMeta the row meta to generate
*/
public static void fieldsToRowMeta(
final CobFileInputField[] fields, final String origin, final RowMetaInterface rowMeta) {
rowMeta.clear(); // Start with a clean slate, eats the input
for (int i = 0; i < fields.length; i++) {
CobFileInputField field = fields[i];
ValueMetaInterface valueMeta = new ValueMeta(field.getName(), field.getType());
valueMeta.setConversionMask(field.getFormat());
valueMeta.setLength(field.getLength());
valueMeta.setPrecision(field.getPrecision());
valueMeta.setConversionMask(field.getFormat());
valueMeta.setDecimalSymbol(field.getDecimalSymbol());
valueMeta.setGroupingSymbol(field.getGroupSymbol());
valueMeta.setCurrencySymbol(field.getCurrencySymbol());
valueMeta.setTrimType(field.getTrimType());
valueMeta.setOrigin(origin);
rowMeta.addValueMeta(valueMeta);
}
}
public void getFields(
RowMetaInterface row,
String name,
RowMetaInterface[] info,
StepMeta nextStep,
VariableSpace space,
Repository repository,
IMetaStore metaStore)
throws KettleStepException {
ValueMetaInterface v = new ValueMetaString(newFieldname);
v.setOrigin(name);
row.addValueMeta(v);
// include row number
if (includeRowNumber) {
v = new ValueMetaInteger(space.environmentSubstitute(rowNumberField));
v.setLength(ValueMetaInterface.DEFAULT_INTEGER_LENGTH, 0);
v.setOrigin(name);
row.addValueMeta(v);
}
}
public void getFields(
RowMetaInterface row,
String origin,
RowMetaInterface[] info,
StepMeta nextStep,
VariableSpace space)
throws KettleStepException {
if (!Const.isEmpty(technicalKeyField)) {
ValueMetaInterface v = new ValueMeta(technicalKeyField, ValueMetaInterface.TYPE_INTEGER);
v.setLength(10);
v.setPrecision(0);
v.setOrigin(origin);
row.addValueMeta(v);
}
if (replaceFields) {
for (int i = 0; i < keyField.length; i++) {
int idx = row.indexOfValue(keyField[i]);
if (idx >= 0) {
row.removeValueMeta(idx);
}
}
}
}
public static final RowMetaAndData buildRow(
RowGeneratorMeta meta, List<CheckResultInterface> remarks, String origin) {
RowMetaInterface rowMeta = new RowMeta();
Object[] rowData = RowDataUtil.allocateRowData(meta.getFieldName().length);
for (int i = 0; i < meta.getFieldName().length; i++) {
int valtype = ValueMeta.getType(meta.getFieldType()[i]);
if (meta.getFieldName()[i] != null) {
ValueMetaInterface valueMeta =
new ValueMeta(meta.getFieldName()[i], valtype); // build a value!
valueMeta.setLength(meta.getFieldLength()[i]);
valueMeta.setPrecision(meta.getFieldPrecision()[i]);
valueMeta.setConversionMask(meta.getFieldFormat()[i]);
valueMeta.setGroupingSymbol(meta.getGroup()[i]);
valueMeta.setDecimalSymbol(meta.getDecimal()[i]);
valueMeta.setOrigin(origin);
ValueMetaInterface stringMeta = valueMeta.clone();
stringMeta.setType(ValueMetaInterface.TYPE_STRING);
String stringValue = meta.getValue()[i];
// If the value is empty: consider it to be NULL.
if (Const.isEmpty(stringValue)) {
rowData[i] = null;
if (valueMeta.getType() == ValueMetaInterface.TYPE_NONE) {
String message =
BaseMessages.getString(
PKG,
"RowGenerator.CheckResult.SpecifyTypeError",
valueMeta.getName(),
stringValue);
remarks.add(new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null));
}
} else {
// Convert the data from String to the specified type ...
//
try {
rowData[i] = valueMeta.convertData(stringMeta, stringValue);
} catch (KettleValueException e) {
switch (valueMeta.getType()) {
case ValueMetaInterface.TYPE_NUMBER:
{
String message =
BaseMessages.getString(
PKG,
"RowGenerator.BuildRow.Error.Parsing.Number",
valueMeta.getName(),
stringValue,
e.toString());
remarks.add(
new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null));
}
break;
case ValueMetaInterface.TYPE_DATE:
{
String message =
BaseMessages.getString(
PKG,
"RowGenerator.BuildRow.Error.Parsing.Date",
valueMeta.getName(),
stringValue,
e.toString());
remarks.add(
new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null));
}
break;
case ValueMetaInterface.TYPE_INTEGER:
{
String message =
BaseMessages.getString(
PKG,
"RowGenerator.BuildRow.Error.Parsing.Integer",
valueMeta.getName(),
stringValue,
e.toString());
remarks.add(
new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null));
}
break;
case ValueMetaInterface.TYPE_BIGNUMBER:
{
String message =
BaseMessages.getString(
PKG,
"RowGenerator.BuildRow.Error.Parsing.BigNumber",
valueMeta.getName(),
stringValue,
e.toString());
remarks.add(
new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null));
}
break;
default:
// Boolean and binary don't throw errors normally, so it's probably an unspecified
// error problem...
{
String message =
BaseMessages.getString(
PKG,
"RowGenerator.CheckResult.SpecifyTypeError",
valueMeta.getName(),
stringValue);
remarks.add(
new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null));
}
break;
}
}
}
// Now add value to the row!
// This is in fact a copy from the fields row, but now with data.
rowMeta.addValueMeta(valueMeta);
}
}
return new RowMetaAndData(rowMeta, rowData);
}
private void writeRowToFile(RowMetaInterface rowMeta, Object[] r) throws KettleException {
try {
if (first) {
data.formatRowMeta = rowMeta.clone();
first = false;
data.fieldnrs = new int[meta.getOutputFields().length];
for (int i = 0; i < meta.getOutputFields().length; i++) {
data.fieldnrs[i] =
data.formatRowMeta.indexOfValue(meta.getOutputFields()[i].getFieldName());
if (data.fieldnrs[i] < 0) {
throw new KettleException(
"Field ["
+ meta.getOutputFields()[i].getFieldName()
+ "] couldn't be found in the input stream!"); //$NON-NLS-1$ //$NON-NLS-2$
}
// Apply the formatting settings to the valueMeta object...
//
ValueMetaInterface valueMeta = data.formatRowMeta.getValueMeta(data.fieldnrs[i]);
XMLField field = meta.getOutputFields()[i];
valueMeta.setConversionMask(field.getFormat());
valueMeta.setLength(field.getLength(), field.getPrecision());
valueMeta.setDecimalSymbol(field.getDecimalSymbol());
valueMeta.setGroupingSymbol(field.getGroupingSymbol());
valueMeta.setCurrencySymbol(field.getCurrencySymbol());
}
}
if (meta.getOutputFields() == null || meta.getOutputFields().length == 0) {
/*
* Write all values in stream to text file.
*/
// OK, write a new row to the XML file:
data.writer.write(
(" <" + meta.getRepeatElement() + ">").toCharArray()); // $NON-NLS-1$ //$NON-NLS-2$
for (int i = 0; i < data.formatRowMeta.size(); i++) {
// Put a space between the XML elements of the row
//
if (i > 0) data.writer.write(' ');
ValueMetaInterface valueMeta = data.formatRowMeta.getValueMeta(i);
Object valueData = r[i];
writeField(valueMeta, valueData, valueMeta.getName());
}
} else {
/*
* Only write the fields specified!
*/
// Write a new row to the XML file:
data.writer.write(
(" <" + meta.getRepeatElement() + ">").toCharArray()); // $NON-NLS-1$ //$NON-NLS-2$
for (int i = 0; i < meta.getOutputFields().length; i++) {
XMLField outputField = meta.getOutputFields()[i];
if (i > 0) data.writer.write(' '); // a space between
// elements
ValueMetaInterface valueMeta = data.formatRowMeta.getValueMeta(data.fieldnrs[i]);
Object valueData = r[data.fieldnrs[i]];
String elementName = outputField.getElementName();
if (Const.isEmpty(elementName)) {
elementName = outputField.getFieldName();
}
if (!(valueMeta.isNull(valueData) && meta.isOmitNullValues())) {
writeField(valueMeta, valueData, elementName);
}
}
}
data.writer.write(
(" </" + meta.getRepeatElement() + ">").toCharArray()); // $NON-NLS-1$ //$NON-NLS-2$
data.writer.write(Const.CR.toCharArray());
} catch (Exception e) {
throw new KettleException(
"Error writing XML row :"
+ e.toString()
+ Const.CR
+ "Row: "
+ getInputRowMeta().getString(r),
e); //$NON-NLS-1$ //$NON-NLS-2$
}
incrementLinesOutput();
}
@Override
public void getFields(
RowMetaInterface rowMeta,
String origin,
RowMetaInterface[] info,
StepMeta nextStep,
VariableSpace space,
Repository repository,
IMetaStore metaStore) {
// re-assemble a new row of metadata
//
RowMetaInterface fields = new RowMeta();
if (!passAllRows) {
// Add the grouping fields in the correct order...
//
for (int i = 0; i < groupField.length; i++) {
ValueMetaInterface valueMeta = rowMeta.searchValueMeta(groupField[i]);
if (valueMeta != null) {
fields.addValueMeta(valueMeta);
}
}
} else {
// Add all the original fields from the incoming row meta
//
fields.addRowMeta(rowMeta);
}
// Re-add aggregates
//
for (int i = 0; i < subjectField.length; i++) {
ValueMetaInterface subj = rowMeta.searchValueMeta(subjectField[i]);
if (subj != null || aggregateType[i] == TYPE_GROUP_COUNT_ANY) {
String valueName = aggregateField[i];
int valueType = ValueMetaInterface.TYPE_NONE;
int length = -1;
int precision = -1;
switch (aggregateType[i]) {
case TYPE_GROUP_SUM:
case TYPE_GROUP_AVERAGE:
case TYPE_GROUP_CUMULATIVE_SUM:
case TYPE_GROUP_CUMULATIVE_AVERAGE:
case TYPE_GROUP_FIRST:
case TYPE_GROUP_LAST:
case TYPE_GROUP_FIRST_INCL_NULL:
case TYPE_GROUP_LAST_INCL_NULL:
case TYPE_GROUP_MIN:
case TYPE_GROUP_MAX:
valueType = subj.getType();
break;
case TYPE_GROUP_COUNT_DISTINCT:
case TYPE_GROUP_COUNT_ANY:
case TYPE_GROUP_COUNT_ALL:
valueType = ValueMetaInterface.TYPE_INTEGER;
break;
case TYPE_GROUP_CONCAT_COMMA:
valueType = ValueMetaInterface.TYPE_STRING;
break;
case TYPE_GROUP_STANDARD_DEVIATION:
case TYPE_GROUP_MEDIAN:
case TYPE_GROUP_PERCENTILE:
valueType = ValueMetaInterface.TYPE_NUMBER;
break;
case TYPE_GROUP_CONCAT_STRING:
valueType = ValueMetaInterface.TYPE_STRING;
break;
default:
break;
}
// Change type from integer to number in case off averages for cumulative average
//
if (aggregateType[i] == TYPE_GROUP_CUMULATIVE_AVERAGE
&& valueType == ValueMetaInterface.TYPE_INTEGER) {
valueType = ValueMetaInterface.TYPE_NUMBER;
precision = -1;
length = -1;
} else if (aggregateType[i] == TYPE_GROUP_COUNT_ALL
|| aggregateType[i] == TYPE_GROUP_COUNT_DISTINCT
|| aggregateType[i] == TYPE_GROUP_COUNT_ANY) {
length = ValueMetaInterface.DEFAULT_INTEGER_LENGTH;
precision = 0;
} else if (aggregateType[i] == TYPE_GROUP_SUM
&& valueType != ValueMetaInterface.TYPE_INTEGER
&& valueType != ValueMetaInterface.TYPE_NUMBER
&& valueType != ValueMetaInterface.TYPE_BIGNUMBER) {
// If it ain't numeric, we change it to Number
//
valueType = ValueMetaInterface.TYPE_NUMBER;
precision = -1;
length = -1;
}
if (valueType != ValueMetaInterface.TYPE_NONE) {
ValueMetaInterface v = new ValueMeta(valueName, valueType);
v.setOrigin(origin);
v.setLength(length, precision);
fields.addValueMeta(v);
}
}
}
if (passAllRows) {
// If we pass all rows, we can add a line nr in the group...
if (addingLineNrInGroup && !Utils.isEmpty(lineNrInGroupField)) {
ValueMetaInterface lineNr = new ValueMetaInteger(lineNrInGroupField);
lineNr.setLength(ValueMetaInterface.DEFAULT_INTEGER_LENGTH, 0);
lineNr.setOrigin(origin);
fields.addValueMeta(lineNr);
}
}
// Now that we have all the fields we want, we should clear the original row and replace the
// values...
//
rowMeta.clear();
rowMeta.addRowMeta(fields);
}
public static final RowMetaInterface getLayout(Table table) throws SQLException {
RowMetaInterface row = new RowMeta();
List<Column> columns = table.getColumns();
for (int i = 0; i < columns.size(); i++) {
Column column = columns.get(i);
int valtype = ValueMetaInterface.TYPE_STRING;
int length = -1;
int precision = -1;
int type = column.getType().getSQLType();
switch (type) {
case java.sql.Types.CHAR:
case java.sql.Types.VARCHAR:
case java.sql.Types.LONGVARCHAR: // Character Large Object
valtype = ValueMetaInterface.TYPE_STRING;
length = column.getLength();
break;
case java.sql.Types.CLOB:
valtype = ValueMetaInterface.TYPE_STRING;
length = DatabaseMeta.CLOB_LENGTH;
break;
case java.sql.Types.BIGINT:
valtype = ValueMetaInterface.TYPE_INTEGER;
precision = 0; // Max 9.223.372.036.854.775.807
length = 15;
break;
case java.sql.Types.INTEGER:
valtype = ValueMetaInterface.TYPE_INTEGER;
precision = 0; // Max 2.147.483.647
length = 9;
break;
case java.sql.Types.SMALLINT:
valtype = ValueMetaInterface.TYPE_INTEGER;
precision = 0; // Max 32.767
length = 4;
break;
case java.sql.Types.TINYINT:
valtype = ValueMetaInterface.TYPE_INTEGER;
precision = 0; // Max 127
length = 2;
break;
case java.sql.Types.DECIMAL:
case java.sql.Types.DOUBLE:
case java.sql.Types.FLOAT:
case java.sql.Types.REAL:
case java.sql.Types.NUMERIC:
valtype = ValueMetaInterface.TYPE_NUMBER;
length = column.getLength();
precision = column.getPrecision();
if (length >= 126) {
length = -1;
}
if (precision >= 126) {
precision = -1;
}
if (type == java.sql.Types.DOUBLE
|| type == java.sql.Types.FLOAT
|| type == java.sql.Types.REAL) {
if (precision == 0) {
precision = -1; // precision is obviously incorrect if the type if Double/Float/Real
}
} else {
if (precision == 0
&& length < 18
&& length > 0) { // Among others Oracle is affected here.
valtype = ValueMetaInterface.TYPE_INTEGER;
}
}
if (length > 18 || precision > 18) {
valtype = ValueMetaInterface.TYPE_BIGNUMBER;
}
break;
case java.sql.Types.DATE:
case java.sql.Types.TIME:
case java.sql.Types.TIMESTAMP:
valtype = ValueMetaInterface.TYPE_DATE;
break;
case java.sql.Types.BOOLEAN:
case java.sql.Types.BIT:
valtype = ValueMetaInterface.TYPE_BOOLEAN;
break;
case java.sql.Types.BINARY:
case java.sql.Types.BLOB:
case java.sql.Types.VARBINARY:
case java.sql.Types.LONGVARBINARY:
valtype = ValueMetaInterface.TYPE_BINARY;
break;
default:
valtype = ValueMetaInterface.TYPE_STRING;
length = column.getLength();
break;
}
ValueMetaInterface v = new ValueMeta(column.getName(), valtype);
v.setLength(length, precision);
row.addValueMeta(v);
}
return row;
}
private ValueMetaInterface getValueMeta(CalculatorMetaFunction fn, String origin) {
ValueMetaInterface v = new ValueMeta(fn.getFieldName(), fn.getValueType());
// What if the user didn't specify a data type?
// In that case we look for the default data type
//
if (fn.getValueType() == ValueMetaInterface.TYPE_NONE) {
int defaultResultType = ValueMetaInterface.TYPE_NONE;
switch (fn.getCalcType()) {
case CalculatorMetaFunction.CALC_NONE:
break;
case CalculatorMetaFunction.CALC_ADD: // A + B
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_SUBTRACT: // A - B
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_MULTIPLY: // A * B
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_DIVIDE: // A / B
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_SQUARE: // A * A
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_SQUARE_ROOT: // SQRT( A )
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_PERCENT_1: // 100 * A / B
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_PERCENT_2: // A - ( A * B / 100 )
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_PERCENT_3: // A + ( A * B / 100 )
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_COMBINATION_1: // A + B * C
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_COMBINATION_2: // SQRT( A*A + B*B )
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_ROUND_1: // ROUND( A )
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_ROUND_2: // ROUND( A , B )
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_ROUND_STD_1: // STDROUND( A )
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_ROUND_STD_2: // STDROUND( A , B )
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_CONSTANT: // Set field to constant value...
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_NVL: // Replace null values with another value
break;
case CalculatorMetaFunction.CALC_ADD_DAYS: // Add B days to date field A
defaultResultType = ValueMetaInterface.TYPE_DATE;
break;
case CalculatorMetaFunction.CALC_YEAR_OF_DATE: // What is the year (Integer) of a date?
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_MONTH_OF_DATE: // What is the month (Integer) of a date?
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction
.CALC_DAY_OF_YEAR: // What is the day of year (Integer) of a date?
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction
.CALC_DAY_OF_MONTH: // What is the day of month (Integer) of a date?
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction
.CALC_DAY_OF_WEEK: // What is the day of week (Integer) of a date?
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction
.CALC_WEEK_OF_YEAR: // What is the week of year (Integer) of a date?
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction
.CALC_WEEK_OF_YEAR_ISO8601: // What is the week of year (Integer) of a date ISO8601
// style?
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction
.CALC_YEAR_OF_DATE_ISO8601: // What is the year (Integer) of a date ISO8601 style?
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_BYTE_TO_HEX_ENCODE: // Byte to Hex encode string field A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_HEX_TO_BYTE_DECODE: // Hex to Byte decode string field A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_CHAR_TO_HEX_ENCODE: // Char to Hex encode string field A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_HEX_TO_CHAR_DECODE: // Hex to Char decode string field A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_CRC32: // CRC32 of a file A
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_ADLER32: // ADLER32 of a file A
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_MD5: // MD5 of a file A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_SHA1: // SHA1 of a file Al
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction
.CALC_LEVENSHTEIN_DISTANCE: // LEVENSHTEIN_DISTANCE of string A and string B
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_METAPHONE: // METAPHONE of string A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_DOUBLE_METAPHONE: // Double METAPHONE of string A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_ABS: // ABS( A )
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_REMOVE_TIME_FROM_DATE: // Remove time from field A
defaultResultType = ValueMetaInterface.TYPE_DATE;
break;
case CalculatorMetaFunction.CALC_DATE_DIFF: // DateA - DateB
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_ADD3: // A + B +C
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_INITCAP: // InitCap(A)
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_UPPER_CASE: // UpperCase(A)
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_LOWER_CASE: // LowerCase(A)
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_MASK_XML: // MaskXML(A)
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_USE_CDATA: // CDATA(A)
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_REMOVE_CR: // REMOVE CR FROM string A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_REMOVE_LF: // REMOVE LF FROM string A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_REMOVE_CRLF: // REMOVE CRLF FROM string A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_REMOVE_TAB: // REMOVE TAB FROM string A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_GET_ONLY_DIGITS: // GET ONLY DIGITS FROM string A
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_REMOVE_DIGITS: // REMOVE DIGITS FROM string A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_STRING_LEN: // LENGTH OF string A
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_LOAD_FILE_CONTENT_BINARY: // LOAD FILE CONTENT IN BLOB
defaultResultType = ValueMetaInterface.TYPE_BINARY;
break;
case CalculatorMetaFunction.CALC_ADD_TIME_TO_DATE: // ADD TIME TO A DATE
defaultResultType = ValueMetaInterface.TYPE_DATE;
break;
case CalculatorMetaFunction
.CALC_QUARTER_OF_DATE: // What is the quarter (Integer) of a date?
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_SUBSTITUTE_VARIABLE: // variable substitution in string
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_ESCAPE_HTML: // escape HTML
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_ESCAPE_SQL: // escape SQL
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_UNESCAPE_HTML: // unEscape HTML
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_UNESCAPE_XML: // unEscape XML
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_DATE_WORKING_DIFF: // Date A - Date B
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_ADD_MONTHS: // Date A - B Months
defaultResultType = ValueMetaInterface.TYPE_DATE;
break;
case CalculatorMetaFunction.CALC_CHECK_XML_FILE_WELL_FORMED: // XML file A well formed
defaultResultType = ValueMetaInterface.TYPE_BOOLEAN;
break;
case CalculatorMetaFunction.CALC_CHECK_XML_WELL_FORMED: // XML string A well formed
defaultResultType = ValueMetaInterface.TYPE_BOOLEAN;
break;
case CalculatorMetaFunction.CALC_GET_FILE_ENCODING: // get file encoding
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_DAMERAU_LEVENSHTEIN:
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_NEEDLEMAN_WUNSH:
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_JARO:
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_JARO_WINKLER:
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_SOUNDEX:
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_REFINED_SOUNDEX:
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_CEIL: // CEIL( A )
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_FLOOR: // FLOOR( A )
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_HOUR_OF_DAY:
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_MINUTE_OF_HOUR:
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_SECOND_OF_MINUTE:
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
default:
break;
}
v = new ValueMeta(fn.getFieldName(), defaultResultType);
}
v.setLength(fn.getValueLength());
v.setPrecision(fn.getValuePrecision());
v.setOrigin(origin);
v.setComments(fn.getCalcTypeDesc());
v.setConversionMask(fn.getConversionMask());
v.setDecimalSymbol(fn.getDecimalSymbol());
v.setGroupingSymbol(fn.getGroupingSymbol());
v.setCurrencySymbol(fn.getCurrencySymbol());
return v;
}
public void getFields(
RowMetaInterface rowMeta,
String origin,
RowMetaInterface[] info,
StepMeta nextStep,
VariableSpace space)
throws KettleStepException {
rowMeta.clear(); // Start with a clean slate, eats the input
for (int i = 0; i < inputFields.length; i++) {
TextFileInputField field = inputFields[i];
ValueMetaInterface valueMeta = new ValueMeta(field.getName(), field.getType());
valueMeta.setConversionMask(field.getFormat());
valueMeta.setLength(field.getLength());
valueMeta.setPrecision(field.getPrecision());
valueMeta.setConversionMask(field.getFormat());
valueMeta.setDecimalSymbol(field.getDecimalSymbol());
valueMeta.setGroupingSymbol(field.getGroupSymbol());
valueMeta.setCurrencySymbol(field.getCurrencySymbol());
valueMeta.setTrimType(field.getTrimType());
if (lazyConversionActive)
valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING);
valueMeta.setStringEncoding(space.environmentSubstitute(encoding));
// In case we want to convert Strings...
// Using a copy of the valueMeta object means that the inner and outer representation format
// is the same.
// Preview will show the data the same way as we read it.
// This layout is then taken further down the road by the metadata through the transformation.
//
ValueMetaInterface storageMetadata = valueMeta.clone();
storageMetadata.setType(ValueMetaInterface.TYPE_STRING);
storageMetadata.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
storageMetadata.setLength(
-1, -1); // we don't really know the lengths of the strings read in advance.
valueMeta.setStorageMetadata(storageMetadata);
valueMeta.setOrigin(origin);
rowMeta.addValueMeta(valueMeta);
}
if (!Const.isEmpty(filenameField) && includingFilename) {
ValueMetaInterface filenameMeta =
new ValueMeta(filenameField, ValueMetaInterface.TYPE_STRING);
filenameMeta.setOrigin(origin);
if (lazyConversionActive) {
filenameMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING);
filenameMeta.setStorageMetadata(
new ValueMeta(filenameField, ValueMetaInterface.TYPE_STRING));
}
rowMeta.addValueMeta(filenameMeta);
}
if (!Const.isEmpty(rowNumField)) {
ValueMetaInterface rowNumMeta = new ValueMeta(rowNumField, ValueMetaInterface.TYPE_INTEGER);
rowNumMeta.setLength(10);
rowNumMeta.setOrigin(origin);
rowMeta.addValueMeta(rowNumMeta);
}
}
public void getFields(
RowMetaInterface r,
String name,
RowMetaInterface[] info,
StepMeta nextStep,
VariableSpace space,
Repository repository,
IMetaStore metaStore)
throws KettleStepException {
int i;
for (i = 0; i < inputFields.length; i++) {
PropertyInputField field = inputFields[i];
int type = field.getType();
if (type == ValueMetaInterface.TYPE_NONE) {
type = ValueMetaInterface.TYPE_STRING;
}
try {
ValueMetaInterface v =
ValueMetaFactory.createValueMeta(space.environmentSubstitute(field.getName()), type);
v.setLength(field.getLength());
v.setPrecision(field.getPrecision());
v.setOrigin(name);
v.setConversionMask(field.getFormat());
v.setDecimalSymbol(field.getDecimalSymbol());
v.setGroupingSymbol(field.getGroupSymbol());
v.setCurrencySymbol(field.getCurrencySymbol());
r.addValueMeta(v);
} catch (Exception e) {
throw new KettleStepException(e);
}
}
String realFilenameField = space.environmentSubstitute(filenameField);
if (includeFilename && !Utils.isEmpty(realFilenameField)) {
ValueMetaInterface v = new ValueMetaString(realFilenameField);
v.setLength(500);
v.setPrecision(-1);
v.setOrigin(name);
r.addValueMeta(v);
}
String realRowNumberField = space.environmentSubstitute(rowNumberField);
if (includeRowNumber && !Utils.isEmpty(realRowNumberField)) {
ValueMetaInterface v = new ValueMetaInteger(realRowNumberField);
v.setLength(ValueMetaInterface.DEFAULT_INTEGER_LENGTH, 0);
v.setOrigin(name);
r.addValueMeta(v);
}
String realSectionField = space.environmentSubstitute(iniSectionField);
if (includeIniSection && !Utils.isEmpty(realSectionField)) {
ValueMetaInterface v = new ValueMetaString(realSectionField);
v.setLength(500);
v.setPrecision(-1);
v.setOrigin(name);
r.addValueMeta(v);
}
// Add additional fields
if (getShortFileNameField() != null && getShortFileNameField().length() > 0) {
ValueMetaInterface v =
new ValueMetaString(space.environmentSubstitute(getShortFileNameField()));
v.setLength(100, -1);
v.setOrigin(name);
r.addValueMeta(v);
}
if (getExtensionField() != null && getExtensionField().length() > 0) {
ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(getExtensionField()));
v.setLength(100, -1);
v.setOrigin(name);
r.addValueMeta(v);
}
if (getPathField() != null && getPathField().length() > 0) {
ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(getPathField()));
v.setLength(100, -1);
v.setOrigin(name);
r.addValueMeta(v);
}
if (getSizeField() != null && getSizeField().length() > 0) {
ValueMetaInterface v = new ValueMetaInteger(space.environmentSubstitute(getSizeField()));
v.setOrigin(name);
v.setLength(9);
r.addValueMeta(v);
}
if (isHiddenField() != null && isHiddenField().length() > 0) {
ValueMetaInterface v = new ValueMetaBoolean(space.environmentSubstitute(isHiddenField()));
v.setOrigin(name);
r.addValueMeta(v);
}
if (getLastModificationDateField() != null && getLastModificationDateField().length() > 0) {
ValueMetaInterface v =
new ValueMetaDate(space.environmentSubstitute(getLastModificationDateField()));
v.setOrigin(name);
r.addValueMeta(v);
}
if (getUriField() != null && getUriField().length() > 0) {
ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(getUriField()));
v.setLength(100, -1);
v.setOrigin(name);
r.addValueMeta(v);
}
if (getRootUriField() != null && getRootUriField().length() > 0) {
ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(getRootUriField()));
v.setLength(100, -1);
v.setOrigin(name);
r.addValueMeta(v);
}
}
public SQLStatement getSQLStatements(
TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev) {
SQLStatement retval =
new SQLStatement(stepMeta.getName(), databaseWriteMeta, null); // default: nothing to do!
int i;
if (databaseWriteMeta != null) {
if (prev != null && prev.size() > 0) {
if (!Const.isEmpty(tablename)) {
String schemaTable =
databaseWriteMeta.getQuotedSchemaTableCombination(schemaName, tablename);
Database db = new Database(databaseWriteMeta);
try {
boolean doHash = false;
String cr_table = null;
db.connect();
// OK, what do we put in the new table??
RowMetaInterface fields = new RowMeta();
ValueMetaInterface vkeyfield = null;
if (!Const.isEmpty(technicalKeyField)) {
// First, the new technical key...
vkeyfield = new ValueMeta(technicalKeyField, ValueMetaInterface.TYPE_INTEGER);
vkeyfield.setLength(10);
vkeyfield.setPrecision(0);
}
// Then the hashcode (optional)
ValueMetaInterface vhashfield = null;
if (useHash && !Const.isEmpty(hashField)) {
vhashfield = new ValueMeta(hashField, ValueMetaInterface.TYPE_INTEGER);
vhashfield.setLength(15);
vhashfield.setPrecision(0);
doHash = true;
}
// Then the last update field (optional)
ValueMetaInterface vLastUpdateField = null;
if (!Const.isEmpty(lastUpdateField)) {
vLastUpdateField = new ValueMeta(lastUpdateField, ValueMetaInterface.TYPE_DATE);
}
if (!db.checkTableExists(schemaTable)) {
if (vkeyfield != null) {
// Add technical key field.
fields.addValueMeta(vkeyfield);
}
// Add the keys only to the table
if (keyField != null && keyLookup != null) {
int cnt = keyField.length;
for (i = 0; i < cnt; i++) {
String error_field = ""; // $NON-NLS-1$
// Find the value in the stream
ValueMetaInterface v = prev.searchValueMeta(keyField[i]);
if (v != null) {
String name = keyLookup[i];
ValueMetaInterface newValue = v.clone();
newValue.setName(name);
if (vkeyfield != null) {
if (name.equals(vkeyfield.getName())
|| (doHash == true && name.equals(vhashfield.getName()))) {
error_field += name;
}
}
if (error_field.length() > 0) {
retval.setError(
Messages.getString(
"ConcurrentCombinationLookupMeta.ReturnValue.NameCollision",
error_field)); //$NON-NLS-1$
} else {
fields.addValueMeta(newValue);
}
}
}
}
if (doHash == true) {
fields.addValueMeta(vhashfield);
}
if (vLastUpdateField != null) {
fields.addValueMeta(vLastUpdateField);
}
} else {
// Table already exists
// Get the fields that are in the table now:
RowMetaInterface tabFields = db.getTableFields(schemaTable);
// Don't forget to quote these as well...
databaseWriteMeta.quoteReservedWords(tabFields);
if (vkeyfield != null && tabFields.searchValueMeta(vkeyfield.getName()) == null) {
// Add technical key field if it didn't exist yet
fields.addValueMeta(vkeyfield);
}
// Add the already existing fields
int cnt = tabFields.size();
for (i = 0; i < cnt; i++) {
ValueMetaInterface v = tabFields.getValueMeta(i);
fields.addValueMeta(v);
}
// Find the missing fields in the real table
String keyLookup[] = getKeyLookup();
String keyField[] = getKeyField();
if (keyField != null && keyLookup != null) {
cnt = keyField.length;
for (i = 0; i < cnt; i++) {
// Find the value in the stream
ValueMetaInterface v = prev.searchValueMeta(keyField[i]);
if (v != null) {
ValueMetaInterface newValue = v.clone();
newValue.setName(keyLookup[i]);
// Does the corresponding name exist in the table
if (tabFields.searchValueMeta(newValue.getName()) == null) {
fields.addValueMeta(newValue); // nope --> add
}
}
}
}
if (doHash == true && tabFields.searchValueMeta(vhashfield.getName()) == null) {
// Add hash field
fields.addValueMeta(vhashfield);
}
if (vLastUpdateField != null
&& tabFields.searchValueMeta(vLastUpdateField.getName()) == null) {
fields.addValueMeta(vLastUpdateField);
}
}
cr_table =
db.getDDL(
schemaTable,
fields,
(CREATION_METHOD_SEQUENCE.equals(getTechKeyCreation())
&& sequenceFrom != null
&& sequenceFrom.length() != 0)
? null
: technicalKeyField,
CREATION_METHOD_AUTOINC.equals(getTechKeyCreation()),
null,
true);
//
// OK, now let's build the index
//
// What fields do we put int the index?
// Only the hashcode or all fields?
String cr_index = ""; // $NON-NLS-1$
String cr_uniq_index = ""; // $NON-NLS-1$
String idx_fields[] = null;
if (useHash) {
if (hashField != null && hashField.length() > 0) {
idx_fields = new String[] {hashField};
} else {
retval.setError(
Messages.getString(
"ConcurrentCombinationLookupMeta.ReturnValue.NotHashFieldSpecified")); //$NON-NLS-1$
}
} else // index on all key fields...
{
if (!Const.isEmpty(keyLookup)) {
int nrfields = keyLookup.length;
if (nrfields > 32
&& databaseWriteMeta.getDatabaseType() == DatabaseMeta.TYPE_DATABASE_ORACLE) {
nrfields = 32; // Oracle indexes are limited to 32 fields...
}
idx_fields = new String[nrfields];
for (i = 0; i < nrfields; i++) idx_fields[i] = keyLookup[i];
} else {
retval.setError(
Messages.getString(
"ConcurrentCombinationLookupMeta.ReturnValue.NotFieldsSpecified")); //$NON-NLS-1$
}
}
// OK, now get the create index statement...
if (!Const.isEmpty(technicalKeyField)) {
String techKeyArr[] = new String[] {technicalKeyField};
if (!db.checkIndexExists(schemaName, tablename, techKeyArr)) {
String indexname = "idx_" + tablename + "_pk"; // $NON-NLS-1$ //$NON-NLS-2$
cr_uniq_index =
db.getCreateIndexStatement(
schemaName, tablename, indexname, techKeyArr, true, true, false, true);
cr_uniq_index += Const.CR;
}
}
// OK, now get the create lookup index statement...
if (!Const.isEmpty(idx_fields)
&& !db.checkIndexExists(schemaName, tablename, idx_fields)) {
String indexname = "idx_" + tablename + "_lookup"; // $NON-NLS-1$ //$NON-NLS-2$
cr_index =
db.getCreateIndexStatement(
schemaName, tablename, indexname, idx_fields, false, false, false, true);
cr_index += Const.CR;
}
//
// Don't forget the sequence (optional)
//
String cr_seq = ""; // $NON-NLS-1$
if (databaseWriteMeta.supportsSequences() && !Const.isEmpty(sequenceFrom)) {
if (!db.checkSequenceExists(schemaName, sequenceFrom)) {
cr_seq +=
db.getCreateSequenceStatement(schemaName, sequenceFrom, 1L, 1L, -1L, true);
cr_seq += Const.CR;
}
}
retval.setSQL(cr_table + cr_uniq_index + cr_index + cr_seq);
} catch (KettleException e) {
retval.setError(
Messages.getString("ConcurrentCombinationLookupMeta.ReturnValue.ErrorOccurred")
+ Const.CR
+ e.getMessage()); // $NON-NLS-1$
}
} else {
retval.setError(
Messages.getString(
"ConcurrentCombinationLookupMeta.ReturnValue.NotTableDefined")); //$NON-NLS-1$
}
} else {
retval.setError(
Messages.getString(
"ConcurrentCombinationLookupMeta.ReturnValue.NotReceivingField")); //$NON-NLS-1$
}
} else {
retval.setError(
Messages.getString(
"ConcurrentCombinationLookupMeta.ReturnValue.NotConnectionDefined")); //$NON-NLS-1$
}
return retval;
}
public void getFields(
RowMetaInterface row,
String origin,
RowMetaInterface[] info,
StepMeta nextStep,
VariableSpace space)
throws KettleStepException {
// Row should normally be empty when we get here.
// That is because there is no previous step to this mapping input step from the viewpoint of
// this single sub-transformation.
// From the viewpoint of the transformation that executes the mapping, it's important to know
// what comes out at the exit points.
// For that reason we need to re-order etc, based on the input specification...
//
if (inputRowMeta != null && !inputRowMeta.isEmpty()) {
// this gets set only in the parent transformation...
// It includes all the renames that needed to be done
//
if (selectingAndSortingUnspecifiedFields) {
// First rename any fields...
if (valueRenames != null) {
for (MappingValueRename valueRename : valueRenames) {
ValueMetaInterface valueMeta =
inputRowMeta.searchValueMeta(valueRename.getSourceValueName());
if (valueMeta == null) {
throw new KettleStepException(
BaseMessages.getString(
PKG,
"MappingInput.Exception.UnableToFindMappedValue",
valueRename.getSourceValueName()));
}
valueMeta.setName(valueRename.getTargetValueName());
}
}
// Select the specified fields from the input, re-order everything and put the other fields
// at the back, sorted...
//
RowMetaInterface newRow = new RowMeta();
for (int i = 0; i < fieldName.length; i++) {
int index = inputRowMeta.indexOfValue(fieldName[i]);
if (index < 0) {
throw new KettleStepException(
BaseMessages.getString(
PKG, "MappingInputMeta.Exception.UnknownField", fieldName[i]));
}
newRow.addValueMeta(inputRowMeta.getValueMeta(index));
}
// Now get the unspecified fields.
// Sort the fields
// Add them after the specified fields...
//
List<String> extra = new ArrayList<String>();
for (int i = 0; i < inputRowMeta.size(); i++) {
String fieldName = inputRowMeta.getValueMeta(i).getName();
if (newRow.indexOfValue(fieldName) < 0) {
extra.add(fieldName);
}
}
Collections.sort(extra);
for (String fieldName : extra) {
ValueMetaInterface extraValue = inputRowMeta.searchValueMeta(fieldName);
newRow.addValueMeta(extraValue);
}
// now merge the new row...
// This is basically the input row meta data with the fields re-ordered.
//
row.mergeRowMeta(newRow);
} else {
row.mergeRowMeta(inputRowMeta);
// Validate the existence of all the specified fields...
//
if (!row.isEmpty()) {
for (int i = 0; i < fieldName.length; i++) {
if (row.indexOfValue(fieldName[i]) < 0) {
throw new KettleStepException(
BaseMessages.getString(
PKG, "MappingInputMeta.Exception.UnknownField", fieldName[i]));
}
}
}
}
} else {
// We'll have to work with the statically provided information
for (int i = 0; i < fieldName.length; i++) {
if (!Const.isEmpty(fieldName[i])) {
ValueMetaInterface v = new ValueMeta(fieldName[i], fieldType[i]);
if (v.getType() == ValueMetaInterface.TYPE_NONE)
v.setType(ValueMetaInterface.TYPE_STRING);
v.setLength(fieldLength[i]);
v.setPrecision(fieldPrecision[i]);
v.setOrigin(origin);
row.addValueMeta(v);
}
}
}
}
private ValueMetaInterface getValueMeta(CalculatorMetaFunction fn, String origin) {
ValueMetaInterface v = new ValueMeta(fn.getFieldName(), fn.getValueType());
v.setLength(fn.getValueLength());
v.setPrecision(fn.getValuePrecision());
v.setOrigin(origin);
v.setComments(fn.getCalcTypeDesc());
v.setConversionMask(fn.getConversionMask());
v.setDecimalSymbol(fn.getDecimalSymbol());
v.setGroupingSymbol(fn.getGroupingSymbol());
v.setCurrencySymbol(fn.getCurrencySymbol());
// What if the user didn't specify a data type?
// In that case we look for the default data type
//
if (fn.getValueType() == ValueMetaInterface.TYPE_NONE) {
int defaultResultType = ValueMetaInterface.TYPE_NONE;
switch (fn.getCalcType()) {
case CalculatorMetaFunction.CALC_NONE:
break;
case CalculatorMetaFunction.CALC_ADD: // A + B
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_SUBTRACT: // A - B
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_MULTIPLY: // A * B
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_DIVIDE: // A / B
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_SQUARE: // A * A
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_SQUARE_ROOT: // SQRT( A )
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_PERCENT_1: // 100 * A / B
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_PERCENT_2: // A - ( A * B / 100 )
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_PERCENT_3: // A + ( A * B / 100 )
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_COMBINATION_1: // A + B * C
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_COMBINATION_2: // SQRT( A*A + B*B )
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_ROUND_1: // ROUND( A )
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_ROUND_2: // ROUND( A , B )
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_CONSTANT: // Set field to constant value...
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_NVL: // Replace null values with another value
break;
case CalculatorMetaFunction.CALC_ADD_DAYS: // Add B days to date field A
defaultResultType = ValueMetaInterface.TYPE_DATE;
break;
case CalculatorMetaFunction.CALC_YEAR_OF_DATE: // What is the year (Integer) of a date?
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_MONTH_OF_DATE: // What is the month (Integer) of a date?
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction
.CALC_DAY_OF_YEAR: // What is the day of year (Integer) of a date?
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction
.CALC_DAY_OF_MONTH: // What is the day of month (Integer) of a date?
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction
.CALC_DAY_OF_WEEK: // What is the day of week (Integer) of a date?
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction
.CALC_WEEK_OF_YEAR: // What is the week of year (Integer) of a date?
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction
.CALC_WEEK_OF_YEAR_ISO8601: // What is the week of year (Integer) of a date ISO8601
// style?
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction
.CALC_YEAR_OF_DATE_ISO8601: // What is the year (Integer) of a date ISO8601 style?
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_BYTE_TO_HEX_ENCODE: // Byte to Hex encode string field A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_HEX_TO_BYTE_DECODE: // Hex to Byte decode string field A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_CHAR_TO_HEX_ENCODE: // Char to Hex encode string field A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_HEX_TO_CHAR_DECODE: // Hex to Char decode string field A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_CRC32: // CRC32 of a file A
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_ADLER32: // ADLER32 of a file A
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_MD5: // MD5 of a file A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_SHA1: // SHA1 of a file Al
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction
.CALC_LEVENSHTEIN_DISTANCE: // LEVENSHTEIN_DISTANCE of string A and string B
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_METAPHONE: // METAPHONE of string A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_DOUBLE_METAPHONE: // Double METAPHONE of string A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_ABS: // ABS( A )
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_REMOVE_TIME_FROM_DATE: // Remove time from field A
defaultResultType = ValueMetaInterface.TYPE_DATE;
break;
case CalculatorMetaFunction.CALC_DATE_DIFF: // DateA - DateB
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_ADD3: // A + B +C
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_INITCAP: // InitCap(A)
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_UPPER_CASE: // UpperCase(A)
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_LOWER_CASE: // LowerCase(A)
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_MASK_XML: // MaskXML(A)
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_USE_CDATA: // CDATA(A)
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_REMOVE_CR: // REMOVE CR FROM string A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_REMOVE_LF: // REMOVE LF FROM string A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_REMOVE_CRLF: // REMOVE CRLF FROM string A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_REMOVE_TAB: // REMOVE TAB FROM string A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_GET_ONLY_DIGITS: // GET ONLY DIGITS FROM string A
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_REMOVE_DIGITS: // REMOVE DIGITS FROM string A
defaultResultType = ValueMetaInterface.TYPE_STRING;
break;
case CalculatorMetaFunction.CALC_STRING_LEN: // LENGTH OF string A
defaultResultType = ValueMetaInterface.TYPE_INTEGER;
break;
case CalculatorMetaFunction.CALC_LOAD_FILE_CONTENT_BINARY: // LOAD FILE CONTENT IN BLOB
defaultResultType = ValueMetaInterface.TYPE_BINARY;
break;
case CalculatorMetaFunction.CALC_ADD_TIME_TO_DATE: // ADD TIME TO A DATE
defaultResultType = ValueMetaInterface.TYPE_DATE;
break;
case CalculatorMetaFunction.CALC_GEOM_UNION: // Calculate geometry union
defaultResultType = ValueMetaInterface.TYPE_GEOMETRY;
break;
case CalculatorMetaFunction.CALC_GEOM_INTERSECTION: // Calculate geometry intersection
defaultResultType = ValueMetaInterface.TYPE_GEOMETRY;
break;
case CalculatorMetaFunction.CALC_GEOM_DIFFERENCE: // Calculate geometry difference
defaultResultType = ValueMetaInterface.TYPE_GEOMETRY;
break;
case CalculatorMetaFunction
.CALC_GEOM_SYMETRIC_DIFFERENCE: // Calculate geometry symetric difference
defaultResultType = ValueMetaInterface.TYPE_GEOMETRY;
break;
case CalculatorMetaFunction.CALC_GEOM_AREA: // Calculate area
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_GEOM_LENGTH: // Calculate length
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
case CalculatorMetaFunction.CALC_GEOM_CENTROID: // Calculate centroid
defaultResultType = ValueMetaInterface.TYPE_GEOMETRY;
break;
case CalculatorMetaFunction.CALC_GEOM_POINT_ON_SURFACE: // Calculate random point on surface
defaultResultType = ValueMetaInterface.TYPE_GEOMETRY;
break;
case CalculatorMetaFunction.CALC_GEOM_REVERSE: // Reverse geometry
defaultResultType = ValueMetaInterface.TYPE_GEOMETRY;
break;
case CalculatorMetaFunction.CALC_GEOM_BOUNDARY: // Calculate geometry boundary
defaultResultType = ValueMetaInterface.TYPE_GEOMETRY;
break;
case CalculatorMetaFunction.CALC_GEOM_ENVELOPE: // Calculate geometry envelope
defaultResultType = ValueMetaInterface.TYPE_GEOMETRY;
break;
case CalculatorMetaFunction.CALC_GEOM_CONVEX_HULL: // Calculate geometry convex hull
defaultResultType = ValueMetaInterface.TYPE_GEOMETRY;
break;
case CalculatorMetaFunction.CALC_GEOM_BUFFER: // Calculate geometry buffer
defaultResultType = ValueMetaInterface.TYPE_GEOMETRY;
break;
case CalculatorMetaFunction.CALC_GEOM_DISTANCE: // Calculate distance between geometries
defaultResultType = ValueMetaInterface.TYPE_NUMBER;
break;
default:
break;
}
v.setType(defaultResultType);
}
return v;
}
