private void sql() {
try {
IngresVectorwiseLoaderMeta info = new IngresVectorwiseLoaderMeta();
getInfo(info);
RowMetaInterface prev = transMeta.getPrevStepFields(stepname);
StepMeta stepMeta = transMeta.findStep(stepname);
// Only use the fields that were specified.
//
RowMetaInterface prevNew = new RowMeta();
for (int i = 0; i < info.getFieldDatabase().length; i++) {
ValueMetaInterface insValue = prev.searchValueMeta(info.getFieldStream()[i]);
if (insValue != null) {
ValueMetaInterface insertValue = insValue.clone();
insertValue.setName(info.getFieldDatabase()[i]);
prevNew.addValueMeta(insertValue);
} else {
throw new KettleStepException(
BaseMessages.getString(
PKG,
"IngresVectorWiseLoaderDialog.FailedToFindField.Message",
info.getFieldStream()[i]));
}
}
prev = prevNew;
SQLStatement sql = info.getSQLStatements(transMeta, stepMeta, prev, repository, metaStore);
if (!sql.hasError()) {
if (sql.hasSQL()) {
SQLEditor sqledit =
new SQLEditor(
transMeta,
shell,
SWT.NONE,
info.getDatabaseMeta(),
transMeta.getDbCache(),
sql.getSQL());
sqledit.open();
} else {
MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_INFORMATION);
mb.setMessage(
BaseMessages.getString(PKG, "IngresVectorWiseLoaderDialog.NoSQL.DialogMessage"));
mb.setText(BaseMessages.getString(PKG, "IngresVectorWiseLoaderDialog.NoSQL.DialogTitle"));
mb.open();
}
} else {
MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR);
mb.setMessage(sql.getError());
mb.setText(BaseMessages.getString(PKG, "System.Dialog.Error.Title"));
mb.open();
}
} catch (KettleException ke) {
new ErrorDialog(
shell,
BaseMessages.getString(PKG, "IngresVectorWiseLoaderDialog.BuildSQLError.DialogTitle"),
BaseMessages.getString(PKG, "IngresVectorWiseLoaderDialog.BuildSQLError.DialogMessage"),
ke);
}
}
public SQLStatement getSQLStatements(
TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev) throws KettleStepException {
SQLStatement retval =
new SQLStatement(stepMeta.getName(), databaseMeta, null); // default: nothing to do!
if (databaseMeta != null) {
if (prev != null && prev.size() > 0) {
// Copy the row
RowMetaInterface tableFields = new RowMeta();
// Now change the field names
for (int i = 0; i < fieldTable.length; i++) {
ValueMetaInterface v = prev.searchValueMeta(fieldStream[i]);
if (v != null) {
ValueMetaInterface tableField = v.clone();
tableField.setName(fieldTable[i]);
tableFields.addValueMeta(tableField);
} else {
throw new KettleStepException(
"Unable to find field [" + fieldStream[i] + "] in the input rows");
}
}
if (!Const.isEmpty(tableName)) {
Database db = new Database(loggingObject, databaseMeta);
db.shareVariablesWith(transMeta);
try {
db.connect();
String schemaTable =
databaseMeta.getQuotedSchemaTableCombination(
transMeta.environmentSubstitute(schemaName),
transMeta.environmentSubstitute(tableName));
String sql = db.getDDL(schemaTable, tableFields, null, false, null, true);
if (sql.length() == 0) retval.setSQL(null);
else retval.setSQL(sql);
} catch (KettleException e) {
retval.setError(
BaseMessages.getString(PKG, "GPBulkLoaderMeta.GetSQL.ErrorOccurred")
+ e.getMessage()); // $NON-NLS-1$
}
} else {
retval.setError(
BaseMessages.getString(
PKG, "GPBulkLoaderMeta.GetSQL.NoTableDefinedOnConnection")); // $NON-NLS-1$
}
} else {
retval.setError(
BaseMessages.getString(
PKG, "GPBulkLoaderMeta.GetSQL.NotReceivingAnyFields")); // $NON-NLS-1$
}
} else {
retval.setError(
BaseMessages.getString(
PKG, "GPBulkLoaderMeta.GetSQL.NoConnectionDefined")); // $NON-NLS-1$
}
return retval;
}
public static ValueMetaInterface cloneValueMeta(ValueMetaInterface source, int targetType)
throws KettlePluginException {
ValueMetaInterface target = null;
// If we're Cloneable and not changing types, call clone()
if (source instanceof Cloneable && source.getType() == targetType) {
target = source.clone();
} else {
target =
createValueMeta(source.getName(), targetType, source.getLength(), source.getPrecision());
}
target.setConversionMask(source.getConversionMask());
target.setDecimalSymbol(source.getDecimalSymbol());
target.setGroupingSymbol(source.getGroupingSymbol());
target.setStorageType(source.getStorageType());
if (source.getStorageMetadata() != null) {
target.setStorageMetadata(
cloneValueMeta(source.getStorageMetadata(), source.getStorageMetadata().getType()));
}
target.setStringEncoding(source.getStringEncoding());
target.setTrimType(source.getTrimType());
target.setDateFormatLenient(source.isDateFormatLenient());
target.setDateFormatLocale(source.getDateFormatLocale());
target.setDateFormatTimeZone(source.getDateFormatTimeZone());
target.setLenientStringToNumber(source.isLenientStringToNumber());
target.setLargeTextField(source.isLargeTextField());
target.setComments(source.getComments());
target.setCaseInsensitive(source.isCaseInsensitive());
target.setIndex(source.getIndex());
target.setOrigin(source.getOrigin());
target.setOriginalAutoIncrement(source.isOriginalAutoIncrement());
target.setOriginalColumnType(source.getOriginalColumnType());
target.setOriginalColumnTypeName(source.getOriginalColumnTypeName());
target.setOriginalNullable(source.isOriginalNullable());
target.setOriginalPrecision(source.getOriginalPrecision());
target.setOriginalScale(source.getOriginalScale());
target.setOriginalSigned(source.isOriginalSigned());
return target;
}
public void getFields(
RowMetaInterface row,
String name,
RowMetaInterface[] info,
StepMeta nextStep,
VariableSpace space)
throws KettleStepException {
// Remember the types of the row.
int fieldnrs[] = new int[fieldName.length];
ValueMetaInterface[] values = new ValueMetaInterface[fieldName.length];
for (int i = 0; i < fieldName.length; i++) {
fieldnrs[i] = row.indexOfValue(fieldName[i]);
ValueMetaInterface v = row.getValueMeta(fieldnrs[i]);
values[i] = v.clone(); // copy value : default settings!
switch (aggregateType[i]) {
case TYPE_AGGREGATE_AVERAGE:
case TYPE_AGGREGATE_COUNT:
case TYPE_AGGREGATE_SUM:
values[i].setType(Value.VALUE_TYPE_NUMBER);
values[i].setLength(-1, -1);
break;
}
}
// Only the aggregate is returned!
row.clear();
for (int i = 0; i < fieldName.length; i++) {
ValueMetaInterface v = values[i];
v.setName(fieldNewName[i]);
v.setOrigin(name);
row.addValueMeta(v);
}
}
public static final RowMetaAndData buildRow(
RowGeneratorMeta meta, List<CheckResultInterface> remarks, String origin) {
RowMetaInterface rowMeta = new RowMeta();
Object[] rowData = RowDataUtil.allocateRowData(meta.getFieldName().length);
for (int i = 0; i < meta.getFieldName().length; i++) {
int valtype = ValueMeta.getType(meta.getFieldType()[i]);
if (meta.getFieldName()[i] != null) {
ValueMetaInterface valueMeta =
new ValueMeta(meta.getFieldName()[i], valtype); // build a value!
valueMeta.setLength(meta.getFieldLength()[i]);
valueMeta.setPrecision(meta.getFieldPrecision()[i]);
valueMeta.setConversionMask(meta.getFieldFormat()[i]);
valueMeta.setGroupingSymbol(meta.getGroup()[i]);
valueMeta.setDecimalSymbol(meta.getDecimal()[i]);
valueMeta.setOrigin(origin);
ValueMetaInterface stringMeta = valueMeta.clone();
stringMeta.setType(ValueMetaInterface.TYPE_STRING);
String stringValue = meta.getValue()[i];
// If the value is empty: consider it to be NULL.
if (Const.isEmpty(stringValue)) {
rowData[i] = null;
if (valueMeta.getType() == ValueMetaInterface.TYPE_NONE) {
String message =
BaseMessages.getString(
PKG,
"RowGenerator.CheckResult.SpecifyTypeError",
valueMeta.getName(),
stringValue);
remarks.add(new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null));
}
} else {
// Convert the data from String to the specified type ...
//
try {
rowData[i] = valueMeta.convertData(stringMeta, stringValue);
} catch (KettleValueException e) {
switch (valueMeta.getType()) {
case ValueMetaInterface.TYPE_NUMBER:
{
String message =
BaseMessages.getString(
PKG,
"RowGenerator.BuildRow.Error.Parsing.Number",
valueMeta.getName(),
stringValue,
e.toString());
remarks.add(
new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null));
}
break;
case ValueMetaInterface.TYPE_DATE:
{
String message =
BaseMessages.getString(
PKG,
"RowGenerator.BuildRow.Error.Parsing.Date",
valueMeta.getName(),
stringValue,
e.toString());
remarks.add(
new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null));
}
break;
case ValueMetaInterface.TYPE_INTEGER:
{
String message =
BaseMessages.getString(
PKG,
"RowGenerator.BuildRow.Error.Parsing.Integer",
valueMeta.getName(),
stringValue,
e.toString());
remarks.add(
new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null));
}
break;
case ValueMetaInterface.TYPE_BIGNUMBER:
{
String message =
BaseMessages.getString(
PKG,
"RowGenerator.BuildRow.Error.Parsing.BigNumber",
valueMeta.getName(),
stringValue,
e.toString());
remarks.add(
new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null));
}
break;
default:
// Boolean and binary don't throw errors normally, so it's probably an unspecified
// error problem...
{
String message =
BaseMessages.getString(
PKG,
"RowGenerator.CheckResult.SpecifyTypeError",
valueMeta.getName(),
stringValue);
remarks.add(
new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, message, null));
}
break;
}
}
}
// Now add value to the row!
// This is in fact a copy from the fields row, but now with data.
rowMeta.addValueMeta(valueMeta);
}
}
return new RowMetaAndData(rowMeta, rowData);
}
public void getFields(
RowMetaInterface rowMeta,
String origin,
RowMetaInterface[] info,
StepMeta nextStep,
VariableSpace space)
throws KettleStepException {
rowMeta.clear(); // Start with a clean slate, eats the input
for (int i = 0; i < inputFields.length; i++) {
TextFileInputField field = inputFields[i];
ValueMetaInterface valueMeta = new ValueMeta(field.getName(), field.getType());
valueMeta.setConversionMask(field.getFormat());
valueMeta.setLength(field.getLength());
valueMeta.setPrecision(field.getPrecision());
valueMeta.setConversionMask(field.getFormat());
valueMeta.setDecimalSymbol(field.getDecimalSymbol());
valueMeta.setGroupingSymbol(field.getGroupSymbol());
valueMeta.setCurrencySymbol(field.getCurrencySymbol());
valueMeta.setTrimType(field.getTrimType());
if (lazyConversionActive)
valueMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING);
valueMeta.setStringEncoding(space.environmentSubstitute(encoding));
// In case we want to convert Strings...
// Using a copy of the valueMeta object means that the inner and outer representation format
// is the same.
// Preview will show the data the same way as we read it.
// This layout is then taken further down the road by the metadata through the transformation.
//
ValueMetaInterface storageMetadata = valueMeta.clone();
storageMetadata.setType(ValueMetaInterface.TYPE_STRING);
storageMetadata.setStorageType(ValueMetaInterface.STORAGE_TYPE_NORMAL);
storageMetadata.setLength(
-1, -1); // we don't really know the lengths of the strings read in advance.
valueMeta.setStorageMetadata(storageMetadata);
valueMeta.setOrigin(origin);
rowMeta.addValueMeta(valueMeta);
}
if (!Const.isEmpty(filenameField) && includingFilename) {
ValueMetaInterface filenameMeta =
new ValueMeta(filenameField, ValueMetaInterface.TYPE_STRING);
filenameMeta.setOrigin(origin);
if (lazyConversionActive) {
filenameMeta.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING);
filenameMeta.setStorageMetadata(
new ValueMeta(filenameField, ValueMetaInterface.TYPE_STRING));
}
rowMeta.addValueMeta(filenameMeta);
}
if (!Const.isEmpty(rowNumField)) {
ValueMetaInterface rowNumMeta = new ValueMeta(rowNumField, ValueMetaInterface.TYPE_INTEGER);
rowNumMeta.setLength(10);
rowNumMeta.setOrigin(origin);
rowMeta.addValueMeta(rowNumMeta);
}
}
public SQLStatement getSQLStatements(
TransMeta transMeta,
StepMeta stepMeta,
RowMetaInterface prev,
Repository repository,
IMetaStore metaStore)
throws KettleStepException {
SQLStatement retval =
new SQLStatement(stepMeta.getName(), databaseMeta, null); // default: nothing to do!
if (databaseMeta != null) {
if (prev != null && prev.size() > 0) {
// Copy the row
RowMetaInterface tableFields = new RowMeta();
// Now change the field names
// the key fields
if (keyLookup != null) {
for (int i = 0; i < keyLookup.length; i++) {
ValueMetaInterface v = prev.searchValueMeta(keyStream[i]);
if (v != null) {
ValueMetaInterface tableField = v.clone();
tableField.setName(keyLookup[i]);
tableFields.addValueMeta(tableField);
} else {
throw new KettleStepException(
"Unable to find field [" + keyStream[i] + "] in the input rows");
}
}
}
// the lookup fields
for (int i = 0; i < updateLookup.length; i++) {
ValueMetaInterface v = prev.searchValueMeta(updateStream[i]);
if (v != null) {
ValueMetaInterface vk = tableFields.searchValueMeta(updateStream[i]);
if (vk == null) { // do not add again when already added as key fields
ValueMetaInterface tableField = v.clone();
tableField.setName(updateLookup[i]);
tableFields.addValueMeta(tableField);
}
} else {
throw new KettleStepException(
"Unable to find field [" + updateStream[i] + "] in the input rows");
}
}
if (!Const.isEmpty(tableName)) {
Database db = new Database(loggingObject, databaseMeta);
db.shareVariablesWith(transMeta);
try {
db.connect();
String schemaTable =
databaseMeta.getQuotedSchemaTableCombination(schemaName, tableName);
String cr_table = db.getDDL(schemaTable, tableFields, null, false, null, true);
String cr_index = "";
String[] idx_fields = null;
if (keyLookup != null && keyLookup.length > 0) {
idx_fields = new String[keyLookup.length];
for (int i = 0; i < keyLookup.length; i++) {
idx_fields[i] = keyLookup[i];
}
} else {
retval.setError(
BaseMessages.getString(PKG, "InsertUpdateMeta.CheckResult.MissingKeyFields"));
}
// Key lookup dimensions...
if (idx_fields != null
&& idx_fields.length > 0
&& !db.checkIndexExists(schemaName, tableName, idx_fields)) {
String indexname = "idx_" + tableName + "_lookup";
cr_index =
db.getCreateIndexStatement(
schemaTable, indexname, idx_fields, false, false, false, true);
}
String sql = cr_table + cr_index;
if (sql.length() == 0) {
retval.setSQL(null);
} else {
retval.setSQL(sql);
}
} catch (KettleException e) {
retval.setError(
BaseMessages.getString(PKG, "InsertUpdateMeta.ReturnValue.ErrorOccurred")
+ e.getMessage());
}
} else {
retval.setError(
BaseMessages.getString(
PKG, "InsertUpdateMeta.ReturnValue.NoTableDefinedOnConnection"));
}
} else {
retval.setError(
BaseMessages.getString(PKG, "InsertUpdateMeta.ReturnValue.NotReceivingAnyFields"));
}
} else {
retval.setError(
BaseMessages.getString(PKG, "InsertUpdateMeta.ReturnValue.NoConnectionDefined"));
}
return retval;
}
public SQLStatement getSQLStatements(
TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev) {
SQLStatement retval =
new SQLStatement(stepMeta.getName(), databaseWriteMeta, null); // default: nothing to do!
int i;
if (databaseWriteMeta != null) {
if (prev != null && prev.size() > 0) {
if (!Const.isEmpty(tablename)) {
String schemaTable =
databaseWriteMeta.getQuotedSchemaTableCombination(schemaName, tablename);
Database db = new Database(databaseWriteMeta);
try {
boolean doHash = false;
String cr_table = null;
db.connect();
// OK, what do we put in the new table??
RowMetaInterface fields = new RowMeta();
ValueMetaInterface vkeyfield = null;
if (!Const.isEmpty(technicalKeyField)) {
// First, the new technical key...
vkeyfield = new ValueMeta(technicalKeyField, ValueMetaInterface.TYPE_INTEGER);
vkeyfield.setLength(10);
vkeyfield.setPrecision(0);
}
// Then the hashcode (optional)
ValueMetaInterface vhashfield = null;
if (useHash && !Const.isEmpty(hashField)) {
vhashfield = new ValueMeta(hashField, ValueMetaInterface.TYPE_INTEGER);
vhashfield.setLength(15);
vhashfield.setPrecision(0);
doHash = true;
}
// Then the last update field (optional)
ValueMetaInterface vLastUpdateField = null;
if (!Const.isEmpty(lastUpdateField)) {
vLastUpdateField = new ValueMeta(lastUpdateField, ValueMetaInterface.TYPE_DATE);
}
if (!db.checkTableExists(schemaTable)) {
if (vkeyfield != null) {
// Add technical key field.
fields.addValueMeta(vkeyfield);
}
// Add the keys only to the table
if (keyField != null && keyLookup != null) {
int cnt = keyField.length;
for (i = 0; i < cnt; i++) {
String error_field = ""; // $NON-NLS-1$
// Find the value in the stream
ValueMetaInterface v = prev.searchValueMeta(keyField[i]);
if (v != null) {
String name = keyLookup[i];
ValueMetaInterface newValue = v.clone();
newValue.setName(name);
if (vkeyfield != null) {
if (name.equals(vkeyfield.getName())
|| (doHash == true && name.equals(vhashfield.getName()))) {
error_field += name;
}
}
if (error_field.length() > 0) {
retval.setError(
Messages.getString(
"ConcurrentCombinationLookupMeta.ReturnValue.NameCollision",
error_field)); //$NON-NLS-1$
} else {
fields.addValueMeta(newValue);
}
}
}
}
if (doHash == true) {
fields.addValueMeta(vhashfield);
}
if (vLastUpdateField != null) {
fields.addValueMeta(vLastUpdateField);
}
} else {
// Table already exists
// Get the fields that are in the table now:
RowMetaInterface tabFields = db.getTableFields(schemaTable);
// Don't forget to quote these as well...
databaseWriteMeta.quoteReservedWords(tabFields);
if (vkeyfield != null && tabFields.searchValueMeta(vkeyfield.getName()) == null) {
// Add technical key field if it didn't exist yet
fields.addValueMeta(vkeyfield);
}
// Add the already existing fields
int cnt = tabFields.size();
for (i = 0; i < cnt; i++) {
ValueMetaInterface v = tabFields.getValueMeta(i);
fields.addValueMeta(v);
}
// Find the missing fields in the real table
String keyLookup[] = getKeyLookup();
String keyField[] = getKeyField();
if (keyField != null && keyLookup != null) {
cnt = keyField.length;
for (i = 0; i < cnt; i++) {
// Find the value in the stream
ValueMetaInterface v = prev.searchValueMeta(keyField[i]);
if (v != null) {
ValueMetaInterface newValue = v.clone();
newValue.setName(keyLookup[i]);
// Does the corresponding name exist in the table
if (tabFields.searchValueMeta(newValue.getName()) == null) {
fields.addValueMeta(newValue); // nope --> add
}
}
}
}
if (doHash == true && tabFields.searchValueMeta(vhashfield.getName()) == null) {
// Add hash field
fields.addValueMeta(vhashfield);
}
if (vLastUpdateField != null
&& tabFields.searchValueMeta(vLastUpdateField.getName()) == null) {
fields.addValueMeta(vLastUpdateField);
}
}
cr_table =
db.getDDL(
schemaTable,
fields,
(CREATION_METHOD_SEQUENCE.equals(getTechKeyCreation())
&& sequenceFrom != null
&& sequenceFrom.length() != 0)
? null
: technicalKeyField,
CREATION_METHOD_AUTOINC.equals(getTechKeyCreation()),
null,
true);
//
// OK, now let's build the index
//
// What fields do we put int the index?
// Only the hashcode or all fields?
String cr_index = ""; // $NON-NLS-1$
String cr_uniq_index = ""; // $NON-NLS-1$
String idx_fields[] = null;
if (useHash) {
if (hashField != null && hashField.length() > 0) {
idx_fields = new String[] {hashField};
} else {
retval.setError(
Messages.getString(
"ConcurrentCombinationLookupMeta.ReturnValue.NotHashFieldSpecified")); //$NON-NLS-1$
}
} else // index on all key fields...
{
if (!Const.isEmpty(keyLookup)) {
int nrfields = keyLookup.length;
if (nrfields > 32
&& databaseWriteMeta.getDatabaseType() == DatabaseMeta.TYPE_DATABASE_ORACLE) {
nrfields = 32; // Oracle indexes are limited to 32 fields...
}
idx_fields = new String[nrfields];
for (i = 0; i < nrfields; i++) idx_fields[i] = keyLookup[i];
} else {
retval.setError(
Messages.getString(
"ConcurrentCombinationLookupMeta.ReturnValue.NotFieldsSpecified")); //$NON-NLS-1$
}
}
// OK, now get the create index statement...
if (!Const.isEmpty(technicalKeyField)) {
String techKeyArr[] = new String[] {technicalKeyField};
if (!db.checkIndexExists(schemaName, tablename, techKeyArr)) {
String indexname = "idx_" + tablename + "_pk"; // $NON-NLS-1$ //$NON-NLS-2$
cr_uniq_index =
db.getCreateIndexStatement(
schemaName, tablename, indexname, techKeyArr, true, true, false, true);
cr_uniq_index += Const.CR;
}
}
// OK, now get the create lookup index statement...
if (!Const.isEmpty(idx_fields)
&& !db.checkIndexExists(schemaName, tablename, idx_fields)) {
String indexname = "idx_" + tablename + "_lookup"; // $NON-NLS-1$ //$NON-NLS-2$
cr_index =
db.getCreateIndexStatement(
schemaName, tablename, indexname, idx_fields, false, false, false, true);
cr_index += Const.CR;
}
//
// Don't forget the sequence (optional)
//
String cr_seq = ""; // $NON-NLS-1$
if (databaseWriteMeta.supportsSequences() && !Const.isEmpty(sequenceFrom)) {
if (!db.checkSequenceExists(schemaName, sequenceFrom)) {
cr_seq +=
db.getCreateSequenceStatement(schemaName, sequenceFrom, 1L, 1L, -1L, true);
cr_seq += Const.CR;
}
}
retval.setSQL(cr_table + cr_uniq_index + cr_index + cr_seq);
} catch (KettleException e) {
retval.setError(
Messages.getString("ConcurrentCombinationLookupMeta.ReturnValue.ErrorOccurred")
+ Const.CR
+ e.getMessage()); // $NON-NLS-1$
}
} else {
retval.setError(
Messages.getString(
"ConcurrentCombinationLookupMeta.ReturnValue.NotTableDefined")); //$NON-NLS-1$
}
} else {
retval.setError(
Messages.getString(
"ConcurrentCombinationLookupMeta.ReturnValue.NotReceivingField")); //$NON-NLS-1$
}
} else {
retval.setError(
Messages.getString(
"ConcurrentCombinationLookupMeta.ReturnValue.NotConnectionDefined")); //$NON-NLS-1$
}
return retval;
}
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
meta = (MySQLTableOutputMeta) smi;
data = (MySQLTableOutputData) sdi;
ArrayList<Object[]> rows = new ArrayList<Object[]>();
for (int counter = 0; counter < data.batchSize; counter++) {
Object[] r = getRow();
; // this also waits for a previous step to be finished.
if (r != null) {
rows.add(r);
}
}
if (rows.isEmpty()) {
return false;
}
if (first) {
first = false;
data.outputRowMeta = getInputRowMeta().clone();
meta.getFields(data.outputRowMeta, getStepname(), null, null, this);
data.insertRowMeta = new RowMeta();
//
// Cache the position of the compare fields in Row row
//
data.valuenrs = new int[meta.getFieldDatabase().length];
for (int i = 0; i < meta.getFieldDatabase().length; i++) {
data.valuenrs[i] = getInputRowMeta().indexOfValue(meta.getFieldStream()[i]);
if (data.valuenrs[i] < 0) {
throw new KettleStepException(
Messages.getString(
"TableOutput.Exception.FieldRequired", meta.getFieldStream()[i])); // $NON-NLS-1$
}
}
for (int i = 0; i < meta.getFieldDatabase().length; i++) {
ValueMetaInterface insValue = getInputRowMeta().searchValueMeta(meta.getFieldStream()[i]);
if (insValue != null) {
ValueMetaInterface insertValue = insValue.clone();
insertValue.setName(meta.getFieldDatabase()[i]);
data.insertRowMeta.addValueMeta(insertValue);
} else {
throw new KettleStepException(
Messages.getString(
"TableOutput.Exception.FailedToFindField",
meta.getFieldStream()[i])); // $NON-NLS-1$
}
}
}
try {
ArrayList<Object[]> outputRowData = writeToTable(getInputRowMeta(), rows);
if (outputRowData != null) {
for (Object[] row : outputRowData) {
putRow(data.outputRowMeta, row); // in case we want it go
// further...
incrementLinesOutput();
}
}
if (checkFeedback(getLinesRead())) {
if (log.isBasic()) logBasic("linenr " + getLinesRead()); // $NON-NLS-1$
}
} catch (KettleException e) {
logError("Because of an error, this step can't continue: ", e);
setErrors(1);
stopAll();
setOutputDone(); // signal end to receiver(s)
return false;
}
return true;
}
private void assertGetFieldDefinition(
ValueMetaInterface valueMetaInterface, String name, String expectedType) {
valueMetaInterface = valueMetaInterface.clone();
valueMetaInterface.setName(name);
assertGetFieldDefinition(valueMetaInterface, expectedType);
}
@Override
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
meta = (VerticaBulkLoaderMeta) smi;
data = (VerticaBulkLoaderData) sdi;
Object[] r = getRow(); // this also waits for a previous step to be
// finished.
if (r == null) // no more input to be expected...
{
try {
data.close();
} catch (IOException ioe) {
throw new KettleStepException("Error releasing resources", ioe);
}
return false;
}
if (first) {
first = false;
data.outputRowMeta = getInputRowMeta().clone();
meta.getFields(data.outputRowMeta, getStepname(), null, null, this);
RowMetaInterface tableMeta = meta.getTableRowMetaInterface();
if (!meta.specifyFields()) {
// Just take the whole input row
data.insertRowMeta = getInputRowMeta().clone();
data.selectedRowFieldIndices = new int[data.insertRowMeta.size()];
data.colSpecs = new ArrayList<ColumnSpec>(data.insertRowMeta.size());
for (int insertFieldIdx = 0; insertFieldIdx < data.insertRowMeta.size(); insertFieldIdx++) {
data.selectedRowFieldIndices[insertFieldIdx] = insertFieldIdx;
ValueMetaInterface inputValueMeta = data.insertRowMeta.getValueMeta(insertFieldIdx);
ValueMetaInterface insertValueMeta = inputValueMeta.clone();
ValueMetaInterface targetValueMeta = tableMeta.getValueMeta(insertFieldIdx);
ColumnSpec cs = getColumnSpecFromField(inputValueMeta, insertValueMeta, targetValueMeta);
data.colSpecs.add(insertFieldIdx, cs);
}
} else {
int numberOfInsertFields = meta.getFieldDatabase().length;
data.insertRowMeta = new RowMeta();
data.colSpecs = new ArrayList<ColumnSpec>(numberOfInsertFields);
// Cache the position of the selected fields in the row array
data.selectedRowFieldIndices = new int[numberOfInsertFields];
for (int insertFieldIdx = 0; insertFieldIdx < numberOfInsertFields; insertFieldIdx++) {
String inputFieldName = meta.getFieldStream()[insertFieldIdx];
int inputFieldIdx = getInputRowMeta().indexOfValue(inputFieldName);
if (inputFieldIdx < 0) {
throw new KettleStepException(
BaseMessages.getString(
PKG,
"VerticaBulkLoader.Exception.FieldRequired",
inputFieldName)); //$NON-NLS-1$
}
data.selectedRowFieldIndices[insertFieldIdx] = inputFieldIdx;
String insertFieldName = meta.getFieldDatabase()[insertFieldIdx];
ValueMetaInterface inputValueMeta = getInputRowMeta().getValueMeta(inputFieldIdx);
if (inputValueMeta == null) {
throw new KettleStepException(
BaseMessages.getString(
PKG,
"VerticaBulkLoader.Exception.FailedToFindField",
meta.getFieldStream()[insertFieldIdx])); // $NON-NLS-1$
}
ValueMetaInterface insertValueMeta = inputValueMeta.clone();
insertValueMeta.setName(insertFieldName);
data.insertRowMeta.addValueMeta(insertValueMeta);
ValueMetaInterface targetValueMeta = tableMeta.searchValueMeta(insertFieldName);
ColumnSpec cs = getColumnSpecFromField(inputValueMeta, insertValueMeta, targetValueMeta);
data.colSpecs.add(insertFieldIdx, cs);
}
}
try {
data.pipedInputStream = new PipedInputStream();
data.encoder = new StreamEncoder(data.colSpecs, data.pipedInputStream);
initializeWorker();
data.encoder.writeHeader();
} catch (IOException ioe) {
throw new KettleStepException("Error creating stream encoder", ioe);
}
}
try {
Object[] outputRowData = writeToOutputStream(r);
if (outputRowData != null) {
putRow(data.outputRowMeta, outputRowData); // in case we want it
// go further...
incrementLinesOutput();
}
if (checkFeedback(getLinesRead())) {
if (log.isBasic()) logBasic("linenr " + getLinesRead()); // $NON-NLS-1$
}
} catch (KettleException e) {
logError("Because of an error, this step can't continue: ", e);
setErrors(1);
stopAll();
setOutputDone(); // signal end to receiver(s)
return false;
} catch (IOException e) {
e.printStackTrace();
}
return true;
}
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
meta = (TableOutputMeta) smi;
data = (TableOutputData) sdi;
Object[] r = getRow(); // this also waits for a previous step to be finished.
if (r == null) { // no more input to be expected...
return false;
}
if (first) {
first = false;
data.outputRowMeta = getInputRowMeta().clone();
meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
if (!meta.specifyFields()) {
// Just take the input row
data.insertRowMeta = getInputRowMeta().clone();
} else {
data.insertRowMeta = new RowMeta();
//
// Cache the position of the compare fields in Row row
//
data.valuenrs = new int[meta.getFieldDatabase().length];
for (int i = 0; i < meta.getFieldDatabase().length; i++) {
data.valuenrs[i] = getInputRowMeta().indexOfValue(meta.getFieldStream()[i]);
if (data.valuenrs[i] < 0) {
throw new KettleStepException(
BaseMessages.getString(
PKG, "TableOutput.Exception.FieldRequired", meta.getFieldStream()[i]));
}
}
for (int i = 0; i < meta.getFieldDatabase().length; i++) {
ValueMetaInterface insValue = getInputRowMeta().searchValueMeta(meta.getFieldStream()[i]);
if (insValue != null) {
ValueMetaInterface insertValue = insValue.clone();
insertValue.setName(meta.getFieldDatabase()[i]);
data.insertRowMeta.addValueMeta(insertValue);
} else {
throw new KettleStepException(
BaseMessages.getString(
PKG, "TableOutput.Exception.FailedToFindField", meta.getFieldStream()[i]));
}
}
}
}
try {
Object[] outputRowData = writeToTable(getInputRowMeta(), r);
if (outputRowData != null) {
putRow(data.outputRowMeta, outputRowData); // in case we want it go further...
incrementLinesOutput();
}
if (checkFeedback(getLinesRead())) {
if (log.isBasic()) {
logBasic("linenr " + getLinesRead());
}
}
} catch (KettleException e) {
logError("Because of an error, this step can't continue: ", e);
setErrors(1);
stopAll();
setOutputDone(); // signal end to receiver(s)
return false;
}
return true;
}