private Object[] getFromCache(RowMetaInterface keyMeta, Object[] keyData)
throws KettleValueException {
if (meta.isMemoryPreservationActive()) {
if (meta.isUsingSortedList()) {
KeyValue keyValue = new KeyValue(keyData, null);
int idx = Collections.binarySearch(data.list, keyValue, data.comparator);
if (idx < 0) return null; // nothing found
keyValue = (KeyValue) data.list.get(idx);
return keyValue.getValue();
} else {
if (meta.isUsingIntegerPair()) {
Long value = data.longIndex.get(keyMeta.getInteger(keyData, 0));
if (value == null) return null;
return new Object[] {
value,
};
} else {
try {
byte[] value = data.hashIndex.get(RowMeta.extractData(keyMeta, keyData));
if (value == null) return null;
return RowMeta.getRow(data.valueMeta, value);
} catch (Exception e) {
logError("Oops", e);
throw new RuntimeException(e);
}
}
}
} else {
return (Object[]) data.look.get(new RowMetaAndData(keyMeta, keyData));
}
}
@Test
public void testQuoteReservedWords() {
DatabaseMeta databaseMeta = mock(DatabaseMeta.class);
doCallRealMethod().when(databaseMeta).quoteReservedWords(any(RowMetaInterface.class));
doCallRealMethod().when(databaseMeta).quoteField(anyString());
doCallRealMethod().when(databaseMeta).setDatabaseInterface(any(DatabaseInterface.class));
doReturn("\"").when(databaseMeta).getStartQuote();
doReturn("\"").when(databaseMeta).getEndQuote();
final DatabaseInterface databaseInterface = mock(DatabaseInterface.class);
doReturn(true).when(databaseInterface).isQuoteAllFields();
databaseMeta.setDatabaseInterface(databaseInterface);
final RowMeta fields = new RowMeta();
for (int i = 0; i < 10; i++) {
final ValueMeta valueMeta = new ValueMeta("test_" + i);
fields.addValueMeta(valueMeta);
}
for (int i = 0; i < 10; i++) {
databaseMeta.quoteReservedWords(fields);
}
for (int i = 0; i < 10; i++) {
databaseMeta.quoteReservedWords(fields);
final String name = fields.getValueMeta(i).getName();
// check valueMeta index in list
assertTrue(name.contains("test_" + i));
// check valueMeta is found by quoted name
assertNotNull(fields.searchValueMeta(name));
}
}
@Test
public void testHTTPResultDefaultRows() throws IOException {
File localFileForUpload = getInputFile("existingFile1", ".tmp");
File tempFileForDownload = File.createTempFile("downloadedFile1", ".tmp");
localFileForUpload.deleteOnExit();
tempFileForDownload.deleteOnExit();
Object[] r =
new Object[] {
HTTP_SERVER_BASEURL + "/uploadFile",
localFileForUpload.getCanonicalPath(),
tempFileForDownload.getCanonicalPath()
};
RowMeta rowMetaDefault = new RowMeta();
rowMetaDefault.addValueMeta(new ValueMetaString("URL"));
rowMetaDefault.addValueMeta(new ValueMetaString("UPLOAD"));
rowMetaDefault.addValueMeta(new ValueMetaString("DESTINATION"));
List<RowMetaAndData> rows = new ArrayList<RowMetaAndData>();
rows.add(new RowMetaAndData(rowMetaDefault, r));
Result previousResult = new Result();
previousResult.setRows(rows);
JobEntryHTTP http = new JobEntryHTTP();
http.setParentJob(new Job());
http.setRunForEveryRow(true);
http.setAddFilenameToResult(false);
http.execute(previousResult, 0);
assertTrue(FileUtils.contentEquals(localFileForUpload, tempFileForDownload));
}
@Test
public void testConditionResolution() throws Exception {
RowMeta rowMeta = new RowMeta();
rowMeta.addValueMeta(new ValueMetaString("aString"));
rowMeta.addValueMeta(new ValueMetaInteger("anInt"));
rowMeta.addValueMeta(new ValueMetaDate("aDate"));
String query =
"SELECT * FROM "
+ DATA_SERVICE_NAME
+ " WHERE anInt = 2 AND aDate IN ('2014-12-05','2008-01-01')";
when(transMeta.getStepFields(DATA_SERVICE_STEP)).thenReturn(rowMeta);
DataServiceExecutor executor =
new DataServiceExecutor.Builder(new SQL(query), dataService, context)
.serviceTrans(transMeta)
.prepareExecution(false)
.build();
Condition condition = executor.getSql().getWhereCondition().getCondition();
Calendar calendar = Calendar.getInstance();
calendar.clear();
calendar.set(2014, Calendar.DECEMBER, 5);
assertThat(
condition.evaluate(rowMeta, new Object[] {"value", 2L, calendar.getTime()}), is(true));
assertThat(condition.evaluate(rowMeta, new Object[] {"value", 2L, new Date()}), is(false));
}
@Test
public void testReturnDigitsOnly() throws KettleException {
RowMeta inputRowMeta = new RowMeta();
ValueMetaString nameMeta = new ValueMetaString("Name");
inputRowMeta.addValueMeta(nameMeta);
ValueMetaString valueMeta = new ValueMetaString("Value");
inputRowMeta.addValueMeta(valueMeta);
RowSet inputRowSet =
smh.getMockInputRowSet(new Object[][] {{"name1", "qwe123asd456zxc"}, {"name2", null}});
inputRowSet.setRowMeta(inputRowMeta);
Calculator calculator =
new Calculator(smh.stepMeta, smh.stepDataInterface, 0, smh.transMeta, smh.trans);
calculator.getInputRowSets().add(inputRowSet);
calculator.setInputRowMeta(inputRowMeta);
calculator.init(smh.initStepMetaInterface, smh.initStepDataInterface);
CalculatorMeta meta = new CalculatorMeta();
meta.setCalculation(
new CalculatorMetaFunction[] {
new CalculatorMetaFunction(
"digits",
CalculatorMetaFunction.CALC_GET_ONLY_DIGITS,
"Value",
null,
null,
ValueMetaInterface.TYPE_STRING,
0,
0,
false,
"",
"",
"",
"")
});
// Verify output
try {
calculator.addRowListener(
new RowAdapter() {
@Override
public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row)
throws KettleStepException {
assertEquals("123456", row[2]);
}
});
calculator.processRow(meta, new CalculatorData());
} catch (KettleException ke) {
ke.printStackTrace();
fail();
}
}
private void addToCache(
RowMetaInterface keyMeta, Object[] keyData, RowMetaInterface valueMeta, Object[] valueData)
throws KettleValueException {
if (meta.isMemoryPreservationActive()) {
if (meta.isUsingSortedList()) {
KeyValue keyValue = new KeyValue(keyData, valueData);
int idx = Collections.binarySearch(data.list, keyValue, data.comparator);
if (idx < 0) {
int index = -idx - 1; // this is the insertion point
data.list.add(index, keyValue); // insert to keep sorted.
} else {
data.list.set(idx, keyValue); // Overwrite to simulate Hashtable behaviour
}
} else {
if (meta.isUsingIntegerPair()) {
if (!data.metadataVerifiedIntegerPair) {
data.metadataVerifiedIntegerPair = true;
if (keyMeta.size() != 1
|| valueMeta.size() != 1
|| !keyMeta.getValueMeta(0).isInteger()
|| !valueMeta.getValueMeta(0).isInteger()) {
throw new KettleValueException(
BaseMessages.getString(
PKG, "StreamLookup.Exception.CanNotUseIntegerPairAlgorithm"));
}
}
Long key = keyMeta.getInteger(keyData, 0);
Long value = valueMeta.getInteger(valueData, 0);
data.longIndex.put(key, value);
} else {
if (data.hashIndex == null) {
data.hashIndex = new ByteArrayHashIndex(keyMeta);
}
data.hashIndex.put(
RowMeta.extractData(keyMeta, keyData), RowMeta.extractData(valueMeta, valueData));
}
}
} else {
// We can't just put Object[] in the map
// The compare function is not in it.
// We need to wrap in and use that.
// Let's use RowMetaAndData for this one.
//
data.look.put(new RowMetaAndData(keyMeta, keyData), valueData);
}
}
@Test
public void calculatorShouldClearDataInstance() throws Exception {
RowMeta inputRowMeta = new RowMeta();
ValueMetaInteger valueMeta = new ValueMetaInteger("Value");
inputRowMeta.addValueMeta(valueMeta);
RowSet inputRowSet = smh.getMockInputRowSet(new Object[] {-1L});
inputRowSet.setRowMeta(inputRowMeta);
Calculator calculator =
new Calculator(smh.stepMeta, smh.stepDataInterface, 0, smh.transMeta, smh.trans);
calculator.getInputRowSets().add(inputRowSet);
calculator.setInputRowMeta(inputRowMeta);
calculator.init(smh.initStepMetaInterface, smh.initStepDataInterface);
CalculatorMeta meta = new CalculatorMeta();
meta.setCalculation(
new CalculatorMetaFunction[] {
new CalculatorMetaFunction(
"test",
CalculatorMetaFunction.CALC_ABS,
"Value",
null,
null,
ValueMetaInterface.TYPE_STRING,
0,
0,
false,
"",
"",
"",
"")
});
CalculatorData data = new CalculatorData();
data = spy(data);
calculator.processRow(meta, data);
verify(data).getValueMetaFor(eq(valueMeta.getType()), anyString());
calculator.processRow(meta, data);
verify(data).clearValuesMetaMapping();
}
@Test
public void testWithLazyConversion() throws Exception {
RowMeta rowMeta = new RowMeta();
ValueMetaInterface vm = new ValueMetaString("aBinaryStoredString");
vm.setStorageType(ValueMetaInterface.STORAGE_TYPE_BINARY_STRING);
vm.setStorageMetadata(new ValueMetaString());
rowMeta.addValueMeta(vm);
String query = "SELECT * FROM " + DATA_SERVICE_NAME + " WHERE aBinaryStoredString = 'value'";
when(transMeta.getStepFields(DATA_SERVICE_STEP)).thenReturn(rowMeta);
DataServiceExecutor executor =
new DataServiceExecutor.Builder(new SQL(query), dataService, context)
.serviceTrans(new Trans(transMeta))
.prepareExecution(false)
.build();
executor
.getSql()
.getWhereCondition()
.getCondition()
.evaluate(rowMeta, new Object[] {"value".getBytes()});
}
private void loadAllTableDataIntoTheCache() throws KettleException {
DatabaseMeta dbMeta = meta.getDatabaseMeta();
try {
// We only want to get the used table fields...
//
String sql = "SELECT ";
for (int i = 0; i < meta.getStreamKeyField1().length; i++) {
if (i > 0) {
sql += ", ";
}
sql += dbMeta.quoteField(meta.getTableKeyField()[i]);
}
// Also grab the return field...
//
for (int i = 0; i < meta.getReturnValueField().length; i++) {
sql += ", " + dbMeta.quoteField(meta.getReturnValueField()[i]);
}
// The schema/table
//
sql +=
" FROM "
+ dbMeta.getQuotedSchemaTableCombination(
environmentSubstitute(meta.getSchemaName()),
environmentSubstitute(meta.getTablename()));
// order by?
if (meta.getOrderByClause() != null && meta.getOrderByClause().length() != 0) {
sql += " ORDER BY " + meta.getOrderByClause();
}
// Now that we have the SQL constructed, let's store the rows...
//
List<Object[]> rows = data.db.getRows(sql, 0);
if (rows != null && rows.size() > 0) {
RowMetaInterface returnRowMeta = data.db.getReturnRowMeta();
// Copy the data into 2 parts: key and value...
//
for (Object[] row : rows) {
int index = 0;
RowMeta keyMeta = new RowMeta();
Object[] keyData = new Object[meta.getStreamKeyField1().length];
for (int i = 0; i < meta.getStreamKeyField1().length; i++) {
keyData[i] = row[index];
keyMeta.addValueMeta(returnRowMeta.getValueMeta(index++));
}
// RowMeta valueMeta = new RowMeta();
Object[] valueData = new Object[data.returnMeta.size()];
for (int i = 0; i < data.returnMeta.size(); i++) {
valueData[i] = row[index++];
// valueMeta.addValueMeta(returnRowMeta.getValueMeta(index++));
}
// Store the data...
//
storeRowInCache(keyMeta, keyData, valueData);
incrementLinesInput();
}
}
} catch (Exception e) {
throw new KettleException(e);
}
}
public ResultSet getColumns(
String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern)
throws SQLException {
log.debug(
"catalog:"
+ catalog
+ " , schemaPattern:"
+ schemaPattern
+ " , tableNamePattern:"
+ tableNamePattern
+ " ,columnNamePattern:"
+ columnNamePattern);
List<RowMetaAndData> rowAndDatas = new ArrayList<RowMetaAndData>();
// if USE_TRANSNAME_AS_SCHEMA is true, then we use the filename or
// transformation name as the schema
if (!isDir) {
log.debug(helper.getRowMeta(tableNamePattern));
RowMeta rm = helper.getRowMeta(tableNamePattern);
ColInfo[] colInfo = KettleHelper.convert(rm);
String[] columns = rm.getFieldNames();
for (int i = 0; columns != null && i < columns.length; i++) {
RowMetaAndData rd = new RowMetaAndData();
rd.addValue("TABLE_CAT", ValueMetaInterface.TYPE_STRING, catalog);
rd.addValue("TABLE_SCHEM", ValueMetaInterface.TYPE_STRING, schemaPattern);
rd.addValue("TABLE_NAME", ValueMetaInterface.TYPE_STRING, tableNamePattern);
rd.addValue("COLUMN_NAME", ValueMetaInterface.TYPE_STRING, columns[i]);
rd.addValue("DATA_TYPE", ValueMetaInterface.TYPE_INTEGER, colInfo[i].jdbcType);
rd.addValue("TYPE_NAME", ValueMetaInterface.TYPE_STRING, "");
rd.addValue("COLUMN_SIZE", ValueMetaInterface.TYPE_INTEGER, columns.length);
rd.addValue("BUFFER_LENGTH", ValueMetaInterface.TYPE_INTEGER, "20");
rd.addValue("DECIMAL_DIGITS", ValueMetaInterface.TYPE_INTEGER, "20");
rd.addValue("NUM_PREC_RADIX", ValueMetaInterface.TYPE_INTEGER, "20");
rd.addValue("NULLABLE", ValueMetaInterface.TYPE_INTEGER, "20");
rd.addValue("REMARKS", ValueMetaInterface.TYPE_STRING, "");
rd.addValue("COLUMN_DEF", ValueMetaInterface.TYPE_STRING, "");
rd.addValue("SQL_DATA_TYPE", ValueMetaInterface.TYPE_INTEGER, "20");
rd.addValue("SQL_DATETIME_SUB", ValueMetaInterface.TYPE_INTEGER, "20");
rd.addValue("CHAR_OCTET_LENGTH", ValueMetaInterface.TYPE_INTEGER, "1");
rd.addValue("ORDINAL_POSITION", ValueMetaInterface.TYPE_INTEGER, "20");
rd.addValue("IS_NULLABLE", ValueMetaInterface.TYPE_STRING, "0");
rd.addValue("SCOPE_CATALOG", ValueMetaInterface.TYPE_STRING, "0");
rd.addValue("SCOPE_SCHEMA", ValueMetaInterface.TYPE_STRING, "0");
rd.addValue("SCOPE_TABLE", ValueMetaInterface.TYPE_STRING, "0");
rd.addValue("SOURCE_DATA_TYPE", ValueMetaInterface.TYPE_INTEGER, "1");
rowAndDatas.add(rd);
}
KettleJDBCResultSet rs = new KettleJDBCResultSet(null, rowAndDatas, "*");
return rs;
}
// log.debug("getRowMeta:" + helper.getRowMeta(tableNamePattern));
RowMeta rm = helper.getRowMeta(tableNamePattern);
ColInfo[] colInfo = KettleHelper.convert(rm);
String[] columns = rm.getFieldNames();
for (int i = 0; columns != null && i < columns.length; i++) {
String name = columns[i];
RowMetaAndData rd = new RowMetaAndData();
rd.addValue("TABLE_CAT", ValueMetaInterface.TYPE_STRING, catalog);
rd.addValue("TABLE_SCHEM", ValueMetaInterface.TYPE_STRING, schemaPattern);
rd.addValue("TABLE_NAME", ValueMetaInterface.TYPE_STRING, tableNamePattern);
rd.addValue("COLUMN_NAME", ValueMetaInterface.TYPE_STRING, name);
rd.addValue("DATA_TYPE", ValueMetaInterface.TYPE_INTEGER, colInfo[i].getJdbcType());
rd.addValue("TYPE_NAME", ValueMetaInterface.TYPE_STRING, "");
rd.addValue("COLUMN_SIZE", ValueMetaInterface.TYPE_INTEGER, columns.length);
rd.addValue("BUFFER_LENGTH", ValueMetaInterface.TYPE_INTEGER, name);
rd.addValue("DECIMAL_DIGITS", ValueMetaInterface.TYPE_INTEGER, "20");
rd.addValue("NUM_PREC_RADIX", ValueMetaInterface.TYPE_INTEGER, "20");
rd.addValue("NULLABLE", ValueMetaInterface.TYPE_INTEGER, "20");
rd.addValue("REMARKS", ValueMetaInterface.TYPE_STRING, name);
rd.addValue("COLUMN_DEF", ValueMetaInterface.TYPE_STRING, name);
rd.addValue("SQL_DATA_TYPE", ValueMetaInterface.TYPE_INTEGER, "20");
rd.addValue("SQL_DATETIME_SUB", ValueMetaInterface.TYPE_INTEGER, "20");
rd.addValue("CHAR_OCTET_LENGTH", ValueMetaInterface.TYPE_INTEGER, "1");
rd.addValue("ORDINAL_POSITION", ValueMetaInterface.TYPE_INTEGER, "20");
rd.addValue("IS_NULLABLE", ValueMetaInterface.TYPE_STRING, "0");
rd.addValue("SCOPE_CATALOG", ValueMetaInterface.TYPE_STRING, "0");
rd.addValue("SCOPE_SCHEMA", ValueMetaInterface.TYPE_STRING, "0");
rd.addValue("SCOPE_TABLE", ValueMetaInterface.TYPE_STRING, "0");
rd.addValue("SOURCE_DATA_TYPE", ValueMetaInterface.TYPE_INTEGER, "1");
rowAndDatas.add(rd);
}
KettleJDBCResultSet rs = new KettleJDBCResultSet(null, rowAndDatas, "*");
return rs;
}
public void assertRoundGeneral(
final Object expectedResult,
final int calcFunction,
final Number value,
final Long precision,
final Long roundingMode,
final int valueDataType,
final int functionDataType)
throws KettleException {
final String msg =
getKettleTypeName(valueDataType) + "->" + getKettleTypeName(functionDataType) + " ";
final RowMeta inputRowMeta = new RowMeta();
final List<Object> inputValues = new ArrayList<Object>(3);
final String fieldValue = "Value";
final ValueMetaInterface valueMeta;
switch (valueDataType) {
case ValueMetaInterface.TYPE_BIGNUMBER:
valueMeta = new ValueMetaBigNumber(fieldValue);
break;
case ValueMetaInterface.TYPE_NUMBER:
valueMeta = new ValueMetaNumber(fieldValue);
break;
case ValueMetaInterface.TYPE_INTEGER:
valueMeta = new ValueMetaInteger(fieldValue);
break;
default:
throw new IllegalArgumentException(
msg
+ "Unexpected value dataType: "
+ value.getClass().getName()
+ ". Long, Double or BigDecimal expected.");
}
inputRowMeta.addValueMeta(valueMeta);
inputValues.add(value);
final String fieldPrecision;
final ValueMetaInteger precisionMeta;
if (precision == null) {
fieldPrecision = null;
precisionMeta = null;
} else {
fieldPrecision = "Precision";
precisionMeta = new ValueMetaInteger(fieldPrecision);
inputRowMeta.addValueMeta(precisionMeta);
inputValues.add(precision);
}
final String fieldRoundingMode;
final ValueMetaInteger roundingModeMeta;
if (roundingMode == null) {
fieldRoundingMode = null;
roundingModeMeta = null;
} else {
fieldRoundingMode = "RoundingMode";
roundingModeMeta = new ValueMetaInteger(fieldRoundingMode);
inputRowMeta.addValueMeta(roundingModeMeta);
inputValues.add(roundingMode);
}
RowSet inputRowSet = smh.getMockInputRowSet(inputValues.toArray());
inputRowSet.setRowMeta(inputRowMeta);
final String fieldA =
inputRowMeta.size() > 0 ? inputRowMeta.getValueMetaList().get(0).getName() : null;
final String fieldB =
inputRowMeta.size() > 1 ? inputRowMeta.getValueMetaList().get(1).getName() : null;
final String fieldC =
inputRowMeta.size() > 2 ? inputRowMeta.getValueMetaList().get(2).getName() : null;
final int resultDataType = functionDataType;
final String fieldResult = "test";
final int expectedResultRowSize = inputRowMeta.size() + 1;
Calculator calculator =
new Calculator(smh.stepMeta, smh.stepDataInterface, 0, smh.transMeta, smh.trans);
calculator.getInputRowSets().add(inputRowSet);
calculator.setInputRowMeta(inputRowMeta);
calculator.init(smh.initStepMetaInterface, smh.initStepDataInterface);
CalculatorMeta meta = new CalculatorMeta();
meta.setCalculation(
new CalculatorMetaFunction[] {
new CalculatorMetaFunction(
fieldResult,
calcFunction,
fieldA,
fieldB,
fieldC,
resultDataType,
2,
0,
false,
"",
"",
"",
"")
});
// Verify output
try {
calculator.addRowListener(
new RowAdapter() {
@Override
public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row)
throws KettleStepException {
assertEquals(msg + " resultRowSize", expectedResultRowSize, rowMeta.size());
final int fieldResultIndex = rowMeta.size() - 1;
assertEquals(
msg + " fieldResult",
fieldResult,
rowMeta.getValueMeta(fieldResultIndex).getName());
assertEquals(msg, expectedResult, row[fieldResultIndex]);
}
});
calculator.processRow(meta, new CalculatorData());
} catch (KettleException ke) {
ke.printStackTrace();
fail(msg + ke.getMessage());
}
}
private ArrayList<Object[]> writeToTable(RowMetaInterface rowMeta, ArrayList<Object[]> rows)
throws KettleException {
if (rows.isEmpty()) // Stop: last line or error encountered
{
if (log.isDetailed()) logDetailed("Last line inserted: stop");
return null;
}
PreparedStatement insertStatement = null;
ArrayList<Object[]> insertRowsData = new ArrayList<Object[]>();
ArrayList<Object[]> outputRowsData = rows;
String tableName = null;
boolean sendToErrorRow = false;
String errorMessage = null;
boolean rowIsSafe = false;
int[] updateCounts = null;
List<Exception> exceptionsList = null;
boolean batchProblem = false;
for (Object[] row : rows) {
if (meta.isTableNameInField()) {
// Cache the position of the table name field
if (data.indexOfTableNameField < 0) {
String realTablename = environmentSubstitute(meta.getTableNameField());
data.indexOfTableNameField = rowMeta.indexOfValue(realTablename);
if (data.indexOfTableNameField < 0) {
String message = "Unable to find table name field [" + realTablename + "] in input row";
logError(message);
throw new KettleStepException(message);
}
if (!meta.isTableNameInTable()) {
data.insertRowMeta.removeValueMeta(data.indexOfTableNameField);
}
}
tableName = rowMeta.getString(rows.get(0), data.indexOfTableNameField);
if (!meta.isTableNameInTable()) {
// If the name of the table should not be inserted itself,
// remove the table name
// from the input row data as well. This forcibly creates a
// copy of r
insertRowsData.add(
RowDataUtil.removeItem(rowMeta.cloneRow(row), data.indexOfTableNameField));
} else {
insertRowsData.add(row);
}
} else if (meta.isPartitioningEnabled()
&& (meta.isPartitioningDaily() || meta.isPartitioningMonthly())
&& (meta.getPartitioningField() != null && meta.getPartitioningField().length() > 0)) {
// Initialize some stuff!
if (data.indexOfPartitioningField < 0) {
data.indexOfPartitioningField =
rowMeta.indexOfValue(environmentSubstitute(meta.getPartitioningField()));
if (data.indexOfPartitioningField < 0) {
throw new KettleStepException(
"Unable to find field [" + meta.getPartitioningField() + "] in the input row!");
}
if (meta.isPartitioningDaily()) {
data.dateFormater = new SimpleDateFormat("yyyyMMdd");
} else {
data.dateFormater = new SimpleDateFormat("yyyyMM");
}
}
ValueMetaInterface partitioningValue = rowMeta.getValueMeta(data.indexOfPartitioningField);
if (!partitioningValue.isDate() || row[data.indexOfPartitioningField] == null) {
throw new KettleStepException(
"Sorry, the partitioning field needs to contain a data value and can't be empty!");
}
Object partitioningValueData = rowMeta.getDate(row, data.indexOfPartitioningField);
tableName =
environmentSubstitute(meta.getTablename())
+ "_"
+ data.dateFormater.format((Date) partitioningValueData);
insertRowsData.add(row);
} else {
tableName = data.tableName;
insertRowsData.add(row);
}
if (Const.isEmpty(tableName)) {
throw new KettleStepException("The tablename is not defined (empty)");
}
}
if (!data.preparedStatements.containsKey(tableName)) {
data.preparedStatements.put(tableName, new Hashtable<Integer, PreparedStatement>());
}
insertStatement = (PreparedStatement) data.preparedStatements.get(tableName).get(rows.size());
if (insertStatement == null) {
String sql =
getInsertStatement(
environmentSubstitute(meta.getSchemaName()),
tableName,
data.insertRowMeta,
rows.size());
if (log.isDetailed()) logDetailed("Prepared statement : " + sql);
insertStatement = data.db.prepareSQL(sql);
if (!data.preparedStatements.containsKey(tableName)) {
data.preparedStatements.put(tableName, new Hashtable<Integer, PreparedStatement>());
}
data.preparedStatements.get(tableName).put(rows.size(), insertStatement);
}
try {
// For PG & GP, we add a savepoint before the row.
// Then revert to the savepoint afterwards... (not a transaction, so
// hopefully still fast)
//
if (data.specialErrorHandling) {
data.savepoint = data.db.setSavepoint();
}
RowMeta insertRowMeta = new RowMeta();
for (int i = 0; i < rows.size(); i++) {
for (int j = 0; j < data.valuenrs.length; j++) {
insertRowMeta.addValueMeta(data.insertRowMeta.getValueMeta(j));
}
}
data.db.setValues(insertRowMeta, toArray(insertRowsData), insertStatement);
data.db.insertRow(insertStatement, data.batchMode, false); // false:
// no
// commit,
// it is
// handled
// in
// this
// step
// different
// Get a commit counter per prepared statement to keep track of
// separate tables, etc.
//
Integer commitCounter = data.commitCounterMap.get(tableName);
if (commitCounter == null) {
commitCounter = Integer.valueOf(1);
} else {
commitCounter++;
}
data.commitCounterMap.put(tableName, Integer.valueOf(commitCounter.intValue()));
// Release the savepoint if needed
//
if (data.specialErrorHandling) {
if (data.releaseSavepoint) {
data.db.releaseSavepoint(data.savepoint);
}
}
// Perform a commit if needed
//
if ((data.commitSize > 0) && ((commitCounter % data.commitSize) == 0)) {
if (data.batchMode) {
try {
insertStatement.executeBatch();
data.db.commit();
insertStatement.clearBatch();
} catch (BatchUpdateException ex) {
KettleDatabaseBatchException kdbe =
new KettleDatabaseBatchException("Error updating batch", ex);
kdbe.setUpdateCounts(ex.getUpdateCounts());
List<Exception> exceptions = new ArrayList<Exception>();
// 'seed' the loop with the root exception
SQLException nextException = ex;
do {
exceptions.add(nextException);
// while current exception has next exception, add
// to list
} while ((nextException = nextException.getNextException()) != null);
kdbe.setExceptionsList(exceptions);
throw kdbe;
} catch (SQLException ex) {
throw new KettleDatabaseException("Error inserting row", ex);
} catch (Exception ex) {
throw new KettleDatabaseException("Unexpected error inserting row", ex);
}
} else {
// insertRow normal commit
data.db.commit();
}
// Clear the batch/commit counter...
//
data.commitCounterMap.put(tableName, Integer.valueOf(0));
rowIsSafe = true;
} else {
rowIsSafe = false;
}
} catch (KettleDatabaseBatchException be) {
errorMessage = be.toString();
batchProblem = true;
sendToErrorRow = true;
updateCounts = be.getUpdateCounts();
exceptionsList = be.getExceptionsList();
if (getStepMeta().isDoingErrorHandling()) {
data.db.clearBatch(insertStatement);
data.db.commit(true);
} else {
data.db.clearBatch(insertStatement);
data.db.rollback();
StringBuffer msg =
new StringBuffer("Error batch inserting rows into table [" + tableName + "].");
msg.append(Const.CR);
msg.append("Errors encountered (first 10):").append(Const.CR);
for (int x = 0; x < be.getExceptionsList().size() && x < 10; x++) {
Exception exception = be.getExceptionsList().get(x);
if (exception.getMessage() != null) msg.append(exception.getMessage()).append(Const.CR);
}
throw new KettleException(msg.toString(), be);
}
} catch (KettleDatabaseException dbe) {
if (getStepMeta().isDoingErrorHandling()) {
if (data.specialErrorHandling) {
data.db.rollback(data.savepoint);
if (data.releaseSavepoint) {
data.db.releaseSavepoint(data.savepoint);
}
// data.db.commit(true); // force a commit on the connection
// too.
}
sendToErrorRow = true;
errorMessage = dbe.toString();
} else {
if (meta.ignoreErrors()) {
if (data.warnings < 20) {
if (log.isBasic())
logBasic("WARNING: Couldn't insert row into table." + Const.CR + dbe.getMessage());
} else if (data.warnings == 20) {
if (log.isBasic())
logBasic(
"FINAL WARNING (no more then 20 displayed): Couldn't insert row into table: "
+ Const.CR
+ dbe.getMessage());
}
data.warnings++;
} else {
setErrors(getErrors() + 1);
data.db.rollback();
throw new KettleException("Error inserting row into table [" + tableName + "]", dbe);
}
}
}
if (data.batchMode) {
if (sendToErrorRow) {
if (batchProblem) {
for (Object[] row : outputRowsData) {
data.batchBuffer.add(row);
}
outputRowsData = null;
processBatchException(errorMessage, updateCounts, exceptionsList);
} else {
// Simply add this row to the error row
for (Object[] row : outputRowsData) {
putError(rowMeta, row, 1L, errorMessage, null, "TOP001");
}
outputRowsData = null;
}
} else {
for (Object[] row : outputRowsData) {
data.batchBuffer.add(row);
}
outputRowsData = null;
if (rowIsSafe) // A commit was done and the rows are all safe
// (no error)
{
for (int i = 0; i < data.batchBuffer.size(); i++) {
Object[] row = (Object[]) data.batchBuffer.get(i);
putRow(data.outputRowMeta, row);
incrementLinesOutput();
}
// Clear the buffer
data.batchBuffer.clear();
}
}
} else {
if (sendToErrorRow) {
for (Object[] row : outputRowsData) {
putError(rowMeta, row, 1L, errorMessage, null, "TOP001");
}
outputRowsData = null;
}
}
return outputRowsData;
}
protected static RowMetaInterface createRowMeta(ValueMetaInterface... valueMetas) {
RowMeta rowMeta = new RowMeta();
rowMeta.setValueMetaList(Arrays.asList(valueMetas));
return rowMeta;
}
