public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
meta = (DatabaseLookupMeta) smi;
data = (DatabaseLookupData) sdi;
boolean sendToErrorRow = false;
String errorMessage = null;
Object[] r = getRow(); // Get row from input rowset & set row busy!
if (r == null) { // no more input to be expected...
setOutputDone();
return false;
}
if (first) {
first = false;
// create the output metadata
data.outputRowMeta = getInputRowMeta().clone();
meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
if (meta.isCached()) {
if (meta.getCacheSize() > 0) {
data.look = new Hashtable<RowMetaAndData, TimedRow>((int) (meta.getCacheSize() * 1.5));
} else {
data.look = new Hashtable<RowMetaAndData, TimedRow>();
}
}
data.db.setLookup(
environmentSubstitute(meta.getSchemaName()),
environmentSubstitute(meta.getTablename()),
meta.getTableKeyField(),
meta.getKeyCondition(),
meta.getReturnValueField(),
meta.getReturnValueNewName(),
meta.getOrderByClause(),
meta.isFailingOnMultipleResults());
// lookup the values!
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(PKG, "DatabaseLookup.Log.CheckingRow")
+ getInputRowMeta().getString(r));
}
data.keynrs = new int[meta.getStreamKeyField1().length];
data.keynrs2 = new int[meta.getStreamKeyField1().length];
for (int i = 0; i < meta.getStreamKeyField1().length; i++) {
data.keynrs[i] = getInputRowMeta().indexOfValue(meta.getStreamKeyField1()[i]);
if (data.keynrs[i] < 0
&& // couldn't find field!
!"IS NULL".equalsIgnoreCase(meta.getKeyCondition()[i])
&& // No field needed!
!"IS NOT NULL".equalsIgnoreCase(meta.getKeyCondition()[i]) // No field needed!
) {
throw new KettleStepException(
BaseMessages.getString(PKG, "DatabaseLookup.ERROR0001.FieldRequired1.Exception")
+ meta.getStreamKeyField1()[i]
+ BaseMessages.getString(
PKG, "DatabaseLookup.ERROR0001.FieldRequired2.Exception"));
}
data.keynrs2[i] = getInputRowMeta().indexOfValue(meta.getStreamKeyField2()[i]);
if (data.keynrs2[i] < 0
&& // couldn't find field!
"BETWEEN".equalsIgnoreCase(meta.getKeyCondition()[i]) // 2 fields needed!
) {
throw new KettleStepException(
BaseMessages.getString(PKG, "DatabaseLookup.ERROR0001.FieldRequired3.Exception")
+ meta.getStreamKeyField2()[i]
+ BaseMessages.getString(
PKG, "DatabaseLookup.ERROR0001.FieldRequired4.Exception"));
}
if (log.isDebug()) {
logDebug(
BaseMessages.getString(PKG, "DatabaseLookup.Log.FieldHasIndex1")
+ meta.getStreamKeyField1()[i]
+ BaseMessages.getString(PKG, "DatabaseLookup.Log.FieldHasIndex2")
+ data.keynrs[i]);
}
}
data.nullif = new Object[meta.getReturnValueField().length];
for (int i = 0; i < meta.getReturnValueField().length; i++) {
ValueMetaInterface stringMeta = new ValueMeta("string", ValueMetaInterface.TYPE_STRING);
ValueMetaInterface returnMeta =
data.outputRowMeta.getValueMeta(i + getInputRowMeta().size());
if (!Const.isEmpty(meta.getReturnValueDefault()[i])) {
data.nullif[i] = returnMeta.convertData(stringMeta, meta.getReturnValueDefault()[i]);
} else {
data.nullif[i] = null;
}
}
// Determine the types...
data.keytypes = new int[meta.getTableKeyField().length];
String schemaTable =
meta.getDatabaseMeta()
.getQuotedSchemaTableCombination(
environmentSubstitute(meta.getSchemaName()),
environmentSubstitute(meta.getTablename()));
RowMetaInterface fields = data.db.getTableFields(schemaTable);
if (fields != null) {
// Fill in the types...
for (int i = 0; i < meta.getTableKeyField().length; i++) {
ValueMetaInterface key = fields.searchValueMeta(meta.getTableKeyField()[i]);
if (key != null) {
data.keytypes[i] = key.getType();
} else {
throw new KettleStepException(
BaseMessages.getString(PKG, "DatabaseLookup.ERROR0001.FieldRequired5.Exception")
+ meta.getTableKeyField()[i]
+ BaseMessages.getString(
PKG, "DatabaseLookup.ERROR0001.FieldRequired6.Exception"));
}
}
} else {
throw new KettleStepException(
BaseMessages.getString(PKG, "DatabaseLookup.ERROR0002.UnableToDetermineFieldsOfTable")
+ schemaTable
+ "]");
}
// Count the number of values in the lookup as well as the metadata to send along with it.
//
data.lookupMeta = new RowMeta();
for (int i = 0; i < meta.getStreamKeyField1().length; i++) {
if (data.keynrs[i] >= 0) {
ValueMetaInterface inputValueMeta = getInputRowMeta().getValueMeta(data.keynrs[i]);
// Try to convert type if needed in a clone, we don't want to
// change the type in the original row
//
ValueMetaInterface value =
ValueMetaFactory.cloneValueMeta(inputValueMeta, data.keytypes[i]);
data.lookupMeta.addValueMeta(value);
}
if (data.keynrs2[i] >= 0) {
ValueMetaInterface inputValueMeta = getInputRowMeta().getValueMeta(data.keynrs2[i]);
// Try to convert type if needed in a clone, we don't want to
// change the type in the original row
//
ValueMetaInterface value =
ValueMetaFactory.cloneValueMeta(inputValueMeta, data.keytypes[i]);
data.lookupMeta.addValueMeta(value);
}
}
// We also want to know the metadata of the return values beforehand (null handling)
data.returnMeta = new RowMeta();
for (int i = 0; i < meta.getReturnValueField().length; i++) {
ValueMetaInterface v =
data.outputRowMeta.getValueMeta(getInputRowMeta().size() + i).clone();
data.returnMeta.addValueMeta(v);
}
// If the user selected to load all data into the cache at startup, that's what we do now...
//
if (meta.isCached() && meta.isLoadingAllDataInCache()) {
loadAllTableDataIntoTheCache();
}
}
if (log.isRowLevel()) {
logRowlevel(
BaseMessages.getString(PKG, "DatabaseLookup.Log.GotRowFromPreviousStep")
+ getInputRowMeta().getString(r));
}
try {
// add new lookup values to the row
Object[] outputRow = lookupValues(getInputRowMeta(), r);
if (outputRow != null) {
// copy row to output rowset(s);
putRow(data.outputRowMeta, outputRow);
if (log.isRowLevel()) {
logRowlevel(
BaseMessages.getString(PKG, "DatabaseLookup.Log.WroteRowToNextStep")
+ getInputRowMeta().getString(r));
}
if (checkFeedback(getLinesRead())) {
logBasic("linenr " + getLinesRead());
}
}
} catch (KettleException e) {
if (getStepMeta().isDoingErrorHandling()) {
sendToErrorRow = true;
errorMessage = e.toString();
} else {
logError(
BaseMessages.getString(PKG, "DatabaseLookup.ERROR003.UnexpectedErrorDuringProcessing")
+ e.getMessage());
setErrors(1);
stopAll();
setOutputDone(); // signal end to receiver(s)
return false;
}
if (sendToErrorRow) {
// Simply add this row to the error row
putError(getInputRowMeta(), r, 1, errorMessage, null, "DBLOOKUPD001");
}
}
return true;
}
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
meta = (GaInputStepMeta) smi;
data = (GaInputStepData) sdi;
if (first) {
first = false;
data.outputRowMeta = new RowMeta();
meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
// stores the indices where to look for the key fields in the input rows
data.conversionMeta = new ValueMetaInterface[meta.getFieldsCount()];
for (int i = 0; i < meta.getFieldsCount(); i++) {
// get output and from-string conversion format for each field
ValueMetaInterface returnMeta = data.outputRowMeta.getValueMeta(i);
ValueMetaInterface conversionMeta;
conversionMeta =
ValueMetaFactory.cloneValueMeta(returnMeta, ValueMetaInterface.TYPE_STRING);
conversionMeta.setConversionMask(meta.getConversionMask()[i]);
conversionMeta.setDecimalSymbol("."); // google analytics is en-US
conversionMeta.setGroupingSymbol(null); // google analytics uses no grouping symbol
data.conversionMeta[i] = conversionMeta;
}
}
// generate output row, make it correct size
Object[] outputRow = RowDataUtil.allocateRowData(data.outputRowMeta.size());
List<String> entry = getNextDataEntry();
if (entry != null
&& (meta.getRowLimit() <= 0
|| getLinesWritten() < meta.getRowLimit())) { // another record to
// fill the output fields with look up data
for (int i = 0, j = 0; i < meta.getFieldsCount(); i++) {
String fieldName = environmentSubstitute(meta.getFeedField()[i]);
Object dataObject;
String type = environmentSubstitute(meta.getFeedFieldType()[i]);
// We handle fields differently depending on whether its a Dimension/Metric, Data Source
// Property, or
// Data Source Field. Also the API doesn't exactly match the concepts anymore (see
// individual comments below),
// so there is quite a bit of special processing.
if (GaInputStepMeta.FIELD_TYPE_DATA_SOURCE_PROPERTY.equals(type)) {
// Account name has to be handled differently, it's in the Accounts API not Profiles API
if (GaInputStepMeta.PROPERTY_DATA_SOURCE_ACCOUNT_NAME.equals(fieldName)) {
// We expect a single account name, and already fetched it during init
dataObject = accountName;
} else {
dataObject = data.feed.getProfileInfo().get(removeClassifier(fieldName));
}
} else if (GaInputStepMeta.FIELD_TYPE_DATA_SOURCE_FIELD.equals(type)) {
// Get tableId or tableName
if (GaInputStepMeta.FIELD_DATA_SOURCE_TABLE_ID.equals(fieldName)) {
dataObject = data.feed.getProfileInfo().get(removeClassifier(fieldName));
} else {
// We only have two Data Source Fields and they're hard-coded, so we handle tableName in
// this else-clause
// since tableId was done in the if-clause. We have to handle the two differently
// because tableName is
// actually the profile name in this version (v3) of the Google Analytics API.
dataObject = data.feed.getProfileInfo().getProfileName();
}
} else if (GaInputStepMeta.DEPRECATED_FIELD_TYPE_CONFIDENCE_INTERVAL.equals(type)) {
dataObject = null;
if (log.isRowLevel()) {
logRowlevel(
BaseMessages.getString(
PKG,
"GoogleAnalytics.Warn.FieldTypeNotSupported",
GaInputStepMeta.DEPRECATED_FIELD_TYPE_CONFIDENCE_INTERVAL));
}
} else {
// Assume it's a Dimension or Metric, we've covered the rest of the cases above.
dataObject = entry.get(j++);
}
outputRow[i] =
data.outputRowMeta.getValueMeta(i).convertData(data.conversionMeta[i], dataObject);
}
// copy row to possible alternate rowset(s)
putRow(data.outputRowMeta, outputRow);
// Some basic logging
if (checkFeedback(getLinesWritten())) {
if (log.isBasic()) {
logBasic("Linenr " + getLinesWritten());
}
}
return true;
} else {
setOutputDone();
return false;
}
}
