public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
meta = (JsonInputMeta) smi;
data = (JsonInputData) sdi;
if (super.init(smi, sdi)) {
data.rownr = 1L;
data.nrInputFields = meta.getInputFields().length;
// Take care of variable substitution
for (int i = 0; i < data.nrInputFields; i++) {
JsonInputField field = meta.getInputFields()[i];
field.setPath(environmentSubstitute(field.getPath()));
}
try {
// Init a new JSON reader
data.jsonReader = new JsonReader();
data.jsonReader.SetIgnoreMissingPath(meta.isIgnoreMissingPath());
} catch (KettleException e) {
logError(e.getMessage());
return false;
}
return true;
}
return false;
}
@Override
public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
meta = (VerticaBulkLoaderMeta) smi;
data = (VerticaBulkLoaderData) sdi;
if (super.init(smi, sdi)) {
try {
data.databaseMeta = meta.getDatabaseMeta();
data.db = new Database(this, meta.getDatabaseMeta());
data.db.shareVariablesWith(this);
if (getTransMeta().isUsingUniqueConnections()) {
synchronized (getTrans()) {
data.db.connect(getTrans().getThreadName(), getPartitionID());
}
} else {
data.db.connect(getPartitionID());
}
if (log.isBasic()) logBasic("Connected to database [" + meta.getDatabaseMeta() + "]");
data.db.setAutoCommit(false);
return true;
} catch (KettleException e) {
logError("An error occurred intialising this step: " + e.getMessage());
stopAll();
setErrors(1);
}
}
return false;
}
public SQLStatement getSQLStatements(
TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev) throws KettleStepException {
SQLStatement retval =
new SQLStatement(stepMeta.getName(), databaseMeta, null); // default: nothing to do!
if (databaseMeta != null) {
if (prev != null && prev.size() > 0) {
// Copy the row
RowMetaInterface tableFields = new RowMeta();
// Now change the field names
for (int i = 0; i < fieldTable.length; i++) {
ValueMetaInterface v = prev.searchValueMeta(fieldStream[i]);
if (v != null) {
ValueMetaInterface tableField = v.clone();
tableField.setName(fieldTable[i]);
tableFields.addValueMeta(tableField);
} else {
throw new KettleStepException(
"Unable to find field [" + fieldStream[i] + "] in the input rows");
}
}
if (!Const.isEmpty(tableName)) {
Database db = new Database(loggingObject, databaseMeta);
db.shareVariablesWith(transMeta);
try {
db.connect();
String schemaTable =
databaseMeta.getQuotedSchemaTableCombination(
transMeta.environmentSubstitute(schemaName),
transMeta.environmentSubstitute(tableName));
String sql = db.getDDL(schemaTable, tableFields, null, false, null, true);
if (sql.length() == 0) retval.setSQL(null);
else retval.setSQL(sql);
} catch (KettleException e) {
retval.setError(
BaseMessages.getString(PKG, "GPBulkLoaderMeta.GetSQL.ErrorOccurred")
+ e.getMessage()); // $NON-NLS-1$
}
} else {
retval.setError(
BaseMessages.getString(
PKG, "GPBulkLoaderMeta.GetSQL.NoTableDefinedOnConnection")); // $NON-NLS-1$
}
} else {
retval.setError(
BaseMessages.getString(
PKG, "GPBulkLoaderMeta.GetSQL.NotReceivingAnyFields")); // $NON-NLS-1$
}
} else {
retval.setError(
BaseMessages.getString(
PKG, "GPBulkLoaderMeta.GetSQL.NoConnectionDefined")); // $NON-NLS-1$
}
return retval;
}
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
try {
// Grab a row
Object[] outputRowData = getOneRow();
if (outputRowData == null) {
setOutputDone(); // signal end to receiver(s)
return false; // end of data or error.
}
if (isRowLevel())
logRowlevel(
BaseMessages.getString(
PKG, "LoadFileInput.Log.ReadRow", data.outputRowMeta.getString(outputRowData)));
putRow(data.outputRowMeta, outputRowData);
if (meta.getRowLimit() > 0
&& data.rownr > meta.getRowLimit()) // limit has been reached: stop now.
{
setOutputDone();
return false;
}
} catch (KettleException e) {
logError(
BaseMessages.getString(
PKG, "LoadFileInput.ErrorInStepRunning", e.getMessage())); // $NON-NLS-1$
logError(Const.getStackTracker(e));
setErrors(1);
stopAll();
setOutputDone(); // signal end to receiver(s)
return false;
}
return true;
}
@Test
public void testReturnDigitsOnly() throws KettleException {
RowMeta inputRowMeta = new RowMeta();
ValueMetaString nameMeta = new ValueMetaString("Name");
inputRowMeta.addValueMeta(nameMeta);
ValueMetaString valueMeta = new ValueMetaString("Value");
inputRowMeta.addValueMeta(valueMeta);
RowSet inputRowSet =
smh.getMockInputRowSet(new Object[][] {{"name1", "qwe123asd456zxc"}, {"name2", null}});
inputRowSet.setRowMeta(inputRowMeta);
Calculator calculator =
new Calculator(smh.stepMeta, smh.stepDataInterface, 0, smh.transMeta, smh.trans);
calculator.getInputRowSets().add(inputRowSet);
calculator.setInputRowMeta(inputRowMeta);
calculator.init(smh.initStepMetaInterface, smh.initStepDataInterface);
CalculatorMeta meta = new CalculatorMeta();
meta.setCalculation(
new CalculatorMetaFunction[] {
new CalculatorMetaFunction(
"digits",
CalculatorMetaFunction.CALC_GET_ONLY_DIGITS,
"Value",
null,
null,
ValueMetaInterface.TYPE_STRING,
0,
0,
false,
"",
"",
"",
"")
});
// Verify output
try {
calculator.addRowListener(
new RowAdapter() {
@Override
public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row)
throws KettleStepException {
assertEquals("123456", row[2]);
}
});
calculator.processRow(meta, new CalculatorData());
} catch (KettleException ke) {
ke.printStackTrace();
fail();
}
}
public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
meta = (SalesforceDeleteMeta) smi;
data = (SalesforceDeleteData) sdi;
if (super.init(smi, sdi)) {
try {
data.realModule = environmentSubstitute(meta.getModule());
// Check if module is specified
if (Const.isEmpty(data.realModule)) {
log.logError(
BaseMessages.getString(PKG, "SalesforceDeleteDialog.ModuleMissing.DialogMessage"));
return false;
}
String realUser = environmentSubstitute(meta.getUserName());
// Check if username is specified
if (Const.isEmpty(realUser)) {
log.logError(
BaseMessages.getString(PKG, "SalesforceDeleteDialog.UsernameMissing.DialogMessage"));
return false;
}
// initialize variables
data.realURL = environmentSubstitute(meta.getTargetURL());
// create a Salesforce connection
data.connection =
new SalesforceConnection(
log, data.realURL, realUser, environmentSubstitute(meta.getPassword()));
// set timeout
data.connection.setTimeOut(Const.toInt(environmentSubstitute(meta.getTimeOut()), 0));
// Do we use compression?
data.connection.setUsingCompression(meta.isUsingCompression());
// Do we rollback all changes on error
data.connection.rollbackAllChangesOnError(meta.isRollbackAllChangesOnError());
// Now connect ...
data.connection.connect();
return true;
} catch (KettleException ke) {
logError(
BaseMessages.getString(PKG, "SalesforceDelete.Log.ErrorOccurredDuringStepInitialize")
+ ke.getMessage()); // $NON-NLS-1$
}
return true;
}
return false;
}
public void loadRep(
Repository rep,
IMetaStore metaStore,
ObjectId id_jobentry,
List<DatabaseMeta> databases,
List<SlaveServer> slaveServers)
throws KettleException {
try {
serverName = rep.getJobEntryAttributeString(id_jobentry, "servername");
userName = rep.getJobEntryAttributeString(id_jobentry, "username");
password =
Encr.decryptPasswordOptionallyEncrypted(
rep.getJobEntryAttributeString(id_jobentry, "password"));
serverPort = rep.getJobEntryAttributeString(id_jobentry, "serverport");
ftpDirectory = rep.getJobEntryAttributeString(id_jobentry, "ftpdirectory");
localDirectory = rep.getJobEntryAttributeString(id_jobentry, "localdirectory");
wildcard = rep.getJobEntryAttributeString(id_jobentry, "wildcard");
onlyGettingNewFiles = rep.getJobEntryAttributeBoolean(id_jobentry, "only_new");
usehttpproxy = rep.getJobEntryAttributeBoolean(id_jobentry, "usehttpproxy");
httpProxyHost = rep.getJobEntryAttributeString(id_jobentry, "httpproxyhost");
httpproxyusername = rep.getJobEntryAttributeString(id_jobentry, "httpproxyusername");
httpProxyPassword = rep.getJobEntryAttributeString(id_jobentry, "httpproxypassword");
publicpublickey = rep.getJobEntryAttributeBoolean(id_jobentry, "publicpublickey");
keyFilename = rep.getJobEntryAttributeString(id_jobentry, "keyfilename");
keyFilePass = rep.getJobEntryAttributeString(id_jobentry, "keyfilepass");
useBasicAuthentication =
rep.getJobEntryAttributeBoolean(id_jobentry, "usebasicauthentication");
afterFtpPut = rep.getJobEntryAttributeString(id_jobentry, "afterftpput");
destinationfolder = rep.getJobEntryAttributeString(id_jobentry, "destinationfolder");
createdestinationfolder =
rep.getJobEntryAttributeBoolean(id_jobentry, "createdestinationfolder");
cachehostkey = rep.getJobEntryAttributeBoolean(id_jobentry, "cachehostkey");
timeout = (int) rep.getJobEntryAttributeInteger(id_jobentry, "timeout");
createtargetfolder = rep.getJobEntryAttributeBoolean(id_jobentry, "createtargetfolder");
includeSubFolders = rep.getJobEntryAttributeBoolean(id_jobentry, "includeSubFolders");
} catch (KettleException dbe) {
throw new KettleException(
BaseMessages.getString(
PKG, "JobSSH2GET.Log.UnableLoadRep", "" + id_jobentry, dbe.getMessage()));
}
}
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
meta = (AccessOutputMeta) smi;
data = (AccessOutputData) sdi;
Object[] r = getRow(); // this also waits for a previous step to be finished.
if (r == null) // no more input to be expected...
{
setOutputDone();
return false;
}
if (first && meta.isDoNotOpenNewFileInit()) {
try {
if (!OpenFile()) {
return false;
}
} catch (Exception e) {
logError("An error occurred intialising this step: " + e.getMessage());
stopAll();
setErrors(1);
}
}
try {
writeToTable(r);
putRow(data.outputRowMeta, r); // in case we want it go further...
if (checkFeedback(getLinesOutput())) {
if (log.isBasic()) {
logBasic("linenr " + getLinesOutput());
}
}
} catch (KettleException e) {
logError("Because of an error, this step can't continue: " + e.getMessage());
setErrors(1);
stopAll();
setOutputDone(); // signal end to receiver(s)
return false;
}
return true;
}
public void getUsedArguments(
JobMeta jobMeta, String[] commandLineArguments, IMetaStore metaStore) {
for (JobEntryCopy jobEntryCopy : jobMeta.jobcopies) {
if (jobEntryCopy.isTransformation()) {
JobEntryTrans jobEntryTrans = (JobEntryTrans) jobEntryCopy.getEntry();
try {
TransMeta transMeta = jobEntryTrans.getTransMeta(repository, metaStore, jobMeta);
Map<String, String> map = transMeta.getUsedArguments(commandLineArguments);
for (String key : map.keySet()) {
String value = map.get(key);
if (!arguments.containsKey(key)) {
arguments.put(key, value);
}
}
} catch (KettleException ke) {
log.logBasic(ke.getMessage(), ke);
}
}
}
}
public static synchronized void initializePentahoSystem(String solutionPath) throws Exception {
// this setting is useful only for running the integration tests from within IntelliJ
// this same property is set for integration tests via the pom when running with mvn
String folderPaths = "target/spoon/plugins";
File f = new File(folderPaths);
System.setProperty("KETTLE_PLUGIN_BASE_FOLDERS", f.getAbsolutePath());
StandaloneApplicationContext appContext = new StandaloneApplicationContext(solutionPath, "");
PentahoSystem.setSystemSettingsService(new PathBasedSystemSettings());
if (solutionPath == null) {
throw new MetaverseException(
Messages.getString("ERROR.MetaverseInit.BadConfigPath", solutionPath));
}
try {
ClassLoader cl = Thread.currentThread().getContextClassLoader();
Thread.currentThread().setContextClassLoader(MetaverseUtil.class.getClassLoader());
IPentahoObjectFactory pentahoObjectFactory = new StandaloneSpringPentahoObjectFactory();
pentahoObjectFactory.init(solutionPath, PentahoSystem.getApplicationContext());
PentahoSystem.registerObjectFactory(pentahoObjectFactory);
// Restore context classloader
Thread.currentThread().setContextClassLoader(cl);
} catch (Exception e) {
throw new MetaverseException(Messages.getString("ERROR.MetaverseInit.CouldNotInit"), e);
}
PentahoSystem.init(appContext);
PentahoSessionHolder.setSession(new StandaloneSession());
registerKettlePlugins();
try {
KettleEnvironment.init();
} catch (KettleException e) {
e.printStackTrace();
}
}
public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
meta = (SalesforceUpdateMeta) smi;
data = (SalesforceUpdateData) sdi;
if (super.init(smi, sdi)) {
try {
// Do we need to rollback all changes on error
data.connection.setRollbackAllChangesOnError(meta.isRollbackAllChangesOnError());
// Now connect ...
data.connection.connect();
return true;
} catch (KettleException ke) {
logError(
BaseMessages.getString(PKG, "SalesforceUpdate.Log.ErrorOccurredDuringStepInitialize")
+ ke.getMessage());
}
return true;
}
return false;
}
public void loadRep(
Repository rep,
IMetaStore metaStore,
ObjectId id_jobentry,
List<DatabaseMeta> databases,
List<SlaveServer> slaveServers)
throws KettleException {
try {
replaceVars = rep.getJobEntryAttributeBoolean(id_jobentry, "replacevars");
filename = rep.getJobEntryAttributeString(id_jobentry, "filename");
fileVariableType =
getVariableType(rep.getJobEntryAttributeString(id_jobentry, "file_variable_type"));
// How many variableName?
int argnr = rep.countNrJobEntryAttributes(id_jobentry, "variable_name");
variableName = new String[argnr];
variableValue = new String[argnr];
variableType = new int[argnr];
// Read them all...
for (int a = 0; a < argnr; a++) {
variableName[a] = rep.getJobEntryAttributeString(id_jobentry, a, "variable_name");
variableValue[a] = rep.getJobEntryAttributeString(id_jobentry, a, "variable_value");
variableType[a] =
getVariableType(rep.getJobEntryAttributeString(id_jobentry, a, "variable_type"));
}
} catch (KettleException dbe) {
throw new KettleException(
BaseMessages.getString(
PKG,
"JobEntrySetVariables.Meta.UnableLoadRep",
String.valueOf(id_jobentry),
dbe.getMessage()),
dbe);
}
}
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
meta = (HTTPPOSTMeta) smi;
data = (HTTPPOSTData) sdi;
boolean sendToErrorRow = false;
String errorMessage = null;
Object[] r = getRow(); // Get row from input rowset & set row busy!
if (r == null) // no more input to be expected...
{
setOutputDone();
return false;
}
if (first) {
first = false;
data.inputRowMeta = getInputRowMeta();
data.outputRowMeta = getInputRowMeta().clone();
meta.getFields(data.outputRowMeta, getStepname(), null, null, this);
if (meta.isUrlInField()) {
if (Const.isEmpty(meta.getUrlField())) {
logError(Messages.getString("HTTPPOST.Log.NoField"));
throw new KettleException(Messages.getString("HTTPPOST.Log.NoField"));
}
// cache the position of the field
if (data.indexOfUrlField < 0) {
String realUrlfieldName = environmentSubstitute(meta.getUrlField());
data.indexOfUrlField = data.inputRowMeta.indexOfValue((realUrlfieldName));
if (data.indexOfUrlField < 0) {
// The field is unreachable !
logError(Messages.getString("HTTPPOST.Log.ErrorFindingField", realUrlfieldName));
throw new KettleException(
Messages.getString("HTTPPOST.Exception.ErrorFindingField", realUrlfieldName));
}
}
} else {
data.realUrl = environmentSubstitute(meta.getUrl());
}
// set body parameters
int nrargs = meta.getArgumentField().length;
if (nrargs > 0) {
data.useBodyParameters = true;
data.body_parameters_nrs = new int[nrargs];
data.bodyParameters = new NameValuePair[nrargs];
for (int i = 0; i < nrargs; i++) {
data.body_parameters_nrs[i] = data.inputRowMeta.indexOfValue(meta.getArgumentField()[i]);
if (data.body_parameters_nrs[i] < 0) {
logError(
Messages.getString("HTTPPOST.Log.ErrorFindingField")
+ meta.getArgumentField()[i]
+ "]"); //$NON-NLS-1$ //$NON-NLS-2$
throw new KettleStepException(
Messages.getString(
"HTTPPOST.Exception.CouldnotFindField",
meta.getArgumentField()[i])); // $NON-NLS-1$ //$NON-NLS-2$
}
data.bodyParameters[i] =
new NameValuePair(
environmentSubstitute(meta.getArgumentParameter()[i]),
data.outputRowMeta.getString(r, data.body_parameters_nrs[i]));
}
}
// set query parameters
int nrQuery = meta.getQueryField().length;
if (nrQuery > 0) {
data.useQueryParameters = true;
data.query_parameters_nrs = new int[nrQuery];
data.queryParameters = new NameValuePair[nrQuery];
for (int i = 0; i < nrQuery; i++) {
data.query_parameters_nrs[i] = data.inputRowMeta.indexOfValue(meta.getQueryField()[i]);
if (data.query_parameters_nrs[i] < 0) {
logError(
Messages.getString("HTTPPOST.Log.ErrorFindingField")
+ meta.getQueryField()[i]
+ "]"); //$NON-NLS-1$ //$NON-NLS-2$
throw new KettleStepException(
Messages.getString(
"HTTPPOST.Exception.CouldnotFindField",
meta.getQueryField()[i])); // $NON-NLS-1$ //$NON-NLS-2$
}
data.queryParameters[i] =
new NameValuePair(
environmentSubstitute(meta.getQueryParameter()[i]),
data.outputRowMeta.getString(r, data.query_parameters_nrs[i]));
}
}
// set request entity?
if (!Const.isEmpty(meta.getRequestEntity())) {
data.indexOfRequestEntity =
data.inputRowMeta.indexOfValue(environmentSubstitute(meta.getRequestEntity()));
if (data.indexOfRequestEntity < 0) {
throw new KettleStepException(
Messages.getString(
"HTTPPOST.Exception.CouldnotFindRequestEntityField",
meta.getRequestEntity())); // $NON-NLS-1$ //$NON-NLS-2$
}
}
data.realEncoding = environmentSubstitute(meta.getEncoding());
} // end if first
try {
Object[] outputRowData = callHTTPPOST(r);
putRow(data.outputRowMeta, outputRowData); // copy row to output rowset(s);
if (checkFeedback(getLinesRead())) {
if (log.isDetailed())
logDetailed(Messages.getString("HTTPPOST.LineNumber") + getLinesRead()); // $NON-NLS-1$
}
} catch (KettleException e) {
if (getStepMeta().isDoingErrorHandling()) {
sendToErrorRow = true;
errorMessage = e.toString();
} else {
logError(Messages.getString("HTTPPOST.ErrorInStepRunning") + e.getMessage()); // $NON-NLS-1$
setErrors(1);
logError(Const.getStackTracker(e));
stopAll();
setOutputDone(); // signal end to receiver(s)
return false;
}
if (sendToErrorRow) {
// Simply add this row to the error row
putError(getInputRowMeta(), r, 1, errorMessage, null, "HTTPPOST001");
}
}
return true;
}
public SQLStatement getSQLStatements(
TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev) {
SQLStatement retval =
new SQLStatement(stepMeta.getName(), databaseMeta, null); // default: nothing to do!
if (databaseMeta != null) {
if (prev != null && prev.size() > 0) {
if (!Const.isEmpty(tableName)) {
Database db = new Database(databaseMeta);
db.shareVariablesWith(transMeta);
try {
db.connect();
String schemaTable =
databaseMeta.getQuotedSchemaTableCombination(schemaName, tableName);
String cr_table = db.getDDL(schemaTable, prev, null, false, null, true);
String cr_index = ""; // $NON-NLS-1$
String idx_fields[] = null;
if (keyLookup != null && keyLookup.length > 0) {
idx_fields = new String[keyLookup.length];
for (int i = 0; i < keyLookup.length; i++) idx_fields[i] = keyLookup[i];
} else {
retval.setError(
Messages.getString("DeleteMeta.CheckResult.KeyFieldsRequired")); // $NON-NLS-1$
}
// Key lookup dimensions...
if (idx_fields != null
&& idx_fields.length > 0
&& !db.checkIndexExists(schemaTable, idx_fields)) {
String indexname = "idx_" + tableName + "_lookup"; // $NON-NLS-1$ //$NON-NLS-2$
cr_index =
db.getCreateIndexStatement(
schemaName, tableName, indexname, idx_fields, false, false, false, true);
}
String sql = cr_table + cr_index;
if (sql.length() == 0) retval.setSQL(null);
else retval.setSQL(sql);
} catch (KettleException e) {
retval.setError(
Messages.getString("DeleteMeta.Returnvalue.ErrorOccurred")
+ e.getMessage()); // $NON-NLS-1$
}
} else {
retval.setError(
Messages.getString(
"DeleteMeta.Returnvalue.NoTableDefinedOnConnection")); //$NON-NLS-1$
}
} else {
retval.setError(
Messages.getString("DeleteMeta.Returnvalue.NoReceivingAnyFields")); // $NON-NLS-1$
}
} else {
retval.setError(
Messages.getString("DeleteMeta.Returnvalue.NoConnectionDefined")); // $NON-NLS-1$
}
return retval;
}
public void check(
List<CheckResultInterface> remarks,
TransMeta transMeta,
StepMeta stepinfo,
RowMetaInterface prev,
String input[],
String output[],
RowMetaInterface info) {
CheckResult cr;
String error_message = ""; // $NON-NLS-1$
if (databaseMeta != null) {
Database db = new Database(databaseMeta);
db.shareVariablesWith(transMeta);
try {
db.connect();
if (!Const.isEmpty(tableName)) {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
Messages.getString("DeleteMeta.CheckResult.TablenameOK"),
stepinfo); //$NON-NLS-1$
remarks.add(cr);
boolean first = true;
boolean error_found = false;
error_message = ""; // $NON-NLS-1$
// Check fields in table
String schemaTable = databaseMeta.getQuotedSchemaTableCombination(schemaName, tableName);
RowMetaInterface r = db.getTableFields(schemaTable);
if (r != null) {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
Messages.getString("DeleteMeta.CheckResult.VisitTableSuccessfully"),
stepinfo); //$NON-NLS-1$
remarks.add(cr);
for (int i = 0; i < keyLookup.length; i++) {
String lufield = keyLookup[i];
ValueMetaInterface v = r.searchValueMeta(lufield);
if (v == null) {
if (first) {
first = false;
error_message +=
Messages.getString("DeleteMeta.CheckResult.MissingCompareFieldsInTargetTable")
+ Const.CR; // $NON-NLS-1$
}
error_found = true;
error_message += "\t\t" + lufield + Const.CR; // $NON-NLS-1$
}
}
if (error_found) {
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepinfo);
} else {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
Messages.getString("DeleteMeta.CheckResult.FoundLookupFields"),
stepinfo); //$NON-NLS-1$
}
remarks.add(cr);
} else {
error_message =
Messages.getString("DeleteMeta.CheckResult.CouldNotReadTableInfo"); // $NON-NLS-1$
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepinfo);
remarks.add(cr);
}
}
// Look up fields in the input stream <prev>
if (prev != null && prev.size() > 0) {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
Messages.getString(
"DeleteMeta.CheckResult.ConnectedStepSuccessfully",
String.valueOf(prev.size())),
stepinfo); //$NON-NLS-1$ //$NON-NLS-2$
remarks.add(cr);
boolean first = true;
error_message = ""; // $NON-NLS-1$
boolean error_found = false;
for (int i = 0; i < keyStream.length; i++) {
ValueMetaInterface v = prev.searchValueMeta(keyStream[i]);
if (v == null) {
if (first) {
first = false;
error_message +=
Messages.getString("DeleteMeta.CheckResult.MissingFields")
+ Const.CR; // $NON-NLS-1$
}
error_found = true;
error_message += "\t\t" + keyStream[i] + Const.CR; // $NON-NLS-1$
}
}
for (int i = 0; i < keyStream2.length; i++) {
if (keyStream2[i] != null && keyStream2[i].length() > 0) {
ValueMetaInterface v = prev.searchValueMeta(keyStream2[i]);
if (v == null) {
if (first) {
first = false;
error_message +=
Messages.getString("DeleteMeta.CheckResult.MissingFields2")
+ Const.CR; // $NON-NLS-1$
}
error_found = true;
error_message += "\t\t" + keyStream[i] + Const.CR; // $NON-NLS-1$
}
}
}
if (error_found) {
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepinfo);
} else {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
Messages.getString("DeleteMeta.CheckResult.AllFieldsFound"),
stepinfo); //$NON-NLS-1$
}
remarks.add(cr);
// How about the fields to insert/update the table with?
first = true;
error_found = false;
error_message = ""; // $NON-NLS-1$
} else {
error_message =
Messages.getString("DeleteMeta.CheckResult.MissingFields3") + Const.CR; // $NON-NLS-1$
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepinfo);
remarks.add(cr);
}
} catch (KettleException e) {
error_message =
Messages.getString("DeleteMeta.CheckResult.DatabaseError")
+ e.getMessage(); // $NON-NLS-1$
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepinfo);
remarks.add(cr);
} finally {
db.disconnect();
}
} else {
error_message = Messages.getString("DeleteMeta.CheckResult.InvalidConnection"); // $NON-NLS-1$
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepinfo);
remarks.add(cr);
}
// See if we have input streams leading to this step!
if (input.length > 0) {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
Messages.getString("DeleteMeta.CheckResult.StepReceivingInfo"),
stepinfo); //$NON-NLS-1$
remarks.add(cr);
} else {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_ERROR,
Messages.getString("DeleteMeta.CheckResult.NoInputReceived"),
stepinfo); //$NON-NLS-1$
remarks.add(cr);
}
}
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
meta = (GPLoadMeta) smi;
data = (GPLoadData) sdi;
try {
Object[] r = getRow(); // Get row from input rowset & set row busy!
if (r == null) // no more input to be expected...
{
setOutputDone();
if (!preview) {
if (output != null) {
// Close the output
try {
output.close();
} catch (IOException e) {
throw new KettleException("Error while closing output", e);
}
output = null;
}
String loadMethod = meta.getLoadMethod();
if (GPLoadMeta.METHOD_AUTO_END.equals(loadMethod)) {
execute(meta, true);
}
// else if ( GPLoadMeta.METHOD_AUTO_CONCURRENT.equals(meta.getLoadMethod()) )
// {
// try
// {
// if ( psqlProcess != null )
// {
// int exitVal = psqlProcess.waitFor();
// logBasic(BaseMessages.getString(PKG, "GPLoad.Log.ExitValueSqlldr", "" + exitVal));
// //$NON-NLS-1$
// }
// else
// {
// throw new KettleException("Internal error: no sqlldr process running");
// }
// }
// catch ( Exception ex )
// {
// throw new KettleException("Error while executing sqlldr", ex);
// }
// }
}
return false;
}
if (!preview) {
if (first) {
first = false;
createControlFile(environmentSubstitute(meta.getControlFile()), r, meta);
output = new GPLoadDataOutput(meta, log.getLogLevel());
// if ( GPLoadMeta.METHOD_AUTO_CONCURRENT.equals(meta.getLoadMethod()) )
// {
// execute(meta, false);
// }
output.open(this, gploadProcess);
}
output.writeLine(getInputRowMeta(), r);
}
putRow(getInputRowMeta(), r);
incrementLinesOutput();
} catch (KettleException e) {
logError(
BaseMessages.getString(PKG, "GPLoad.Log.ErrorInStep") + e.getMessage()); // $NON-NLS-1$
setErrors(1);
stopAll();
setOutputDone(); // signal end to receiver(s)
return false;
}
return true;
}
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
meta = (OraBulkLoaderMeta) smi;
data = (OraBulkLoaderData) sdi;
try {
Object[] r = getRow(); // Get row from input rowset & set row busy!
if (r == null) {
// no more input to be expected...
setOutputDone();
if (!preview) {
if (output != null) {
// Close the output
try {
output.close();
} catch (IOException e) {
throw new KettleException("Error while closing output", e);
}
output = null;
}
String loadMethod = meta.getLoadMethod();
if (OraBulkLoaderMeta.METHOD_AUTO_END.equals(loadMethod)) {
// if this is the first line, we do not need to execute loader
// control file may not exists
if (!first) {
execute(meta, true);
sqlldrProcess = null;
}
} else if (OraBulkLoaderMeta.METHOD_AUTO_CONCURRENT.equals(meta.getLoadMethod())) {
try {
if (sqlldrProcess != null) {
int exitVal = sqlldrProcess.waitFor();
sqlldrProcess = null;
logBasic(
BaseMessages.getString(PKG, "OraBulkLoader.Log.ExitValueSqlldr", "" + exitVal));
checkExitVal(exitVal);
} else if (!first) {
throw new KettleException("Internal error: no sqlldr process running");
}
} catch (Exception ex) {
throw new KettleException("Error while executing sqlldr", ex);
}
}
}
return false;
}
if (!preview) {
if (first) {
first = false;
String recTerm = Const.CR;
if (!Utils.isEmpty(meta.getAltRecordTerm())) {
recTerm = substituteRecordTerminator(meta.getAltRecordTerm());
}
createControlFile(environmentSubstitute(meta.getControlFile()), r, meta);
output = new OraBulkDataOutput(meta, recTerm);
if (OraBulkLoaderMeta.METHOD_AUTO_CONCURRENT.equals(meta.getLoadMethod())) {
execute(meta, false);
}
output.open(this, sqlldrProcess);
}
output.writeLine(getInputRowMeta(), r);
}
putRow(getInputRowMeta(), r);
incrementLinesOutput();
} catch (KettleException e) {
logError(BaseMessages.getString(PKG, "OraBulkLoader.Log.ErrorInStep") + e.getMessage());
setErrors(1);
stopAll();
setOutputDone(); // signal end to receiver(s)
return false;
}
return true;
}
public SQLStatement getSQLStatements(
TransMeta transMeta,
StepMeta stepMeta,
RowMetaInterface prev,
Repository repository,
IMetaStore metaStore)
throws KettleStepException {
SQLStatement retval =
new SQLStatement(stepMeta.getName(), databaseMeta, null); // default: nothing to do!
if (databaseMeta != null) {
if (prev != null && prev.size() > 0) {
// Copy the row
RowMetaInterface tableFields = new RowMeta();
// Now change the field names
// the key fields
if (keyLookup != null) {
for (int i = 0; i < keyLookup.length; i++) {
ValueMetaInterface v = prev.searchValueMeta(keyStream[i]);
if (v != null) {
ValueMetaInterface tableField = v.clone();
tableField.setName(keyLookup[i]);
tableFields.addValueMeta(tableField);
} else {
throw new KettleStepException(
"Unable to find field [" + keyStream[i] + "] in the input rows");
}
}
}
// the lookup fields
for (int i = 0; i < updateLookup.length; i++) {
ValueMetaInterface v = prev.searchValueMeta(updateStream[i]);
if (v != null) {
ValueMetaInterface vk = tableFields.searchValueMeta(updateStream[i]);
if (vk == null) { // do not add again when already added as key fields
ValueMetaInterface tableField = v.clone();
tableField.setName(updateLookup[i]);
tableFields.addValueMeta(tableField);
}
} else {
throw new KettleStepException(
"Unable to find field [" + updateStream[i] + "] in the input rows");
}
}
if (!Const.isEmpty(tableName)) {
Database db = new Database(loggingObject, databaseMeta);
db.shareVariablesWith(transMeta);
try {
db.connect();
String schemaTable =
databaseMeta.getQuotedSchemaTableCombination(schemaName, tableName);
String cr_table = db.getDDL(schemaTable, tableFields, null, false, null, true);
String cr_index = "";
String[] idx_fields = null;
if (keyLookup != null && keyLookup.length > 0) {
idx_fields = new String[keyLookup.length];
for (int i = 0; i < keyLookup.length; i++) {
idx_fields[i] = keyLookup[i];
}
} else {
retval.setError(
BaseMessages.getString(PKG, "InsertUpdateMeta.CheckResult.MissingKeyFields"));
}
// Key lookup dimensions...
if (idx_fields != null
&& idx_fields.length > 0
&& !db.checkIndexExists(schemaName, tableName, idx_fields)) {
String indexname = "idx_" + tableName + "_lookup";
cr_index =
db.getCreateIndexStatement(
schemaTable, indexname, idx_fields, false, false, false, true);
}
String sql = cr_table + cr_index;
if (sql.length() == 0) {
retval.setSQL(null);
} else {
retval.setSQL(sql);
}
} catch (KettleException e) {
retval.setError(
BaseMessages.getString(PKG, "InsertUpdateMeta.ReturnValue.ErrorOccurred")
+ e.getMessage());
}
} else {
retval.setError(
BaseMessages.getString(
PKG, "InsertUpdateMeta.ReturnValue.NoTableDefinedOnConnection"));
}
} else {
retval.setError(
BaseMessages.getString(PKG, "InsertUpdateMeta.ReturnValue.NotReceivingAnyFields"));
}
} else {
retval.setError(
BaseMessages.getString(PKG, "InsertUpdateMeta.ReturnValue.NoConnectionDefined"));
}
return retval;
}
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
meta = (ExecProcessMeta) smi;
data = (ExecProcessData) sdi;
Object[] r = getRow(); // Get row from input rowset & set row busy!
if (r == null) { // no more input to be expected...
setOutputDone();
return false;
}
if (first) {
first = false;
// get the RowMeta
data.previousRowMeta = getInputRowMeta().clone();
data.NrPrevFields = data.previousRowMeta.size();
data.outputRowMeta = data.previousRowMeta;
meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
// Check is process field is provided
if (Const.isEmpty(meta.getProcessField())) {
logError(BaseMessages.getString(PKG, "ExecProcess.Error.ProcessFieldMissing"));
throw new KettleException(
BaseMessages.getString(PKG, "ExecProcess.Error.ProcessFieldMissing"));
}
// cache the position of the field
if (data.indexOfProcess < 0) {
data.indexOfProcess = data.previousRowMeta.indexOfValue(meta.getProcessField());
if (data.indexOfProcess < 0) {
// The field is unreachable !
logError(
BaseMessages.getString(PKG, "ExecProcess.Exception.CouldnotFindField")
+ "["
+ meta.getProcessField()
+ "]");
throw new KettleException(
BaseMessages.getString(
PKG, "ExecProcess.Exception.CouldnotFindField", meta.getProcessField()));
}
}
} // End If first
Object[] outputRow = RowDataUtil.allocateRowData(data.outputRowMeta.size());
for (int i = 0; i < data.NrPrevFields; i++) {
outputRow[i] = r[i];
}
// get process to execute
String processString = data.previousRowMeta.getString(r, data.indexOfProcess);
ProcessResult processResult = new ProcessResult();
try {
if (Const.isEmpty(processString)) {
throw new KettleException(BaseMessages.getString(PKG, "ExecProcess.ProcessEmpty"));
}
// execute and return result
execProcess(processString, processResult);
if (meta.isFailWhenNotSuccess()) {
if (processResult.getExistStatus() != 0) {
String errorString = processResult.getErrorStream();
if (Const.isEmpty(errorString)) {
errorString = processResult.getOutputStream();
}
throw new KettleException(errorString);
}
}
// Add result field to input stream
int rowIndex = data.NrPrevFields;
outputRow[rowIndex++] = processResult.getOutputStream();
// Add result field to input stream
outputRow[rowIndex++] = processResult.getErrorStream();
// Add result field to input stream
outputRow[rowIndex++] = processResult.getExistStatus();
// add new values to the row.
putRow(data.outputRowMeta, outputRow); // copy row to output rowset(s);
if (log.isRowLevel()) {
logRowlevel(
BaseMessages.getString(
PKG,
"ExecProcess.LineNumber",
getLinesRead() + " : " + getInputRowMeta().getString(r)));
}
} catch (KettleException e) {
boolean sendToErrorRow = false;
String errorMessage = null;
if (getStepMeta().isDoingErrorHandling()) {
sendToErrorRow = true;
errorMessage = e.toString();
} else {
logError(BaseMessages.getString(PKG, "ExecProcess.ErrorInStepRunning") + e.getMessage());
setErrors(1);
stopAll();
setOutputDone(); // signal end to receiver(s)
return false;
}
if (sendToErrorRow) {
// Simply add this row to the error row
putError(
getInputRowMeta(), r, 1, errorMessage, meta.getResultFieldName(), "ExecProcess001");
}
}
return true;
}
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
meta = (DatabaseLookupMeta) smi;
data = (DatabaseLookupData) sdi;
boolean sendToErrorRow = false;
String errorMessage = null;
Object[] r = getRow(); // Get row from input rowset & set row busy!
if (r == null) { // no more input to be expected...
setOutputDone();
return false;
}
if (first) {
first = false;
// create the output metadata
data.outputRowMeta = getInputRowMeta().clone();
meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
if (meta.isCached()) {
if (meta.getCacheSize() > 0) {
data.look = new Hashtable<RowMetaAndData, TimedRow>((int) (meta.getCacheSize() * 1.5));
} else {
data.look = new Hashtable<RowMetaAndData, TimedRow>();
}
}
data.db.setLookup(
environmentSubstitute(meta.getSchemaName()),
environmentSubstitute(meta.getTablename()),
meta.getTableKeyField(),
meta.getKeyCondition(),
meta.getReturnValueField(),
meta.getReturnValueNewName(),
meta.getOrderByClause(),
meta.isFailingOnMultipleResults());
// lookup the values!
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(PKG, "DatabaseLookup.Log.CheckingRow")
+ getInputRowMeta().getString(r));
}
data.keynrs = new int[meta.getStreamKeyField1().length];
data.keynrs2 = new int[meta.getStreamKeyField1().length];
for (int i = 0; i < meta.getStreamKeyField1().length; i++) {
data.keynrs[i] = getInputRowMeta().indexOfValue(meta.getStreamKeyField1()[i]);
if (data.keynrs[i] < 0
&& // couldn't find field!
!"IS NULL".equalsIgnoreCase(meta.getKeyCondition()[i])
&& // No field needed!
!"IS NOT NULL".equalsIgnoreCase(meta.getKeyCondition()[i]) // No field needed!
) {
throw new KettleStepException(
BaseMessages.getString(PKG, "DatabaseLookup.ERROR0001.FieldRequired1.Exception")
+ meta.getStreamKeyField1()[i]
+ BaseMessages.getString(
PKG, "DatabaseLookup.ERROR0001.FieldRequired2.Exception"));
}
data.keynrs2[i] = getInputRowMeta().indexOfValue(meta.getStreamKeyField2()[i]);
if (data.keynrs2[i] < 0
&& // couldn't find field!
"BETWEEN".equalsIgnoreCase(meta.getKeyCondition()[i]) // 2 fields needed!
) {
throw new KettleStepException(
BaseMessages.getString(PKG, "DatabaseLookup.ERROR0001.FieldRequired3.Exception")
+ meta.getStreamKeyField2()[i]
+ BaseMessages.getString(
PKG, "DatabaseLookup.ERROR0001.FieldRequired4.Exception"));
}
if (log.isDebug()) {
logDebug(
BaseMessages.getString(PKG, "DatabaseLookup.Log.FieldHasIndex1")
+ meta.getStreamKeyField1()[i]
+ BaseMessages.getString(PKG, "DatabaseLookup.Log.FieldHasIndex2")
+ data.keynrs[i]);
}
}
data.nullif = new Object[meta.getReturnValueField().length];
for (int i = 0; i < meta.getReturnValueField().length; i++) {
ValueMetaInterface stringMeta = new ValueMeta("string", ValueMetaInterface.TYPE_STRING);
ValueMetaInterface returnMeta =
data.outputRowMeta.getValueMeta(i + getInputRowMeta().size());
if (!Const.isEmpty(meta.getReturnValueDefault()[i])) {
data.nullif[i] = returnMeta.convertData(stringMeta, meta.getReturnValueDefault()[i]);
} else {
data.nullif[i] = null;
}
}
// Determine the types...
data.keytypes = new int[meta.getTableKeyField().length];
String schemaTable =
meta.getDatabaseMeta()
.getQuotedSchemaTableCombination(
environmentSubstitute(meta.getSchemaName()),
environmentSubstitute(meta.getTablename()));
RowMetaInterface fields = data.db.getTableFields(schemaTable);
if (fields != null) {
// Fill in the types...
for (int i = 0; i < meta.getTableKeyField().length; i++) {
ValueMetaInterface key = fields.searchValueMeta(meta.getTableKeyField()[i]);
if (key != null) {
data.keytypes[i] = key.getType();
} else {
throw new KettleStepException(
BaseMessages.getString(PKG, "DatabaseLookup.ERROR0001.FieldRequired5.Exception")
+ meta.getTableKeyField()[i]
+ BaseMessages.getString(
PKG, "DatabaseLookup.ERROR0001.FieldRequired6.Exception"));
}
}
} else {
throw new KettleStepException(
BaseMessages.getString(PKG, "DatabaseLookup.ERROR0002.UnableToDetermineFieldsOfTable")
+ schemaTable
+ "]");
}
// Count the number of values in the lookup as well as the metadata to send along with it.
//
data.lookupMeta = new RowMeta();
for (int i = 0; i < meta.getStreamKeyField1().length; i++) {
if (data.keynrs[i] >= 0) {
ValueMetaInterface inputValueMeta = getInputRowMeta().getValueMeta(data.keynrs[i]);
// Try to convert type if needed in a clone, we don't want to
// change the type in the original row
//
ValueMetaInterface value =
ValueMetaFactory.cloneValueMeta(inputValueMeta, data.keytypes[i]);
data.lookupMeta.addValueMeta(value);
}
if (data.keynrs2[i] >= 0) {
ValueMetaInterface inputValueMeta = getInputRowMeta().getValueMeta(data.keynrs2[i]);
// Try to convert type if needed in a clone, we don't want to
// change the type in the original row
//
ValueMetaInterface value =
ValueMetaFactory.cloneValueMeta(inputValueMeta, data.keytypes[i]);
data.lookupMeta.addValueMeta(value);
}
}
// We also want to know the metadata of the return values beforehand (null handling)
data.returnMeta = new RowMeta();
for (int i = 0; i < meta.getReturnValueField().length; i++) {
ValueMetaInterface v =
data.outputRowMeta.getValueMeta(getInputRowMeta().size() + i).clone();
data.returnMeta.addValueMeta(v);
}
// If the user selected to load all data into the cache at startup, that's what we do now...
//
if (meta.isCached() && meta.isLoadingAllDataInCache()) {
loadAllTableDataIntoTheCache();
}
}
if (log.isRowLevel()) {
logRowlevel(
BaseMessages.getString(PKG, "DatabaseLookup.Log.GotRowFromPreviousStep")
+ getInputRowMeta().getString(r));
}
try {
// add new lookup values to the row
Object[] outputRow = lookupValues(getInputRowMeta(), r);
if (outputRow != null) {
// copy row to output rowset(s);
putRow(data.outputRowMeta, outputRow);
if (log.isRowLevel()) {
logRowlevel(
BaseMessages.getString(PKG, "DatabaseLookup.Log.WroteRowToNextStep")
+ getInputRowMeta().getString(r));
}
if (checkFeedback(getLinesRead())) {
logBasic("linenr " + getLinesRead());
}
}
} catch (KettleException e) {
if (getStepMeta().isDoingErrorHandling()) {
sendToErrorRow = true;
errorMessage = e.toString();
} else {
logError(
BaseMessages.getString(PKG, "DatabaseLookup.ERROR003.UnexpectedErrorDuringProcessing")
+ e.getMessage());
setErrors(1);
stopAll();
setOutputDone(); // signal end to receiver(s)
return false;
}
if (sendToErrorRow) {
// Simply add this row to the error row
putError(getInputRowMeta(), r, 1, errorMessage, null, "DBLOOKUPD001");
}
}
return true;
}
public void check(
List<CheckResultInterface> remarks,
TransMeta transMeta,
StepMeta stepMeta,
RowMetaInterface prev,
String input[],
String output[],
RowMetaInterface info) {
CheckResult cr;
String error_message = ""; // $NON-NLS-1$
if (databaseMeta != null) {
Database db = new Database(loggingObject, databaseMeta);
db.shareVariablesWith(transMeta);
try {
db.connect();
if (!Const.isEmpty(tableName)) {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
BaseMessages.getString(PKG, "GPBulkLoaderMeta.CheckResult.TableNameOK"),
stepMeta); //$NON-NLS-1$
remarks.add(cr);
boolean first = true;
boolean error_found = false;
error_message = ""; // $NON-NLS-1$
// Check fields in table
String schemaTable =
databaseMeta.getQuotedSchemaTableCombination(
transMeta.environmentSubstitute(schemaName),
transMeta.environmentSubstitute(tableName));
RowMetaInterface r = db.getTableFields(schemaTable);
if (r != null) {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
BaseMessages.getString(PKG, "GPBulkLoaderMeta.CheckResult.TableExists"),
stepMeta); //$NON-NLS-1$
remarks.add(cr);
// How about the fields to insert/dateMask in the table?
first = true;
error_found = false;
error_message = ""; // $NON-NLS-1$
for (int i = 0; i < fieldTable.length; i++) {
String field = fieldTable[i];
ValueMetaInterface v = r.searchValueMeta(field);
if (v == null) {
if (first) {
first = false;
error_message +=
BaseMessages.getString(
PKG, "GPBulkLoaderMeta.CheckResult.MissingFieldsToLoadInTargetTable")
+ Const.CR; // $NON-NLS-1$
}
error_found = true;
error_message += "\t\t" + field + Const.CR; // $NON-NLS-1$
}
}
if (error_found) {
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
} else {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
BaseMessages.getString(
PKG, "GPBulkLoaderMeta.CheckResult.AllFieldsFoundInTargetTable"),
stepMeta); //$NON-NLS-1$
}
remarks.add(cr);
} else {
error_message =
BaseMessages.getString(
PKG, "GPBulkLoaderMeta.CheckResult.CouldNotReadTableInfo"); // $NON-NLS-1$
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
remarks.add(cr);
}
}
// Look up fields in the input stream <prev>
if (prev != null && prev.size() > 0) {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
BaseMessages.getString(
PKG, "GPBulkLoaderMeta.CheckResult.StepReceivingDatas", prev.size() + ""),
stepMeta); //$NON-NLS-1$ //$NON-NLS-2$
remarks.add(cr);
boolean first = true;
error_message = ""; // $NON-NLS-1$
boolean error_found = false;
for (int i = 0; i < fieldStream.length; i++) {
ValueMetaInterface v = prev.searchValueMeta(fieldStream[i]);
if (v == null) {
if (first) {
first = false;
error_message +=
BaseMessages.getString(PKG, "GPBulkLoaderMeta.CheckResult.MissingFieldsInInput")
+ Const.CR; // $NON-NLS-1$
}
error_found = true;
error_message += "\t\t" + fieldStream[i] + Const.CR; // $NON-NLS-1$
}
}
if (error_found) {
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
} else {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
BaseMessages.getString(
PKG, "GPBulkLoaderMeta.CheckResult.AllFieldsFoundInInput"),
stepMeta); //$NON-NLS-1$
}
remarks.add(cr);
} else {
error_message =
BaseMessages.getString(PKG, "GPBulkLoaderMeta.CheckResult.MissingFieldsInInput3")
+ Const.CR; // $NON-NLS-1$
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
remarks.add(cr);
}
} catch (KettleException e) {
error_message =
BaseMessages.getString(PKG, "GPBulkLoaderMeta.CheckResult.DatabaseErrorOccurred")
+ e.getMessage(); // $NON-NLS-1$
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
remarks.add(cr);
} finally {
db.disconnect();
}
} else {
error_message =
BaseMessages.getString(
PKG, "GPBulkLoaderMeta.CheckResult.InvalidConnection"); // $NON-NLS-1$
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
remarks.add(cr);
}
// See if we have input streams leading to this step!
if (input.length > 0) {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
BaseMessages.getString(
PKG, "GPBulkLoaderMeta.CheckResult.StepReceivingInfoFromOtherSteps"),
stepMeta); //$NON-NLS-1$
remarks.add(cr);
} else {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_ERROR,
BaseMessages.getString(PKG, "GPBulkLoaderMeta.CheckResult.NoInputError"),
stepMeta); //$NON-NLS-1$
remarks.add(cr);
}
}
public SQLStatement getSQLStatements(
TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev) {
SQLStatement retval =
new SQLStatement(stepMeta.getName(), databaseWriteMeta, null); // default: nothing to do!
int i;
if (databaseWriteMeta != null) {
if (prev != null && prev.size() > 0) {
if (!Const.isEmpty(tablename)) {
String schemaTable =
databaseWriteMeta.getQuotedSchemaTableCombination(schemaName, tablename);
Database db = new Database(databaseWriteMeta);
try {
boolean doHash = false;
String cr_table = null;
db.connect();
// OK, what do we put in the new table??
RowMetaInterface fields = new RowMeta();
ValueMetaInterface vkeyfield = null;
if (!Const.isEmpty(technicalKeyField)) {
// First, the new technical key...
vkeyfield = new ValueMeta(technicalKeyField, ValueMetaInterface.TYPE_INTEGER);
vkeyfield.setLength(10);
vkeyfield.setPrecision(0);
}
// Then the hashcode (optional)
ValueMetaInterface vhashfield = null;
if (useHash && !Const.isEmpty(hashField)) {
vhashfield = new ValueMeta(hashField, ValueMetaInterface.TYPE_INTEGER);
vhashfield.setLength(15);
vhashfield.setPrecision(0);
doHash = true;
}
// Then the last update field (optional)
ValueMetaInterface vLastUpdateField = null;
if (!Const.isEmpty(lastUpdateField)) {
vLastUpdateField = new ValueMeta(lastUpdateField, ValueMetaInterface.TYPE_DATE);
}
if (!db.checkTableExists(schemaTable)) {
if (vkeyfield != null) {
// Add technical key field.
fields.addValueMeta(vkeyfield);
}
// Add the keys only to the table
if (keyField != null && keyLookup != null) {
int cnt = keyField.length;
for (i = 0; i < cnt; i++) {
String error_field = ""; // $NON-NLS-1$
// Find the value in the stream
ValueMetaInterface v = prev.searchValueMeta(keyField[i]);
if (v != null) {
String name = keyLookup[i];
ValueMetaInterface newValue = v.clone();
newValue.setName(name);
if (vkeyfield != null) {
if (name.equals(vkeyfield.getName())
|| (doHash == true && name.equals(vhashfield.getName()))) {
error_field += name;
}
}
if (error_field.length() > 0) {
retval.setError(
Messages.getString(
"ConcurrentCombinationLookupMeta.ReturnValue.NameCollision",
error_field)); //$NON-NLS-1$
} else {
fields.addValueMeta(newValue);
}
}
}
}
if (doHash == true) {
fields.addValueMeta(vhashfield);
}
if (vLastUpdateField != null) {
fields.addValueMeta(vLastUpdateField);
}
} else {
// Table already exists
// Get the fields that are in the table now:
RowMetaInterface tabFields = db.getTableFields(schemaTable);
// Don't forget to quote these as well...
databaseWriteMeta.quoteReservedWords(tabFields);
if (vkeyfield != null && tabFields.searchValueMeta(vkeyfield.getName()) == null) {
// Add technical key field if it didn't exist yet
fields.addValueMeta(vkeyfield);
}
// Add the already existing fields
int cnt = tabFields.size();
for (i = 0; i < cnt; i++) {
ValueMetaInterface v = tabFields.getValueMeta(i);
fields.addValueMeta(v);
}
// Find the missing fields in the real table
String keyLookup[] = getKeyLookup();
String keyField[] = getKeyField();
if (keyField != null && keyLookup != null) {
cnt = keyField.length;
for (i = 0; i < cnt; i++) {
// Find the value in the stream
ValueMetaInterface v = prev.searchValueMeta(keyField[i]);
if (v != null) {
ValueMetaInterface newValue = v.clone();
newValue.setName(keyLookup[i]);
// Does the corresponding name exist in the table
if (tabFields.searchValueMeta(newValue.getName()) == null) {
fields.addValueMeta(newValue); // nope --> add
}
}
}
}
if (doHash == true && tabFields.searchValueMeta(vhashfield.getName()) == null) {
// Add hash field
fields.addValueMeta(vhashfield);
}
if (vLastUpdateField != null
&& tabFields.searchValueMeta(vLastUpdateField.getName()) == null) {
fields.addValueMeta(vLastUpdateField);
}
}
cr_table =
db.getDDL(
schemaTable,
fields,
(CREATION_METHOD_SEQUENCE.equals(getTechKeyCreation())
&& sequenceFrom != null
&& sequenceFrom.length() != 0)
? null
: technicalKeyField,
CREATION_METHOD_AUTOINC.equals(getTechKeyCreation()),
null,
true);
//
// OK, now let's build the index
//
// What fields do we put int the index?
// Only the hashcode or all fields?
String cr_index = ""; // $NON-NLS-1$
String cr_uniq_index = ""; // $NON-NLS-1$
String idx_fields[] = null;
if (useHash) {
if (hashField != null && hashField.length() > 0) {
idx_fields = new String[] {hashField};
} else {
retval.setError(
Messages.getString(
"ConcurrentCombinationLookupMeta.ReturnValue.NotHashFieldSpecified")); //$NON-NLS-1$
}
} else // index on all key fields...
{
if (!Const.isEmpty(keyLookup)) {
int nrfields = keyLookup.length;
if (nrfields > 32
&& databaseWriteMeta.getDatabaseType() == DatabaseMeta.TYPE_DATABASE_ORACLE) {
nrfields = 32; // Oracle indexes are limited to 32 fields...
}
idx_fields = new String[nrfields];
for (i = 0; i < nrfields; i++) idx_fields[i] = keyLookup[i];
} else {
retval.setError(
Messages.getString(
"ConcurrentCombinationLookupMeta.ReturnValue.NotFieldsSpecified")); //$NON-NLS-1$
}
}
// OK, now get the create index statement...
if (!Const.isEmpty(technicalKeyField)) {
String techKeyArr[] = new String[] {technicalKeyField};
if (!db.checkIndexExists(schemaName, tablename, techKeyArr)) {
String indexname = "idx_" + tablename + "_pk"; // $NON-NLS-1$ //$NON-NLS-2$
cr_uniq_index =
db.getCreateIndexStatement(
schemaName, tablename, indexname, techKeyArr, true, true, false, true);
cr_uniq_index += Const.CR;
}
}
// OK, now get the create lookup index statement...
if (!Const.isEmpty(idx_fields)
&& !db.checkIndexExists(schemaName, tablename, idx_fields)) {
String indexname = "idx_" + tablename + "_lookup"; // $NON-NLS-1$ //$NON-NLS-2$
cr_index =
db.getCreateIndexStatement(
schemaName, tablename, indexname, idx_fields, false, false, false, true);
cr_index += Const.CR;
}
//
// Don't forget the sequence (optional)
//
String cr_seq = ""; // $NON-NLS-1$
if (databaseWriteMeta.supportsSequences() && !Const.isEmpty(sequenceFrom)) {
if (!db.checkSequenceExists(schemaName, sequenceFrom)) {
cr_seq +=
db.getCreateSequenceStatement(schemaName, sequenceFrom, 1L, 1L, -1L, true);
cr_seq += Const.CR;
}
}
retval.setSQL(cr_table + cr_uniq_index + cr_index + cr_seq);
} catch (KettleException e) {
retval.setError(
Messages.getString("ConcurrentCombinationLookupMeta.ReturnValue.ErrorOccurred")
+ Const.CR
+ e.getMessage()); // $NON-NLS-1$
}
} else {
retval.setError(
Messages.getString(
"ConcurrentCombinationLookupMeta.ReturnValue.NotTableDefined")); //$NON-NLS-1$
}
} else {
retval.setError(
Messages.getString(
"ConcurrentCombinationLookupMeta.ReturnValue.NotReceivingField")); //$NON-NLS-1$
}
} else {
retval.setError(
Messages.getString(
"ConcurrentCombinationLookupMeta.ReturnValue.NotConnectionDefined")); //$NON-NLS-1$
}
return retval;
}
private void check(
List<CheckResultInterface> remarks,
StepMeta stepMeta,
RowMetaInterface prev,
String[] input,
DatabaseMeta daMeta) {
CheckResult cr;
String error_message;
if (daMeta != null) {
Database db = new Database(daMeta);
try {
db.connect();
if (!Const.isEmpty(tablename)) {
boolean first = true;
boolean error_found = false;
error_message = ""; // $NON-NLS-1$
String schemaTable = daMeta.getQuotedSchemaTableCombination(schemaName, tablename);
RowMetaInterface r = db.getTableFields(schemaTable);
if (r != null) {
for (int i = 0; i < keyLookup.length; i++) {
String lufield = keyLookup[i];
ValueMetaInterface v = r.searchValueMeta(lufield);
if (v == null) {
if (first) {
first = false;
error_message +=
Messages.getString(
"ConcurrentCombinationLookupMeta.CheckResult.MissingCompareFields")
+ Const.CR; // $NON-NLS-1$
}
error_found = true;
error_message += "\t\t" + lufield + Const.CR; // $NON-NLS-1$
}
}
if (error_found) {
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
} else {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
Messages.getString(
"ConcurrentCombinationLookupMeta.CheckResult.AllFieldsFound"),
stepMeta); //$NON-NLS-1$
}
remarks.add(cr);
/* Also, check the fields: tk, version, from-to, ... */
if (!Const.isEmpty(technicalKeyField) && r.indexOfValue(technicalKeyField) < 0) {
error_message =
Messages.getString(
"ConcurrentCombinationLookupMeta.CheckResult.TechnicalKeyNotFound",
technicalKeyField)
+ Const.CR; // $NON-NLS-1$ //$NON-NLS-2$
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
} else {
error_message =
Messages.getString(
"ConcurrentCombinationLookupMeta.CheckResult.TechnicalKeyFound",
technicalKeyField)
+ Const.CR; // $NON-NLS-1$ //$NON-NLS-2$
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_OK, error_message, stepMeta);
}
remarks.add(cr);
} else {
error_message =
Messages.getString(
"ConcurrentCombinationLookupMeta.CheckResult.CouldNotReadTableInfo"); //$NON-NLS-1$
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
remarks.add(cr);
}
}
// Look up fields in the input stream <prev>
if (prev != null && prev.size() > 0) {
boolean first = true;
error_message = ""; // $NON-NLS-1$
boolean error_found = false;
for (int i = 0; i < keyField.length; i++) {
ValueMetaInterface v = prev.searchValueMeta(keyField[i]);
if (v == null) {
if (first) {
first = false;
error_message +=
Messages.getString("ConcurrentCombinationLookupMeta.CheckResult.MissingFields")
+ Const.CR; // $NON-NLS-1$
}
error_found = true;
error_message += "\t\t" + keyField[i] + Const.CR; // $NON-NLS-1$
}
}
if (error_found) {
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
} else {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
Messages.getString(
"ConcurrentCombinationLookupMeta.CheckResult.AllFieldsFoundInInputStream"),
stepMeta); //$NON-NLS-1$
}
remarks.add(cr);
} else {
error_message =
Messages.getString("ConcurrentCombinationLookupMeta.CheckResult.CouldNotReadFields")
+ Const.CR; // $NON-NLS-1$
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
remarks.add(cr);
}
// Check sequence
if (daMeta.supportsSequences() && CREATION_METHOD_SEQUENCE.equals(getTechKeyCreation())) {
if (Const.isEmpty(sequenceFrom)) {
error_message +=
Messages.getString(
"ConcurrentCombinationLookupMeta.CheckResult.ErrorNoSequenceName")
+ "!"; //$NON-NLS-1$ //$NON-NLS-2$
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
remarks.add(cr);
} else {
// It doesn't make sense to check the sequence name
// if it's not filled in.
if (db.checkSequenceExists(sequenceFrom)) {
error_message =
Messages.getString(
"ConcurrentCombinationLookupMeta.CheckResult.ReadingSequenceOK",
sequenceFrom); //$NON-NLS-1$ //$NON-NLS-2$
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_OK, error_message, stepMeta);
remarks.add(cr);
} else {
error_message +=
Messages.getString(
"ConcurrentCombinationLookupMeta.CheckResult.ErrorReadingSequence")
+ sequenceFrom
+ "!"; //$NON-NLS-1$ //$NON-NLS-2$
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
remarks.add(cr);
}
}
}
if (techKeyCreation != null) {
// post 2.2 version
if (!(CREATION_METHOD_AUTOINC.equals(techKeyCreation)
|| CREATION_METHOD_SEQUENCE.equals(techKeyCreation)
|| CREATION_METHOD_TABLEMAX.equals(techKeyCreation))) {
error_message +=
Messages.getString(
"ConcurrentCombinationLookupMeta.CheckResult.ErrorTechKeyCreation")
+ ": "
+ techKeyCreation
+ "!"; //$NON-NLS-1$ //$NON-NLS-2$
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
remarks.add(cr);
}
}
} catch (KettleException e) {
error_message =
Messages.getString("ConcurrentCombinationLookupMeta.CheckResult.ErrorOccurred")
+ e.getMessage(); // $NON-NLS-1$
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
remarks.add(cr);
} finally {
db.disconnect();
}
} else {
error_message =
Messages.getString(
"ConcurrentCombinationLookupMeta.CheckResult.InvalidConnection"); //$NON-NLS-1$
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
remarks.add(cr);
}
// See if we have input streams leading to this step!
if (input.length > 0) {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
Messages.getString(
"ConcurrentCombinationLookupMeta.CheckResult.ReceivingInfoFromOtherSteps"),
stepMeta); //$NON-NLS-1$
remarks.add(cr);
} else {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_ERROR,
Messages.getString("ConcurrentCombinationLookupMeta.CheckResult.NoInputReceived"),
stepMeta); //$NON-NLS-1$
remarks.add(cr);
}
}
public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
meta = (MySQLTableOutputMeta) smi;
data = (MySQLTableOutputData) sdi;
if (super.init(smi, sdi)) {
try {
data.databaseMeta = meta.getDatabaseMeta();
// Batch updates are not supported on PostgreSQL (and
// look-a-likes) together with error handling (PDI-366)
//
data.specialErrorHandling =
getStepMeta().isDoingErrorHandling()
&& (meta.getDatabaseMeta().getDatabaseType() == DatabaseMeta.TYPE_DATABASE_POSTGRES
|| meta.getDatabaseMeta().getDatabaseType()
== DatabaseMeta.TYPE_DATABASE_GREENPLUM);
// get the boolean that indicates whether or not we can release
// savepoints
// and set in in data.
data.releaseSavepoint = data.specialErrorHandling;
data.commitSize = Integer.parseInt(environmentSubstitute(meta.getCommitSize()));
data.batchSize = Integer.parseInt(environmentSubstitute(meta.getBatchSize()));
data.batchMode = data.commitSize > 0 && meta.useBatchUpdate();
// Batch updates are not supported in case we are running with
// transactions in the transformation.
// It is also disabled when we return keys...
//
data.specialErrorHandling |= getTransMeta().isUsingUniqueConnections();
if (data.batchMode && data.specialErrorHandling) {
data.batchMode = false;
if (log.isBasic()) logBasic(Messages.getString("TableOutput.Log.BatchModeDisabled"));
}
if (meta.getDatabaseMeta() == null) {
throw new KettleException(
Messages.getString("TableOutput.Exception.DatabaseNeedsToBeSelected"));
}
if (meta.getDatabaseMeta() == null) {
logError(Messages.getString("TableOutput.Init.ConnectionMissing", getStepname()));
return false;
}
data.db = new Database(meta.getDatabaseMeta());
data.db.shareVariablesWith(this);
if (getTransMeta().isUsingUniqueConnections()) {
synchronized (getTrans()) {
data.db.connect(getTrans().getThreadName(), getPartitionID());
}
} else {
data.db.connect(getPartitionID());
}
if (log.isBasic())
logBasic(
"Connected to database ["
+ meta.getDatabaseMeta()
+ "] (commit="
+ data.commitSize
+ ")");
// Postpone commit as long as possible. PDI-2091
//
if (data.commitSize == 0) {
data.commitSize = Integer.MAX_VALUE;
}
data.db.setCommit(data.commitSize);
if (!meta.isPartitioningEnabled() && !meta.isTableNameInField()) {
data.tableName = environmentSubstitute(meta.getTablename());
// Only the first one truncates in a non-partitioned step
// copy
//
if (meta.truncateTable()
&& ((getCopy() == 0 && getUniqueStepNrAcrossSlaves() == 0)
|| !Const.isEmpty(getPartitionID()))) {
data.db.truncateTable(
environmentSubstitute(meta.getSchemaName()),
environmentSubstitute(meta.getTablename()));
}
}
return true;
} catch (KettleException e) {
logError("An error occurred intialising this step: " + e.getMessage());
stopAll();
setErrors(1);
}
}
return false;
}
public void assertRoundGeneral(
final Object expectedResult,
final int calcFunction,
final Number value,
final Long precision,
final Long roundingMode,
final int valueDataType,
final int functionDataType)
throws KettleException {
final String msg =
getKettleTypeName(valueDataType) + "->" + getKettleTypeName(functionDataType) + " ";
final RowMeta inputRowMeta = new RowMeta();
final List<Object> inputValues = new ArrayList<Object>(3);
final String fieldValue = "Value";
final ValueMetaInterface valueMeta;
switch (valueDataType) {
case ValueMetaInterface.TYPE_BIGNUMBER:
valueMeta = new ValueMetaBigNumber(fieldValue);
break;
case ValueMetaInterface.TYPE_NUMBER:
valueMeta = new ValueMetaNumber(fieldValue);
break;
case ValueMetaInterface.TYPE_INTEGER:
valueMeta = new ValueMetaInteger(fieldValue);
break;
default:
throw new IllegalArgumentException(
msg
+ "Unexpected value dataType: "
+ value.getClass().getName()
+ ". Long, Double or BigDecimal expected.");
}
inputRowMeta.addValueMeta(valueMeta);
inputValues.add(value);
final String fieldPrecision;
final ValueMetaInteger precisionMeta;
if (precision == null) {
fieldPrecision = null;
precisionMeta = null;
} else {
fieldPrecision = "Precision";
precisionMeta = new ValueMetaInteger(fieldPrecision);
inputRowMeta.addValueMeta(precisionMeta);
inputValues.add(precision);
}
final String fieldRoundingMode;
final ValueMetaInteger roundingModeMeta;
if (roundingMode == null) {
fieldRoundingMode = null;
roundingModeMeta = null;
} else {
fieldRoundingMode = "RoundingMode";
roundingModeMeta = new ValueMetaInteger(fieldRoundingMode);
inputRowMeta.addValueMeta(roundingModeMeta);
inputValues.add(roundingMode);
}
RowSet inputRowSet = smh.getMockInputRowSet(inputValues.toArray());
inputRowSet.setRowMeta(inputRowMeta);
final String fieldA =
inputRowMeta.size() > 0 ? inputRowMeta.getValueMetaList().get(0).getName() : null;
final String fieldB =
inputRowMeta.size() > 1 ? inputRowMeta.getValueMetaList().get(1).getName() : null;
final String fieldC =
inputRowMeta.size() > 2 ? inputRowMeta.getValueMetaList().get(2).getName() : null;
final int resultDataType = functionDataType;
final String fieldResult = "test";
final int expectedResultRowSize = inputRowMeta.size() + 1;
Calculator calculator =
new Calculator(smh.stepMeta, smh.stepDataInterface, 0, smh.transMeta, smh.trans);
calculator.getInputRowSets().add(inputRowSet);
calculator.setInputRowMeta(inputRowMeta);
calculator.init(smh.initStepMetaInterface, smh.initStepDataInterface);
CalculatorMeta meta = new CalculatorMeta();
meta.setCalculation(
new CalculatorMetaFunction[] {
new CalculatorMetaFunction(
fieldResult,
calcFunction,
fieldA,
fieldB,
fieldC,
resultDataType,
2,
0,
false,
"",
"",
"",
"")
});
// Verify output
try {
calculator.addRowListener(
new RowAdapter() {
@Override
public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row)
throws KettleStepException {
assertEquals(msg + " resultRowSize", expectedResultRowSize, rowMeta.size());
final int fieldResultIndex = rowMeta.size() - 1;
assertEquals(
msg + " fieldResult",
fieldResult,
rowMeta.getValueMeta(fieldResultIndex).getName());
assertEquals(msg, expectedResult, row[fieldResultIndex]);
}
});
calculator.processRow(meta, new CalculatorData());
} catch (KettleException ke) {
ke.printStackTrace();
fail(msg + ke.getMessage());
}
}
public void check(
List<CheckResultInterface> remarks,
TransMeta transMeta,
StepMeta stepMeta,
RowMetaInterface prev,
String[] input,
String[] output,
RowMetaInterface info,
VariableSpace space,
Repository repository,
IMetaStore metaStore) {
CheckResult cr;
if (databaseMeta != null) {
cr =
new CheckResult(
CheckResult.TYPE_RESULT_OK,
BaseMessages.getString(PKG, "ExecSQLRowMeta.CheckResult.ConnectionExists"),
stepMeta);
remarks.add(cr);
Database db = new Database(loggingObject, databaseMeta);
databases = new Database[] {db}; // keep track of it for cancelling purposes...
try {
db.connect();
cr =
new CheckResult(
CheckResult.TYPE_RESULT_OK,
BaseMessages.getString(PKG, "ExecSQLRowMeta.CheckResult.DBConnectionOK"),
stepMeta);
remarks.add(cr);
if (sqlField != null && sqlField.length() != 0) {
cr =
new CheckResult(
CheckResult.TYPE_RESULT_OK,
BaseMessages.getString(PKG, "ExecSQLRowMeta.CheckResult.SQLFieldNameEntered"),
stepMeta);
} else {
cr =
new CheckResult(
CheckResult.TYPE_RESULT_ERROR,
BaseMessages.getString(PKG, "ExecSQLRowMeta.CheckResult.SQLFieldNameMissing"),
stepMeta);
}
remarks.add(cr);
} catch (KettleException e) {
cr =
new CheckResult(
CheckResult.TYPE_RESULT_ERROR,
BaseMessages.getString(PKG, "ExecSQLRowMeta.CheckResult.ErrorOccurred")
+ e.getMessage(),
stepMeta);
remarks.add(cr);
} finally {
db.disconnect();
}
} else {
cr =
new CheckResult(
CheckResult.TYPE_RESULT_ERROR,
BaseMessages.getString(PKG, "ExecSQLRowMeta.CheckResult.ConnectionNeeded"),
stepMeta);
remarks.add(cr);
}
if (input.length > 0) {
cr =
new CheckResult(
CheckResult.TYPE_RESULT_OK,
BaseMessages.getString(PKG, "ExecSQLRowMeta.CheckResult.StepReceivingInfoOK"),
stepMeta);
remarks.add(cr);
} else {
cr =
new CheckResult(
CheckResult.TYPE_RESULT_ERROR,
BaseMessages.getString(PKG, "ExecSQLRowMeta.CheckResult.NoInputReceivedError"),
stepMeta);
remarks.add(cr);
}
}
public Result execute(Result result, int nr) throws KettleException {
FileLoggingEventListener loggingEventListener = null;
LogLevel shellLogLevel = parentJob.getLogLevel();
if (setLogfile) {
String realLogFilename = environmentSubstitute(getLogFilename());
// We need to check here the log filename
// if we do not have one, we must fail
if (Const.isEmpty(realLogFilename)) {
logError(BaseMessages.getString(PKG, "JobEntryShell.Exception.LogFilenameMissing"));
result.setNrErrors(1);
result.setResult(false);
return result;
}
try {
loggingEventListener =
new FileLoggingEventListener(getLogChannelId(), realLogFilename, setAppendLogfile);
KettleLogStore.getAppender().addLoggingEventListener(loggingEventListener);
} catch (KettleException e) {
logError(
BaseMessages.getString(
PKG, "JobEntryShell.Error.UnableopenAppenderFile", getLogFilename(), e.toString()));
logError(Const.getStackTracker(e));
result.setNrErrors(1);
result.setResult(false);
return result;
}
shellLogLevel = logFileLevel;
}
log.setLogLevel(shellLogLevel);
result.setEntryNr(nr);
// "Translate" the arguments for later
String[] substArgs = null;
if (arguments != null) {
substArgs = new String[arguments.length];
for (int idx = 0; idx < arguments.length; idx++) {
substArgs[idx] = environmentSubstitute(arguments[idx]);
}
}
int iteration = 0;
String[] args = substArgs;
RowMetaAndData resultRow = null;
boolean first = true;
List<RowMetaAndData> rows = result.getRows();
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntryShell.Log.FoundPreviousRows", "" + (rows != null ? rows.size() : 0)));
}
while ((first && !execPerRow)
|| (execPerRow && rows != null && iteration < rows.size() && result.getNrErrors() == 0)) {
first = false;
if (rows != null && execPerRow) {
resultRow = rows.get(iteration);
} else {
resultRow = null;
}
List<RowMetaAndData> cmdRows = null;
if (execPerRow) {
// Execute for each input row
if (argFromPrevious) {
// Copy the input row to the (command line) arguments
if (resultRow != null) {
args = new String[resultRow.size()];
for (int i = 0; i < resultRow.size(); i++) {
args[i] = resultRow.getString(i, null);
}
}
} else {
// Just pass a single row
List<RowMetaAndData> newList = new ArrayList<RowMetaAndData>();
newList.add(resultRow);
cmdRows = newList;
}
} else {
if (argFromPrevious) {
// Only put the first Row on the arguments
args = null;
if (resultRow != null) {
args = new String[resultRow.size()];
for (int i = 0; i < resultRow.size(); i++) {
args[i] = resultRow.getString(i, null);
}
} else {
cmdRows = rows;
}
} else {
// Keep it as it was...
cmdRows = rows;
}
}
executeShell(result, cmdRows, args);
iteration++;
}
if (setLogfile) {
if (loggingEventListener != null) {
KettleLogStore.getAppender().removeLoggingEventListener(loggingEventListener);
loggingEventListener.close();
ResultFile resultFile =
new ResultFile(
ResultFile.FILE_TYPE_LOG,
loggingEventListener.getFile(),
parentJob.getJobname(),
getName());
result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
}
}
return result;
}
public void check(
List<CheckResultInterface> remarks,
TransMeta transMeta,
StepMeta stepMeta,
RowMetaInterface prev,
String[] input,
String[] output,
RowMetaInterface info,
VariableSpace space,
Repository repository,
IMetaStore metaStore) {
CheckResult cr;
String error_message = "";
if (databaseMeta != null) {
Database db = new Database(loggingObject, databaseMeta);
db.shareVariablesWith(transMeta);
try {
db.connect();
if (!Const.isEmpty(tableName)) {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
BaseMessages.getString(PKG, "InsertUpdateMeta.CheckResult.TableNameOK"),
stepMeta);
remarks.add(cr);
boolean first = true;
boolean error_found = false;
error_message = "";
// Check fields in table
String schemaTable = databaseMeta.getQuotedSchemaTableCombination(schemaName, tableName);
RowMetaInterface r = db.getTableFields(schemaTable);
if (r != null) {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
BaseMessages.getString(PKG, "InsertUpdateMeta.CheckResult.TableExists"),
stepMeta);
remarks.add(cr);
for (int i = 0; i < keyLookup.length; i++) {
String lufield = keyLookup[i];
ValueMetaInterface v = r.searchValueMeta(lufield);
if (v == null) {
if (first) {
first = false;
error_message +=
BaseMessages.getString(
PKG, "InsertUpdateMeta.CheckResult.MissingCompareFieldsInTargetTable")
+ Const.CR;
}
error_found = true;
error_message += "\t\t" + lufield + Const.CR;
}
}
if (error_found) {
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
} else {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
BaseMessages.getString(
PKG, "InsertUpdateMeta.CheckResult.AllLookupFieldsFound"),
stepMeta);
}
remarks.add(cr);
// How about the fields to insert/update in the table?
first = true;
error_found = false;
error_message = "";
for (int i = 0; i < updateLookup.length; i++) {
String lufield = updateLookup[i];
ValueMetaInterface v = r.searchValueMeta(lufield);
if (v == null) {
if (first) {
first = false;
error_message +=
BaseMessages.getString(
PKG,
"InsertUpdateMeta.CheckResult.MissingFieldsToUpdateInTargetTable")
+ Const.CR;
}
error_found = true;
error_message += "\t\t" + lufield + Const.CR;
}
}
if (error_found) {
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
} else {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
BaseMessages.getString(
PKG, "InsertUpdateMeta.CheckResult.AllFieldsToUpdateFoundInTargetTable"),
stepMeta);
}
remarks.add(cr);
} else {
error_message =
BaseMessages.getString(PKG, "InsertUpdateMeta.CheckResult.CouldNotReadTableInfo");
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
remarks.add(cr);
}
}
// Look up fields in the input stream <prev>
if (prev != null && prev.size() > 0) {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
BaseMessages.getString(
PKG, "InsertUpdateMeta.CheckResult.StepReceivingDatas", prev.size() + ""),
stepMeta);
remarks.add(cr);
boolean first = true;
error_message = "";
boolean error_found = false;
for (int i = 0; i < keyStream.length; i++) {
ValueMetaInterface v = prev.searchValueMeta(keyStream[i]);
if (v == null) {
if (first) {
first = false;
error_message +=
BaseMessages.getString(PKG, "InsertUpdateMeta.CheckResult.MissingFieldsInInput")
+ Const.CR;
}
error_found = true;
error_message += "\t\t" + keyStream[i] + Const.CR;
}
}
for (int i = 0; i < keyStream2.length; i++) {
if (keyStream2[i] != null && keyStream2[i].length() > 0) {
ValueMetaInterface v = prev.searchValueMeta(keyStream2[i]);
if (v == null) {
if (first) {
first = false;
error_message +=
BaseMessages.getString(
PKG, "InsertUpdateMeta.CheckResult.MissingFieldsInInput")
+ Const.CR;
}
error_found = true;
error_message += "\t\t" + keyStream[i] + Const.CR;
}
}
}
if (error_found) {
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
} else {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
BaseMessages.getString(
PKG, "InsertUpdateMeta.CheckResult.AllFieldsFoundInInput"),
stepMeta);
}
remarks.add(cr);
// How about the fields to insert/update the table with?
first = true;
error_found = false;
error_message = "";
for (int i = 0; i < updateStream.length; i++) {
String lufield = updateStream[i];
ValueMetaInterface v = prev.searchValueMeta(lufield);
if (v == null) {
if (first) {
first = false;
error_message +=
BaseMessages.getString(
PKG, "InsertUpdateMeta.CheckResult.MissingInputStreamFields")
+ Const.CR;
}
error_found = true;
error_message += "\t\t" + lufield + Const.CR;
}
}
if (error_found) {
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
} else {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
BaseMessages.getString(
PKG, "InsertUpdateMeta.CheckResult.AllFieldsFoundInInput2"),
stepMeta);
}
remarks.add(cr);
} else {
error_message =
BaseMessages.getString(PKG, "InsertUpdateMeta.CheckResult.MissingFieldsInInput3")
+ Const.CR;
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
remarks.add(cr);
}
} catch (KettleException e) {
error_message =
BaseMessages.getString(PKG, "InsertUpdateMeta.CheckResult.DatabaseErrorOccurred")
+ e.getMessage();
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
remarks.add(cr);
} finally {
db.disconnect();
}
} else {
error_message = BaseMessages.getString(PKG, "InsertUpdateMeta.CheckResult.InvalidConnection");
cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
remarks.add(cr);
}
// See if we have input streams leading to this step!
if (input.length > 0) {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
BaseMessages.getString(
PKG, "InsertUpdateMeta.CheckResult.StepReceivingInfoFromOtherSteps"),
stepMeta);
remarks.add(cr);
} else {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_ERROR,
BaseMessages.getString(PKG, "InsertUpdateMeta.CheckResult.NoInputError"),
stepMeta);
remarks.add(cr);
}
}
public synchronized BlockingRowSet openReaderSocket(final BaseStep baseStep)
throws IOException, KettleException {
this.baseStep = baseStep;
final BlockingRowSet rowSet = new BlockingRowSet(baseStep.getTransMeta().getSizeRowset());
// Make sure we handle the case with multiple step copies running on a
// slave...
//
rowSet.setThreadNameFromToCopy(sourceStep, sourceStepCopyNr, targetStep, targetStepCopyNr);
rowSet.setRemoteSlaveServerName(targetSlaveServerName);
final int portNumber = Integer.parseInt(baseStep.environmentSubstitute(port));
final String realHostname = baseStep.environmentSubstitute(hostname);
// Connect to the server socket (started during BaseStep.init())
// Because the accept() call on the server socket can be called after we
// reached this code
// it is best to build in a retry loop with a time-out here.
//
long startTime = System.currentTimeMillis();
boolean connected = false;
KettleException lastException = null;
// // timeout with retry until connected
while (!connected
&& (TIMEOUT_IN_SECONDS > (System.currentTimeMillis() - startTime) / 1000)
&& !baseStep.isStopped()) {
try {
socket = new Socket();
socket.setReuseAddress(true);
baseStep.logDetailed(
"Step variable MASTER_HOST : [" + baseStep.getVariable("MASTER_HOST") + "]");
baseStep.logDetailed(
"Opening client (reader) socket to server ["
+ Const.NVL(realHostname, "")
+ ":"
+ port
+ "]");
socket.connect(new InetSocketAddress(realHostname, portNumber), 5000);
connected = true;
if (compressingStreams) {
gzipInputStream = new GZIPInputStream(socket.getInputStream());
bufferedInputStream = new BufferedInputStream(gzipInputStream, bufferSize);
} else {
bufferedInputStream = new BufferedInputStream(socket.getInputStream(), bufferSize);
}
inputStream = new DataInputStream(bufferedInputStream);
lastException = null;
} catch (Exception e) {
lastException =
new KettleException(
"Unable to open socket to server " + realHostname + " port " + portNumber, e);
}
if (lastException != null) {
// Sleep for a while
try {
Thread.sleep(250);
} catch (InterruptedException e) {
if (socket != null) {
socket.shutdownInput();
socket.shutdownOutput();
socket.close();
baseStep.logDetailed(
"Closed connection to server socket to read rows from remote step on server "
+ realHostname
+ " port "
+ portNumber
+ " - Local port="
+ socket.getLocalPort());
}
throw new KettleException(
"Interrupted while trying to connect to server socket: " + e.toString());
}
}
}
// See if all was OK...
if (lastException != null) {
baseStep.logError("Error initialising step: " + lastException.toString());
if (socket != null) {
socket.shutdownInput();
socket.shutdownOutput();
socket.close();
baseStep.logDetailed(
"Closed connection to server socket to read rows from remote step on server "
+ realHostname
+ " port "
+ portNumber
+ " - Local port="
+ socket.getLocalPort());
}
throw lastException;
} else {
if (inputStream == null)
throw new KettleException(
"Unable to connect to the SocketWriter in the "
+ TIMEOUT_IN_SECONDS
+ "s timeout period.");
}
baseStep.logDetailed(
"Opened connection to server socket to read rows from remote step on server "
+ realHostname
+ " port "
+ portNumber
+ " - Local port="
+ socket.getLocalPort());
// Create a thread to take care of the reading from the client socket.
// The rows read will be put in a RowSet buffer.
// That buffer will hand over the rows to the step that has this RemoteStep
// object defined
// as a remote input step.
//
Runnable runnable =
new Runnable() {
public void run() {
try {
// First read the row meta data from the socket...
//
RowMetaInterface rowMeta = null;
while (!baseStep.isStopped() && rowMeta == null) {
try {
rowMeta = new RowMeta(inputStream);
} catch (SocketTimeoutException e) {
rowMeta = null;
}
}
if (rowMeta == null) {
throw new KettleEOFException(); // leave now.
}
// And a first row of data...
//
Object[] rowData = getRowOfData(rowMeta);
// Now get the data itself, row by row...
//
while (rowData != null && !baseStep.isStopped()) {
baseStep.incrementLinesInput();
baseStep.decrementLinesRead();
if (baseStep.log.isDebug())
baseStep.logDebug("Received row from remote step: " + rowMeta.getString(rowData));
baseStep.putRowTo(rowMeta, rowData, rowSet);
baseStep.decrementLinesWritten();
rowData = getRowOfData(rowMeta);
}
} catch (KettleEOFException e) {
// Nothing, we're simply done reading...
//
if (baseStep.log.isDebug())
baseStep.logDebug(
"Finished reading from remote step on server "
+ hostname
+ " port "
+ portNumber);
} catch (Exception e) {
baseStep.logError("Error reading from client socket to remote step", e);
baseStep.setErrors(1);
baseStep.stopAll();
} finally {
// Close the input socket
if (socket != null && !socket.isClosed() && !socket.isInputShutdown()) {
try {
socket.shutdownInput();
} catch (Exception e) {
baseStep.logError(
"Error shutting down input channel on client socket connection to remote step",
e);
}
}
if (socket != null && !socket.isClosed() && !socket.isOutputShutdown()) {
try {
socket.shutdownOutput();
} catch (Exception e) {
baseStep.logError(
"Error shutting down output channel on client socket connection to remote step",
e);
}
}
if (socket != null && !socket.isClosed()) {
try {
socket.close();
} catch (Exception e) {
baseStep.logError(
"Error shutting down client socket connection to remote step", e);
}
}
if (inputStream != null) {
try {
inputStream.close();
} catch (Exception e) {
baseStep.logError(
"Error closing input stream on socket connection to remote step", e);
}
inputStream = null;
}
if (bufferedInputStream != null) {
try {
bufferedInputStream.close();
} catch (Exception e) {
baseStep.logError(
"Error closing input stream on socket connection to remote step", e);
}
}
bufferedInputStream = null;
if (gzipInputStream != null) {
try {
gzipInputStream.close();
} catch (Exception e) {
baseStep.logError(
"Error closing input stream on socket connection to remote step", e);
}
}
gzipInputStream = null;
baseStep.logDetailed(
"Closed connection to server socket to read rows from remote step on server "
+ realHostname
+ " port "
+ portNumber
+ " - Local port="
+ socket.getLocalPort());
}
// signal baseStep that nothing else comes from this step.
//
rowSet.setDone();
}
};
new Thread(runnable).start();
return rowSet;
}
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
result.setResult(false);
// see PDI-10270, PDI-10644 for details
boolean oldBehavior =
"Y"
.equalsIgnoreCase(
getVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "N"));
String countSQLStatement = null;
long rowsCount = 0;
long errCount = 0;
boolean successOK = false;
int nrRowsLimit = Const.toInt(environmentSubstitute(limit), 0);
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntryEvalTableContent.Log.nrRowsLimit", "" + nrRowsLimit));
}
if (connection != null) {
Database db = new Database(this, connection);
db.shareVariablesWith(this);
try {
db.connect(parentJob.getTransactionId(), null);
if (iscustomSQL) {
String realCustomSQL = customSQL;
if (isUseVars) {
realCustomSQL = environmentSubstitute(realCustomSQL);
}
if (log.isDebug()) {
logDebug(
BaseMessages.getString(
PKG, "JobEntryEvalTableContent.Log.EnteredCustomSQL", realCustomSQL));
}
if (!Const.isEmpty(realCustomSQL)) {
countSQLStatement = realCustomSQL;
} else {
errCount++;
logError(BaseMessages.getString(PKG, "JobEntryEvalTableContent.Error.NoCustomSQL"));
}
} else {
String realTablename = environmentSubstitute(tablename);
String realSchemaname = environmentSubstitute(schemaname);
if (!Const.isEmpty(realTablename)) {
if (!Const.isEmpty(realSchemaname)) {
countSQLStatement =
selectCount
+ db.getDatabaseMeta()
.getQuotedSchemaTableCombination(realSchemaname, realTablename);
} else {
countSQLStatement = selectCount + db.getDatabaseMeta().quoteField(realTablename);
}
} else {
errCount++;
logError(BaseMessages.getString(PKG, "JobEntryEvalTableContent.Error.NoTableName"));
}
}
if (countSQLStatement != null) {
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntryEvalTableContent.Log.RunSQLStatement", countSQLStatement));
}
if (iscustomSQL) {
if (isClearResultList) {
result.getRows().clear();
}
List<Object[]> ar = db.getRows(countSQLStatement, 0);
if (ar != null) {
rowsCount = ar.size();
// ad rows to result
RowMetaInterface rowMeta = db.getQueryFields(countSQLStatement, false);
List<RowMetaAndData> rows = new ArrayList<RowMetaAndData>();
for (int i = 0; i < ar.size(); i++) {
rows.add(new RowMetaAndData(rowMeta, ar.get(i)));
}
if (isAddRowsResult && iscustomSQL) {
if (rows != null) {
result.getRows().addAll(rows);
}
}
} else {
if (log.isDebug()) {
logDebug(
BaseMessages.getString(
PKG,
"JobEntryEvalTableContent.Log.customSQLreturnedNothing",
countSQLStatement));
}
}
} else {
RowMetaAndData row = db.getOneRow(countSQLStatement);
if (row != null) {
rowsCount = row.getInteger(0);
}
}
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntryEvalTableContent.Log.NrRowsReturned", "" + rowsCount));
}
switch (successCondition) {
case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_EQUAL:
successOK = (rowsCount == nrRowsLimit);
break;
case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_DIFFERENT:
successOK = (rowsCount != nrRowsLimit);
break;
case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_SMALLER:
successOK = (rowsCount < nrRowsLimit);
break;
case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_SMALLER_EQUAL:
successOK = (rowsCount <= nrRowsLimit);
break;
case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_GREATER:
successOK = (rowsCount > nrRowsLimit);
break;
case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_GREATER_EQUAL:
successOK = (rowsCount >= nrRowsLimit);
break;
default:
break;
}
if (!successOK && oldBehavior) {
errCount++;
}
} // end if countSQLStatement!=null
} catch (KettleException dbe) {
errCount++;
logError(
BaseMessages.getString(
PKG, "JobEntryEvalTableContent.Error.RunningEntry", dbe.getMessage()));
} finally {
if (db != null) {
db.disconnect();
}
}
} else {
errCount++;
logError(BaseMessages.getString(PKG, "JobEntryEvalTableContent.NoDbConnection"));
}
result.setResult(successOK);
result.setNrLinesRead(rowsCount);
result.setNrErrors(errCount);
return result;
}