public Result execute(Result previousResult, int nr) throws KettleJobException {
Result result = previousResult;
if (isStart()) {
try {
long sleepTime = getNextExecutionTime();
if (sleepTime > 0) {
parentJob
.getLogChannel()
.logBasic(
parentJob.getJobname(),
"Sleeping: "
+ (sleepTime / 1000 / 60)
+ " minutes (sleep time="
+ sleepTime
+ ")");
long totalSleep = 0L;
while (totalSleep < sleepTime && !parentJob.isStopped()) {
Thread.sleep(1000L);
totalSleep += 1000L;
}
}
} catch (InterruptedException e) {
throw new KettleJobException(e);
}
result = previousResult;
result.setResult(true);
} else if (isDummy()) {
result = previousResult;
}
return result;
}
@Override
public Result execute(Result prevResult, int k) throws KettleException {
Result result = prevResult;
ML_Classify direct = new ML_Classify();
direct.setName(this.getRecordsetName());
direct.setRecordsetName(this.getRecordsetName());
direct.setModel(model);
direct.setIndependentVar(independentVar);
direct.setClassifyType(classifyType);
direct.setDataType(dataType);
direct.setRidge(ridge);
direct.setEpsilon(epsilon);
direct.setMaxIter(maxIter);
direct.setPasses(passes);
direct.setAlpha(alpha);
// private Text algType; //NaiveBayes, Logistic
// private Text dependentVar; // 1
// private Text independentVar; // 2
// ml.setIterations(this.getIterations());
// ml.setThreshold(this.getThreshold());
// ml.setThreshold(this.getThreshold());
logBasic("{Iterate Job} Execute = " + direct.ecl());
logBasic("{Iterate Job} Previous =" + result.getLogText());
result.setResult(true);
RowMetaAndData data = new RowMetaAndData();
data.addValue("ecl", Value.VALUE_TYPE_STRING, direct.ecl());
List list = result.getRows();
list.add(data);
String eclCode = "";
if (list == null) {
list = new ArrayList();
} else {
for (int i = 0; i < list.size(); i++) {
RowMetaAndData rowData = (RowMetaAndData) list.get(i);
String code = rowData.getString("ecl", null);
if (code != null) {
eclCode += code;
}
}
logBasic("{Iterate Job} ECL Code =" + eclCode);
}
result.setRows(list);
return result;
}
@Override
public Result execute(Result prevResult, int k) throws KettleException {
Result result = modifyResults(prevResult);
if (result.isStopped()) {
return result;
}
Rollup rollup = new Rollup();
rollup.setName(this.getRecordsetName());
rollup.setRecordset(this.getRecordset());
rollup.setRecordFormat(this.getRecordset());
rollup.setRunLocal(this.getRunLocal());
rollup.setCondition(this.getCondition());
rollup.setFieldlist(this.getFieldlist());
if (this.group.equalsIgnoreCase("yes")) {
rollup.setGroup("GROUP");
} else {
rollup.setGroup("");
}
rollup.setTransformName(this.getTransformName());
rollup.setTransform(generateEclForMapperGrid());
logBasic("{rollup Job} Execute = " + rollup.ecl());
logBasic("{rollup Job} Previous =" + result.getLogText());
result.setResult(true);
RowMetaAndData data = new RowMetaAndData();
data.addValue("ecl", Value.VALUE_TYPE_STRING, rollup.ecl());
List list = result.getRows();
list.add(data);
String eclCode = parseEclFromRowData(list);
/*
String eclCode = "";
if (list == null) {
list = new ArrayList();
} else {
for (int i = 0; i < list.size(); i++) {
RowMetaAndData rowData = (RowMetaAndData) list.get(i);
String code = rowData.getString("ecl", null);
if (code != null) {
eclCode += code;
}
}
logBasic("{Iterate Job} ECL Code =" + eclCode);
}
*/
result.setRows(list);
return result;
}
public Result execute(Result previousResult, int nr, Repository rep, Job parentJob) {
LogWriter log = LogWriter.getInstance();
Result result = previousResult;
result.setResult(false);
if (filename != null) {
String realFilename = getRealFilename();
try {
FileObject file = KettleVFS.getFileObject(realFilename);
if (file.exists() && file.isReadable()) {
log.logDetailed(
toString(),
Messages.getString("JobEntryFileExists.File_Exists", realFilename)); // $NON-NLS-1$
result.setResult(true);
} else {
log.logDetailed(
toString(),
Messages.getString(
"JobEntryFileExists.File_Does_Not_Exist", realFilename)); // $NON-NLS-1$
}
} catch (IOException e) {
result.setNrErrors(1);
log.logError(
toString(),
Messages.getString(
"JobEntryFileExists.ERROR_0004_IO_Exception", e.toString())); // $NON-NLS-1$
}
} else {
result.setNrErrors(1);
log.logError(
toString(),
Messages.getString("JobEntryFileExists.ERROR_0005_No_Filename_Defined")); // $NON-NLS-1$
}
return result;
}
@Override
public Result execute(Result prevResult, int k) throws KettleException {
Result result = prevResult;
logBasic("{Group job} Creating Group object");
Group group = new Group();
logBasic("{Group job} Group object created");
group.setName(this.getRecordSetName());
group.setRecordSet(this.getRecordSet());
group.setBreakCriteria(this.getBreakCriteria());
group.setIsAll(this.getIsAll());
group.setRunLocal(this.getIsRunLocal());
logBasic("{Group job} Execute = " + group.ecl());
logBasic("{Group job} Previous = " + result.getLogText());
result.setResult(true);
RowMetaAndData data = new RowMetaAndData();
data.addValue("ecl", Value.VALUE_TYPE_STRING, group.ecl());
List list = result.getRows();
list.add(data);
String eclCode = "";
if (list == null) {
list = new ArrayList();
} else {
for (int i = 0; i < list.size(); i++) {
RowMetaAndData rowData = (RowMetaAndData) list.get(i);
String code = rowData.getString("ecl", null);
if (code != null) eclCode += code;
}
logBasic("{Group job} ECL Code = " + eclCode);
}
result.setRows(list);
return result;
}
@Override
public Result execute(Result prevResult, int k) throws KettleException {
Result result = prevResult;
Sort sort = new Sort();
sort.setFields(getFields());
sort.setDatasetName(getDatasetName());
sort.setName(getRecordsetName());
logBasic("{Sort Job} Execute = " + sort.ecl());
logBasic("{Sort Job} Previous =" + result.getLogText());
result.setResult(true);
RowMetaAndData data = new RowMetaAndData();
data.addValue("ecl", Value.VALUE_TYPE_STRING, sort.ecl());
List list = result.getRows();
list.add(data);
String eclCode = "";
if (list == null) {
list = new ArrayList();
} else {
for (int i = 0; i < list.size(); i++) {
RowMetaAndData rowData = (RowMetaAndData) list.get(i);
String code = rowData.getString("ecl", null);
if (code != null) {
eclCode += code;
}
}
logBasic("{Sort Job} ECL Code =" + eclCode);
}
result.setRows(list);
return result;
}
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
result.setNrErrors(1);
result.setResult(false);
String servername = environmentSubstitute(serverName);
int nrPort = Const.toInt(environmentSubstitute("" + port), DEFAULT_PORT);
String Oid = environmentSubstitute(oid);
int timeOut = Const.toInt(environmentSubstitute("" + timeout), DEFAULT_TIME_OUT);
int retry = Const.toInt(environmentSubstitute("" + nrretry), 1);
String messageString = environmentSubstitute(message);
Snmp snmp = null;
try {
TransportMapping transMap = new DefaultUdpTransportMapping();
snmp = new Snmp(transMap);
UdpAddress udpAddress = new UdpAddress(InetAddress.getByName(servername), nrPort);
ResponseEvent response = null;
if (targettype.equals(target_type_Code[0])) {
// Community target
String community = environmentSubstitute(comString);
CommunityTarget target = new CommunityTarget();
PDUv1 pdu1 = new PDUv1();
transMap.listen();
target.setCommunity(new OctetString(community));
target.setVersion(SnmpConstants.version1);
target.setAddress(udpAddress);
if (target.getAddress().isValid()) {
if (log.isDebug()) {
logDebug("Valid IP address");
}
} else {
throw new KettleException("Invalid IP address");
}
target.setRetries(retry);
target.setTimeout(timeOut);
// create the PDU
pdu1.setGenericTrap(6);
pdu1.setSpecificTrap(PDUv1.ENTERPRISE_SPECIFIC);
pdu1.setEnterprise(new OID(Oid));
pdu1.add(new VariableBinding(new OID(Oid), new OctetString(messageString)));
response = snmp.send(pdu1, target);
} else {
// User target
String userName = environmentSubstitute(user);
String passPhrase = environmentSubstitute(passphrase);
String engineID = environmentSubstitute(engineid);
UserTarget usertarget = new UserTarget();
transMap.listen();
usertarget.setAddress(udpAddress);
if (usertarget.getAddress().isValid()) {
if (log.isDebug()) {
logDebug("Valid IP address");
}
} else {
throw new KettleException("Invalid IP address");
}
usertarget.setRetries(retry);
usertarget.setTimeout(timeOut);
usertarget.setVersion(SnmpConstants.version3);
usertarget.setSecurityLevel(SecurityLevel.AUTH_PRIV);
usertarget.setSecurityName(new OctetString("MD5DES"));
// Since we are using SNMPv3 we use authenticated users
// this is handled by the UsmUser and USM class
UsmUser uu =
new UsmUser(
new OctetString(userName),
AuthMD5.ID,
new OctetString(passPhrase),
PrivDES.ID,
new OctetString(passPhrase));
USM usm = snmp.getUSM();
if (usm == null) {
throw new KettleException("Null Usm");
} else {
usm =
new USM(
SecurityProtocols.getInstance(), new OctetString(MPv3.createLocalEngineID()), 0);
usm.addUser(new OctetString(userName), uu);
if (log.isDebug()) {
logDebug("Valid Usm");
}
}
// create the PDU
ScopedPDU pdu = new ScopedPDU();
pdu.add(new VariableBinding(new OID(Oid), new OctetString(messageString)));
pdu.setType(PDU.TRAP);
if (!Const.isEmpty(engineID)) {
pdu.setContextEngineID(new OctetString(engineID));
}
// send the PDU
response = snmp.send(pdu, usertarget);
}
if (response != null) {
if (log.isDebug()) {
logDebug("Received response from: " + response.getPeerAddress() + response.toString());
}
}
result.setNrErrors(0);
result.setResult(true);
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "JobEntrySNMPTrap.ErrorGetting", e.getMessage()));
} finally {
try {
if (snmp != null) {
snmp.close();
}
} catch (Exception e) {
/* Ignore */
}
}
return result;
}
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
result.setResult(false);
if (log.isRowLevel()) {
logRowlevel(BaseMessages.getString(PKG, "JobSSH2GET.Log.GettingFieldsValue"));
}
// Get real variable value
String realServerName = environmentSubstitute(serverName);
int realServerPort = Const.toInt(environmentSubstitute(serverPort), 22);
String realUserName = environmentSubstitute(userName);
String realServerPassword =
Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(password));
// Proxy Host
String realProxyHost = environmentSubstitute(httpProxyHost);
int realProxyPort = Const.toInt(environmentSubstitute(httpproxyport), 22);
String realproxyUserName = environmentSubstitute(httpproxyusername);
String realProxyPassword =
Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(httpProxyPassword));
// Key file
String realKeyFilename = environmentSubstitute(keyFilename);
String relKeyFilepass = environmentSubstitute(keyFilePass);
// target files
String realLocalDirectory = environmentSubstitute(localDirectory);
String realwildcard = environmentSubstitute(wildcard);
// Remote source
String realftpDirectory = environmentSubstitute(ftpDirectory);
// Destination folder (Move to)
String realDestinationFolder = environmentSubstitute(destinationfolder);
try {
// Remote source
realftpDirectory = FTPUtils.normalizePath(realftpDirectory);
// Destination folder (Move to)
realDestinationFolder = FTPUtils.normalizePath(realDestinationFolder);
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.CanNotNormalizePath", e.getMessage()));
result.setNrErrors(1);
return result;
}
// Check for mandatory fields
if (log.isRowLevel()) {
logRowlevel(BaseMessages.getString(PKG, "JobSSH2GET.Log.CheckingMandatoryFields"));
}
boolean mandatoryok = true;
if (Const.isEmpty(realServerName)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.ServernameMissing"));
}
if (usehttpproxy) {
if (Const.isEmpty(realProxyHost)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.HttpProxyhostMissing"));
}
}
if (publicpublickey) {
if (Const.isEmpty(realKeyFilename)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.KeyFileMissing"));
} else {
// Let's check if key file exists...
if (!new File(realKeyFilename).exists()) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.KeyFileNotExist"));
}
}
}
if (Const.isEmpty(realLocalDirectory)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.LocalFolderMissing"));
} else {
// Check if target folder exists...
if (!new File(realLocalDirectory).exists()) {
if (createtargetfolder) {
// Create Target folder
if (!CreateFolder(realLocalDirectory)) {
mandatoryok = false;
}
} else {
mandatoryok = false;
logError(
BaseMessages.getString(
PKG, "JobSSH2GET.Log.LocalFolderNotExists", realLocalDirectory));
}
} else {
if (!new File(realLocalDirectory).isDirectory()) {
mandatoryok = false;
logError(
BaseMessages.getString(
PKG, "JobSSH2GET.Log.LocalFolderNotFolder", realLocalDirectory));
}
}
}
if (afterFtpPut.equals("move_file")) {
if (Const.isEmpty(realDestinationFolder)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.DestinatFolderMissing"));
}
}
if (mandatoryok) {
Connection conn = null;
SFTPv3Client client = null;
boolean good = true;
try {
// Create a connection instance
conn =
getConnection(
realServerName,
realServerPort,
realProxyHost,
realProxyPort,
realproxyUserName,
realProxyPassword);
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSSH2GET.Log.ConnectionInstanceCreated"));
}
if (timeout > 0) {
// Use timeout
// Cache Host Key
if (cachehostkey) {
conn.connect(new SimpleVerifier(database), 0, timeout * 1000);
} else {
conn.connect(null, 0, timeout * 1000);
}
} else {
// Cache Host Key
if (cachehostkey) {
conn.connect(new SimpleVerifier(database));
} else {
conn.connect();
}
}
// Authenticate
boolean isAuthenticated = false;
if (publicpublickey) {
isAuthenticated =
conn.authenticateWithPublicKey(
realUserName, new File(realKeyFilename), relKeyFilepass);
} else {
isAuthenticated = conn.authenticateWithPassword(realUserName, realServerPassword);
}
// LET'S CHECK AUTHENTICATION ...
if (isAuthenticated == false) {
logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.AuthenticationFailed"));
} else {
if (log.isBasic()) {
logBasic(BaseMessages.getString(PKG, "JobSSH2GET.Log.Connected", serverName, userName));
}
client = new SFTPv3Client(conn);
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2GET.Log.ProtocolVersion", "" + client.getProtocolVersion()));
}
// Check if ftp (source) directory exists
if (!Const.isEmpty(realftpDirectory)) {
if (!sshDirectoryExists(client, realftpDirectory)) {
good = false;
logError(
BaseMessages.getString(
PKG, "JobSSH2GET.Log.RemoteDirectoryNotExist", realftpDirectory));
} else if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2GET.Log.RemoteDirectoryExist", realftpDirectory));
}
}
if (realDestinationFolder != null) {
// Check now destination folder
if (!sshDirectoryExists(client, realDestinationFolder)) {
if (createdestinationfolder) {
if (!CreateRemoteFolder(client, realDestinationFolder)) {
good = false;
}
} else {
good = false;
logError(
BaseMessages.getString(
PKG, "JobSSH2GET.Log.DestinatFolderNotExist", realDestinationFolder));
}
}
}
if (good) {
Pattern pattern = null;
if (!Const.isEmpty(realwildcard)) {
pattern = Pattern.compile(realwildcard);
}
if (includeSubFolders) {
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSSH2GET.Log.RecursiveModeOn"));
}
copyRecursive(realftpDirectory, realLocalDirectory, client, pattern, parentJob);
} else {
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSSH2GET.Log.RecursiveModeOff"));
}
GetFiles(realftpDirectory, realLocalDirectory, client, pattern, parentJob);
}
/** ****************************** RESULT ******************* */
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSSH2GET.Log.Result.JobEntryEnd1"));
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2GET.Log.Result.TotalFiles", "" + nbfilestoget));
logDetailed(
BaseMessages.getString(PKG, "JobSSH2GET.Log.Result.TotalFilesPut", "" + nbgot));
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2GET.Log.Result.TotalFilesError", "" + nbrerror));
logDetailed(BaseMessages.getString(PKG, "JobSSH2GET.Log.Result.JobEntryEnd2"));
}
if (nbrerror == 0) {
result.setResult(true);
/** ****************************** RESULT ******************* */
}
}
}
} catch (Exception e) {
result.setNrErrors(nbrerror);
logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.Error.ErrorFTP", e.getMessage()));
} finally {
if (conn != null) {
conn.close();
}
if (client != null) {
client.close();
}
}
}
return result;
}
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
List<RowMetaAndData> rows = result.getRows();
RowMetaAndData resultRow = null;
oneFileLocked = false;
result.setResult(true);
try {
if (argFromPrevious) {
if (isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG,
"JobEntryCheckFilesLocked.FoundPreviousRows",
String.valueOf((rows != null ? rows.size() : 0))));
}
}
if (argFromPrevious && rows != null) {
// Copy the input row to the (command line) arguments
for (int iteration = 0; iteration < rows.size() && !parentJob.isStopped(); iteration++) {
resultRow = rows.get(iteration);
// Get values from previous result
String filefolder_previous = resultRow.getString(0, "");
String fmasks_previous = resultRow.getString(1, "");
// ok we can process this file/folder
if (isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG,
"JobEntryCheckFilesLocked.ProcessingRow",
filefolder_previous,
fmasks_previous));
}
ProcessFile(filefolder_previous, fmasks_previous);
}
} else if (arguments != null) {
for (int i = 0; i < arguments.length && !parentJob.isStopped(); i++) {
// ok we can process this file/folder
if (isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntryCheckFilesLocked.ProcessingArg", arguments[i], filemasks[i]));
}
ProcessFile(arguments[i], filemasks[i]);
}
}
if (oneFileLocked) {
result.setResult(false);
result.setNrErrors(1);
}
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "JobEntryCheckFilesLocked.ErrorRunningJobEntry", e));
}
return result;
}
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
result.setResult(false);
// see PDI-10270, PDI-10644 for details
boolean oldBehavior =
"Y"
.equalsIgnoreCase(
getVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "N"));
String countSQLStatement = null;
long rowsCount = 0;
long errCount = 0;
boolean successOK = false;
int nrRowsLimit = Const.toInt(environmentSubstitute(limit), 0);
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntryEvalTableContent.Log.nrRowsLimit", "" + nrRowsLimit));
}
if (connection != null) {
Database db = new Database(this, connection);
db.shareVariablesWith(this);
try {
db.connect(parentJob.getTransactionId(), null);
if (iscustomSQL) {
String realCustomSQL = customSQL;
if (isUseVars) {
realCustomSQL = environmentSubstitute(realCustomSQL);
}
if (log.isDebug()) {
logDebug(
BaseMessages.getString(
PKG, "JobEntryEvalTableContent.Log.EnteredCustomSQL", realCustomSQL));
}
if (!Const.isEmpty(realCustomSQL)) {
countSQLStatement = realCustomSQL;
} else {
errCount++;
logError(BaseMessages.getString(PKG, "JobEntryEvalTableContent.Error.NoCustomSQL"));
}
} else {
String realTablename = environmentSubstitute(tablename);
String realSchemaname = environmentSubstitute(schemaname);
if (!Const.isEmpty(realTablename)) {
if (!Const.isEmpty(realSchemaname)) {
countSQLStatement =
selectCount
+ db.getDatabaseMeta()
.getQuotedSchemaTableCombination(realSchemaname, realTablename);
} else {
countSQLStatement = selectCount + db.getDatabaseMeta().quoteField(realTablename);
}
} else {
errCount++;
logError(BaseMessages.getString(PKG, "JobEntryEvalTableContent.Error.NoTableName"));
}
}
if (countSQLStatement != null) {
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntryEvalTableContent.Log.RunSQLStatement", countSQLStatement));
}
if (iscustomSQL) {
if (isClearResultList) {
result.getRows().clear();
}
List<Object[]> ar = db.getRows(countSQLStatement, 0);
if (ar != null) {
rowsCount = ar.size();
// ad rows to result
RowMetaInterface rowMeta = db.getQueryFields(countSQLStatement, false);
List<RowMetaAndData> rows = new ArrayList<RowMetaAndData>();
for (int i = 0; i < ar.size(); i++) {
rows.add(new RowMetaAndData(rowMeta, ar.get(i)));
}
if (isAddRowsResult && iscustomSQL) {
if (rows != null) {
result.getRows().addAll(rows);
}
}
} else {
if (log.isDebug()) {
logDebug(
BaseMessages.getString(
PKG,
"JobEntryEvalTableContent.Log.customSQLreturnedNothing",
countSQLStatement));
}
}
} else {
RowMetaAndData row = db.getOneRow(countSQLStatement);
if (row != null) {
rowsCount = row.getInteger(0);
}
}
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntryEvalTableContent.Log.NrRowsReturned", "" + rowsCount));
}
switch (successCondition) {
case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_EQUAL:
successOK = (rowsCount == nrRowsLimit);
break;
case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_DIFFERENT:
successOK = (rowsCount != nrRowsLimit);
break;
case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_SMALLER:
successOK = (rowsCount < nrRowsLimit);
break;
case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_SMALLER_EQUAL:
successOK = (rowsCount <= nrRowsLimit);
break;
case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_GREATER:
successOK = (rowsCount > nrRowsLimit);
break;
case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_GREATER_EQUAL:
successOK = (rowsCount >= nrRowsLimit);
break;
default:
break;
}
if (!successOK && oldBehavior) {
errCount++;
}
} // end if countSQLStatement!=null
} catch (KettleException dbe) {
errCount++;
logError(
BaseMessages.getString(
PKG, "JobEntryEvalTableContent.Error.RunningEntry", dbe.getMessage()));
} finally {
if (db != null) {
db.disconnect();
}
}
} else {
errCount++;
logError(BaseMessages.getString(PKG, "JobEntryEvalTableContent.NoDbConnection"));
}
result.setResult(successOK);
result.setNrLinesRead(rowsCount);
result.setNrErrors(errCount);
return result;
}
public Result execute(Result result, int nr) throws KettleException {
result.setResult(true);
result.setNrErrors(0);
try {
List<String> variables = new ArrayList<String>();
List<String> variableValues = new ArrayList<String>();
List<Integer> variableTypes = new ArrayList<Integer>();
String realFilename = environmentSubstitute(filename);
try {
if (!Const.isEmpty(realFilename)) {
Properties properties = new Properties();
properties.load(KettleVFS.getInputStream(realFilename));
for (Object key : properties.keySet()) {
variables.add((String) key);
variableValues.add((String) properties.get(key));
variableTypes.add(fileVariableType);
}
}
} catch (Exception e) {
throw new KettleException(
BaseMessages.getString(
PKG, "JobEntrySetVariables.Error.UnableReadPropertiesFile", realFilename));
}
for (int i = 0; i < variableName.length; i++) {
variables.add(variableName[i]);
variableValues.add(variableValue[i]);
variableTypes.add(variableType[i]);
}
for (int i = 0; i < variables.size(); i++) {
String varname = variables.get(i);
String value = variableValues.get(i);
int type = variableTypes.get(i);
if (replaceVars) {
varname = environmentSubstitute(varname);
value = environmentSubstitute(value);
}
// OK, where do we set this value...
switch (type) {
case VARIABLE_TYPE_JVM:
{
System.setProperty(varname, value);
setVariable(varname, value);
Job parentJobTraverse = parentJob;
while (parentJobTraverse != null) {
parentJobTraverse.setVariable(varname, value);
parentJobTraverse = parentJobTraverse.getParentJob();
}
}
break;
case VARIABLE_TYPE_ROOT_JOB:
{
// set variable in this job entry
setVariable(varname, value);
Job rootJob = parentJob;
while (rootJob != null) {
rootJob.setVariable(varname, value);
rootJob = rootJob.getParentJob();
}
}
break;
case VARIABLE_TYPE_CURRENT_JOB:
{
setVariable(varname, value);
if (parentJob != null) {
parentJob.setVariable(varname, value);
} else {
throw new KettleJobException(
BaseMessages.getString(
PKG, "JobEntrySetVariables.Error.UnableSetVariableCurrentJob", varname));
}
}
break;
case VARIABLE_TYPE_PARENT_JOB:
{
setVariable(varname, value);
if (parentJob != null) {
parentJob.setVariable(varname, value);
Job gpJob = parentJob.getParentJob();
if (gpJob != null) {
gpJob.setVariable(varname, value);
} else {
throw new KettleJobException(
BaseMessages.getString(
PKG, "JobEntrySetVariables.Error.UnableSetVariableParentJob", varname));
}
} else {
throw new KettleJobException(
BaseMessages.getString(
PKG, "JobEntrySetVariables.Error.UnableSetVariableCurrentJob", varname));
}
}
break;
default:
break;
}
// ok we can process this line
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntrySetVariables.Log.SetVariableToValue", varname, value));
}
}
} catch (Exception e) {
result.setResult(false);
result.setNrErrors(1);
logError(
BaseMessages.getString(PKG, "JobEntrySetVariables.UnExcpectedError", e.getMessage()));
}
return result;
}
public Result execute(Result previousResult, int nr) {
String LimitNbrLignes = "";
String ListOfColumn = "*";
String strHighPriority = "";
String OutDumpText = "";
String OptionEnclosed = "";
String FieldSeparator = "";
String LinesTerminated = "";
Result result = previousResult;
result.setResult(false);
// Let's check the filename ...
if (filename != null) {
// User has specified a file, We can continue ...
String realFilename = getRealFilename();
File file = new File(realFilename);
if (file.exists() && iffileexists == 2) {
// the file exists and user want to Fail
result.setResult(false);
result.setNrErrors(1);
logError(
BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists1.Label")
+ realFilename
+ BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists2.Label"));
} else if (file.exists() && iffileexists == 1) {
// the file exists and user want to do nothing
result.setResult(true);
if (log.isDetailed())
logDetailed(
BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists1.Label")
+ realFilename
+ BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists2.Label"));
} else {
if (file.exists() && iffileexists == 0) {
// File exists and user want to renamme it with unique name
// Format Date
// Try to clean filename (without wildcard)
String wildcard =
realFilename.substring(realFilename.length() - 4, realFilename.length());
if (wildcard.substring(0, 1).equals(".")) {
// Find wildcard
realFilename =
realFilename.substring(0, realFilename.length() - 4)
+ "_"
+ StringUtil.getFormattedDateTimeNow(true)
+ wildcard;
} else {
// did not find wildcard
realFilename = realFilename + "_" + StringUtil.getFormattedDateTimeNow(true);
}
logDebug(
BaseMessages.getString(PKG, "JobMysqlBulkFile.FileNameChange1.Label")
+ realFilename
+ BaseMessages.getString(PKG, "JobMysqlBulkFile.FileNameChange1.Label"));
}
// User has specified an existing file, We can continue ...
if (log.isDetailed())
logDetailed(
BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists1.Label")
+ realFilename
+ BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists2.Label"));
if (connection != null) {
// User has specified a connection, We can continue ...
Database db = new Database(this, connection);
db.shareVariablesWith(this);
try {
db.connect(parentJob.getTransactionId(), null);
// Get schemaname
String realSchemaname = environmentSubstitute(schemaname);
// Get tablename
String realTablename = environmentSubstitute(tablename);
if (db.checkTableExists(realTablename)) {
// The table existe, We can continue ...
if (log.isDetailed())
logDetailed(
BaseMessages.getString(PKG, "JobMysqlBulkFile.TableExists1.Label")
+ realTablename
+ BaseMessages.getString(PKG, "JobMysqlBulkFile.TableExists2.Label"));
// Add schemaname (Most the time Schemaname.Tablename)
if (schemaname != null) {
realTablename = realSchemaname + "." + realTablename;
}
// Set the Limit lines
if (Const.toInt(getRealLimitlines(), 0) > 0) {
LimitNbrLignes = "LIMIT " + getRealLimitlines();
}
// Set list of Column, if null get all columns (*)
if (getRealListColumn() != null) {
ListOfColumn = MysqlString(getRealListColumn());
}
// Fields separator
if (getRealSeparator() != null && outdumpvalue == 0) {
FieldSeparator =
"FIELDS TERMINATED BY '" + Const.replace(getRealSeparator(), "'", "''") + "'";
}
// Lines Terminated by
if (getRealLineterminated() != null && outdumpvalue == 0) {
LinesTerminated =
"LINES TERMINATED BY '"
+ Const.replace(getRealLineterminated(), "'", "''")
+ "'";
}
// High Priority ?
if (isHighPriority()) {
strHighPriority = "HIGH_PRIORITY";
}
if (getRealEnclosed() != null && outdumpvalue == 0) {
if (isOptionEnclosed()) {
OptionEnclosed = "OPTIONALLY ";
}
OptionEnclosed =
OptionEnclosed
+ "ENCLOSED BY '"
+ Const.replace(getRealEnclosed(), "'", "''")
+ "'";
}
// OutFile or Dumpfile
if (outdumpvalue == 0) {
OutDumpText = "INTO OUTFILE";
} else {
OutDumpText = "INTO DUMPFILE";
}
String FILEBulkFile =
"SELECT "
+ strHighPriority
+ " "
+ ListOfColumn
+ " "
+ OutDumpText
+ " '"
+ realFilename
+ "' "
+ FieldSeparator
+ " "
+ OptionEnclosed
+ " "
+ LinesTerminated
+ " FROM "
+ realTablename
+ " "
+ LimitNbrLignes
+ " LOCK IN SHARE MODE";
try {
if (log.isDetailed()) logDetailed(FILEBulkFile);
// Run the SQL
PreparedStatement ps = db.prepareSQL(FILEBulkFile);
ps.execute();
// Everything is OK...we can disconnect now
db.disconnect();
if (isAddFileToResult()) {
// Add filename to output files
ResultFile resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL,
KettleVFS.getFileObject(realFilename, this),
parentJob.getJobname(),
toString());
result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
}
result.setResult(true);
} catch (SQLException je) {
db.disconnect();
result.setNrErrors(1);
logError(
BaseMessages.getString(PKG, "JobMysqlBulkFile.Error.Label")
+ " "
+ je.getMessage());
} catch (KettleFileException e) {
logError(
BaseMessages.getString(PKG, "JobMysqlBulkFile.Error.Label") + e.getMessage());
result.setNrErrors(1);
}
} else {
// Of course, the table should have been created already before the bulk load
// operation
db.disconnect();
result.setNrErrors(1);
if (log.isDetailed())
logDetailed(
BaseMessages.getString(PKG, "JobMysqlBulkFile.TableNotExists1.Label")
+ realTablename
+ BaseMessages.getString(PKG, "JobMysqlBulkFile.TableNotExists2.Label"));
}
} catch (KettleDatabaseException dbe) {
db.disconnect();
result.setNrErrors(1);
logError(
BaseMessages.getString(PKG, "JobMysqlBulkFile.Error.Label")
+ " "
+ dbe.getMessage());
}
} else {
// No database connection is defined
result.setNrErrors(1);
logError(BaseMessages.getString(PKG, "JobMysqlBulkFile.Nodatabase.Label"));
}
}
} else {
// No file was specified
result.setNrErrors(1);
logError(BaseMessages.getString(PKG, "JobMysqlBulkFile.Nofilename.Label"));
}
return result;
}
@Override
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
result.setResult(false);
result.setNrErrors(1);
String realCustomSQL = null;
String realTablename = environmentSubstitute(tablename);
String realSchemaname = environmentSubstitute(schemaname);
if (connection == null) {
logError(BaseMessages.getString(PKG, "JobEntryWaitForSQL.NoDbConnection"));
return result;
}
if (iscustomSQL) {
// clear result list rows
if (isClearResultList) {
result.getRows().clear();
}
realCustomSQL = customSQL;
if (isUseVars) {
realCustomSQL = environmentSubstitute(realCustomSQL);
}
if (log.isDebug()) {
logDebug(
BaseMessages.getString(PKG, "JobEntryWaitForSQL.Log.EnteredCustomSQL", realCustomSQL));
}
if (Const.isEmpty(realCustomSQL)) {
logError(BaseMessages.getString(PKG, "JobEntryWaitForSQL.Error.NoCustomSQL"));
return result;
}
} else {
if (Const.isEmpty(realTablename)) {
logError(BaseMessages.getString(PKG, "JobEntryWaitForSQL.Error.NoTableName"));
return result;
}
}
try {
// check connection
// connect and disconnect
checkConnection();
// starttime (in seconds)
long timeStart = System.currentTimeMillis() / 1000;
int nrRowsLimit = Const.toInt(environmentSubstitute(rowsCountValue), 0);
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(PKG, "JobEntryWaitForSQL.Log.nrRowsLimit", "" + nrRowsLimit));
}
long iMaximumTimeout =
Const.toInt(
environmentSubstitute(maximumTimeout), Const.toInt(DEFAULT_MAXIMUM_TIMEOUT, 0));
long iCycleTime =
Const.toInt(
environmentSubstitute(checkCycleTime), Const.toInt(DEFAULT_CHECK_CYCLE_TIME, 0));
//
// Sanity check on some values, and complain on insanity
//
if (iMaximumTimeout < 0) {
iMaximumTimeout = Const.toInt(DEFAULT_MAXIMUM_TIMEOUT, 0);
logBasic("Maximum timeout invalid, reset to " + iMaximumTimeout);
}
if (iCycleTime < 1) {
// If lower than 1 set to the default
iCycleTime = Const.toInt(DEFAULT_CHECK_CYCLE_TIME, 1);
logBasic("Check cycle time invalid, reset to " + iCycleTime);
}
if (iMaximumTimeout == 0) {
logBasic("Waiting indefinitely for SQL data");
} else {
logBasic("Waiting " + iMaximumTimeout + " seconds for SQL data");
}
boolean continueLoop = true;
while (continueLoop && !parentJob.isStopped()) {
if (SQLDataOK(result, nrRowsLimit, realSchemaname, realTablename, realCustomSQL)) {
// SQL data exists, we're happy to exit
logBasic("Detected SQL data within timeout");
result.setResult(true);
continueLoop = false;
} else {
long now = System.currentTimeMillis() / 1000;
if ((iMaximumTimeout > 0) && (now > (timeStart + iMaximumTimeout))) {
continueLoop = false;
// SQL data doesn't exist after timeout, either true or false
if (isSuccessOnTimeout()) {
logBasic("Didn't detect SQL data before timeout, success");
result.setResult(true);
} else {
logBasic("Didn't detect SQL data before timeout, failure");
result.setResult(false);
}
}
// sleep algorithm
long sleepTime = 0;
if (iMaximumTimeout == 0) {
sleepTime = iCycleTime;
} else {
if ((now + iCycleTime) < (timeStart + iMaximumTimeout)) {
sleepTime = iCycleTime;
} else {
sleepTime = iCycleTime - ((now + iCycleTime) - (timeStart + iMaximumTimeout));
}
}
try {
if (sleepTime > 0) {
if (log.isDetailed()) {
logDetailed("Sleeping " + sleepTime + " seconds before next check for SQL data");
}
Thread.sleep(sleepTime * 1000);
}
} catch (InterruptedException e) {
// something strange happened
result.setResult(false);
continueLoop = false;
}
}
}
} catch (Exception e) {
logBasic("Exception while waiting for SQL data: " + e.getMessage());
}
if (result.getResult()) {
// Remove error count set at the beginning of the method
// PDI-15437
result.setNrErrors(0);
}
return result;
}
private void executeShell(Result result, List<RowMetaAndData> cmdRows, String[] args) {
FileObject fileObject = null;
String realScript = null;
FileObject tempFile = null;
try {
// What's the exact command?
String[] base = null;
List<String> cmds = new ArrayList<String>();
if (log.isBasic()) {
logBasic(BaseMessages.getString(PKG, "JobShell.RunningOn", Const.getOS()));
}
if (insertScript) {
realScript = environmentSubstitute(script);
} else {
String realFilename = environmentSubstitute(getFilename());
fileObject = KettleVFS.getFileObject(realFilename, this);
}
if (Const.getOS().equals("Windows 95")) {
base = new String[] {"command.com", "/C"};
if (insertScript) {
tempFile =
KettleVFS.createTempFile(
"kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this);
fileObject = createTemporaryShellFile(tempFile, realScript);
}
} else if (Const.getOS().startsWith("Windows")) {
base = new String[] {"cmd.exe", "/C"};
if (insertScript) {
tempFile =
KettleVFS.createTempFile(
"kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this);
fileObject = createTemporaryShellFile(tempFile, realScript);
}
} else {
if (insertScript) {
tempFile =
KettleVFS.createTempFile(
"kettle", "shell", System.getProperty("java.io.tmpdir"), this);
fileObject = createTemporaryShellFile(tempFile, realScript);
}
base = new String[] {KettleVFS.getFilename(fileObject)};
}
// Construct the arguments...
if (argFromPrevious && cmdRows != null) {
// Add the base command...
for (int i = 0; i < base.length; i++) {
cmds.add(base[i]);
}
if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) {
// for windows all arguments including the command itself
// need to be
// included in 1 argument to cmd/command.
StringBuffer cmdline = new StringBuffer(300);
cmdline.append('"');
cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject)));
// Add the arguments from previous results...
for (int i = 0; i < cmdRows.size(); i++) {
// Normally just one row, but once in a while to remain compatible we have multiple.
RowMetaAndData r = cmdRows.get(i);
for (int j = 0; j < r.size(); j++) {
cmdline.append(' ');
cmdline.append(Const.optionallyQuoteStringByOS(r.getString(j, null)));
}
}
cmdline.append('"');
cmds.add(cmdline.toString());
} else {
// Add the arguments from previous results...
for (int i = 0; i < cmdRows.size(); i++) {
// Normally just one row, but once in a while to remain compatible we have multiple.
RowMetaAndData r = cmdRows.get(i);
for (int j = 0; j < r.size(); j++) {
cmds.add(Const.optionallyQuoteStringByOS(r.getString(j, null)));
}
}
}
} else if (args != null) {
// Add the base command...
for (int i = 0; i < base.length; i++) {
cmds.add(base[i]);
}
if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) {
// for windows all arguments including the command itself
// need to be
// included in 1 argument to cmd/command.
StringBuffer cmdline = new StringBuffer(300);
cmdline.append('"');
cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject)));
for (int i = 0; i < args.length; i++) {
cmdline.append(' ');
cmdline.append(Const.optionallyQuoteStringByOS(args[i]));
}
cmdline.append('"');
cmds.add(cmdline.toString());
} else {
for (int i = 0; i < args.length; i++) {
cmds.add(args[i]);
}
}
}
StringBuffer command = new StringBuffer();
Iterator<String> it = cmds.iterator();
boolean first = true;
while (it.hasNext()) {
if (!first) {
command.append(' ');
} else {
first = false;
}
command.append(it.next());
}
if (log.isBasic()) {
logBasic(BaseMessages.getString(PKG, "JobShell.ExecCommand", command.toString()));
}
// Build the environment variable list...
ProcessBuilder procBuilder = new ProcessBuilder(cmds);
Map<String, String> env = procBuilder.environment();
String[] variables = listVariables();
for (int i = 0; i < variables.length; i++) {
env.put(variables[i], getVariable(variables[i]));
}
if (getWorkDirectory() != null && !Const.isEmpty(Const.rtrim(getWorkDirectory()))) {
String vfsFilename = environmentSubstitute(getWorkDirectory());
File file = new File(KettleVFS.getFilename(KettleVFS.getFileObject(vfsFilename, this)));
procBuilder.directory(file);
}
Process proc = procBuilder.start();
// any error message?
StreamLogger errorLogger = new StreamLogger(log, proc.getErrorStream(), "(stderr)", true);
// any output?
StreamLogger outputLogger = new StreamLogger(log, proc.getInputStream(), "(stdout)");
// kick them off
Thread errorLoggerThread = new Thread(errorLogger);
errorLoggerThread.start();
Thread outputLoggerThread = new Thread(outputLogger);
outputLoggerThread.start();
proc.waitFor();
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobShell.CommandFinished", command.toString()));
}
// What's the exit status?
result.setExitStatus(proc.exitValue());
if (result.getExitStatus() != 0) {
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG,
"JobShell.ExitStatus",
environmentSubstitute(getFilename()),
"" + result.getExitStatus()));
}
result.setNrErrors(1);
}
// wait until loggers read all data from stdout and stderr
errorLoggerThread.join();
outputLoggerThread.join();
// close the streams
// otherwise you get "Too many open files, java.io.IOException" after a lot of iterations
proc.getErrorStream().close();
proc.getOutputStream().close();
} catch (IOException ioe) {
logError(
BaseMessages.getString(
PKG,
"JobShell.ErrorRunningShell",
environmentSubstitute(getFilename()),
ioe.toString()),
ioe);
result.setNrErrors(1);
} catch (InterruptedException ie) {
logError(
BaseMessages.getString(
PKG, "JobShell.Shellinterupted", environmentSubstitute(getFilename()), ie.toString()),
ie);
result.setNrErrors(1);
} catch (Exception e) {
logError(
BaseMessages.getString(
PKG, "JobShell.UnexpectedError", environmentSubstitute(getFilename()), e.toString()),
e);
result.setNrErrors(1);
} finally {
// If we created a temporary file, remove it...
//
if (tempFile != null) {
try {
tempFile.delete();
} catch (Exception e) {
BaseMessages.getString(
PKG, "JobShell.UnexpectedError", tempFile.toString(), e.toString());
}
}
}
if (result.getNrErrors() > 0) {
result.setResult(false);
} else {
result.setResult(true);
}
}
public Result execute(Result result, int nr) throws KettleException {
FileLoggingEventListener loggingEventListener = null;
LogLevel shellLogLevel = parentJob.getLogLevel();
if (setLogfile) {
String realLogFilename = environmentSubstitute(getLogFilename());
// We need to check here the log filename
// if we do not have one, we must fail
if (Const.isEmpty(realLogFilename)) {
logError(BaseMessages.getString(PKG, "JobEntryShell.Exception.LogFilenameMissing"));
result.setNrErrors(1);
result.setResult(false);
return result;
}
try {
loggingEventListener =
new FileLoggingEventListener(getLogChannelId(), realLogFilename, setAppendLogfile);
KettleLogStore.getAppender().addLoggingEventListener(loggingEventListener);
} catch (KettleException e) {
logError(
BaseMessages.getString(
PKG, "JobEntryShell.Error.UnableopenAppenderFile", getLogFilename(), e.toString()));
logError(Const.getStackTracker(e));
result.setNrErrors(1);
result.setResult(false);
return result;
}
shellLogLevel = logFileLevel;
}
log.setLogLevel(shellLogLevel);
result.setEntryNr(nr);
// "Translate" the arguments for later
String[] substArgs = null;
if (arguments != null) {
substArgs = new String[arguments.length];
for (int idx = 0; idx < arguments.length; idx++) {
substArgs[idx] = environmentSubstitute(arguments[idx]);
}
}
int iteration = 0;
String[] args = substArgs;
RowMetaAndData resultRow = null;
boolean first = true;
List<RowMetaAndData> rows = result.getRows();
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntryShell.Log.FoundPreviousRows", "" + (rows != null ? rows.size() : 0)));
}
while ((first && !execPerRow)
|| (execPerRow && rows != null && iteration < rows.size() && result.getNrErrors() == 0)) {
first = false;
if (rows != null && execPerRow) {
resultRow = rows.get(iteration);
} else {
resultRow = null;
}
List<RowMetaAndData> cmdRows = null;
if (execPerRow) {
// Execute for each input row
if (argFromPrevious) {
// Copy the input row to the (command line) arguments
if (resultRow != null) {
args = new String[resultRow.size()];
for (int i = 0; i < resultRow.size(); i++) {
args[i] = resultRow.getString(i, null);
}
}
} else {
// Just pass a single row
List<RowMetaAndData> newList = new ArrayList<RowMetaAndData>();
newList.add(resultRow);
cmdRows = newList;
}
} else {
if (argFromPrevious) {
// Only put the first Row on the arguments
args = null;
if (resultRow != null) {
args = new String[resultRow.size()];
for (int i = 0; i < resultRow.size(); i++) {
args[i] = resultRow.getString(i, null);
}
} else {
cmdRows = rows;
}
} else {
// Keep it as it was...
cmdRows = rows;
}
}
executeShell(result, cmdRows, args);
iteration++;
}
if (setLogfile) {
if (loggingEventListener != null) {
KettleLogStore.getAppender().removeLoggingEventListener(loggingEventListener);
loggingEventListener.close();
ResultFile resultFile =
new ResultFile(
ResultFile.FILE_TYPE_LOG,
loggingEventListener.getFile(),
parentJob.getJobname(),
getName());
result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
}
}
return result;
}
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
result.setNrErrors(1);
result.setResult(false);
String hostname = getRealHostname();
int timeoutInt = Const.toInt(getRealTimeOut(), 300);
int packets = Const.toInt(getRealNbrPackets(), 2);
boolean status = false;
if (Const.isEmpty(hostname)) {
// No Host was specified
logError(BaseMessages.getString(PKG, "JobPing.SpecifyHost.Label"));
return result;
}
try {
if (ipingtype == isystemPing || ipingtype == ibothPings) {
// Perform a system (Java) ping ...
status = systemPing(hostname, timeoutInt);
if (status) {
if (log.isDetailed()) {
log.logDetailed(
BaseMessages.getString(PKG, "JobPing.SystemPing"),
BaseMessages.getString(PKG, "JobPing.OK.Label", hostname));
}
} else {
log.logError(
BaseMessages.getString(PKG, "JobPing.SystemPing"),
BaseMessages.getString(PKG, "JobPing.NOK.Label", hostname));
}
}
if ((ipingtype == iclassicPing) || (ipingtype == ibothPings && !status)) {
// Perform a classic ping ..
status = classicPing(hostname, packets);
if (status) {
if (log.isDetailed()) {
log.logDetailed(
BaseMessages.getString(PKG, "JobPing.ClassicPing"),
BaseMessages.getString(PKG, "JobPing.OK.Label", hostname));
}
} else {
log.logError(
BaseMessages.getString(PKG, "JobPing.ClassicPing"),
BaseMessages.getString(PKG, "JobPing.NOK.Label", hostname));
}
}
} catch (Exception ex) {
logError(BaseMessages.getString(PKG, "JobPing.Error.Label") + ex.getMessage());
}
if (status) {
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobPing.OK.Label", hostname));
}
result.setNrErrors(0);
result.setResult(true);
} else {
logError(BaseMessages.getString(PKG, "JobPing.NOK.Label", hostname));
}
return result;
}
public Result execute(Result previousResult, int nr) throws KettleException {
Result result = previousResult;
int NrErrors = 0;
int NrSuccess = 0;
// Check output parameters
int nrOutputProps = getOutputPropertyName() == null ? 0 : getOutputPropertyName().length;
if (nrOutputProps > 0) {
outputProperties = new Properties();
for (int i = 0; i < nrOutputProps; i++) {
outputProperties.put(
getOutputPropertyName()[i], environmentSubstitute(getOutputPropertyValue()[i]));
}
setOutputProperties = true;
}
// Check parameters
nrParams = getParameterField() == null ? 0 : getParameterField().length;
if (nrParams > 0) {
nameOfParams = new String[nrParams];
valueOfParams = new String[nrParams];
for (int i = 0; i < nrParams; i++) {
String name = environmentSubstitute(getParameterName()[i]);
String value = environmentSubstitute(getParameterField()[i]);
if (Const.isEmpty(value)) {
throw new KettleStepException(
BaseMessages.getString(PKG, "Xslt.Exception.ParameterFieldMissing", name, i));
}
nameOfParams[i] = name;
valueOfParams[i] = value;
}
useParameters = true;
}
List<RowMetaAndData> rows = result.getRows();
if (isFilenamesFromPrevious()) {
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG,
"JobEntryXSLT.Log.ArgFromPrevious.Found",
(rows != null ? rows.size() : 0) + ""));
}
}
if (isFilenamesFromPrevious() && rows != null) {
// Copy the input row to the (command line) arguments
RowMetaAndData resultRow = null;
for (int iteration = 0; iteration < rows.size() && !parentJob.isStopped(); iteration++) {
resultRow = rows.get(iteration);
// Get filenames (xml, xsl, output filename)
String xmlfilename_previous = resultRow.getString(0, null);
String xslfilename_previous = resultRow.getString(1, null);
String ouputfilename_previous = resultRow.getString(2, null);
if (!Const.isEmpty(xmlfilename_previous)
&& !Const.isEmpty(xslfilename_previous)
&& !Const.isEmpty(ouputfilename_previous)) {
if (processOneXMLFile(
xmlfilename_previous,
xslfilename_previous,
ouputfilename_previous,
result,
parentJob)) {
NrSuccess++;
} else {
NrErrors++;
}
} else {
// We failed!
logError(BaseMessages.getString(PKG, "JobEntryXSLT.AllFilesNotNull.Label"));
NrErrors++;
}
}
} else {
String realxmlfilename = getRealxmlfilename();
String realxslfilename = getRealxslfilename();
String realoutputfilename = getoutputfilename();
if (!Const.isEmpty(realxmlfilename)
&& !Const.isEmpty(realxslfilename)
&& !Const.isEmpty(realoutputfilename)) {
if (processOneXMLFile(
realxmlfilename, realxslfilename, realoutputfilename, result, parentJob)) {
NrSuccess++;
} else {
NrErrors++;
}
} else {
// We failed!
logError(BaseMessages.getString(PKG, "JobEntryXSLT.AllFilesNotNull.Label"));
NrErrors++;
}
}
result.setResult(NrErrors == 0);
result.setNrErrors(NrErrors);
result.setNrLinesWritten(NrSuccess);
return result;
}
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
result.setResult(false);
// starttime (in seconds)
long timeStart = System.currentTimeMillis() / 1000;
if (filename != null) {
FileObject fileObject = null;
String realFilename = getRealFilename();
try {
fileObject = KettleVFS.getFileObject(realFilename, this);
long iMaximumTimeout =
Const.toInt(getRealMaximumTimeout(), Const.toInt(DEFAULT_MAXIMUM_TIMEOUT, 0));
long iCycleTime =
Const.toInt(getRealCheckCycleTime(), Const.toInt(DEFAULT_CHECK_CYCLE_TIME, 0));
//
// Sanity check on some values, and complain on insanity
//
if (iMaximumTimeout < 0) {
iMaximumTimeout = Const.toInt(DEFAULT_MAXIMUM_TIMEOUT, 0);
if (log.isBasic()) logBasic("Maximum timeout invalid, reset to " + iMaximumTimeout);
}
if (iCycleTime < 1) {
// If lower than 1 set to the default
iCycleTime = Const.toInt(DEFAULT_CHECK_CYCLE_TIME, 1);
if (log.isBasic()) logBasic("Check cycle time invalid, reset to " + iCycleTime);
}
if (iMaximumTimeout == 0) {
if (log.isBasic()) logBasic("Waiting indefinitely for file [" + realFilename + "]");
} else {
if (log.isBasic())
logBasic("Waiting " + iMaximumTimeout + " seconds for file [" + realFilename + "]");
}
boolean continueLoop = true;
while (continueLoop && !parentJob.isStopped()) {
fileObject = KettleVFS.getFileObject(realFilename, this);
if (fileObject.exists()) {
// file exists, we're happy to exit
if (log.isBasic()) logBasic("Detected file [" + realFilename + "] within timeout");
result.setResult(true);
continueLoop = false;
// add filename to result filenames
if (addFilenameToResult && fileObject.getType() == FileType.FILE) {
ResultFile resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL, fileObject, parentJob.getJobname(), toString());
resultFile.setComment(BaseMessages.getString(PKG, "JobWaitForFile.FilenameAdded"));
result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
}
} else {
long now = System.currentTimeMillis() / 1000;
if ((iMaximumTimeout > 0) && (now > (timeStart + iMaximumTimeout))) {
continueLoop = false;
// file doesn't exist after timeout, either true or false
if (isSuccessOnTimeout()) {
if (log.isBasic())
logBasic("Didn't detect file [" + realFilename + "] before timeout, success");
result.setResult(true);
} else {
if (log.isBasic())
logBasic("Didn't detect file [" + realFilename + "] before timeout, failure");
result.setResult(false);
}
}
// sleep algorithm
long sleepTime = 0;
if (iMaximumTimeout == 0) {
sleepTime = iCycleTime;
} else {
if ((now + iCycleTime) < (timeStart + iMaximumTimeout)) {
sleepTime = iCycleTime;
} else {
sleepTime = iCycleTime - ((now + iCycleTime) - (timeStart + iMaximumTimeout));
}
}
try {
if (sleepTime > 0) {
if (log.isDetailed()) {
logDetailed(
"Sleeping "
+ sleepTime
+ " seconds before next check for file ["
+ realFilename
+ "]");
}
Thread.sleep(sleepTime * 1000);
}
} catch (InterruptedException e) {
// something strange happened
result.setResult(false);
continueLoop = false;
}
}
}
if (!parentJob.isStopped() && fileObject.exists() && isFileSizeCheck()) {
long oldSize = -1;
long newSize = fileObject.getContent().getSize();
if (log.isDetailed())
logDetailed("File [" + realFilename + "] is " + newSize + " bytes long");
if (log.isBasic())
logBasic(
"Waiting until file ["
+ realFilename
+ "] stops growing for "
+ iCycleTime
+ " seconds");
while (oldSize != newSize && !parentJob.isStopped()) {
try {
if (log.isDetailed()) {
logDetailed(
"Sleeping "
+ iCycleTime
+ " seconds, waiting for file ["
+ realFilename
+ "] to stop growing");
}
Thread.sleep(iCycleTime * 1000);
} catch (InterruptedException e) {
// something strange happened
result.setResult(false);
continueLoop = false;
}
oldSize = newSize;
newSize = fileObject.getContent().getSize();
if (log.isDetailed()) {
logDetailed("File [" + realFilename + "] is " + newSize + " bytes long");
}
}
if (log.isBasic())
logBasic("Stopped waiting for file [" + realFilename + "] to stop growing");
}
if (parentJob.isStopped()) {
result.setResult(false);
}
} catch (Exception e) {
logBasic("Exception while waiting for file [" + realFilename + "] to stop growing", e);
} finally {
if (fileObject != null) {
try {
fileObject.close();
} catch (Exception e) {
}
}
}
} else {
logError("No filename is defined.");
}
return result;
}
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
result.setResult(false);
try {
// Get real variable value
String realServerName = environmentSubstitute(serverName);
int realServerPort = Const.toInt(environmentSubstitute(serverPort), 22);
String realUserName = environmentSubstitute(userName);
String realServerPassword =
Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(password));
// Proxy Host
String realProxyHost = environmentSubstitute(httpproxyhost);
int realProxyPort = Const.toInt(environmentSubstitute(httpproxyport), 22);
String realproxyUserName = environmentSubstitute(httpproxyusername);
String realProxyPassword =
Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(httpProxyPassword));
// Key file
String realKeyFilename = environmentSubstitute(keyFilename);
String relKeyFilepass = environmentSubstitute(keyFilePass);
// Source files
String realLocalDirectory = environmentSubstitute(localDirectory);
String realwildcard = environmentSubstitute(wildcard);
// Remote destination
String realftpDirectory = environmentSubstitute(ftpDirectory);
// Destination folder (Move to)
String realDestinationFolder = environmentSubstitute(destinationfolder);
try {
// Remote source
realftpDirectory = FTPUtils.normalizePath(realftpDirectory);
// Destination folder (Move to)
realDestinationFolder = FTPUtils.normalizePath(realDestinationFolder);
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.CanNotNormalizePath", e.getMessage()));
result.setNrErrors(1);
return result;
}
// Check for mandatory fields
boolean mandatoryok = true;
if (Const.isEmpty(realServerName)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.ServernameMissing"));
}
if (usehttpproxy) {
if (Const.isEmpty(realProxyHost)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.HttpProxyhostMissing"));
}
}
if (publicpublickey) {
if (Const.isEmpty(realKeyFilename)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.KeyFileMissing"));
} else {
// Let's check if folder exists...
if (!KettleVFS.fileExists(realKeyFilename, this)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.KeyFileNotExist"));
}
}
}
if (Const.isEmpty(realLocalDirectory)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.LocalFolderMissing"));
}
if (afterFtpPut.equals("move_file")) {
if (Const.isEmpty(realDestinationFolder)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.DestinatFolderMissing"));
} else {
FileObject folder = null;
try {
folder = KettleVFS.getFileObject(realDestinationFolder, this);
// Let's check if folder exists...
if (!folder.exists()) {
// Do we need to create it?
if (createDestinationFolder) {
folder.createFolder();
} else {
logError(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.DestinatFolderNotExist", realDestinationFolder));
}
}
} catch (Exception e) {
throw new KettleException(e);
} finally {
if (folder != null) {
try {
folder.close();
folder = null;
} catch (Exception e) {
/* Ignore */
}
}
}
}
}
if (mandatoryok) {
Connection conn = null;
SFTPv3Client client = null;
boolean good = true;
int nbfilestoput = 0;
int nbput = 0;
int nbrerror = 0;
try {
// Create a connection instance
conn =
getConnection(
realServerName,
realServerPort,
realProxyHost,
realProxyPort,
realproxyUserName,
realProxyPassword);
if (timeout > 0) {
// Use timeout
// Cache Host Key
if (cachehostkey) {
conn.connect(new SimpleVerifier(database), 0, timeout * 1000);
} else {
conn.connect(null, 0, timeout * 1000);
}
} else {
// Cache Host Key
if (cachehostkey) {
conn.connect(new SimpleVerifier(database));
} else {
conn.connect();
}
}
// Authenticate
boolean isAuthenticated = false;
if (publicpublickey) {
String keyContent =
KettleVFS.getTextFileContent(realKeyFilename, this, Const.XML_ENCODING);
isAuthenticated =
conn.authenticateWithPublicKey(
realUserName, keyContent.toCharArray(), relKeyFilepass);
} else {
isAuthenticated = conn.authenticateWithPassword(realUserName, realServerPassword);
}
// LET'S CHECK AUTHENTICATION ...
if (isAuthenticated == false) {
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.AuthenticationFailed"));
} else {
if (log.isBasic()) {
logBasic(
BaseMessages.getString(PKG, "JobSSH2PUT.Log.Connected", serverName, userName));
}
client = new SFTPv3Client(conn);
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.ProtocolVersion", "" + client.getProtocolVersion()));
}
// Check if remote directory exists
if (!Const.isEmpty(realftpDirectory)) {
if (!sshDirectoryExists(client, realftpDirectory)) {
good = false;
if (createRemoteFolder) {
good = CreateRemoteFolder(client, realftpDirectory);
if (good) {
logBasic(BaseMessages.getString(PKG, "JobSSH2PUT.Log.RemoteDirectoryCreated"));
}
} else {
logError(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.RemoteDirectoryNotExist", realftpDirectory));
}
} else if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.RemoteDirectoryExist", realftpDirectory));
}
}
if (good) {
// Get files list from local folder (source)
List<FileObject> myFileList = getFiles(realLocalDirectory);
// Prepare Pattern for wildcard
Pattern pattern = null;
if (!Const.isEmpty(realwildcard)) {
pattern = Pattern.compile(realwildcard);
}
// Let's put files now ...
// Get the files in the list
for (int i = 0; i < myFileList.size() && !parentJob.isStopped(); i++) {
FileObject myFile = myFileList.get(i);
String localFilename = myFile.toString();
String remoteFilename = myFile.getName().getBaseName();
boolean getIt = true;
// First see if the file matches the regular expression!
if (pattern != null) {
Matcher matcher = pattern.matcher(remoteFilename);
getIt = matcher.matches();
}
// do we have a target directory?
if (!Const.isEmpty(realftpDirectory)) {
remoteFilename = realftpDirectory + FTPUtils.FILE_SEPARATOR + remoteFilename;
}
if (onlyGettingNewFiles) {
// We get only new files
// ie not exist on the remote server
getIt = !sshFileExists(client, remoteFilename);
}
if (getIt) {
nbfilestoput++;
boolean putok = putFile(myFile, remoteFilename, client);
if (!putok) {
nbrerror++;
logError(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.Error.CanNotPutFile", localFilename));
} else {
nbput++;
}
if (putok && !afterFtpPut.equals("do_nothing")) {
deleteOrMoveFiles(myFile, realDestinationFolder);
}
}
}
/** ****************************** RESULT ******************* */
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.JobEntryEnd1"));
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.Result.TotalFiles", "" + nbfilestoput));
logDetailed(
BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.TotalFilesPut", "" + nbput));
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.Result.TotalFilesError", "" + nbrerror));
logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.JobEntryEnd2"));
}
if (nbrerror == 0) {
result.setResult(true);
/** ****************************** RESULT ******************* */
}
}
}
} catch (Exception e) {
result.setNrErrors(nbrerror);
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Error.ErrorFTP", e.getMessage()));
} finally {
if (conn != null) {
conn.close();
}
if (client != null) {
client.close();
}
}
}
} catch (Exception e) {
result.setResult(false);
result.setNrErrors(1L);
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Error.UnexpectedError"), e);
}
return result;
}
/**
* Execute this job entry and return the result. In this case it means, just set the result
* boolean in the Result class.
*
* @param previousResult The result of the previous execution
* @return The Result of the execution.
*/
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
result.setResult(false);
int Multiple;
String Waitscale;
// Scale time
switch (scaleTime) {
case 0:
// Second
Multiple = 1000;
Waitscale = BaseMessages.getString(PKG, "JobEntryDelay.SScaleTime.Label");
break;
case 1:
// Minute
Multiple = 60000;
Waitscale = BaseMessages.getString(PKG, "JobEntryDelay.MnScaleTime.Label");
break;
default:
// Hour
Multiple = 3600000;
Waitscale = BaseMessages.getString(PKG, "JobEntryDelay.HrScaleTime.Label");
break;
}
try {
// starttime (in seconds ,Minutes or Hours)
long timeStart = System.currentTimeMillis() / Multiple;
long iMaximumTimeout =
Const.toInt(getrealMaximumTimeout(), Const.toInt(DEFAULT_MAXIMUM_TIMEOUT, 0));
if (isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntryDelay.LetsWaitFor.Label", iMaximumTimeout, Waitscale));
}
boolean continueLoop = true;
//
// Sanity check on some values, and complain on insanity
//
if (iMaximumTimeout < 0) {
iMaximumTimeout = Const.toInt(DEFAULT_MAXIMUM_TIMEOUT, 0);
logBasic(
BaseMessages.getString(
PKG,
"JobEntryDelay.MaximumTimeReset.Label",
String.valueOf(iMaximumTimeout),
String.valueOf(Waitscale)));
}
// Loop until the delay time has expired.
//
while (continueLoop && !parentJob.isStopped()) {
// Update Time value
long now = System.currentTimeMillis() / Multiple;
// Let's check the limit time
if ((iMaximumTimeout > 0) && (now >= (timeStart + iMaximumTimeout))) {
// We have reached the time limit
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobEntryDelay.WaitTimeIsElapsed.Label"));
}
continueLoop = false;
result.setResult(true);
} else {
Thread.sleep(100);
}
}
} catch (Exception e) {
// We get an exception
result.setResult(false);
logError("Error : " + e.getMessage());
}
return result;
}