@Test
public void testHTTPResultDefaultRows() throws IOException {
File localFileForUpload = getInputFile("existingFile1", ".tmp");
File tempFileForDownload = File.createTempFile("downloadedFile1", ".tmp");
localFileForUpload.deleteOnExit();
tempFileForDownload.deleteOnExit();
Object[] r =
new Object[] {
HTTP_SERVER_BASEURL + "/uploadFile",
localFileForUpload.getCanonicalPath(),
tempFileForDownload.getCanonicalPath()
};
RowMeta rowMetaDefault = new RowMeta();
rowMetaDefault.addValueMeta(new ValueMetaString("URL"));
rowMetaDefault.addValueMeta(new ValueMetaString("UPLOAD"));
rowMetaDefault.addValueMeta(new ValueMetaString("DESTINATION"));
List<RowMetaAndData> rows = new ArrayList<RowMetaAndData>();
rows.add(new RowMetaAndData(rowMetaDefault, r));
Result previousResult = new Result();
previousResult.setRows(rows);
JobEntryHTTP http = new JobEntryHTTP();
http.setParentJob(new Job());
http.setRunForEveryRow(true);
http.setAddFilenameToResult(false);
http.execute(previousResult, 0);
assertTrue(FileUtils.contentEquals(localFileForUpload, tempFileForDownload));
}
@Test
public void execute_blocking_FAIL() throws KettleException {
final long waitTime = 0;
OozieJobExecutorConfig config = new OozieJobExecutorConfig();
config.setOozieUrl(
"http://localhost:11000/oozie"); // don't worry if it isn't running, we fake out our test
// connection to it anyway
config.setOozieWorkflowConfig("test-src/job.properties");
config.setJobEntryName("name");
config.setBlockingPollingInterval("1000");
TestOozieJobExecutorJobEntry je = new TestOozieJobExecutorJobEntry(getFailingTest());
je.setParentJob(new Job("test", null, null));
je.setJobConfig(config);
Result result = new Result();
long start = System.currentTimeMillis();
je.execute(result, 0);
long end = System.currentTimeMillis();
assertTrue(
"Total runtime should be >= the wait time if we are blocking", (end - start) >= waitTime);
Assert.assertEquals(1, result.getNrErrors());
assertFalse(result.getResult());
}
public Result execute(Result previousResult, int nr) throws KettleJobException {
Result result = previousResult;
if (isStart()) {
try {
long sleepTime = getNextExecutionTime();
if (sleepTime > 0) {
parentJob
.getLogChannel()
.logBasic(
parentJob.getJobname(),
"Sleeping: "
+ (sleepTime / 1000 / 60)
+ " minutes (sleep time="
+ sleepTime
+ ")");
long totalSleep = 0L;
while (totalSleep < sleepTime && !parentJob.isStopped()) {
Thread.sleep(1000L);
totalSleep += 1000L;
}
}
} catch (InterruptedException e) {
throw new KettleJobException(e);
}
result = previousResult;
result.setResult(true);
} else if (isDummy()) {
result = previousResult;
}
return result;
}
@Override
public Result execute(Result prevResult, int k) throws KettleException {
Result result = prevResult;
ML_Classify direct = new ML_Classify();
direct.setName(this.getRecordsetName());
direct.setRecordsetName(this.getRecordsetName());
direct.setModel(model);
direct.setIndependentVar(independentVar);
direct.setClassifyType(classifyType);
direct.setDataType(dataType);
direct.setRidge(ridge);
direct.setEpsilon(epsilon);
direct.setMaxIter(maxIter);
direct.setPasses(passes);
direct.setAlpha(alpha);
// private Text algType; //NaiveBayes, Logistic
// private Text dependentVar; // 1
// private Text independentVar; // 2
// ml.setIterations(this.getIterations());
// ml.setThreshold(this.getThreshold());
// ml.setThreshold(this.getThreshold());
logBasic("{Iterate Job} Execute = " + direct.ecl());
logBasic("{Iterate Job} Previous =" + result.getLogText());
result.setResult(true);
RowMetaAndData data = new RowMetaAndData();
data.addValue("ecl", Value.VALUE_TYPE_STRING, direct.ecl());
List list = result.getRows();
list.add(data);
String eclCode = "";
if (list == null) {
list = new ArrayList();
} else {
for (int i = 0; i < list.size(); i++) {
RowMetaAndData rowData = (RowMetaAndData) list.get(i);
String code = rowData.getString("ecl", null);
if (code != null) {
eclCode += code;
}
}
logBasic("{Iterate Job} ECL Code =" + eclCode);
}
result.setRows(list);
return result;
}
@Test
public void testExecuteSuccess() throws Exception {
entry.arguments = new String[] {existingFile1, existingFile2};
Result res = entry.execute(new Result(), 0);
assertTrue("Entry failed", res.getResult());
}
@Test
public void testExecuteFail() throws Exception {
entry.arguments =
new String[] {existingFile1, existingFile2, "nonExistingFile1.ext", "nonExistingFile2.ext"};
Result res = entry.execute(new Result(), 0);
assertFalse("Entry should fail", res.getResult());
}
@Test
public void testExecuteWithException() throws Exception {
entry.arguments = new String[] {null};
Result res = entry.execute(new Result(), 0);
assertFalse("Entry should fail", res.getResult());
assertEquals(
"File with wrong name was specified. One error should be reported", 1, res.getNrErrors());
}
@Test
public void testSetNrErrorsNewBehaviorFalseResult() throws Exception {
// this tests fix for PDI-10270
entry.arguments = new String[] {"nonExistingFile.ext"};
Result res = entry.execute(new Result(), 0);
assertFalse("Entry should fail", res.getResult());
assertEquals(
"Files not found. Result is false. But... No of errors should be zero",
0,
res.getNrErrors());
}
@Test
public void testNrErrorsFailureNewBehavior() throws Exception {
entry.limit = "1";
entry.successCondition = JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_EQUAL;
entry.tablename = "table";
Result res = entry.execute(new Result(), 0);
assertFalse("Eval number of rows should fail", res.getResult());
assertEquals(
"No errors should be reported in result object accoding to the new behavior",
res.getNrErrors(),
0);
}
@Test
public void testSetNrErrorsOldBehaviorFalseResult() throws Exception {
// this tests backward compatibility settings for PDI-10270
entry.arguments = new String[] {"nonExistingFile1.ext", "nonExistingFile2.ext"};
entry.setVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "Y");
Result res = entry.execute(new Result(), 0);
assertFalse("Entry should fail", res.getResult());
assertEquals(
"Files not found. Result is false. And... Number of errors should be the same as number of not found files",
entry.arguments.length,
res.getNrErrors());
}
@Test
public void testNrErrorsFailureOldBehavior() throws Exception {
entry.limit = "1";
entry.successCondition = JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_EQUAL;
entry.tablename = "table";
entry.setVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "Y");
Result res = entry.execute(new Result(), 0);
assertFalse("Eval number of rows should fail", res.getResult());
assertEquals(
"An error should be reported in result object accoding to the old behavior",
res.getNrErrors(),
1);
}
@Test
public void testNrErrorsSuccess() throws Exception {
entry.limit = "5";
entry.successCondition = JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_EQUAL;
entry.tablename = "table";
Result res = entry.execute(new Result(), 0);
assertTrue("Eval number of rows should be suceeded", res.getResult());
assertEquals("Apparently there should no error", res.getNrErrors(), 0);
// that should work regardless of old/new behavior flag
entry.setVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "Y");
res = entry.execute(new Result(), 0);
assertTrue("Eval number of rows should be suceeded", res.getResult());
assertEquals("Apparently there should no error", res.getNrErrors(), 0);
}
@Override
public Result execute(Result prevResult, int k) throws KettleException {
Result result = prevResult;
logBasic("{Group job} Creating Group object");
Group group = new Group();
logBasic("{Group job} Group object created");
group.setName(this.getRecordSetName());
group.setRecordSet(this.getRecordSet());
group.setBreakCriteria(this.getBreakCriteria());
group.setIsAll(this.getIsAll());
group.setRunLocal(this.getIsRunLocal());
logBasic("{Group job} Execute = " + group.ecl());
logBasic("{Group job} Previous = " + result.getLogText());
result.setResult(true);
RowMetaAndData data = new RowMetaAndData();
data.addValue("ecl", Value.VALUE_TYPE_STRING, group.ecl());
List list = result.getRows();
list.add(data);
String eclCode = "";
if (list == null) {
list = new ArrayList();
} else {
for (int i = 0; i < list.size(); i++) {
RowMetaAndData rowData = (RowMetaAndData) list.get(i);
String code = rowData.getString("ecl", null);
if (code != null) eclCode += code;
}
logBasic("{Group job} ECL Code = " + eclCode);
}
result.setRows(list);
return result;
}
@Test
public void testNrErrorsNoCustomSql() throws Exception {
entry.limit = "5";
entry.successCondition = JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_EQUAL;
entry.iscustomSQL = true;
entry.customSQL = null;
Result res = entry.execute(new Result(), 0);
assertFalse("Eval number of rows should fail", res.getResult());
assertEquals("Apparently there should be an error", res.getNrErrors(), 1);
// that should work regardless of old/new behavior flag
entry.setVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "Y");
res = entry.execute(new Result(), 0);
assertFalse("Eval number of rows should fail", res.getResult());
assertEquals("Apparently there should be an error", res.getNrErrors(), 1);
}
@Override
public Result execute(Result prevResult, int k) throws KettleException {
Result result = prevResult;
Sort sort = new Sort();
sort.setFields(getFields());
sort.setDatasetName(getDatasetName());
sort.setName(getRecordsetName());
logBasic("{Sort Job} Execute = " + sort.ecl());
logBasic("{Sort Job} Previous =" + result.getLogText());
result.setResult(true);
RowMetaAndData data = new RowMetaAndData();
data.addValue("ecl", Value.VALUE_TYPE_STRING, sort.ecl());
List list = result.getRows();
list.add(data);
String eclCode = "";
if (list == null) {
list = new ArrayList();
} else {
for (int i = 0; i < list.size(); i++) {
RowMetaAndData rowData = (RowMetaAndData) list.get(i);
String code = rowData.getString("ecl", null);
if (code != null) {
eclCode += code;
}
}
logBasic("{Sort Job} ECL Code =" + eclCode);
}
result.setRows(list);
return result;
}
@Override
public Result execute(Result prevResult, int k) throws KettleException {
Result result = modifyResults(prevResult);
if (result.isStopped()) {
return result;
}
Rollup rollup = new Rollup();
rollup.setName(this.getRecordsetName());
rollup.setRecordset(this.getRecordset());
rollup.setRecordFormat(this.getRecordset());
rollup.setRunLocal(this.getRunLocal());
rollup.setCondition(this.getCondition());
rollup.setFieldlist(this.getFieldlist());
if (this.group.equalsIgnoreCase("yes")) {
rollup.setGroup("GROUP");
} else {
rollup.setGroup("");
}
rollup.setTransformName(this.getTransformName());
rollup.setTransform(generateEclForMapperGrid());
logBasic("{rollup Job} Execute = " + rollup.ecl());
logBasic("{rollup Job} Previous =" + result.getLogText());
result.setResult(true);
RowMetaAndData data = new RowMetaAndData();
data.addValue("ecl", Value.VALUE_TYPE_STRING, rollup.ecl());
List list = result.getRows();
list.add(data);
String eclCode = parseEclFromRowData(list);
/*
String eclCode = "";
if (list == null) {
list = new ArrayList();
} else {
for (int i = 0; i < list.size(); i++) {
RowMetaAndData rowData = (RowMetaAndData) list.get(i);
String code = rowData.getString("ecl", null);
if (code != null) {
eclCode += code;
}
}
logBasic("{Iterate Job} ECL Code =" + eclCode);
}
*/
result.setRows(list);
return result;
}
public Result execute(Result previousResult, int nr, Repository rep, Job parentJob) {
LogWriter log = LogWriter.getInstance();
Result result = previousResult;
result.setResult(false);
if (filename != null) {
String realFilename = getRealFilename();
try {
FileObject file = KettleVFS.getFileObject(realFilename);
if (file.exists() && file.isReadable()) {
log.logDetailed(
toString(),
Messages.getString("JobEntryFileExists.File_Exists", realFilename)); // $NON-NLS-1$
result.setResult(true);
} else {
log.logDetailed(
toString(),
Messages.getString(
"JobEntryFileExists.File_Does_Not_Exist", realFilename)); // $NON-NLS-1$
}
} catch (IOException e) {
result.setNrErrors(1);
log.logError(
toString(),
Messages.getString(
"JobEntryFileExists.ERROR_0004_IO_Exception", e.toString())); // $NON-NLS-1$
}
} else {
result.setNrErrors(1);
log.logError(
toString(),
Messages.getString("JobEntryFileExists.ERROR_0005_No_Filename_Defined")); // $NON-NLS-1$
}
return result;
}
public Object clone() {
try {
JobExecutionConfiguration configuration = (JobExecutionConfiguration) super.clone();
configuration.params = new HashMap<String, String>();
configuration.params.putAll(params);
configuration.arguments = new HashMap<String, String>();
configuration.arguments.putAll(arguments);
configuration.variables = new HashMap<String, String>();
configuration.variables.putAll(variables);
if (previousResult != null) {
configuration.previousResult = previousResult.clone();
}
return configuration;
} catch (CloneNotSupportedException e) {
return null;
}
}
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
result.setResult(false);
try {
// Get real variable value
String realServerName = environmentSubstitute(serverName);
int realServerPort = Const.toInt(environmentSubstitute(serverPort), 22);
String realUserName = environmentSubstitute(userName);
String realServerPassword =
Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(password));
// Proxy Host
String realProxyHost = environmentSubstitute(httpproxyhost);
int realProxyPort = Const.toInt(environmentSubstitute(httpproxyport), 22);
String realproxyUserName = environmentSubstitute(httpproxyusername);
String realProxyPassword =
Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(httpProxyPassword));
// Key file
String realKeyFilename = environmentSubstitute(keyFilename);
String relKeyFilepass = environmentSubstitute(keyFilePass);
// Source files
String realLocalDirectory = environmentSubstitute(localDirectory);
String realwildcard = environmentSubstitute(wildcard);
// Remote destination
String realftpDirectory = environmentSubstitute(ftpDirectory);
// Destination folder (Move to)
String realDestinationFolder = environmentSubstitute(destinationfolder);
try {
// Remote source
realftpDirectory = FTPUtils.normalizePath(realftpDirectory);
// Destination folder (Move to)
realDestinationFolder = FTPUtils.normalizePath(realDestinationFolder);
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.CanNotNormalizePath", e.getMessage()));
result.setNrErrors(1);
return result;
}
// Check for mandatory fields
boolean mandatoryok = true;
if (Const.isEmpty(realServerName)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.ServernameMissing"));
}
if (usehttpproxy) {
if (Const.isEmpty(realProxyHost)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.HttpProxyhostMissing"));
}
}
if (publicpublickey) {
if (Const.isEmpty(realKeyFilename)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.KeyFileMissing"));
} else {
// Let's check if folder exists...
if (!KettleVFS.fileExists(realKeyFilename, this)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.KeyFileNotExist"));
}
}
}
if (Const.isEmpty(realLocalDirectory)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.LocalFolderMissing"));
}
if (afterFtpPut.equals("move_file")) {
if (Const.isEmpty(realDestinationFolder)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.DestinatFolderMissing"));
} else {
FileObject folder = null;
try {
folder = KettleVFS.getFileObject(realDestinationFolder, this);
// Let's check if folder exists...
if (!folder.exists()) {
// Do we need to create it?
if (createDestinationFolder) {
folder.createFolder();
} else {
logError(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.DestinatFolderNotExist", realDestinationFolder));
}
}
} catch (Exception e) {
throw new KettleException(e);
} finally {
if (folder != null) {
try {
folder.close();
folder = null;
} catch (Exception e) {
/* Ignore */
}
}
}
}
}
if (mandatoryok) {
Connection conn = null;
SFTPv3Client client = null;
boolean good = true;
int nbfilestoput = 0;
int nbput = 0;
int nbrerror = 0;
try {
// Create a connection instance
conn =
getConnection(
realServerName,
realServerPort,
realProxyHost,
realProxyPort,
realproxyUserName,
realProxyPassword);
if (timeout > 0) {
// Use timeout
// Cache Host Key
if (cachehostkey) {
conn.connect(new SimpleVerifier(database), 0, timeout * 1000);
} else {
conn.connect(null, 0, timeout * 1000);
}
} else {
// Cache Host Key
if (cachehostkey) {
conn.connect(new SimpleVerifier(database));
} else {
conn.connect();
}
}
// Authenticate
boolean isAuthenticated = false;
if (publicpublickey) {
String keyContent =
KettleVFS.getTextFileContent(realKeyFilename, this, Const.XML_ENCODING);
isAuthenticated =
conn.authenticateWithPublicKey(
realUserName, keyContent.toCharArray(), relKeyFilepass);
} else {
isAuthenticated = conn.authenticateWithPassword(realUserName, realServerPassword);
}
// LET'S CHECK AUTHENTICATION ...
if (isAuthenticated == false) {
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.AuthenticationFailed"));
} else {
if (log.isBasic()) {
logBasic(
BaseMessages.getString(PKG, "JobSSH2PUT.Log.Connected", serverName, userName));
}
client = new SFTPv3Client(conn);
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.ProtocolVersion", "" + client.getProtocolVersion()));
}
// Check if remote directory exists
if (!Const.isEmpty(realftpDirectory)) {
if (!sshDirectoryExists(client, realftpDirectory)) {
good = false;
if (createRemoteFolder) {
good = CreateRemoteFolder(client, realftpDirectory);
if (good) {
logBasic(BaseMessages.getString(PKG, "JobSSH2PUT.Log.RemoteDirectoryCreated"));
}
} else {
logError(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.RemoteDirectoryNotExist", realftpDirectory));
}
} else if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.RemoteDirectoryExist", realftpDirectory));
}
}
if (good) {
// Get files list from local folder (source)
List<FileObject> myFileList = getFiles(realLocalDirectory);
// Prepare Pattern for wildcard
Pattern pattern = null;
if (!Const.isEmpty(realwildcard)) {
pattern = Pattern.compile(realwildcard);
}
// Let's put files now ...
// Get the files in the list
for (int i = 0; i < myFileList.size() && !parentJob.isStopped(); i++) {
FileObject myFile = myFileList.get(i);
String localFilename = myFile.toString();
String remoteFilename = myFile.getName().getBaseName();
boolean getIt = true;
// First see if the file matches the regular expression!
if (pattern != null) {
Matcher matcher = pattern.matcher(remoteFilename);
getIt = matcher.matches();
}
// do we have a target directory?
if (!Const.isEmpty(realftpDirectory)) {
remoteFilename = realftpDirectory + FTPUtils.FILE_SEPARATOR + remoteFilename;
}
if (onlyGettingNewFiles) {
// We get only new files
// ie not exist on the remote server
getIt = !sshFileExists(client, remoteFilename);
}
if (getIt) {
nbfilestoput++;
boolean putok = putFile(myFile, remoteFilename, client);
if (!putok) {
nbrerror++;
logError(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.Error.CanNotPutFile", localFilename));
} else {
nbput++;
}
if (putok && !afterFtpPut.equals("do_nothing")) {
deleteOrMoveFiles(myFile, realDestinationFolder);
}
}
}
/** ****************************** RESULT ******************* */
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.JobEntryEnd1"));
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.Result.TotalFiles", "" + nbfilestoput));
logDetailed(
BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.TotalFilesPut", "" + nbput));
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.Result.TotalFilesError", "" + nbrerror));
logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.JobEntryEnd2"));
}
if (nbrerror == 0) {
result.setResult(true);
/** ****************************** RESULT ******************* */
}
}
}
} catch (Exception e) {
result.setNrErrors(nbrerror);
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Error.ErrorFTP", e.getMessage()));
} finally {
if (conn != null) {
conn.close();
}
if (client != null) {
client.close();
}
}
}
} catch (Exception e) {
result.setResult(false);
result.setNrErrors(1L);
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Error.UnexpectedError"), e);
}
return result;
}
protected boolean SQLDataOK(
Result result,
long nrRowsLimit,
String realSchemaName,
String realTableName,
String customSQL)
throws KettleException {
String countStatement = null;
long rowsCount = 0;
boolean successOK = false;
List<Object[]> ar = null;
RowMetaInterface rowMeta = null;
Database db = new Database(this, connection);
db.shareVariablesWith(this);
try {
db.connect(parentJob.getTransactionId(), null);
if (iscustomSQL) {
countStatement = customSQL;
} else {
if (!Const.isEmpty(realSchemaName)) {
countStatement =
selectCount
+ db.getDatabaseMeta()
.getQuotedSchemaTableCombination(realSchemaName, realTableName);
} else {
countStatement = selectCount + db.getDatabaseMeta().quoteField(realTableName);
}
}
if (countStatement != null) {
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntryWaitForSQL.Log.RunSQLStatement", countStatement));
}
if (iscustomSQL) {
ar = db.getRows(countStatement, 0);
if (ar != null) {
rowsCount = ar.size();
} else {
if (log.isDebug()) {
logDebug(
BaseMessages.getString(
PKG, "JobEntryWaitForSQL.Log.customSQLreturnedNothing", countStatement));
}
}
} else {
RowMetaAndData row = db.getOneRow(countStatement);
if (row != null) {
rowsCount = row.getInteger(0);
}
}
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(PKG, "JobEntryWaitForSQL.Log.NrRowsReturned", "" + rowsCount));
}
switch (successCondition) {
case JobEntryWaitForSQL.SUCCESS_CONDITION_ROWS_COUNT_EQUAL:
successOK = (rowsCount == nrRowsLimit);
break;
case JobEntryWaitForSQL.SUCCESS_CONDITION_ROWS_COUNT_DIFFERENT:
successOK = (rowsCount != nrRowsLimit);
break;
case JobEntryWaitForSQL.SUCCESS_CONDITION_ROWS_COUNT_SMALLER:
successOK = (rowsCount < nrRowsLimit);
break;
case JobEntryWaitForSQL.SUCCESS_CONDITION_ROWS_COUNT_SMALLER_EQUAL:
successOK = (rowsCount <= nrRowsLimit);
break;
case JobEntryWaitForSQL.SUCCESS_CONDITION_ROWS_COUNT_GREATER:
successOK = (rowsCount > nrRowsLimit);
break;
case JobEntryWaitForSQL.SUCCESS_CONDITION_ROWS_COUNT_GREATER_EQUAL:
successOK = (rowsCount >= nrRowsLimit);
break;
default:
break;
}
} // end if countStatement!=null
} catch (KettleDatabaseException dbe) {
logError(
BaseMessages.getString(PKG, "JobEntryWaitForSQL.Error.RunningEntry", dbe.getMessage()));
} finally {
if (db != null) {
if (isAddRowsResult && iscustomSQL && ar != null) {
rowMeta = db.getQueryFields(countStatement, false);
}
db.disconnect();
}
}
if (successOK) {
// ad rows to result
if (isAddRowsResult && iscustomSQL && ar != null) {
List<RowMetaAndData> rows = new ArrayList<RowMetaAndData>();
for (int i = 0; i < ar.size(); i++) {
rows.add(new RowMetaAndData(rowMeta, ar.get(i)));
}
if (rows != null) {
result.getRows().addAll(rows);
}
}
}
return successOK;
}
public Result execute(Result previousResult, int nr) {
String LimitNbrLignes = "";
String ListOfColumn = "*";
String strHighPriority = "";
String OutDumpText = "";
String OptionEnclosed = "";
String FieldSeparator = "";
String LinesTerminated = "";
Result result = previousResult;
result.setResult(false);
// Let's check the filename ...
if (filename != null) {
// User has specified a file, We can continue ...
String realFilename = getRealFilename();
File file = new File(realFilename);
if (file.exists() && iffileexists == 2) {
// the file exists and user want to Fail
result.setResult(false);
result.setNrErrors(1);
logError(
BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists1.Label")
+ realFilename
+ BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists2.Label"));
} else if (file.exists() && iffileexists == 1) {
// the file exists and user want to do nothing
result.setResult(true);
if (log.isDetailed())
logDetailed(
BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists1.Label")
+ realFilename
+ BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists2.Label"));
} else {
if (file.exists() && iffileexists == 0) {
// File exists and user want to renamme it with unique name
// Format Date
// Try to clean filename (without wildcard)
String wildcard =
realFilename.substring(realFilename.length() - 4, realFilename.length());
if (wildcard.substring(0, 1).equals(".")) {
// Find wildcard
realFilename =
realFilename.substring(0, realFilename.length() - 4)
+ "_"
+ StringUtil.getFormattedDateTimeNow(true)
+ wildcard;
} else {
// did not find wildcard
realFilename = realFilename + "_" + StringUtil.getFormattedDateTimeNow(true);
}
logDebug(
BaseMessages.getString(PKG, "JobMysqlBulkFile.FileNameChange1.Label")
+ realFilename
+ BaseMessages.getString(PKG, "JobMysqlBulkFile.FileNameChange1.Label"));
}
// User has specified an existing file, We can continue ...
if (log.isDetailed())
logDetailed(
BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists1.Label")
+ realFilename
+ BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists2.Label"));
if (connection != null) {
// User has specified a connection, We can continue ...
Database db = new Database(this, connection);
db.shareVariablesWith(this);
try {
db.connect(parentJob.getTransactionId(), null);
// Get schemaname
String realSchemaname = environmentSubstitute(schemaname);
// Get tablename
String realTablename = environmentSubstitute(tablename);
if (db.checkTableExists(realTablename)) {
// The table existe, We can continue ...
if (log.isDetailed())
logDetailed(
BaseMessages.getString(PKG, "JobMysqlBulkFile.TableExists1.Label")
+ realTablename
+ BaseMessages.getString(PKG, "JobMysqlBulkFile.TableExists2.Label"));
// Add schemaname (Most the time Schemaname.Tablename)
if (schemaname != null) {
realTablename = realSchemaname + "." + realTablename;
}
// Set the Limit lines
if (Const.toInt(getRealLimitlines(), 0) > 0) {
LimitNbrLignes = "LIMIT " + getRealLimitlines();
}
// Set list of Column, if null get all columns (*)
if (getRealListColumn() != null) {
ListOfColumn = MysqlString(getRealListColumn());
}
// Fields separator
if (getRealSeparator() != null && outdumpvalue == 0) {
FieldSeparator =
"FIELDS TERMINATED BY '" + Const.replace(getRealSeparator(), "'", "''") + "'";
}
// Lines Terminated by
if (getRealLineterminated() != null && outdumpvalue == 0) {
LinesTerminated =
"LINES TERMINATED BY '"
+ Const.replace(getRealLineterminated(), "'", "''")
+ "'";
}
// High Priority ?
if (isHighPriority()) {
strHighPriority = "HIGH_PRIORITY";
}
if (getRealEnclosed() != null && outdumpvalue == 0) {
if (isOptionEnclosed()) {
OptionEnclosed = "OPTIONALLY ";
}
OptionEnclosed =
OptionEnclosed
+ "ENCLOSED BY '"
+ Const.replace(getRealEnclosed(), "'", "''")
+ "'";
}
// OutFile or Dumpfile
if (outdumpvalue == 0) {
OutDumpText = "INTO OUTFILE";
} else {
OutDumpText = "INTO DUMPFILE";
}
String FILEBulkFile =
"SELECT "
+ strHighPriority
+ " "
+ ListOfColumn
+ " "
+ OutDumpText
+ " '"
+ realFilename
+ "' "
+ FieldSeparator
+ " "
+ OptionEnclosed
+ " "
+ LinesTerminated
+ " FROM "
+ realTablename
+ " "
+ LimitNbrLignes
+ " LOCK IN SHARE MODE";
try {
if (log.isDetailed()) logDetailed(FILEBulkFile);
// Run the SQL
PreparedStatement ps = db.prepareSQL(FILEBulkFile);
ps.execute();
// Everything is OK...we can disconnect now
db.disconnect();
if (isAddFileToResult()) {
// Add filename to output files
ResultFile resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL,
KettleVFS.getFileObject(realFilename, this),
parentJob.getJobname(),
toString());
result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
}
result.setResult(true);
} catch (SQLException je) {
db.disconnect();
result.setNrErrors(1);
logError(
BaseMessages.getString(PKG, "JobMysqlBulkFile.Error.Label")
+ " "
+ je.getMessage());
} catch (KettleFileException e) {
logError(
BaseMessages.getString(PKG, "JobMysqlBulkFile.Error.Label") + e.getMessage());
result.setNrErrors(1);
}
} else {
// Of course, the table should have been created already before the bulk load
// operation
db.disconnect();
result.setNrErrors(1);
if (log.isDetailed())
logDetailed(
BaseMessages.getString(PKG, "JobMysqlBulkFile.TableNotExists1.Label")
+ realTablename
+ BaseMessages.getString(PKG, "JobMysqlBulkFile.TableNotExists2.Label"));
}
} catch (KettleDatabaseException dbe) {
db.disconnect();
result.setNrErrors(1);
logError(
BaseMessages.getString(PKG, "JobMysqlBulkFile.Error.Label")
+ " "
+ dbe.getMessage());
}
} else {
// No database connection is defined
result.setNrErrors(1);
logError(BaseMessages.getString(PKG, "JobMysqlBulkFile.Nodatabase.Label"));
}
}
} else {
// No file was specified
result.setNrErrors(1);
logError(BaseMessages.getString(PKG, "JobMysqlBulkFile.Nofilename.Label"));
}
return result;
}
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
result.setNrErrors(1);
result.setResult(false);
String hostname = getRealHostname();
int timeoutInt = Const.toInt(getRealTimeOut(), 300);
int packets = Const.toInt(getRealNbrPackets(), 2);
boolean status = false;
if (Const.isEmpty(hostname)) {
// No Host was specified
logError(BaseMessages.getString(PKG, "JobPing.SpecifyHost.Label"));
return result;
}
try {
if (ipingtype == isystemPing || ipingtype == ibothPings) {
// Perform a system (Java) ping ...
status = systemPing(hostname, timeoutInt);
if (status) {
if (log.isDetailed()) {
log.logDetailed(
BaseMessages.getString(PKG, "JobPing.SystemPing"),
BaseMessages.getString(PKG, "JobPing.OK.Label", hostname));
}
} else {
log.logError(
BaseMessages.getString(PKG, "JobPing.SystemPing"),
BaseMessages.getString(PKG, "JobPing.NOK.Label", hostname));
}
}
if ((ipingtype == iclassicPing) || (ipingtype == ibothPings && !status)) {
// Perform a classic ping ..
status = classicPing(hostname, packets);
if (status) {
if (log.isDetailed()) {
log.logDetailed(
BaseMessages.getString(PKG, "JobPing.ClassicPing"),
BaseMessages.getString(PKG, "JobPing.OK.Label", hostname));
}
} else {
log.logError(
BaseMessages.getString(PKG, "JobPing.ClassicPing"),
BaseMessages.getString(PKG, "JobPing.NOK.Label", hostname));
}
}
} catch (Exception ex) {
logError(BaseMessages.getString(PKG, "JobPing.Error.Label") + ex.getMessage());
}
if (status) {
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobPing.OK.Label", hostname));
}
result.setNrErrors(0);
result.setResult(true);
} else {
logError(BaseMessages.getString(PKG, "JobPing.NOK.Label", hostname));
}
return result;
}
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
result.setResult(false);
// starttime (in seconds)
long timeStart = System.currentTimeMillis() / 1000;
if (filename != null) {
FileObject fileObject = null;
String realFilename = getRealFilename();
try {
fileObject = KettleVFS.getFileObject(realFilename, this);
long iMaximumTimeout =
Const.toInt(getRealMaximumTimeout(), Const.toInt(DEFAULT_MAXIMUM_TIMEOUT, 0));
long iCycleTime =
Const.toInt(getRealCheckCycleTime(), Const.toInt(DEFAULT_CHECK_CYCLE_TIME, 0));
//
// Sanity check on some values, and complain on insanity
//
if (iMaximumTimeout < 0) {
iMaximumTimeout = Const.toInt(DEFAULT_MAXIMUM_TIMEOUT, 0);
if (log.isBasic()) logBasic("Maximum timeout invalid, reset to " + iMaximumTimeout);
}
if (iCycleTime < 1) {
// If lower than 1 set to the default
iCycleTime = Const.toInt(DEFAULT_CHECK_CYCLE_TIME, 1);
if (log.isBasic()) logBasic("Check cycle time invalid, reset to " + iCycleTime);
}
if (iMaximumTimeout == 0) {
if (log.isBasic()) logBasic("Waiting indefinitely for file [" + realFilename + "]");
} else {
if (log.isBasic())
logBasic("Waiting " + iMaximumTimeout + " seconds for file [" + realFilename + "]");
}
boolean continueLoop = true;
while (continueLoop && !parentJob.isStopped()) {
fileObject = KettleVFS.getFileObject(realFilename, this);
if (fileObject.exists()) {
// file exists, we're happy to exit
if (log.isBasic()) logBasic("Detected file [" + realFilename + "] within timeout");
result.setResult(true);
continueLoop = false;
// add filename to result filenames
if (addFilenameToResult && fileObject.getType() == FileType.FILE) {
ResultFile resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL, fileObject, parentJob.getJobname(), toString());
resultFile.setComment(BaseMessages.getString(PKG, "JobWaitForFile.FilenameAdded"));
result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
}
} else {
long now = System.currentTimeMillis() / 1000;
if ((iMaximumTimeout > 0) && (now > (timeStart + iMaximumTimeout))) {
continueLoop = false;
// file doesn't exist after timeout, either true or false
if (isSuccessOnTimeout()) {
if (log.isBasic())
logBasic("Didn't detect file [" + realFilename + "] before timeout, success");
result.setResult(true);
} else {
if (log.isBasic())
logBasic("Didn't detect file [" + realFilename + "] before timeout, failure");
result.setResult(false);
}
}
// sleep algorithm
long sleepTime = 0;
if (iMaximumTimeout == 0) {
sleepTime = iCycleTime;
} else {
if ((now + iCycleTime) < (timeStart + iMaximumTimeout)) {
sleepTime = iCycleTime;
} else {
sleepTime = iCycleTime - ((now + iCycleTime) - (timeStart + iMaximumTimeout));
}
}
try {
if (sleepTime > 0) {
if (log.isDetailed()) {
logDetailed(
"Sleeping "
+ sleepTime
+ " seconds before next check for file ["
+ realFilename
+ "]");
}
Thread.sleep(sleepTime * 1000);
}
} catch (InterruptedException e) {
// something strange happened
result.setResult(false);
continueLoop = false;
}
}
}
if (!parentJob.isStopped() && fileObject.exists() && isFileSizeCheck()) {
long oldSize = -1;
long newSize = fileObject.getContent().getSize();
if (log.isDetailed())
logDetailed("File [" + realFilename + "] is " + newSize + " bytes long");
if (log.isBasic())
logBasic(
"Waiting until file ["
+ realFilename
+ "] stops growing for "
+ iCycleTime
+ " seconds");
while (oldSize != newSize && !parentJob.isStopped()) {
try {
if (log.isDetailed()) {
logDetailed(
"Sleeping "
+ iCycleTime
+ " seconds, waiting for file ["
+ realFilename
+ "] to stop growing");
}
Thread.sleep(iCycleTime * 1000);
} catch (InterruptedException e) {
// something strange happened
result.setResult(false);
continueLoop = false;
}
oldSize = newSize;
newSize = fileObject.getContent().getSize();
if (log.isDetailed()) {
logDetailed("File [" + realFilename + "] is " + newSize + " bytes long");
}
}
if (log.isBasic())
logBasic("Stopped waiting for file [" + realFilename + "] to stop growing");
}
if (parentJob.isStopped()) {
result.setResult(false);
}
} catch (Exception e) {
logBasic("Exception while waiting for file [" + realFilename + "] to stop growing", e);
} finally {
if (fileObject != null) {
try {
fileObject.close();
} catch (Exception e) {
}
}
}
} else {
logError("No filename is defined.");
}
return result;
}
private boolean processOneXMLFile(
String xmlfilename, String xslfilename, String outputfilename, Result result, Job parentJob) {
boolean retval = false;
FileObject xmlfile = null;
FileObject xslfile = null;
FileObject outputfile = null;
try {
xmlfile = KettleVFS.getFileObject(xmlfilename, this);
xslfile = KettleVFS.getFileObject(xslfilename, this);
outputfile = KettleVFS.getFileObject(outputfilename, this);
if (xmlfile.exists() && xslfile.exists()) {
if (outputfile.exists() && iffileexists == 2) {
// Output file exists
// User want to fail
logError(
BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists1.Label")
+ outputfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists2.Label"));
return retval;
} else if (outputfile.exists() && iffileexists == 1) {
// Do nothing
if (log.isDebug()) {
logDebug(
BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists1.Label")
+ outputfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists2.Label"));
}
retval = true;
return retval;
} else {
if (outputfile.exists() && iffileexists == 0) {
// the output file exists and user want to create new one with unique name
// Format Date
// Try to clean filename (without wildcard)
String wildcard =
outputfilename.substring(outputfilename.length() - 4, outputfilename.length());
if (wildcard.substring(0, 1).equals(".")) {
// Find wildcard
outputfilename =
outputfilename.substring(0, outputfilename.length() - 4)
+ "_"
+ StringUtil.getFormattedDateTimeNow(true)
+ wildcard;
} else {
// did not find wildcard
outputfilename = outputfilename + "_" + StringUtil.getFormattedDateTimeNow(true);
}
if (log.isDebug()) {
logDebug(
BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists1.Label")
+ outputfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists2.Label"));
logDebug(
BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileNameChange1.Label")
+ outputfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileNameChange2.Label"));
}
}
// Create transformer factory
TransformerFactory factory = TransformerFactory.newInstance();
if (xsltfactory.equals(FACTORY_SAXON)) {
// Set the TransformerFactory to the SAXON implementation.
factory = new net.sf.saxon.TransformerFactoryImpl();
}
if (log.isDetailed()) {
log.logDetailed(
BaseMessages.getString(PKG, "JobEntryXSL.Log.TransformerFactoryInfos"),
BaseMessages.getString(
PKG, "JobEntryXSL.Log.TransformerFactory", factory.getClass().getName()));
}
InputStream xslInputStream = KettleVFS.getInputStream(xslfile);
InputStream xmlInputStream = KettleVFS.getInputStream(xmlfile);
OutputStream os = null;
try {
// Use the factory to create a template containing the xsl file
Templates template = factory.newTemplates(new StreamSource(xslInputStream));
// Use the template to create a transformer
Transformer xformer = template.newTransformer();
if (log.isDetailed()) {
log.logDetailed(
BaseMessages.getString(PKG, "JobEntryXSL.Log.TransformerClassInfos"),
BaseMessages.getString(
PKG, "JobEntryXSL.Log.TransformerClass", xformer.getClass().getName()));
}
// Do we need to set output properties?
if (setOutputProperties) {
xformer.setOutputProperties(outputProperties);
}
// Do we need to pass parameters?
if (useParameters) {
for (int i = 0; i < nrParams; i++) {
xformer.setParameter(nameOfParams[i], valueOfParams[i]);
}
}
// Prepare the input and output files
Source source = new StreamSource(xmlInputStream);
os = KettleVFS.getOutputStream(outputfile, false);
StreamResult resultat = new StreamResult(os);
// Apply the xsl file to the source file and write the result to the output file
xformer.transform(source, resultat);
if (isAddFileToResult()) {
// Add output filename to output files
ResultFile resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL,
KettleVFS.getFileObject(outputfilename, this),
parentJob.getJobname(),
toString());
result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
}
// Everything is OK
retval = true;
} finally {
try {
xslInputStream.close();
} catch (IOException ignored) {
// ignore IO Exception on close
}
try {
xmlInputStream.close();
} catch (IOException ignored) {
// ignore IO Exception on close
}
try {
if (os != null) {
os.close();
}
} catch (IOException ignored) {
// ignore IO Exception on close
}
}
}
} else {
if (!xmlfile.exists()) {
logError(
BaseMessages.getString(PKG, "JobEntryXSLT.FileDoesNotExist1.Label")
+ xmlfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.FileDoesNotExist2.Label"));
}
if (!xslfile.exists()) {
logError(
BaseMessages.getString(PKG, "JobEntryXSLT.FileDoesNotExist1.Label")
+ xmlfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.FileDoesNotExist2.Label"));
}
}
} catch (Exception e) {
logError(
BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLST.Label")
+ BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLSTXML1.Label")
+ xmlfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLSTXML2.Label")
+ BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLSTXSL1.Label")
+ xslfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLSTXSL2.Label")
+ e.getMessage());
} finally {
try {
if (xmlfile != null) {
xmlfile.close();
}
if (xslfile != null) {
xslfile.close();
}
if (outputfile != null) {
outputfile.close();
}
} catch (IOException e) {
logError("Unable to close file", e);
}
}
return retval;
}
public Result execute(Result previousResult, int nr) throws KettleException {
Result result = previousResult;
int NrErrors = 0;
int NrSuccess = 0;
// Check output parameters
int nrOutputProps = getOutputPropertyName() == null ? 0 : getOutputPropertyName().length;
if (nrOutputProps > 0) {
outputProperties = new Properties();
for (int i = 0; i < nrOutputProps; i++) {
outputProperties.put(
getOutputPropertyName()[i], environmentSubstitute(getOutputPropertyValue()[i]));
}
setOutputProperties = true;
}
// Check parameters
nrParams = getParameterField() == null ? 0 : getParameterField().length;
if (nrParams > 0) {
nameOfParams = new String[nrParams];
valueOfParams = new String[nrParams];
for (int i = 0; i < nrParams; i++) {
String name = environmentSubstitute(getParameterName()[i]);
String value = environmentSubstitute(getParameterField()[i]);
if (Const.isEmpty(value)) {
throw new KettleStepException(
BaseMessages.getString(PKG, "Xslt.Exception.ParameterFieldMissing", name, i));
}
nameOfParams[i] = name;
valueOfParams[i] = value;
}
useParameters = true;
}
List<RowMetaAndData> rows = result.getRows();
if (isFilenamesFromPrevious()) {
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG,
"JobEntryXSLT.Log.ArgFromPrevious.Found",
(rows != null ? rows.size() : 0) + ""));
}
}
if (isFilenamesFromPrevious() && rows != null) {
// Copy the input row to the (command line) arguments
RowMetaAndData resultRow = null;
for (int iteration = 0; iteration < rows.size() && !parentJob.isStopped(); iteration++) {
resultRow = rows.get(iteration);
// Get filenames (xml, xsl, output filename)
String xmlfilename_previous = resultRow.getString(0, null);
String xslfilename_previous = resultRow.getString(1, null);
String ouputfilename_previous = resultRow.getString(2, null);
if (!Const.isEmpty(xmlfilename_previous)
&& !Const.isEmpty(xslfilename_previous)
&& !Const.isEmpty(ouputfilename_previous)) {
if (processOneXMLFile(
xmlfilename_previous,
xslfilename_previous,
ouputfilename_previous,
result,
parentJob)) {
NrSuccess++;
} else {
NrErrors++;
}
} else {
// We failed!
logError(BaseMessages.getString(PKG, "JobEntryXSLT.AllFilesNotNull.Label"));
NrErrors++;
}
}
} else {
String realxmlfilename = getRealxmlfilename();
String realxslfilename = getRealxslfilename();
String realoutputfilename = getoutputfilename();
if (!Const.isEmpty(realxmlfilename)
&& !Const.isEmpty(realxslfilename)
&& !Const.isEmpty(realoutputfilename)) {
if (processOneXMLFile(
realxmlfilename, realxslfilename, realoutputfilename, result, parentJob)) {
NrSuccess++;
} else {
NrErrors++;
}
} else {
// We failed!
logError(BaseMessages.getString(PKG, "JobEntryXSLT.AllFilesNotNull.Label"));
NrErrors++;
}
}
result.setResult(NrErrors == 0);
result.setNrErrors(NrErrors);
result.setNrLinesWritten(NrSuccess);
return result;
}
public String getXML() throws IOException {
StringBuilder xml = new StringBuilder(160);
xml.append(" <" + XML_TAG + ">").append(Const.CR);
xml.append(" ").append(XMLHandler.addTagValue("exec_local", executingLocally));
xml.append(" ").append(XMLHandler.addTagValue("exec_remote", executingRemotely));
if (remoteServer != null) {
xml.append(" ").append(remoteServer.getXML()).append(Const.CR);
}
xml.append(" ").append(XMLHandler.addTagValue("pass_export", passingExport));
// Serialize the parameters...
//
xml.append(" <parameters>").append(Const.CR);
List<String> paramNames = new ArrayList<String>(params.keySet());
Collections.sort(paramNames);
for (String name : paramNames) {
String value = params.get(name);
xml.append(" <parameter>");
xml.append(XMLHandler.addTagValue("name", name, false));
xml.append(XMLHandler.addTagValue("value", value, false));
xml.append("</parameter>").append(Const.CR);
}
xml.append(" </parameters>").append(Const.CR);
// Serialize the variables...
//
xml.append(" <variables>").append(Const.CR);
List<String> variableNames = new ArrayList<String>(variables.keySet());
Collections.sort(variableNames);
for (String name : variableNames) {
String value = variables.get(name);
xml.append(" <variable>");
xml.append(XMLHandler.addTagValue("name", name, false));
xml.append(XMLHandler.addTagValue("value", value, false));
xml.append("</variable>").append(Const.CR);
}
xml.append(" </variables>").append(Const.CR);
// Serialize the variables...
//
xml.append(" <arguments>").append(Const.CR);
List<String> argumentNames = new ArrayList<String>(arguments.keySet());
Collections.sort(argumentNames);
for (String name : argumentNames) {
String value = arguments.get(name);
xml.append(" <argument>");
xml.append(XMLHandler.addTagValue("name", name, false));
xml.append(XMLHandler.addTagValue("value", value, false));
xml.append("</argument>").append(Const.CR);
}
xml.append(" </arguments>").append(Const.CR);
xml.append(" ").append(XMLHandler.addTagValue("replay_date", replayDate));
xml.append(" ").append(XMLHandler.addTagValue("safe_mode", safeModeEnabled));
xml.append(" ").append(XMLHandler.addTagValue("log_level", logLevel.getCode()));
xml.append(" ").append(XMLHandler.addTagValue("clear_log", clearingLog));
xml.append(" ").append(XMLHandler.addTagValue("start_copy_name", startCopyName));
xml.append(" ").append(XMLHandler.addTagValue("start_copy_nr", startCopyNr));
xml.append(" ").append(XMLHandler.addTagValue("gather_metrics", gatheringMetrics));
xml.append(" ").append(XMLHandler.addTagValue("expand_remote_job", expandingRemoteJob));
if (passedBatchId != null) {
xml.append(" ").append(XMLHandler.addTagValue("passedBatchId", passedBatchId));
}
// The source rows...
//
if (previousResult != null) {
xml.append(previousResult.getXML());
}
// Send the repository name and user to the remote site...
//
if (repository != null) {
xml.append(XMLHandler.openTag("repository"));
xml.append(XMLHandler.addTagValue("name", repository.getName()));
// File base repositories doesn't have user info
if (repository.getUserInfo() != null) {
xml.append(XMLHandler.addTagValue("login", repository.getUserInfo().getLogin()));
xml.append(
XMLHandler.addTagValue(
"password", Encr.encryptPassword(repository.getUserInfo().getPassword())));
}
xml.append(XMLHandler.closeTag("repository"));
}
xml.append("</" + XML_TAG + ">").append(Const.CR);
return xml.toString();
}
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
result.setNrErrors(1);
result.setResult(false);
String servername = environmentSubstitute(serverName);
int nrPort = Const.toInt(environmentSubstitute("" + port), DEFAULT_PORT);
String Oid = environmentSubstitute(oid);
int timeOut = Const.toInt(environmentSubstitute("" + timeout), DEFAULT_TIME_OUT);
int retry = Const.toInt(environmentSubstitute("" + nrretry), 1);
String messageString = environmentSubstitute(message);
Snmp snmp = null;
try {
TransportMapping transMap = new DefaultUdpTransportMapping();
snmp = new Snmp(transMap);
UdpAddress udpAddress = new UdpAddress(InetAddress.getByName(servername), nrPort);
ResponseEvent response = null;
if (targettype.equals(target_type_Code[0])) {
// Community target
String community = environmentSubstitute(comString);
CommunityTarget target = new CommunityTarget();
PDUv1 pdu1 = new PDUv1();
transMap.listen();
target.setCommunity(new OctetString(community));
target.setVersion(SnmpConstants.version1);
target.setAddress(udpAddress);
if (target.getAddress().isValid()) {
if (log.isDebug()) {
logDebug("Valid IP address");
}
} else {
throw new KettleException("Invalid IP address");
}
target.setRetries(retry);
target.setTimeout(timeOut);
// create the PDU
pdu1.setGenericTrap(6);
pdu1.setSpecificTrap(PDUv1.ENTERPRISE_SPECIFIC);
pdu1.setEnterprise(new OID(Oid));
pdu1.add(new VariableBinding(new OID(Oid), new OctetString(messageString)));
response = snmp.send(pdu1, target);
} else {
// User target
String userName = environmentSubstitute(user);
String passPhrase = environmentSubstitute(passphrase);
String engineID = environmentSubstitute(engineid);
UserTarget usertarget = new UserTarget();
transMap.listen();
usertarget.setAddress(udpAddress);
if (usertarget.getAddress().isValid()) {
if (log.isDebug()) {
logDebug("Valid IP address");
}
} else {
throw new KettleException("Invalid IP address");
}
usertarget.setRetries(retry);
usertarget.setTimeout(timeOut);
usertarget.setVersion(SnmpConstants.version3);
usertarget.setSecurityLevel(SecurityLevel.AUTH_PRIV);
usertarget.setSecurityName(new OctetString("MD5DES"));
// Since we are using SNMPv3 we use authenticated users
// this is handled by the UsmUser and USM class
UsmUser uu =
new UsmUser(
new OctetString(userName),
AuthMD5.ID,
new OctetString(passPhrase),
PrivDES.ID,
new OctetString(passPhrase));
USM usm = snmp.getUSM();
if (usm == null) {
throw new KettleException("Null Usm");
} else {
usm =
new USM(
SecurityProtocols.getInstance(), new OctetString(MPv3.createLocalEngineID()), 0);
usm.addUser(new OctetString(userName), uu);
if (log.isDebug()) {
logDebug("Valid Usm");
}
}
// create the PDU
ScopedPDU pdu = new ScopedPDU();
pdu.add(new VariableBinding(new OID(Oid), new OctetString(messageString)));
pdu.setType(PDU.TRAP);
if (!Const.isEmpty(engineID)) {
pdu.setContextEngineID(new OctetString(engineID));
}
// send the PDU
response = snmp.send(pdu, usertarget);
}
if (response != null) {
if (log.isDebug()) {
logDebug("Received response from: " + response.getPeerAddress() + response.toString());
}
}
result.setNrErrors(0);
result.setResult(true);
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "JobEntrySNMPTrap.ErrorGetting", e.getMessage()));
} finally {
try {
if (snmp != null) {
snmp.close();
}
} catch (Exception e) {
/* Ignore */
}
}
return result;
}
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
result.setResult(false);
// see PDI-10270, PDI-10644 for details
boolean oldBehavior =
"Y"
.equalsIgnoreCase(
getVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "N"));
String countSQLStatement = null;
long rowsCount = 0;
long errCount = 0;
boolean successOK = false;
int nrRowsLimit = Const.toInt(environmentSubstitute(limit), 0);
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntryEvalTableContent.Log.nrRowsLimit", "" + nrRowsLimit));
}
if (connection != null) {
Database db = new Database(this, connection);
db.shareVariablesWith(this);
try {
db.connect(parentJob.getTransactionId(), null);
if (iscustomSQL) {
String realCustomSQL = customSQL;
if (isUseVars) {
realCustomSQL = environmentSubstitute(realCustomSQL);
}
if (log.isDebug()) {
logDebug(
BaseMessages.getString(
PKG, "JobEntryEvalTableContent.Log.EnteredCustomSQL", realCustomSQL));
}
if (!Const.isEmpty(realCustomSQL)) {
countSQLStatement = realCustomSQL;
} else {
errCount++;
logError(BaseMessages.getString(PKG, "JobEntryEvalTableContent.Error.NoCustomSQL"));
}
} else {
String realTablename = environmentSubstitute(tablename);
String realSchemaname = environmentSubstitute(schemaname);
if (!Const.isEmpty(realTablename)) {
if (!Const.isEmpty(realSchemaname)) {
countSQLStatement =
selectCount
+ db.getDatabaseMeta()
.getQuotedSchemaTableCombination(realSchemaname, realTablename);
} else {
countSQLStatement = selectCount + db.getDatabaseMeta().quoteField(realTablename);
}
} else {
errCount++;
logError(BaseMessages.getString(PKG, "JobEntryEvalTableContent.Error.NoTableName"));
}
}
if (countSQLStatement != null) {
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntryEvalTableContent.Log.RunSQLStatement", countSQLStatement));
}
if (iscustomSQL) {
if (isClearResultList) {
result.getRows().clear();
}
List<Object[]> ar = db.getRows(countSQLStatement, 0);
if (ar != null) {
rowsCount = ar.size();
// ad rows to result
RowMetaInterface rowMeta = db.getQueryFields(countSQLStatement, false);
List<RowMetaAndData> rows = new ArrayList<RowMetaAndData>();
for (int i = 0; i < ar.size(); i++) {
rows.add(new RowMetaAndData(rowMeta, ar.get(i)));
}
if (isAddRowsResult && iscustomSQL) {
if (rows != null) {
result.getRows().addAll(rows);
}
}
} else {
if (log.isDebug()) {
logDebug(
BaseMessages.getString(
PKG,
"JobEntryEvalTableContent.Log.customSQLreturnedNothing",
countSQLStatement));
}
}
} else {
RowMetaAndData row = db.getOneRow(countSQLStatement);
if (row != null) {
rowsCount = row.getInteger(0);
}
}
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntryEvalTableContent.Log.NrRowsReturned", "" + rowsCount));
}
switch (successCondition) {
case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_EQUAL:
successOK = (rowsCount == nrRowsLimit);
break;
case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_DIFFERENT:
successOK = (rowsCount != nrRowsLimit);
break;
case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_SMALLER:
successOK = (rowsCount < nrRowsLimit);
break;
case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_SMALLER_EQUAL:
successOK = (rowsCount <= nrRowsLimit);
break;
case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_GREATER:
successOK = (rowsCount > nrRowsLimit);
break;
case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_GREATER_EQUAL:
successOK = (rowsCount >= nrRowsLimit);
break;
default:
break;
}
if (!successOK && oldBehavior) {
errCount++;
}
} // end if countSQLStatement!=null
} catch (KettleException dbe) {
errCount++;
logError(
BaseMessages.getString(
PKG, "JobEntryEvalTableContent.Error.RunningEntry", dbe.getMessage()));
} finally {
if (db != null) {
db.disconnect();
}
}
} else {
errCount++;
logError(BaseMessages.getString(PKG, "JobEntryEvalTableContent.NoDbConnection"));
}
result.setResult(successOK);
result.setNrLinesRead(rowsCount);
result.setNrErrors(errCount);
return result;
}
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
result.setResult(false);
if (log.isRowLevel()) {
logRowlevel(BaseMessages.getString(PKG, "JobSSH2GET.Log.GettingFieldsValue"));
}
// Get real variable value
String realServerName = environmentSubstitute(serverName);
int realServerPort = Const.toInt(environmentSubstitute(serverPort), 22);
String realUserName = environmentSubstitute(userName);
String realServerPassword =
Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(password));
// Proxy Host
String realProxyHost = environmentSubstitute(httpProxyHost);
int realProxyPort = Const.toInt(environmentSubstitute(httpproxyport), 22);
String realproxyUserName = environmentSubstitute(httpproxyusername);
String realProxyPassword =
Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(httpProxyPassword));
// Key file
String realKeyFilename = environmentSubstitute(keyFilename);
String relKeyFilepass = environmentSubstitute(keyFilePass);
// target files
String realLocalDirectory = environmentSubstitute(localDirectory);
String realwildcard = environmentSubstitute(wildcard);
// Remote source
String realftpDirectory = environmentSubstitute(ftpDirectory);
// Destination folder (Move to)
String realDestinationFolder = environmentSubstitute(destinationfolder);
try {
// Remote source
realftpDirectory = FTPUtils.normalizePath(realftpDirectory);
// Destination folder (Move to)
realDestinationFolder = FTPUtils.normalizePath(realDestinationFolder);
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.CanNotNormalizePath", e.getMessage()));
result.setNrErrors(1);
return result;
}
// Check for mandatory fields
if (log.isRowLevel()) {
logRowlevel(BaseMessages.getString(PKG, "JobSSH2GET.Log.CheckingMandatoryFields"));
}
boolean mandatoryok = true;
if (Const.isEmpty(realServerName)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.ServernameMissing"));
}
if (usehttpproxy) {
if (Const.isEmpty(realProxyHost)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.HttpProxyhostMissing"));
}
}
if (publicpublickey) {
if (Const.isEmpty(realKeyFilename)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.KeyFileMissing"));
} else {
// Let's check if key file exists...
if (!new File(realKeyFilename).exists()) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.KeyFileNotExist"));
}
}
}
if (Const.isEmpty(realLocalDirectory)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.LocalFolderMissing"));
} else {
// Check if target folder exists...
if (!new File(realLocalDirectory).exists()) {
if (createtargetfolder) {
// Create Target folder
if (!CreateFolder(realLocalDirectory)) {
mandatoryok = false;
}
} else {
mandatoryok = false;
logError(
BaseMessages.getString(
PKG, "JobSSH2GET.Log.LocalFolderNotExists", realLocalDirectory));
}
} else {
if (!new File(realLocalDirectory).isDirectory()) {
mandatoryok = false;
logError(
BaseMessages.getString(
PKG, "JobSSH2GET.Log.LocalFolderNotFolder", realLocalDirectory));
}
}
}
if (afterFtpPut.equals("move_file")) {
if (Const.isEmpty(realDestinationFolder)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.DestinatFolderMissing"));
}
}
if (mandatoryok) {
Connection conn = null;
SFTPv3Client client = null;
boolean good = true;
try {
// Create a connection instance
conn =
getConnection(
realServerName,
realServerPort,
realProxyHost,
realProxyPort,
realproxyUserName,
realProxyPassword);
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSSH2GET.Log.ConnectionInstanceCreated"));
}
if (timeout > 0) {
// Use timeout
// Cache Host Key
if (cachehostkey) {
conn.connect(new SimpleVerifier(database), 0, timeout * 1000);
} else {
conn.connect(null, 0, timeout * 1000);
}
} else {
// Cache Host Key
if (cachehostkey) {
conn.connect(new SimpleVerifier(database));
} else {
conn.connect();
}
}
// Authenticate
boolean isAuthenticated = false;
if (publicpublickey) {
isAuthenticated =
conn.authenticateWithPublicKey(
realUserName, new File(realKeyFilename), relKeyFilepass);
} else {
isAuthenticated = conn.authenticateWithPassword(realUserName, realServerPassword);
}
// LET'S CHECK AUTHENTICATION ...
if (isAuthenticated == false) {
logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.AuthenticationFailed"));
} else {
if (log.isBasic()) {
logBasic(BaseMessages.getString(PKG, "JobSSH2GET.Log.Connected", serverName, userName));
}
client = new SFTPv3Client(conn);
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2GET.Log.ProtocolVersion", "" + client.getProtocolVersion()));
}
// Check if ftp (source) directory exists
if (!Const.isEmpty(realftpDirectory)) {
if (!sshDirectoryExists(client, realftpDirectory)) {
good = false;
logError(
BaseMessages.getString(
PKG, "JobSSH2GET.Log.RemoteDirectoryNotExist", realftpDirectory));
} else if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2GET.Log.RemoteDirectoryExist", realftpDirectory));
}
}
if (realDestinationFolder != null) {
// Check now destination folder
if (!sshDirectoryExists(client, realDestinationFolder)) {
if (createdestinationfolder) {
if (!CreateRemoteFolder(client, realDestinationFolder)) {
good = false;
}
} else {
good = false;
logError(
BaseMessages.getString(
PKG, "JobSSH2GET.Log.DestinatFolderNotExist", realDestinationFolder));
}
}
}
if (good) {
Pattern pattern = null;
if (!Const.isEmpty(realwildcard)) {
pattern = Pattern.compile(realwildcard);
}
if (includeSubFolders) {
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSSH2GET.Log.RecursiveModeOn"));
}
copyRecursive(realftpDirectory, realLocalDirectory, client, pattern, parentJob);
} else {
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSSH2GET.Log.RecursiveModeOff"));
}
GetFiles(realftpDirectory, realLocalDirectory, client, pattern, parentJob);
}
/** ****************************** RESULT ******************* */
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSSH2GET.Log.Result.JobEntryEnd1"));
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2GET.Log.Result.TotalFiles", "" + nbfilestoget));
logDetailed(
BaseMessages.getString(PKG, "JobSSH2GET.Log.Result.TotalFilesPut", "" + nbgot));
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2GET.Log.Result.TotalFilesError", "" + nbrerror));
logDetailed(BaseMessages.getString(PKG, "JobSSH2GET.Log.Result.JobEntryEnd2"));
}
if (nbrerror == 0) {
result.setResult(true);
/** ****************************** RESULT ******************* */
}
}
}
} catch (Exception e) {
result.setNrErrors(nbrerror);
logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.Error.ErrorFTP", e.getMessage()));
} finally {
if (conn != null) {
conn.close();
}
if (client != null) {
client.close();
}
}
}
return result;
}
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
List<RowMetaAndData> rows = result.getRows();
RowMetaAndData resultRow = null;
oneFileLocked = false;
result.setResult(true);
try {
if (argFromPrevious) {
if (isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG,
"JobEntryCheckFilesLocked.FoundPreviousRows",
String.valueOf((rows != null ? rows.size() : 0))));
}
}
if (argFromPrevious && rows != null) {
// Copy the input row to the (command line) arguments
for (int iteration = 0; iteration < rows.size() && !parentJob.isStopped(); iteration++) {
resultRow = rows.get(iteration);
// Get values from previous result
String filefolder_previous = resultRow.getString(0, "");
String fmasks_previous = resultRow.getString(1, "");
// ok we can process this file/folder
if (isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG,
"JobEntryCheckFilesLocked.ProcessingRow",
filefolder_previous,
fmasks_previous));
}
ProcessFile(filefolder_previous, fmasks_previous);
}
} else if (arguments != null) {
for (int i = 0; i < arguments.length && !parentJob.isStopped(); i++) {
// ok we can process this file/folder
if (isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntryCheckFilesLocked.ProcessingArg", arguments[i], filemasks[i]));
}
ProcessFile(arguments[i], filemasks[i]);
}
}
if (oneFileLocked) {
result.setResult(false);
result.setNrErrors(1);
}
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "JobEntryCheckFilesLocked.ErrorRunningJobEntry", e));
}
return result;
}