@Before
public void setUp() throws Exception {
MockDriver.registerInstance();
Job job = new Job(null, new JobMeta());
entry = new JobEntryEvalTableContent();
job.getJobMeta().addJobEntry(new JobEntryCopy(entry));
entry.setParentJob(job);
job.setStopped(false);
DatabaseMeta dbMeta = new DatabaseMeta();
dbMeta.setDatabaseType("mock-db");
entry.setDatabase(dbMeta);
}
@Before
public void setUp() throws Exception {
job = new Job(null, new JobMeta());
entry = new JobEntryFilesExist();
job.getJobMeta().addJobEntry(new JobEntryCopy(entry));
entry.setParentJob(job);
job.setStopped(false);
File f = File.createTempFile("existingFile", "ext");
f.deleteOnExit();
existingFile1 = f.getPath();
f = File.createTempFile("existingFile", "ext");
f.deleteOnExit();
existingFile2 = f.getPath();
}
@Before
public void before() throws KettleException {
MockitoAnnotations.initMocks(this);
Mockito.when(parentJob.getLogLevel()).thenReturn(LogLevel.BASIC);
entry.setParentJob(parentJob);
entry.setSaveMessage(true);
Mockito.when(message.getMessageNumber()).thenReturn(1);
Mockito.when(mailConn.getMessage()).thenReturn(message);
Mockito.doNothing().when(mailConn).openFolder(Mockito.anyBoolean());
Mockito.doNothing().when(mailConn).openFolder(Mockito.anyString(), Mockito.anyBoolean());
Mockito.when(mailConn.getMessagesCount()).thenReturn(1);
}
/**
* copy a directory from the remote host to the local one.
*
* @param sourceLocation the source directory on the remote host
* @param targetLocation the target directory on the local host
* @param sftpClient is an instance of SFTPv3Client that makes SFTP client connection over SSH-2
* @return the number of files successfully copied
* @throws Exception
*/
@SuppressWarnings("unchecked")
private void GetFiles(
String sourceLocation,
String targetLocation,
SFTPv3Client sftpClient,
Pattern pattern,
Job parentJob)
throws Exception {
String sourceFolder = ".";
if (!Const.isEmpty(sourceLocation)) {
sourceFolder = sourceLocation + FTPUtils.FILE_SEPARATOR;
} else {
sourceFolder += FTPUtils.FILE_SEPARATOR;
}
Vector<SFTPv3DirectoryEntry> filelist = sftpClient.ls(sourceFolder);
if (filelist != null) {
Iterator<SFTPv3DirectoryEntry> iterator = filelist.iterator();
while (iterator.hasNext() && !parentJob.isStopped()) {
SFTPv3DirectoryEntry dirEntry = iterator.next();
if (dirEntry == null) {
continue;
}
if (dirEntry.filename.equals(".")
|| dirEntry.filename.equals("..")
|| isDirectory(sftpClient, sourceFolder + dirEntry.filename)) {
continue;
}
if (getFileWildcard(dirEntry.filename, pattern)) {
// Copy file from remote host
copyFile(
sourceFolder + dirEntry.filename,
targetLocation + FTPUtils.FILE_SEPARATOR + dirEntry.filename,
sftpClient);
}
}
}
}
private boolean processOneXMLFile(
String xmlfilename, String xslfilename, String outputfilename, Result result, Job parentJob) {
boolean retval = false;
FileObject xmlfile = null;
FileObject xslfile = null;
FileObject outputfile = null;
try {
xmlfile = KettleVFS.getFileObject(xmlfilename, this);
xslfile = KettleVFS.getFileObject(xslfilename, this);
outputfile = KettleVFS.getFileObject(outputfilename, this);
if (xmlfile.exists() && xslfile.exists()) {
if (outputfile.exists() && iffileexists == 2) {
// Output file exists
// User want to fail
logError(
BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists1.Label")
+ outputfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists2.Label"));
return retval;
} else if (outputfile.exists() && iffileexists == 1) {
// Do nothing
if (log.isDebug()) {
logDebug(
BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists1.Label")
+ outputfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists2.Label"));
}
retval = true;
return retval;
} else {
if (outputfile.exists() && iffileexists == 0) {
// the output file exists and user want to create new one with unique name
// Format Date
// Try to clean filename (without wildcard)
String wildcard =
outputfilename.substring(outputfilename.length() - 4, outputfilename.length());
if (wildcard.substring(0, 1).equals(".")) {
// Find wildcard
outputfilename =
outputfilename.substring(0, outputfilename.length() - 4)
+ "_"
+ StringUtil.getFormattedDateTimeNow(true)
+ wildcard;
} else {
// did not find wildcard
outputfilename = outputfilename + "_" + StringUtil.getFormattedDateTimeNow(true);
}
if (log.isDebug()) {
logDebug(
BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists1.Label")
+ outputfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists2.Label"));
logDebug(
BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileNameChange1.Label")
+ outputfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileNameChange2.Label"));
}
}
// Create transformer factory
TransformerFactory factory = TransformerFactory.newInstance();
if (xsltfactory.equals(FACTORY_SAXON)) {
// Set the TransformerFactory to the SAXON implementation.
factory = new net.sf.saxon.TransformerFactoryImpl();
}
if (log.isDetailed()) {
log.logDetailed(
BaseMessages.getString(PKG, "JobEntryXSL.Log.TransformerFactoryInfos"),
BaseMessages.getString(
PKG, "JobEntryXSL.Log.TransformerFactory", factory.getClass().getName()));
}
InputStream xslInputStream = KettleVFS.getInputStream(xslfile);
InputStream xmlInputStream = KettleVFS.getInputStream(xmlfile);
OutputStream os = null;
try {
// Use the factory to create a template containing the xsl file
Templates template = factory.newTemplates(new StreamSource(xslInputStream));
// Use the template to create a transformer
Transformer xformer = template.newTransformer();
if (log.isDetailed()) {
log.logDetailed(
BaseMessages.getString(PKG, "JobEntryXSL.Log.TransformerClassInfos"),
BaseMessages.getString(
PKG, "JobEntryXSL.Log.TransformerClass", xformer.getClass().getName()));
}
// Do we need to set output properties?
if (setOutputProperties) {
xformer.setOutputProperties(outputProperties);
}
// Do we need to pass parameters?
if (useParameters) {
for (int i = 0; i < nrParams; i++) {
xformer.setParameter(nameOfParams[i], valueOfParams[i]);
}
}
// Prepare the input and output files
Source source = new StreamSource(xmlInputStream);
os = KettleVFS.getOutputStream(outputfile, false);
StreamResult resultat = new StreamResult(os);
// Apply the xsl file to the source file and write the result to the output file
xformer.transform(source, resultat);
if (isAddFileToResult()) {
// Add output filename to output files
ResultFile resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL,
KettleVFS.getFileObject(outputfilename, this),
parentJob.getJobname(),
toString());
result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
}
// Everything is OK
retval = true;
} finally {
try {
xslInputStream.close();
} catch (IOException ignored) {
// ignore IO Exception on close
}
try {
xmlInputStream.close();
} catch (IOException ignored) {
// ignore IO Exception on close
}
try {
if (os != null) {
os.close();
}
} catch (IOException ignored) {
// ignore IO Exception on close
}
}
}
} else {
if (!xmlfile.exists()) {
logError(
BaseMessages.getString(PKG, "JobEntryXSLT.FileDoesNotExist1.Label")
+ xmlfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.FileDoesNotExist2.Label"));
}
if (!xslfile.exists()) {
logError(
BaseMessages.getString(PKG, "JobEntryXSLT.FileDoesNotExist1.Label")
+ xmlfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.FileDoesNotExist2.Label"));
}
}
} catch (Exception e) {
logError(
BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLST.Label")
+ BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLSTXML1.Label")
+ xmlfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLSTXML2.Label")
+ BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLSTXSL1.Label")
+ xslfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLSTXSL2.Label")
+ e.getMessage());
} finally {
try {
if (xmlfile != null) {
xmlfile.close();
}
if (xslfile != null) {
xslfile.close();
}
if (outputfile != null) {
outputfile.close();
}
} catch (IOException e) {
logError("Unable to close file", e);
}
}
return retval;
}
protected void runJob(Job job) throws KettleException {
job.start(); // runs the thread in the background...
}
/**
* <div id="mindtouch">
*
* <h1>/kettle/startJob</h1>
*
* <a name="GET"></a>
*
* <h2>GET</h2>
*
* <p>Starts the job. If the job cannot be started, an error is returned.
*
* <p><b>Example Request:</b><br>
*
* <pre function="syntax.xml">
* GET /kettle/startJob/?name=dummy_job&xml=Y
* </pre>
*
* <h3>Parameters</h3>
*
* <table class="pentaho-table">
* <tbody>
* <tr>
* <th>name</th>
* <th>description</th>
* <th>type</th>
* </tr>
* <tr>
* <td>name</td>
* <td>Name of the job to be executed.</td>
* <td>query</td>
* </tr>
* <tr>
* <td>xml</td>
* <td>Boolean flag which sets the output format required. Use <code>Y</code> to receive XML response.</td>
* <td>boolean, optional</td>
* </tr>
* <tr>
* <td>id</td>
* <td>Carte job ID of the job to be executed. This parameter is optional when xml=Y is used.</td>
* <td>query, optional</td>
* </tr>
* </tbody>
* </table>
*
* <h3>Response Body</h3>
*
* <table class="pentaho-table">
* <tbody>
* <tr>
* <td align="right">text:</td>
* <td>HTML</td>
* </tr>
* <tr>
* <td align="right">media types:</td>
* <td>text/xml, text/html</td>
* </tr>
* </tbody>
* </table>
*
* <p>Response XML or HTML containing operation result. When using xml=Y <code>result</code> field
* indicates whether operation was successful (<code>OK</code>) or not (<code>ERROR</code>).
*
* <p><b>Example Response:</b>
*
* <pre function="syntax.xml">
* <?xml version="1.0" encoding="UTF-8"?>
* <webresult>
* <result>OK</result>
* <message>Job [dummy_job] was started.</message>
* <id>abd61143-8174-4f27-9037-6b22fbd3e229</id>
* </webresult>
* </pre>
*
* <h3>Status Codes</h3>
*
* <table class="pentaho-table">
* <tbody>
* <tr>
* <th>code</th>
* <th>description</th>
* </tr>
* <tr>
* <td>200</td>
* <td>Request was processed.</td>
* </tr>
* <tr>
* <td>500</td>
* <td>Internal server error occurs during request processing.</td>
* </tr>
* </tbody>
* </table>
*
* </div>
*/
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) {
return;
}
if (log.isDebug()) {
logDebug(BaseMessages.getString(PKG, "StartJobServlet.Log.StartJobRequested"));
}
String jobName = request.getParameter("name");
String id = request.getParameter("id");
boolean useXML = "Y".equalsIgnoreCase(request.getParameter("xml"));
response.setStatus(HttpServletResponse.SC_OK);
Encoder encoder = ESAPI.encoder();
PrintWriter out = response.getWriter();
if (useXML) {
response.setContentType("text/xml");
response.setCharacterEncoding(Const.XML_ENCODING);
out.print(XMLHandler.getXMLHeader(Const.XML_ENCODING));
} else {
response.setContentType("text/html;charset=UTF-8");
out.println("<HTML>");
out.println("<HEAD>");
out.println("<TITLE>Start job</TITLE>");
out.println(
"<META http-equiv=\"Refresh\" content=\"2;url="
+ convertContextPath(GetStatusServlet.CONTEXT_PATH)
+ "?name="
+ URLEncoder.encode(jobName, "UTF-8")
+ "\">");
out.println("<META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">");
out.println("</HEAD>");
out.println("<BODY>");
}
try {
// ID is optional...
//
Job job;
CarteObjectEntry entry;
if (Const.isEmpty(id)) {
// get the first job that matches...
//
entry = getJobMap().getFirstCarteObjectEntry(jobName);
if (entry == null) {
job = null;
} else {
id = entry.getId();
job = getJobMap().getJob(entry);
}
} else {
// Take the ID into account!
//
entry = new CarteObjectEntry(jobName, id);
job = getJobMap().getJob(entry);
}
if (job != null) {
// First see if this job already ran to completion.
// If so, we get an exception is we try to start() the job thread
//
if (job.isInitialized() && !job.isActive()) {
// Re-create the job from the jobMeta
//
// We might need to re-connect to the repository
//
if (job.getRep() != null && !job.getRep().isConnected()) {
if (job.getRep().getUserInfo() != null) {
job.getRep()
.connect(
job.getRep().getUserInfo().getLogin(),
job.getRep().getUserInfo().getPassword());
} else {
job.getRep().connect(null, null);
}
}
// Create a new job object to start from a sane state. Then replace
// the new job in the job map
//
synchronized (getJobMap()) {
JobConfiguration jobConfiguration = getJobMap().getConfiguration(jobName);
String carteObjectId = UUID.randomUUID().toString();
SimpleLoggingObject servletLoggingObject =
new SimpleLoggingObject(CONTEXT_PATH, LoggingObjectType.CARTE, null);
servletLoggingObject.setContainerObjectId(carteObjectId);
Job newJob = new Job(job.getRep(), job.getJobMeta(), servletLoggingObject);
newJob.setLogLevel(job.getLogLevel());
// Discard old log lines from the old job
//
KettleLogStore.discardLines(job.getLogChannelId(), true);
getJobMap().replaceJob(entry, newJob, jobConfiguration);
job = newJob;
}
}
runJob(job);
String message = BaseMessages.getString(PKG, "StartJobServlet.Log.JobStarted", jobName);
if (useXML) {
out.println(new WebResult(WebResult.STRING_OK, message, id).getXML());
} else {
out.println("<H1>" + encoder.encodeForHTML(message) + "</H1>");
out.println(
"<a href=\""
+ convertContextPath(GetJobStatusServlet.CONTEXT_PATH)
+ "?name="
+ URLEncoder.encode(jobName, "UTF-8")
+ "&id="
+ URLEncoder.encode(id, "UTF-8")
+ "\">"
+ BaseMessages.getString(PKG, "JobStatusServlet.BackToJobStatusPage")
+ "</a><p>");
}
} else {
String message =
BaseMessages.getString(PKG, "StartJobServlet.Log.SpecifiedJobNotFound", jobName);
if (useXML) {
out.println(new WebResult(WebResult.STRING_ERROR, message));
} else {
out.println("<H1>" + encoder.encodeForHTML(message) + "</H1>");
out.println(
"<a href=\""
+ convertContextPath(GetStatusServlet.CONTEXT_PATH)
+ "\">"
+ BaseMessages.getString(PKG, "TransStatusServlet.BackToStatusPage")
+ "</a><p>");
}
}
} catch (Exception ex) {
if (useXML) {
out.println(
new WebResult(
WebResult.STRING_ERROR,
BaseMessages.getString(
PKG,
"StartJobServlet.Error.UnexpectedError",
Const.CR + Const.getStackTracker(ex))));
} else {
out.println("<p>");
out.println("<pre>");
out.println(encoder.encodeForHTML(Const.getStackTracker(ex)));
out.println("</pre>");
}
}
if (!useXML) {
out.println("<p>");
out.println("</BODY>");
out.println("</HTML>");
}
}
public Result execute(Result result, int nr) throws KettleException {
result.setResult(true);
result.setNrErrors(0);
try {
List<String> variables = new ArrayList<String>();
List<String> variableValues = new ArrayList<String>();
List<Integer> variableTypes = new ArrayList<Integer>();
String realFilename = environmentSubstitute(filename);
try {
if (!Const.isEmpty(realFilename)) {
Properties properties = new Properties();
properties.load(KettleVFS.getInputStream(realFilename));
for (Object key : properties.keySet()) {
variables.add((String) key);
variableValues.add((String) properties.get(key));
variableTypes.add(fileVariableType);
}
}
} catch (Exception e) {
throw new KettleException(
BaseMessages.getString(
PKG, "JobEntrySetVariables.Error.UnableReadPropertiesFile", realFilename));
}
for (int i = 0; i < variableName.length; i++) {
variables.add(variableName[i]);
variableValues.add(variableValue[i]);
variableTypes.add(variableType[i]);
}
for (int i = 0; i < variables.size(); i++) {
String varname = variables.get(i);
String value = variableValues.get(i);
int type = variableTypes.get(i);
if (replaceVars) {
varname = environmentSubstitute(varname);
value = environmentSubstitute(value);
}
// OK, where do we set this value...
switch (type) {
case VARIABLE_TYPE_JVM:
{
System.setProperty(varname, value);
setVariable(varname, value);
Job parentJobTraverse = parentJob;
while (parentJobTraverse != null) {
parentJobTraverse.setVariable(varname, value);
parentJobTraverse = parentJobTraverse.getParentJob();
}
}
break;
case VARIABLE_TYPE_ROOT_JOB:
{
// set variable in this job entry
setVariable(varname, value);
Job rootJob = parentJob;
while (rootJob != null) {
rootJob.setVariable(varname, value);
rootJob = rootJob.getParentJob();
}
}
break;
case VARIABLE_TYPE_CURRENT_JOB:
{
setVariable(varname, value);
if (parentJob != null) {
parentJob.setVariable(varname, value);
} else {
throw new KettleJobException(
BaseMessages.getString(
PKG, "JobEntrySetVariables.Error.UnableSetVariableCurrentJob", varname));
}
}
break;
case VARIABLE_TYPE_PARENT_JOB:
{
setVariable(varname, value);
if (parentJob != null) {
parentJob.setVariable(varname, value);
Job gpJob = parentJob.getParentJob();
if (gpJob != null) {
gpJob.setVariable(varname, value);
} else {
throw new KettleJobException(
BaseMessages.getString(
PKG, "JobEntrySetVariables.Error.UnableSetVariableParentJob", varname));
}
} else {
throw new KettleJobException(
BaseMessages.getString(
PKG, "JobEntrySetVariables.Error.UnableSetVariableCurrentJob", varname));
}
}
break;
default:
break;
}
// ok we can process this line
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntrySetVariables.Log.SetVariableToValue", varname, value));
}
}
} catch (Exception e) {
result.setResult(false);
result.setNrErrors(1);
logError(
BaseMessages.getString(PKG, "JobEntrySetVariables.UnExcpectedError", e.getMessage()));
}
return result;
}
