@Before
  public void setUp() throws Exception {
    MockDriver.registerInstance();
    Job job = new Job(null, new JobMeta());
    entry = new JobEntryEvalTableContent();

    job.getJobMeta().addJobEntry(new JobEntryCopy(entry));
    entry.setParentJob(job);

    job.setStopped(false);

    DatabaseMeta dbMeta = new DatabaseMeta();
    dbMeta.setDatabaseType("mock-db");

    entry.setDatabase(dbMeta);
  }
  @Before
  public void setUp() throws Exception {
    job = new Job(null, new JobMeta());
    entry = new JobEntryFilesExist();

    job.getJobMeta().addJobEntry(new JobEntryCopy(entry));
    entry.setParentJob(job);

    job.setStopped(false);

    File f = File.createTempFile("existingFile", "ext");
    f.deleteOnExit();
    existingFile1 = f.getPath();

    f = File.createTempFile("existingFile", "ext");
    f.deleteOnExit();
    existingFile2 = f.getPath();
  }
  @Before
  public void before() throws KettleException {
    MockitoAnnotations.initMocks(this);

    Mockito.when(parentJob.getLogLevel()).thenReturn(LogLevel.BASIC);
    entry.setParentJob(parentJob);
    entry.setSaveMessage(true);

    Mockito.when(message.getMessageNumber()).thenReturn(1);
    Mockito.when(mailConn.getMessage()).thenReturn(message);

    Mockito.doNothing().when(mailConn).openFolder(Mockito.anyBoolean());
    Mockito.doNothing().when(mailConn).openFolder(Mockito.anyString(), Mockito.anyBoolean());

    Mockito.when(mailConn.getMessagesCount()).thenReturn(1);
  }
示例#4
0
  /**
   * copy a directory from the remote host to the local one.
   *
   * @param sourceLocation the source directory on the remote host
   * @param targetLocation the target directory on the local host
   * @param sftpClient is an instance of SFTPv3Client that makes SFTP client connection over SSH-2
   * @return the number of files successfully copied
   * @throws Exception
   */
  @SuppressWarnings("unchecked")
  private void GetFiles(
      String sourceLocation,
      String targetLocation,
      SFTPv3Client sftpClient,
      Pattern pattern,
      Job parentJob)
      throws Exception {

    String sourceFolder = ".";
    if (!Const.isEmpty(sourceLocation)) {
      sourceFolder = sourceLocation + FTPUtils.FILE_SEPARATOR;
    } else {
      sourceFolder += FTPUtils.FILE_SEPARATOR;
    }

    Vector<SFTPv3DirectoryEntry> filelist = sftpClient.ls(sourceFolder);

    if (filelist != null) {
      Iterator<SFTPv3DirectoryEntry> iterator = filelist.iterator();

      while (iterator.hasNext() && !parentJob.isStopped()) {
        SFTPv3DirectoryEntry dirEntry = iterator.next();

        if (dirEntry == null) {
          continue;
        }

        if (dirEntry.filename.equals(".")
            || dirEntry.filename.equals("..")
            || isDirectory(sftpClient, sourceFolder + dirEntry.filename)) {
          continue;
        }

        if (getFileWildcard(dirEntry.filename, pattern)) {
          // Copy file from remote host
          copyFile(
              sourceFolder + dirEntry.filename,
              targetLocation + FTPUtils.FILE_SEPARATOR + dirEntry.filename,
              sftpClient);
        }
      }
    }
  }
  private boolean processOneXMLFile(
      String xmlfilename, String xslfilename, String outputfilename, Result result, Job parentJob) {
    boolean retval = false;
    FileObject xmlfile = null;
    FileObject xslfile = null;
    FileObject outputfile = null;

    try {
      xmlfile = KettleVFS.getFileObject(xmlfilename, this);
      xslfile = KettleVFS.getFileObject(xslfilename, this);
      outputfile = KettleVFS.getFileObject(outputfilename, this);

      if (xmlfile.exists() && xslfile.exists()) {
        if (outputfile.exists() && iffileexists == 2) {
          // Output file exists
          // User want to fail
          logError(
              BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists1.Label")
                  + outputfilename
                  + BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists2.Label"));
          return retval;

        } else if (outputfile.exists() && iffileexists == 1) {
          // Do nothing
          if (log.isDebug()) {
            logDebug(
                BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists1.Label")
                    + outputfilename
                    + BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists2.Label"));
          }
          retval = true;
          return retval;

        } else {
          if (outputfile.exists() && iffileexists == 0) {
            // the output file exists and user want to create new one with unique name
            // Format Date

            // Try to clean filename (without wildcard)
            String wildcard =
                outputfilename.substring(outputfilename.length() - 4, outputfilename.length());
            if (wildcard.substring(0, 1).equals(".")) {
              // Find wildcard
              outputfilename =
                  outputfilename.substring(0, outputfilename.length() - 4)
                      + "_"
                      + StringUtil.getFormattedDateTimeNow(true)
                      + wildcard;
            } else {
              // did not find wildcard
              outputfilename = outputfilename + "_" + StringUtil.getFormattedDateTimeNow(true);
            }
            if (log.isDebug()) {
              logDebug(
                  BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists1.Label")
                      + outputfilename
                      + BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists2.Label"));
              logDebug(
                  BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileNameChange1.Label")
                      + outputfilename
                      + BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileNameChange2.Label"));
            }
          }

          // Create transformer factory
          TransformerFactory factory = TransformerFactory.newInstance();

          if (xsltfactory.equals(FACTORY_SAXON)) {
            // Set the TransformerFactory to the SAXON implementation.
            factory = new net.sf.saxon.TransformerFactoryImpl();
          }

          if (log.isDetailed()) {
            log.logDetailed(
                BaseMessages.getString(PKG, "JobEntryXSL.Log.TransformerFactoryInfos"),
                BaseMessages.getString(
                    PKG, "JobEntryXSL.Log.TransformerFactory", factory.getClass().getName()));
          }

          InputStream xslInputStream = KettleVFS.getInputStream(xslfile);
          InputStream xmlInputStream = KettleVFS.getInputStream(xmlfile);
          OutputStream os = null;
          try {
            // Use the factory to create a template containing the xsl file
            Templates template = factory.newTemplates(new StreamSource(xslInputStream));

            // Use the template to create a transformer
            Transformer xformer = template.newTransformer();

            if (log.isDetailed()) {
              log.logDetailed(
                  BaseMessages.getString(PKG, "JobEntryXSL.Log.TransformerClassInfos"),
                  BaseMessages.getString(
                      PKG, "JobEntryXSL.Log.TransformerClass", xformer.getClass().getName()));
            }

            // Do we need to set output properties?
            if (setOutputProperties) {
              xformer.setOutputProperties(outputProperties);
            }

            // Do we need to pass parameters?
            if (useParameters) {
              for (int i = 0; i < nrParams; i++) {
                xformer.setParameter(nameOfParams[i], valueOfParams[i]);
              }
            }

            // Prepare the input and output files
            Source source = new StreamSource(xmlInputStream);
            os = KettleVFS.getOutputStream(outputfile, false);
            StreamResult resultat = new StreamResult(os);

            // Apply the xsl file to the source file and write the result to the output file
            xformer.transform(source, resultat);

            if (isAddFileToResult()) {
              // Add output filename to output files
              ResultFile resultFile =
                  new ResultFile(
                      ResultFile.FILE_TYPE_GENERAL,
                      KettleVFS.getFileObject(outputfilename, this),
                      parentJob.getJobname(),
                      toString());
              result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
            }

            // Everything is OK
            retval = true;
          } finally {
            try {
              xslInputStream.close();
            } catch (IOException ignored) {
              // ignore IO Exception on close
            }
            try {
              xmlInputStream.close();
            } catch (IOException ignored) {
              // ignore IO Exception on close
            }
            try {
              if (os != null) {
                os.close();
              }
            } catch (IOException ignored) {
              // ignore IO Exception on close
            }
          }
        }
      } else {

        if (!xmlfile.exists()) {
          logError(
              BaseMessages.getString(PKG, "JobEntryXSLT.FileDoesNotExist1.Label")
                  + xmlfilename
                  + BaseMessages.getString(PKG, "JobEntryXSLT.FileDoesNotExist2.Label"));
        }
        if (!xslfile.exists()) {
          logError(
              BaseMessages.getString(PKG, "JobEntryXSLT.FileDoesNotExist1.Label")
                  + xmlfilename
                  + BaseMessages.getString(PKG, "JobEntryXSLT.FileDoesNotExist2.Label"));
        }
      }
    } catch (Exception e) {
      logError(
          BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLST.Label")
              + BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLSTXML1.Label")
              + xmlfilename
              + BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLSTXML2.Label")
              + BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLSTXSL1.Label")
              + xslfilename
              + BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLSTXSL2.Label")
              + e.getMessage());
    } finally {
      try {
        if (xmlfile != null) {
          xmlfile.close();
        }

        if (xslfile != null) {
          xslfile.close();
        }
        if (outputfile != null) {
          outputfile.close();
        }
      } catch (IOException e) {
        logError("Unable to close file", e);
      }
    }

    return retval;
  }
 protected void runJob(Job job) throws KettleException {
   job.start(); // runs the thread in the background...
 }
  /**
   * <div id="mindtouch">
   *
   * <h1>/kettle/startJob</h1>
   *
   * <a name="GET"></a>
   *
   * <h2>GET</h2>
   *
   * <p>Starts the job. If the job cannot be started, an error is returned.
   *
   * <p><b>Example Request:</b><br>
   *
   * <pre function="syntax.xml">
   * GET /kettle/startJob/?name=dummy_job&xml=Y
   * </pre>
   *
   * <h3>Parameters</h3>
   *
   * <table class="pentaho-table">
   * <tbody>
   * <tr>
   * <th>name</th>
   * <th>description</th>
   * <th>type</th>
   * </tr>
   * <tr>
   * <td>name</td>
   * <td>Name of the job to be executed.</td>
   * <td>query</td>
   * </tr>
   * <tr>
   * <td>xml</td>
   * <td>Boolean flag which sets the output format required. Use <code>Y</code> to receive XML response.</td>
   * <td>boolean, optional</td>
   * </tr>
   * <tr>
   * <td>id</td>
   * <td>Carte job ID of the job to be executed. This parameter is optional when xml=Y is used.</td>
   * <td>query, optional</td>
   * </tr>
   * </tbody>
   * </table>
   *
   * <h3>Response Body</h3>
   *
   * <table class="pentaho-table">
   * <tbody>
   * <tr>
   * <td align="right">text:</td>
   * <td>HTML</td>
   * </tr>
   * <tr>
   * <td align="right">media types:</td>
   * <td>text/xml, text/html</td>
   * </tr>
   * </tbody>
   * </table>
   *
   * <p>Response XML or HTML containing operation result. When using xml=Y <code>result</code> field
   * indicates whether operation was successful (<code>OK</code>) or not (<code>ERROR</code>).
   *
   * <p><b>Example Response:</b>
   *
   * <pre function="syntax.xml">
   * <?xml version="1.0" encoding="UTF-8"?>
   * <webresult>
   * <result>OK</result>
   * <message>Job &#x5b;dummy_job&#x5d; was started.</message>
   * <id>abd61143-8174-4f27-9037-6b22fbd3e229</id>
   * </webresult>
   * </pre>
   *
   * <h3>Status Codes</h3>
   *
   * <table class="pentaho-table">
   * <tbody>
   * <tr>
   * <th>code</th>
   * <th>description</th>
   * </tr>
   * <tr>
   * <td>200</td>
   * <td>Request was processed.</td>
   * </tr>
   * <tr>
   * <td>500</td>
   * <td>Internal server error occurs during request processing.</td>
   * </tr>
   * </tbody>
   * </table>
   *
   * </div>
   */
  public void doGet(HttpServletRequest request, HttpServletResponse response)
      throws ServletException, IOException {
    if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) {
      return;
    }

    if (log.isDebug()) {
      logDebug(BaseMessages.getString(PKG, "StartJobServlet.Log.StartJobRequested"));
    }

    String jobName = request.getParameter("name");
    String id = request.getParameter("id");
    boolean useXML = "Y".equalsIgnoreCase(request.getParameter("xml"));

    response.setStatus(HttpServletResponse.SC_OK);

    Encoder encoder = ESAPI.encoder();

    PrintWriter out = response.getWriter();
    if (useXML) {
      response.setContentType("text/xml");
      response.setCharacterEncoding(Const.XML_ENCODING);
      out.print(XMLHandler.getXMLHeader(Const.XML_ENCODING));
    } else {
      response.setContentType("text/html;charset=UTF-8");
      out.println("<HTML>");
      out.println("<HEAD>");
      out.println("<TITLE>Start job</TITLE>");
      out.println(
          "<META http-equiv=\"Refresh\" content=\"2;url="
              + convertContextPath(GetStatusServlet.CONTEXT_PATH)
              + "?name="
              + URLEncoder.encode(jobName, "UTF-8")
              + "\">");
      out.println("<META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">");
      out.println("</HEAD>");
      out.println("<BODY>");
    }

    try {
      // ID is optional...
      //
      Job job;
      CarteObjectEntry entry;
      if (Const.isEmpty(id)) {
        // get the first job that matches...
        //
        entry = getJobMap().getFirstCarteObjectEntry(jobName);
        if (entry == null) {
          job = null;
        } else {
          id = entry.getId();
          job = getJobMap().getJob(entry);
        }
      } else {
        // Take the ID into account!
        //
        entry = new CarteObjectEntry(jobName, id);
        job = getJobMap().getJob(entry);
      }

      if (job != null) {
        // First see if this job already ran to completion.
        // If so, we get an exception is we try to start() the job thread
        //
        if (job.isInitialized() && !job.isActive()) {
          // Re-create the job from the jobMeta
          //
          // We might need to re-connect to the repository
          //
          if (job.getRep() != null && !job.getRep().isConnected()) {
            if (job.getRep().getUserInfo() != null) {
              job.getRep()
                  .connect(
                      job.getRep().getUserInfo().getLogin(),
                      job.getRep().getUserInfo().getPassword());
            } else {
              job.getRep().connect(null, null);
            }
          }

          // Create a new job object to start from a sane state. Then replace
          // the new job in the job map
          //
          synchronized (getJobMap()) {
            JobConfiguration jobConfiguration = getJobMap().getConfiguration(jobName);

            String carteObjectId = UUID.randomUUID().toString();
            SimpleLoggingObject servletLoggingObject =
                new SimpleLoggingObject(CONTEXT_PATH, LoggingObjectType.CARTE, null);
            servletLoggingObject.setContainerObjectId(carteObjectId);

            Job newJob = new Job(job.getRep(), job.getJobMeta(), servletLoggingObject);
            newJob.setLogLevel(job.getLogLevel());

            // Discard old log lines from the old job
            //
            KettleLogStore.discardLines(job.getLogChannelId(), true);

            getJobMap().replaceJob(entry, newJob, jobConfiguration);
            job = newJob;
          }
        }

        runJob(job);

        String message = BaseMessages.getString(PKG, "StartJobServlet.Log.JobStarted", jobName);
        if (useXML) {
          out.println(new WebResult(WebResult.STRING_OK, message, id).getXML());
        } else {

          out.println("<H1>" + encoder.encodeForHTML(message) + "</H1>");
          out.println(
              "<a href=\""
                  + convertContextPath(GetJobStatusServlet.CONTEXT_PATH)
                  + "?name="
                  + URLEncoder.encode(jobName, "UTF-8")
                  + "&id="
                  + URLEncoder.encode(id, "UTF-8")
                  + "\">"
                  + BaseMessages.getString(PKG, "JobStatusServlet.BackToJobStatusPage")
                  + "</a><p>");
        }
      } else {
        String message =
            BaseMessages.getString(PKG, "StartJobServlet.Log.SpecifiedJobNotFound", jobName);
        if (useXML) {
          out.println(new WebResult(WebResult.STRING_ERROR, message));
        } else {
          out.println("<H1>" + encoder.encodeForHTML(message) + "</H1>");
          out.println(
              "<a href=\""
                  + convertContextPath(GetStatusServlet.CONTEXT_PATH)
                  + "\">"
                  + BaseMessages.getString(PKG, "TransStatusServlet.BackToStatusPage")
                  + "</a><p>");
        }
      }
    } catch (Exception ex) {
      if (useXML) {
        out.println(
            new WebResult(
                WebResult.STRING_ERROR,
                BaseMessages.getString(
                    PKG,
                    "StartJobServlet.Error.UnexpectedError",
                    Const.CR + Const.getStackTracker(ex))));
      } else {
        out.println("<p>");
        out.println("<pre>");
        out.println(encoder.encodeForHTML(Const.getStackTracker(ex)));
        out.println("</pre>");
      }
    }

    if (!useXML) {
      out.println("<p>");
      out.println("</BODY>");
      out.println("</HTML>");
    }
  }
  public Result execute(Result result, int nr) throws KettleException {
    result.setResult(true);
    result.setNrErrors(0);
    try {

      List<String> variables = new ArrayList<String>();
      List<String> variableValues = new ArrayList<String>();
      List<Integer> variableTypes = new ArrayList<Integer>();

      String realFilename = environmentSubstitute(filename);
      try {
        if (!Const.isEmpty(realFilename)) {
          Properties properties = new Properties();
          properties.load(KettleVFS.getInputStream(realFilename));
          for (Object key : properties.keySet()) {
            variables.add((String) key);
            variableValues.add((String) properties.get(key));
            variableTypes.add(fileVariableType);
          }
        }
      } catch (Exception e) {
        throw new KettleException(
            BaseMessages.getString(
                PKG, "JobEntrySetVariables.Error.UnableReadPropertiesFile", realFilename));
      }

      for (int i = 0; i < variableName.length; i++) {
        variables.add(variableName[i]);
        variableValues.add(variableValue[i]);
        variableTypes.add(variableType[i]);
      }

      for (int i = 0; i < variables.size(); i++) {
        String varname = variables.get(i);
        String value = variableValues.get(i);
        int type = variableTypes.get(i);

        if (replaceVars) {
          varname = environmentSubstitute(varname);
          value = environmentSubstitute(value);
        }

        // OK, where do we set this value...
        switch (type) {
          case VARIABLE_TYPE_JVM:
            {
              System.setProperty(varname, value);
              setVariable(varname, value);
              Job parentJobTraverse = parentJob;
              while (parentJobTraverse != null) {
                parentJobTraverse.setVariable(varname, value);
                parentJobTraverse = parentJobTraverse.getParentJob();
              }
            }
            break;
          case VARIABLE_TYPE_ROOT_JOB:
            {
              // set variable in this job entry
              setVariable(varname, value);
              Job rootJob = parentJob;
              while (rootJob != null) {
                rootJob.setVariable(varname, value);
                rootJob = rootJob.getParentJob();
              }
            }
            break;
          case VARIABLE_TYPE_CURRENT_JOB:
            {
              setVariable(varname, value);
              if (parentJob != null) {
                parentJob.setVariable(varname, value);
              } else {
                throw new KettleJobException(
                    BaseMessages.getString(
                        PKG, "JobEntrySetVariables.Error.UnableSetVariableCurrentJob", varname));
              }
            }
            break;
          case VARIABLE_TYPE_PARENT_JOB:
            {
              setVariable(varname, value);

              if (parentJob != null) {
                parentJob.setVariable(varname, value);
                Job gpJob = parentJob.getParentJob();
                if (gpJob != null) {
                  gpJob.setVariable(varname, value);
                } else {
                  throw new KettleJobException(
                      BaseMessages.getString(
                          PKG, "JobEntrySetVariables.Error.UnableSetVariableParentJob", varname));
                }
              } else {
                throw new KettleJobException(
                    BaseMessages.getString(
                        PKG, "JobEntrySetVariables.Error.UnableSetVariableCurrentJob", varname));
              }
            }
            break;
          default:
            break;
        }

        // ok we can process this line
        if (log.isDetailed()) {
          logDetailed(
              BaseMessages.getString(
                  PKG, "JobEntrySetVariables.Log.SetVariableToValue", varname, value));
        }
      }
    } catch (Exception e) {
      result.setResult(false);
      result.setNrErrors(1);

      logError(
          BaseMessages.getString(PKG, "JobEntrySetVariables.UnExcpectedError", e.getMessage()));
    }

    return result;
  }