public Result execute(Result previousResult, int nr, Repository rep, Job parentJob) {
    LogWriter log = LogWriter.getInstance();
    Result result = previousResult;
    result.setResult(false);

    if (filename != null) {
      String realFilename = getRealFilename();
      try {
        FileObject file = KettleVFS.getFileObject(realFilename);
        if (file.exists() && file.isReadable()) {
          log.logDetailed(
              toString(),
              Messages.getString("JobEntryFileExists.File_Exists", realFilename)); // $NON-NLS-1$
          result.setResult(true);
        } else {
          log.logDetailed(
              toString(),
              Messages.getString(
                  "JobEntryFileExists.File_Does_Not_Exist", realFilename)); // $NON-NLS-1$
        }
      } catch (IOException e) {
        result.setNrErrors(1);
        log.logError(
            toString(),
            Messages.getString(
                "JobEntryFileExists.ERROR_0004_IO_Exception", e.toString())); // $NON-NLS-1$
      }
    } else {
      result.setNrErrors(1);
      log.logError(
          toString(),
          Messages.getString("JobEntryFileExists.ERROR_0005_No_Filename_Defined")); // $NON-NLS-1$
    }

    return result;
  }
Ejemplo n.º 2
0
  public Result execute(Result previousResult, int nr) {
    Result result = previousResult;
    result.setResult(false);

    try {

      // Get real variable value
      String realServerName = environmentSubstitute(serverName);
      int realServerPort = Const.toInt(environmentSubstitute(serverPort), 22);
      String realUserName = environmentSubstitute(userName);
      String realServerPassword =
          Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(password));
      // Proxy Host
      String realProxyHost = environmentSubstitute(httpproxyhost);
      int realProxyPort = Const.toInt(environmentSubstitute(httpproxyport), 22);
      String realproxyUserName = environmentSubstitute(httpproxyusername);
      String realProxyPassword =
          Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(httpProxyPassword));
      // Key file
      String realKeyFilename = environmentSubstitute(keyFilename);
      String relKeyFilepass = environmentSubstitute(keyFilePass);
      // Source files
      String realLocalDirectory = environmentSubstitute(localDirectory);
      String realwildcard = environmentSubstitute(wildcard);
      // Remote destination
      String realftpDirectory = environmentSubstitute(ftpDirectory);
      // Destination folder (Move to)
      String realDestinationFolder = environmentSubstitute(destinationfolder);

      try {
        // Remote source
        realftpDirectory = FTPUtils.normalizePath(realftpDirectory);
        // Destination folder (Move to)
        realDestinationFolder = FTPUtils.normalizePath(realDestinationFolder);
      } catch (Exception e) {
        logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.CanNotNormalizePath", e.getMessage()));
        result.setNrErrors(1);
        return result;
      }

      // Check for mandatory fields
      boolean mandatoryok = true;
      if (Const.isEmpty(realServerName)) {
        mandatoryok = false;
        logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.ServernameMissing"));
      }
      if (usehttpproxy) {
        if (Const.isEmpty(realProxyHost)) {
          mandatoryok = false;
          logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.HttpProxyhostMissing"));
        }
      }
      if (publicpublickey) {
        if (Const.isEmpty(realKeyFilename)) {
          mandatoryok = false;
          logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.KeyFileMissing"));
        } else {
          // Let's check if folder exists...
          if (!KettleVFS.fileExists(realKeyFilename, this)) {
            mandatoryok = false;
            logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.KeyFileNotExist"));
          }
        }
      }

      if (Const.isEmpty(realLocalDirectory)) {
        mandatoryok = false;
        logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.LocalFolderMissing"));
      }
      if (afterFtpPut.equals("move_file")) {
        if (Const.isEmpty(realDestinationFolder)) {
          mandatoryok = false;
          logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.DestinatFolderMissing"));
        } else {
          FileObject folder = null;
          try {
            folder = KettleVFS.getFileObject(realDestinationFolder, this);
            // Let's check if folder exists...
            if (!folder.exists()) {
              // Do we need to create it?
              if (createDestinationFolder) {
                folder.createFolder();
              } else {
                logError(
                    BaseMessages.getString(
                        PKG, "JobSSH2PUT.Log.DestinatFolderNotExist", realDestinationFolder));
              }
            }
          } catch (Exception e) {
            throw new KettleException(e);
          } finally {
            if (folder != null) {
              try {
                folder.close();
                folder = null;
              } catch (Exception e) {
                /* Ignore */
              }
            }
          }
        }
      }

      if (mandatoryok) {
        Connection conn = null;
        SFTPv3Client client = null;
        boolean good = true;

        int nbfilestoput = 0;
        int nbput = 0;
        int nbrerror = 0;

        try {
          // Create a connection instance
          conn =
              getConnection(
                  realServerName,
                  realServerPort,
                  realProxyHost,
                  realProxyPort,
                  realproxyUserName,
                  realProxyPassword);

          if (timeout > 0) {
            // Use timeout
            // Cache Host Key
            if (cachehostkey) {
              conn.connect(new SimpleVerifier(database), 0, timeout * 1000);
            } else {
              conn.connect(null, 0, timeout * 1000);
            }

          } else {
            // Cache Host Key
            if (cachehostkey) {
              conn.connect(new SimpleVerifier(database));
            } else {
              conn.connect();
            }
          }

          // Authenticate

          boolean isAuthenticated = false;
          if (publicpublickey) {
            String keyContent =
                KettleVFS.getTextFileContent(realKeyFilename, this, Const.XML_ENCODING);
            isAuthenticated =
                conn.authenticateWithPublicKey(
                    realUserName, keyContent.toCharArray(), relKeyFilepass);
          } else {
            isAuthenticated = conn.authenticateWithPassword(realUserName, realServerPassword);
          }

          // LET'S CHECK AUTHENTICATION ...
          if (isAuthenticated == false) {
            logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.AuthenticationFailed"));
          } else {
            if (log.isBasic()) {
              logBasic(
                  BaseMessages.getString(PKG, "JobSSH2PUT.Log.Connected", serverName, userName));
            }

            client = new SFTPv3Client(conn);

            if (log.isDetailed()) {
              logDetailed(
                  BaseMessages.getString(
                      PKG, "JobSSH2PUT.Log.ProtocolVersion", "" + client.getProtocolVersion()));
            }

            // Check if remote directory exists
            if (!Const.isEmpty(realftpDirectory)) {
              if (!sshDirectoryExists(client, realftpDirectory)) {
                good = false;
                if (createRemoteFolder) {
                  good = CreateRemoteFolder(client, realftpDirectory);
                  if (good) {
                    logBasic(BaseMessages.getString(PKG, "JobSSH2PUT.Log.RemoteDirectoryCreated"));
                  }

                } else {
                  logError(
                      BaseMessages.getString(
                          PKG, "JobSSH2PUT.Log.RemoteDirectoryNotExist", realftpDirectory));
                }
              } else if (log.isDetailed()) {
                logDetailed(
                    BaseMessages.getString(
                        PKG, "JobSSH2PUT.Log.RemoteDirectoryExist", realftpDirectory));
              }
            }

            if (good) {
              // Get files list from local folder (source)
              List<FileObject> myFileList = getFiles(realLocalDirectory);

              // Prepare Pattern for wildcard
              Pattern pattern = null;
              if (!Const.isEmpty(realwildcard)) {
                pattern = Pattern.compile(realwildcard);
              }

              // Let's put files now ...
              // Get the files in the list
              for (int i = 0; i < myFileList.size() && !parentJob.isStopped(); i++) {
                FileObject myFile = myFileList.get(i);
                String localFilename = myFile.toString();
                String remoteFilename = myFile.getName().getBaseName();

                boolean getIt = true;

                // First see if the file matches the regular expression!
                if (pattern != null) {
                  Matcher matcher = pattern.matcher(remoteFilename);
                  getIt = matcher.matches();
                }

                // do we have a target directory?
                if (!Const.isEmpty(realftpDirectory)) {
                  remoteFilename = realftpDirectory + FTPUtils.FILE_SEPARATOR + remoteFilename;
                }

                if (onlyGettingNewFiles) {
                  // We get only new files
                  // ie not exist on the remote server
                  getIt = !sshFileExists(client, remoteFilename);
                }

                if (getIt) {
                  nbfilestoput++;

                  boolean putok = putFile(myFile, remoteFilename, client);
                  if (!putok) {
                    nbrerror++;
                    logError(
                        BaseMessages.getString(
                            PKG, "JobSSH2PUT.Log.Error.CanNotPutFile", localFilename));
                  } else {
                    nbput++;
                  }

                  if (putok && !afterFtpPut.equals("do_nothing")) {
                    deleteOrMoveFiles(myFile, realDestinationFolder);
                  }
                }
              }
              /** ****************************** RESULT ******************* */
              if (log.isDetailed()) {
                logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.JobEntryEnd1"));
                logDetailed(
                    BaseMessages.getString(
                        PKG, "JobSSH2PUT.Log.Result.TotalFiles", "" + nbfilestoput));
                logDetailed(
                    BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.TotalFilesPut", "" + nbput));
                logDetailed(
                    BaseMessages.getString(
                        PKG, "JobSSH2PUT.Log.Result.TotalFilesError", "" + nbrerror));
                logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.JobEntryEnd2"));
              }
              if (nbrerror == 0) {
                result.setResult(true);
                /** ****************************** RESULT ******************* */
              }
            }
          }

        } catch (Exception e) {
          result.setNrErrors(nbrerror);
          logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Error.ErrorFTP", e.getMessage()));
        } finally {
          if (conn != null) {
            conn.close();
          }
          if (client != null) {
            client.close();
          }
        }
      }
    } catch (Exception e) {
      result.setResult(false);
      result.setNrErrors(1L);
      logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Error.UnexpectedError"), e);
    }

    return result;
  }
  public Result execute(Result previousResult, int nr) throws KettleException {
    Result result = previousResult;
    int NrErrors = 0;
    int NrSuccess = 0;

    // Check output parameters
    int nrOutputProps = getOutputPropertyName() == null ? 0 : getOutputPropertyName().length;
    if (nrOutputProps > 0) {
      outputProperties = new Properties();
      for (int i = 0; i < nrOutputProps; i++) {
        outputProperties.put(
            getOutputPropertyName()[i], environmentSubstitute(getOutputPropertyValue()[i]));
      }
      setOutputProperties = true;
    }

    // Check parameters
    nrParams = getParameterField() == null ? 0 : getParameterField().length;
    if (nrParams > 0) {
      nameOfParams = new String[nrParams];
      valueOfParams = new String[nrParams];
      for (int i = 0; i < nrParams; i++) {
        String name = environmentSubstitute(getParameterName()[i]);
        String value = environmentSubstitute(getParameterField()[i]);
        if (Const.isEmpty(value)) {
          throw new KettleStepException(
              BaseMessages.getString(PKG, "Xslt.Exception.ParameterFieldMissing", name, i));
        }
        nameOfParams[i] = name;
        valueOfParams[i] = value;
      }
      useParameters = true;
    }

    List<RowMetaAndData> rows = result.getRows();
    if (isFilenamesFromPrevious()) {
      if (log.isDetailed()) {
        logDetailed(
            BaseMessages.getString(
                PKG,
                "JobEntryXSLT.Log.ArgFromPrevious.Found",
                (rows != null ? rows.size() : 0) + ""));
      }
    }

    if (isFilenamesFromPrevious() && rows != null) {
      // Copy the input row to the (command line) arguments
      RowMetaAndData resultRow = null;
      for (int iteration = 0; iteration < rows.size() && !parentJob.isStopped(); iteration++) {
        resultRow = rows.get(iteration);

        // Get filenames (xml, xsl, output filename)
        String xmlfilename_previous = resultRow.getString(0, null);
        String xslfilename_previous = resultRow.getString(1, null);
        String ouputfilename_previous = resultRow.getString(2, null);

        if (!Const.isEmpty(xmlfilename_previous)
            && !Const.isEmpty(xslfilename_previous)
            && !Const.isEmpty(ouputfilename_previous)) {
          if (processOneXMLFile(
              xmlfilename_previous,
              xslfilename_previous,
              ouputfilename_previous,
              result,
              parentJob)) {
            NrSuccess++;
          } else {
            NrErrors++;
          }
        } else {
          // We failed!
          logError(BaseMessages.getString(PKG, "JobEntryXSLT.AllFilesNotNull.Label"));
          NrErrors++;
        }
      }
    } else {
      String realxmlfilename = getRealxmlfilename();
      String realxslfilename = getRealxslfilename();
      String realoutputfilename = getoutputfilename();
      if (!Const.isEmpty(realxmlfilename)
          && !Const.isEmpty(realxslfilename)
          && !Const.isEmpty(realoutputfilename)) {
        if (processOneXMLFile(
            realxmlfilename, realxslfilename, realoutputfilename, result, parentJob)) {
          NrSuccess++;
        } else {
          NrErrors++;
        }
      } else {
        // We failed!
        logError(BaseMessages.getString(PKG, "JobEntryXSLT.AllFilesNotNull.Label"));
        NrErrors++;
      }
    }

    result.setResult(NrErrors == 0);
    result.setNrErrors(NrErrors);
    result.setNrLinesWritten(NrSuccess);

    return result;
  }
Ejemplo n.º 4
0
  public Result execute(Result previousResult, int nr) {
    Result result = previousResult;
    result.setNrErrors(1);
    result.setResult(false);

    String servername = environmentSubstitute(serverName);
    int nrPort = Const.toInt(environmentSubstitute("" + port), DEFAULT_PORT);
    String Oid = environmentSubstitute(oid);
    int timeOut = Const.toInt(environmentSubstitute("" + timeout), DEFAULT_TIME_OUT);
    int retry = Const.toInt(environmentSubstitute("" + nrretry), 1);
    String messageString = environmentSubstitute(message);

    Snmp snmp = null;

    try {
      TransportMapping transMap = new DefaultUdpTransportMapping();
      snmp = new Snmp(transMap);

      UdpAddress udpAddress = new UdpAddress(InetAddress.getByName(servername), nrPort);
      ResponseEvent response = null;
      if (targettype.equals(target_type_Code[0])) {
        // Community target
        String community = environmentSubstitute(comString);

        CommunityTarget target = new CommunityTarget();
        PDUv1 pdu1 = new PDUv1();
        transMap.listen();

        target.setCommunity(new OctetString(community));
        target.setVersion(SnmpConstants.version1);
        target.setAddress(udpAddress);
        if (target.getAddress().isValid()) {
          if (log.isDebug()) {
            logDebug("Valid IP address");
          }
        } else {
          throw new KettleException("Invalid IP address");
        }
        target.setRetries(retry);
        target.setTimeout(timeOut);

        // create the PDU
        pdu1.setGenericTrap(6);
        pdu1.setSpecificTrap(PDUv1.ENTERPRISE_SPECIFIC);
        pdu1.setEnterprise(new OID(Oid));
        pdu1.add(new VariableBinding(new OID(Oid), new OctetString(messageString)));

        response = snmp.send(pdu1, target);

      } else {
        // User target
        String userName = environmentSubstitute(user);
        String passPhrase = environmentSubstitute(passphrase);
        String engineID = environmentSubstitute(engineid);

        UserTarget usertarget = new UserTarget();
        transMap.listen();
        usertarget.setAddress(udpAddress);
        if (usertarget.getAddress().isValid()) {
          if (log.isDebug()) {
            logDebug("Valid IP address");
          }
        } else {
          throw new KettleException("Invalid IP address");
        }

        usertarget.setRetries(retry);
        usertarget.setTimeout(timeOut);
        usertarget.setVersion(SnmpConstants.version3);
        usertarget.setSecurityLevel(SecurityLevel.AUTH_PRIV);
        usertarget.setSecurityName(new OctetString("MD5DES"));

        // Since we are using SNMPv3 we use authenticated users
        // this is handled by the UsmUser and USM class

        UsmUser uu =
            new UsmUser(
                new OctetString(userName),
                AuthMD5.ID,
                new OctetString(passPhrase),
                PrivDES.ID,
                new OctetString(passPhrase));

        USM usm = snmp.getUSM();

        if (usm == null) {
          throw new KettleException("Null Usm");
        } else {
          usm =
              new USM(
                  SecurityProtocols.getInstance(), new OctetString(MPv3.createLocalEngineID()), 0);
          usm.addUser(new OctetString(userName), uu);
          if (log.isDebug()) {
            logDebug("Valid Usm");
          }
        }

        // create the PDU
        ScopedPDU pdu = new ScopedPDU();
        pdu.add(new VariableBinding(new OID(Oid), new OctetString(messageString)));
        pdu.setType(PDU.TRAP);
        if (!Const.isEmpty(engineID)) {
          pdu.setContextEngineID(new OctetString(engineID));
        }

        // send the PDU
        response = snmp.send(pdu, usertarget);
      }

      if (response != null) {
        if (log.isDebug()) {
          logDebug("Received response from: " + response.getPeerAddress() + response.toString());
        }
      }

      result.setNrErrors(0);
      result.setResult(true);
    } catch (Exception e) {
      logError(BaseMessages.getString(PKG, "JobEntrySNMPTrap.ErrorGetting", e.getMessage()));
    } finally {
      try {
        if (snmp != null) {
          snmp.close();
        }
      } catch (Exception e) {
        /* Ignore */
      }
    }

    return result;
  }
Ejemplo n.º 5
0
  public Result execute(Result previousResult, int nr) {
    Result result = previousResult;
    result.setResult(false);

    if (log.isRowLevel()) {
      logRowlevel(BaseMessages.getString(PKG, "JobSSH2GET.Log.GettingFieldsValue"));
    }

    // Get real variable value
    String realServerName = environmentSubstitute(serverName);
    int realServerPort = Const.toInt(environmentSubstitute(serverPort), 22);
    String realUserName = environmentSubstitute(userName);
    String realServerPassword =
        Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(password));
    // Proxy Host
    String realProxyHost = environmentSubstitute(httpProxyHost);
    int realProxyPort = Const.toInt(environmentSubstitute(httpproxyport), 22);
    String realproxyUserName = environmentSubstitute(httpproxyusername);
    String realProxyPassword =
        Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(httpProxyPassword));
    // Key file
    String realKeyFilename = environmentSubstitute(keyFilename);
    String relKeyFilepass = environmentSubstitute(keyFilePass);
    // target files
    String realLocalDirectory = environmentSubstitute(localDirectory);
    String realwildcard = environmentSubstitute(wildcard);
    // Remote source
    String realftpDirectory = environmentSubstitute(ftpDirectory);
    // Destination folder (Move to)
    String realDestinationFolder = environmentSubstitute(destinationfolder);

    try {
      // Remote source
      realftpDirectory = FTPUtils.normalizePath(realftpDirectory);
      // Destination folder (Move to)
      realDestinationFolder = FTPUtils.normalizePath(realDestinationFolder);
    } catch (Exception e) {
      logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.CanNotNormalizePath", e.getMessage()));
      result.setNrErrors(1);
      return result;
    }

    // Check for mandatory fields
    if (log.isRowLevel()) {
      logRowlevel(BaseMessages.getString(PKG, "JobSSH2GET.Log.CheckingMandatoryFields"));
    }

    boolean mandatoryok = true;
    if (Const.isEmpty(realServerName)) {
      mandatoryok = false;
      logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.ServernameMissing"));
    }
    if (usehttpproxy) {
      if (Const.isEmpty(realProxyHost)) {
        mandatoryok = false;
        logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.HttpProxyhostMissing"));
      }
    }
    if (publicpublickey) {
      if (Const.isEmpty(realKeyFilename)) {
        mandatoryok = false;
        logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.KeyFileMissing"));
      } else {
        // Let's check if key file exists...
        if (!new File(realKeyFilename).exists()) {
          mandatoryok = false;
          logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.KeyFileNotExist"));
        }
      }
    }

    if (Const.isEmpty(realLocalDirectory)) {
      mandatoryok = false;
      logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.LocalFolderMissing"));
    } else {
      // Check if target folder exists...
      if (!new File(realLocalDirectory).exists()) {

        if (createtargetfolder) {
          // Create Target folder
          if (!CreateFolder(realLocalDirectory)) {
            mandatoryok = false;
          }

        } else {
          mandatoryok = false;
          logError(
              BaseMessages.getString(
                  PKG, "JobSSH2GET.Log.LocalFolderNotExists", realLocalDirectory));
        }
      } else {
        if (!new File(realLocalDirectory).isDirectory()) {
          mandatoryok = false;
          logError(
              BaseMessages.getString(
                  PKG, "JobSSH2GET.Log.LocalFolderNotFolder", realLocalDirectory));
        }
      }
    }
    if (afterFtpPut.equals("move_file")) {
      if (Const.isEmpty(realDestinationFolder)) {
        mandatoryok = false;
        logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.DestinatFolderMissing"));
      }
    }

    if (mandatoryok) {

      Connection conn = null;
      SFTPv3Client client = null;
      boolean good = true;

      try {
        // Create a connection instance
        conn =
            getConnection(
                realServerName,
                realServerPort,
                realProxyHost,
                realProxyPort,
                realproxyUserName,
                realProxyPassword);
        if (log.isDetailed()) {
          logDetailed(BaseMessages.getString(PKG, "JobSSH2GET.Log.ConnectionInstanceCreated"));
        }
        if (timeout > 0) {
          // Use timeout
          // Cache Host Key
          if (cachehostkey) {
            conn.connect(new SimpleVerifier(database), 0, timeout * 1000);
          } else {
            conn.connect(null, 0, timeout * 1000);
          }

        } else {
          // Cache Host Key
          if (cachehostkey) {
            conn.connect(new SimpleVerifier(database));
          } else {
            conn.connect();
          }
        }

        // Authenticate

        boolean isAuthenticated = false;
        if (publicpublickey) {
          isAuthenticated =
              conn.authenticateWithPublicKey(
                  realUserName, new File(realKeyFilename), relKeyFilepass);
        } else {
          isAuthenticated = conn.authenticateWithPassword(realUserName, realServerPassword);
        }

        // LET'S CHECK AUTHENTICATION ...
        if (isAuthenticated == false) {
          logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.AuthenticationFailed"));
        } else {
          if (log.isBasic()) {
            logBasic(BaseMessages.getString(PKG, "JobSSH2GET.Log.Connected", serverName, userName));
          }

          client = new SFTPv3Client(conn);

          if (log.isDetailed()) {
            logDetailed(
                BaseMessages.getString(
                    PKG, "JobSSH2GET.Log.ProtocolVersion", "" + client.getProtocolVersion()));
          }

          // Check if ftp (source) directory exists
          if (!Const.isEmpty(realftpDirectory)) {
            if (!sshDirectoryExists(client, realftpDirectory)) {
              good = false;
              logError(
                  BaseMessages.getString(
                      PKG, "JobSSH2GET.Log.RemoteDirectoryNotExist", realftpDirectory));
            } else if (log.isDetailed()) {
              logDetailed(
                  BaseMessages.getString(
                      PKG, "JobSSH2GET.Log.RemoteDirectoryExist", realftpDirectory));
            }
          }

          if (realDestinationFolder != null) {
            // Check now destination folder
            if (!sshDirectoryExists(client, realDestinationFolder)) {
              if (createdestinationfolder) {
                if (!CreateRemoteFolder(client, realDestinationFolder)) {
                  good = false;
                }
              } else {
                good = false;
                logError(
                    BaseMessages.getString(
                        PKG, "JobSSH2GET.Log.DestinatFolderNotExist", realDestinationFolder));
              }
            }
          }

          if (good) {
            Pattern pattern = null;
            if (!Const.isEmpty(realwildcard)) {
              pattern = Pattern.compile(realwildcard);
            }

            if (includeSubFolders) {
              if (log.isDetailed()) {
                logDetailed(BaseMessages.getString(PKG, "JobSSH2GET.Log.RecursiveModeOn"));
              }
              copyRecursive(realftpDirectory, realLocalDirectory, client, pattern, parentJob);
            } else {
              if (log.isDetailed()) {
                logDetailed(BaseMessages.getString(PKG, "JobSSH2GET.Log.RecursiveModeOff"));
              }
              GetFiles(realftpDirectory, realLocalDirectory, client, pattern, parentJob);
            }

            /** ****************************** RESULT ******************* */
            if (log.isDetailed()) {
              logDetailed(BaseMessages.getString(PKG, "JobSSH2GET.Log.Result.JobEntryEnd1"));
              logDetailed(
                  BaseMessages.getString(
                      PKG, "JobSSH2GET.Log.Result.TotalFiles", "" + nbfilestoget));
              logDetailed(
                  BaseMessages.getString(PKG, "JobSSH2GET.Log.Result.TotalFilesPut", "" + nbgot));
              logDetailed(
                  BaseMessages.getString(
                      PKG, "JobSSH2GET.Log.Result.TotalFilesError", "" + nbrerror));
              logDetailed(BaseMessages.getString(PKG, "JobSSH2GET.Log.Result.JobEntryEnd2"));
            }
            if (nbrerror == 0) {
              result.setResult(true);
              /** ****************************** RESULT ******************* */
            }
          }
        }

      } catch (Exception e) {
        result.setNrErrors(nbrerror);
        logError(BaseMessages.getString(PKG, "JobSSH2GET.Log.Error.ErrorFTP", e.getMessage()));
      } finally {
        if (conn != null) {
          conn.close();
        }
        if (client != null) {
          client.close();
        }
      }
    }

    return result;
  }
  public Result execute(Result previousResult, int nr) {

    Result result = previousResult;
    List<RowMetaAndData> rows = result.getRows();
    RowMetaAndData resultRow = null;

    oneFileLocked = false;
    result.setResult(true);

    try {
      if (argFromPrevious) {
        if (isDetailed()) {
          logDetailed(
              BaseMessages.getString(
                  PKG,
                  "JobEntryCheckFilesLocked.FoundPreviousRows",
                  String.valueOf((rows != null ? rows.size() : 0))));
        }
      }

      if (argFromPrevious && rows != null) {
        // Copy the input row to the (command line) arguments
        for (int iteration = 0; iteration < rows.size() && !parentJob.isStopped(); iteration++) {
          resultRow = rows.get(iteration);

          // Get values from previous result
          String filefolder_previous = resultRow.getString(0, "");
          String fmasks_previous = resultRow.getString(1, "");

          // ok we can process this file/folder
          if (isDetailed()) {
            logDetailed(
                BaseMessages.getString(
                    PKG,
                    "JobEntryCheckFilesLocked.ProcessingRow",
                    filefolder_previous,
                    fmasks_previous));
          }

          ProcessFile(filefolder_previous, fmasks_previous);
        }
      } else if (arguments != null) {

        for (int i = 0; i < arguments.length && !parentJob.isStopped(); i++) {
          // ok we can process this file/folder
          if (isDetailed()) {
            logDetailed(
                BaseMessages.getString(
                    PKG, "JobEntryCheckFilesLocked.ProcessingArg", arguments[i], filemasks[i]));
          }

          ProcessFile(arguments[i], filemasks[i]);
        }
      }

      if (oneFileLocked) {
        result.setResult(false);
        result.setNrErrors(1);
      }
    } catch (Exception e) {
      logError(BaseMessages.getString(PKG, "JobEntryCheckFilesLocked.ErrorRunningJobEntry", e));
    }

    return result;
  }
  public Result execute(Result previousResult, int nr) {
    Result result = previousResult;
    result.setResult(false);

    // see PDI-10270, PDI-10644 for details
    boolean oldBehavior =
        "Y"
            .equalsIgnoreCase(
                getVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "N"));

    String countSQLStatement = null;
    long rowsCount = 0;
    long errCount = 0;

    boolean successOK = false;

    int nrRowsLimit = Const.toInt(environmentSubstitute(limit), 0);
    if (log.isDetailed()) {
      logDetailed(
          BaseMessages.getString(
              PKG, "JobEntryEvalTableContent.Log.nrRowsLimit", "" + nrRowsLimit));
    }

    if (connection != null) {
      Database db = new Database(this, connection);
      db.shareVariablesWith(this);
      try {
        db.connect(parentJob.getTransactionId(), null);

        if (iscustomSQL) {
          String realCustomSQL = customSQL;
          if (isUseVars) {
            realCustomSQL = environmentSubstitute(realCustomSQL);
          }
          if (log.isDebug()) {
            logDebug(
                BaseMessages.getString(
                    PKG, "JobEntryEvalTableContent.Log.EnteredCustomSQL", realCustomSQL));
          }

          if (!Const.isEmpty(realCustomSQL)) {
            countSQLStatement = realCustomSQL;
          } else {
            errCount++;
            logError(BaseMessages.getString(PKG, "JobEntryEvalTableContent.Error.NoCustomSQL"));
          }

        } else {
          String realTablename = environmentSubstitute(tablename);
          String realSchemaname = environmentSubstitute(schemaname);

          if (!Const.isEmpty(realTablename)) {
            if (!Const.isEmpty(realSchemaname)) {
              countSQLStatement =
                  selectCount
                      + db.getDatabaseMeta()
                          .getQuotedSchemaTableCombination(realSchemaname, realTablename);
            } else {
              countSQLStatement = selectCount + db.getDatabaseMeta().quoteField(realTablename);
            }
          } else {
            errCount++;
            logError(BaseMessages.getString(PKG, "JobEntryEvalTableContent.Error.NoTableName"));
          }
        }

        if (countSQLStatement != null) {
          if (log.isDetailed()) {
            logDetailed(
                BaseMessages.getString(
                    PKG, "JobEntryEvalTableContent.Log.RunSQLStatement", countSQLStatement));
          }

          if (iscustomSQL) {
            if (isClearResultList) {
              result.getRows().clear();
            }

            List<Object[]> ar = db.getRows(countSQLStatement, 0);
            if (ar != null) {
              rowsCount = ar.size();

              // ad rows to result
              RowMetaInterface rowMeta = db.getQueryFields(countSQLStatement, false);

              List<RowMetaAndData> rows = new ArrayList<RowMetaAndData>();
              for (int i = 0; i < ar.size(); i++) {
                rows.add(new RowMetaAndData(rowMeta, ar.get(i)));
              }
              if (isAddRowsResult && iscustomSQL) {
                if (rows != null) {
                  result.getRows().addAll(rows);
                }
              }
            } else {
              if (log.isDebug()) {
                logDebug(
                    BaseMessages.getString(
                        PKG,
                        "JobEntryEvalTableContent.Log.customSQLreturnedNothing",
                        countSQLStatement));
              }
            }

          } else {
            RowMetaAndData row = db.getOneRow(countSQLStatement);
            if (row != null) {
              rowsCount = row.getInteger(0);
            }
          }
          if (log.isDetailed()) {
            logDetailed(
                BaseMessages.getString(
                    PKG, "JobEntryEvalTableContent.Log.NrRowsReturned", "" + rowsCount));
          }
          switch (successCondition) {
            case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_EQUAL:
              successOK = (rowsCount == nrRowsLimit);
              break;
            case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_DIFFERENT:
              successOK = (rowsCount != nrRowsLimit);
              break;
            case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_SMALLER:
              successOK = (rowsCount < nrRowsLimit);
              break;
            case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_SMALLER_EQUAL:
              successOK = (rowsCount <= nrRowsLimit);
              break;
            case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_GREATER:
              successOK = (rowsCount > nrRowsLimit);
              break;
            case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_GREATER_EQUAL:
              successOK = (rowsCount >= nrRowsLimit);
              break;
            default:
              break;
          }

          if (!successOK && oldBehavior) {
            errCount++;
          }
        } // end if countSQLStatement!=null
      } catch (KettleException dbe) {
        errCount++;
        logError(
            BaseMessages.getString(
                PKG, "JobEntryEvalTableContent.Error.RunningEntry", dbe.getMessage()));
      } finally {
        if (db != null) {
          db.disconnect();
        }
      }
    } else {
      errCount++;
      logError(BaseMessages.getString(PKG, "JobEntryEvalTableContent.NoDbConnection"));
    }

    result.setResult(successOK);
    result.setNrLinesRead(rowsCount);
    result.setNrErrors(errCount);

    return result;
  }
  public Result execute(Result previousResult, int nr) {
    Result result = previousResult;

    result.setNrErrors(1);
    result.setResult(false);

    String hostname = getRealHostname();
    int timeoutInt = Const.toInt(getRealTimeOut(), 300);
    int packets = Const.toInt(getRealNbrPackets(), 2);
    boolean status = false;

    if (Const.isEmpty(hostname)) {
      // No Host was specified
      logError(BaseMessages.getString(PKG, "JobPing.SpecifyHost.Label"));
      return result;
    }

    try {
      if (ipingtype == isystemPing || ipingtype == ibothPings) {
        // Perform a system (Java) ping ...
        status = systemPing(hostname, timeoutInt);
        if (status) {
          if (log.isDetailed()) {
            log.logDetailed(
                BaseMessages.getString(PKG, "JobPing.SystemPing"),
                BaseMessages.getString(PKG, "JobPing.OK.Label", hostname));
          }
        } else {
          log.logError(
              BaseMessages.getString(PKG, "JobPing.SystemPing"),
              BaseMessages.getString(PKG, "JobPing.NOK.Label", hostname));
        }
      }
      if ((ipingtype == iclassicPing) || (ipingtype == ibothPings && !status)) {
        // Perform a classic ping ..
        status = classicPing(hostname, packets);
        if (status) {
          if (log.isDetailed()) {
            log.logDetailed(
                BaseMessages.getString(PKG, "JobPing.ClassicPing"),
                BaseMessages.getString(PKG, "JobPing.OK.Label", hostname));
          }
        } else {
          log.logError(
              BaseMessages.getString(PKG, "JobPing.ClassicPing"),
              BaseMessages.getString(PKG, "JobPing.NOK.Label", hostname));
        }
      }
    } catch (Exception ex) {
      logError(BaseMessages.getString(PKG, "JobPing.Error.Label") + ex.getMessage());
    }
    if (status) {
      if (log.isDetailed()) {
        logDetailed(BaseMessages.getString(PKG, "JobPing.OK.Label", hostname));
      }
      result.setNrErrors(0);
      result.setResult(true);
    } else {
      logError(BaseMessages.getString(PKG, "JobPing.NOK.Label", hostname));
    }
    return result;
  }
  public Result execute(Result previousResult, int nr) {

    String LimitNbrLignes = "";
    String ListOfColumn = "*";
    String strHighPriority = "";
    String OutDumpText = "";
    String OptionEnclosed = "";
    String FieldSeparator = "";
    String LinesTerminated = "";

    Result result = previousResult;
    result.setResult(false);

    // Let's check  the filename ...
    if (filename != null) {
      // User has specified a file, We can continue ...
      String realFilename = getRealFilename();
      File file = new File(realFilename);

      if (file.exists() && iffileexists == 2) {
        // the file exists and user want to Fail
        result.setResult(false);
        result.setNrErrors(1);
        logError(
            BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists1.Label")
                + realFilename
                + BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists2.Label"));

      } else if (file.exists() && iffileexists == 1) {
        // the file exists and user want to do nothing
        result.setResult(true);
        if (log.isDetailed())
          logDetailed(
              BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists1.Label")
                  + realFilename
                  + BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists2.Label"));

      } else {

        if (file.exists() && iffileexists == 0) {
          // File exists and user want to renamme it with unique name

          // Format Date

          // Try to clean filename (without wildcard)
          String wildcard =
              realFilename.substring(realFilename.length() - 4, realFilename.length());
          if (wildcard.substring(0, 1).equals(".")) {
            // Find wildcard
            realFilename =
                realFilename.substring(0, realFilename.length() - 4)
                    + "_"
                    + StringUtil.getFormattedDateTimeNow(true)
                    + wildcard;
          } else {
            // did not find wildcard
            realFilename = realFilename + "_" + StringUtil.getFormattedDateTimeNow(true);
          }

          logDebug(
              BaseMessages.getString(PKG, "JobMysqlBulkFile.FileNameChange1.Label")
                  + realFilename
                  + BaseMessages.getString(PKG, "JobMysqlBulkFile.FileNameChange1.Label"));
        }

        // User has specified an existing file, We can continue ...
        if (log.isDetailed())
          logDetailed(
              BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists1.Label")
                  + realFilename
                  + BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists2.Label"));

        if (connection != null) {
          // User has specified a connection, We can continue ...
          Database db = new Database(this, connection);
          db.shareVariablesWith(this);
          try {
            db.connect(parentJob.getTransactionId(), null);
            // Get schemaname
            String realSchemaname = environmentSubstitute(schemaname);
            // Get tablename
            String realTablename = environmentSubstitute(tablename);

            if (db.checkTableExists(realTablename)) {
              // The table existe, We can continue ...
              if (log.isDetailed())
                logDetailed(
                    BaseMessages.getString(PKG, "JobMysqlBulkFile.TableExists1.Label")
                        + realTablename
                        + BaseMessages.getString(PKG, "JobMysqlBulkFile.TableExists2.Label"));

              // Add schemaname (Most the time Schemaname.Tablename)
              if (schemaname != null) {
                realTablename = realSchemaname + "." + realTablename;
              }

              // Set the Limit lines
              if (Const.toInt(getRealLimitlines(), 0) > 0) {
                LimitNbrLignes = "LIMIT " + getRealLimitlines();
              }

              // Set list of Column, if null get all columns (*)
              if (getRealListColumn() != null) {
                ListOfColumn = MysqlString(getRealListColumn());
              }

              // Fields separator
              if (getRealSeparator() != null && outdumpvalue == 0) {
                FieldSeparator =
                    "FIELDS TERMINATED BY '" + Const.replace(getRealSeparator(), "'", "''") + "'";
              }

              // Lines Terminated by
              if (getRealLineterminated() != null && outdumpvalue == 0) {
                LinesTerminated =
                    "LINES TERMINATED BY '"
                        + Const.replace(getRealLineterminated(), "'", "''")
                        + "'";
              }

              // High Priority ?
              if (isHighPriority()) {
                strHighPriority = "HIGH_PRIORITY";
              }

              if (getRealEnclosed() != null && outdumpvalue == 0) {
                if (isOptionEnclosed()) {
                  OptionEnclosed = "OPTIONALLY ";
                }
                OptionEnclosed =
                    OptionEnclosed
                        + "ENCLOSED BY '"
                        + Const.replace(getRealEnclosed(), "'", "''")
                        + "'";
              }

              // OutFile or Dumpfile
              if (outdumpvalue == 0) {
                OutDumpText = "INTO OUTFILE";
              } else {
                OutDumpText = "INTO DUMPFILE";
              }

              String FILEBulkFile =
                  "SELECT "
                      + strHighPriority
                      + " "
                      + ListOfColumn
                      + " "
                      + OutDumpText
                      + " '"
                      + realFilename
                      + "' "
                      + FieldSeparator
                      + " "
                      + OptionEnclosed
                      + " "
                      + LinesTerminated
                      + " FROM "
                      + realTablename
                      + " "
                      + LimitNbrLignes
                      + " LOCK IN SHARE MODE";

              try {
                if (log.isDetailed()) logDetailed(FILEBulkFile);
                // Run the SQL
                PreparedStatement ps = db.prepareSQL(FILEBulkFile);
                ps.execute();

                // Everything is OK...we can disconnect now
                db.disconnect();

                if (isAddFileToResult()) {
                  // Add filename to output files
                  ResultFile resultFile =
                      new ResultFile(
                          ResultFile.FILE_TYPE_GENERAL,
                          KettleVFS.getFileObject(realFilename, this),
                          parentJob.getJobname(),
                          toString());
                  result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
                }

                result.setResult(true);

              } catch (SQLException je) {
                db.disconnect();
                result.setNrErrors(1);
                logError(
                    BaseMessages.getString(PKG, "JobMysqlBulkFile.Error.Label")
                        + " "
                        + je.getMessage());
              } catch (KettleFileException e) {
                logError(
                    BaseMessages.getString(PKG, "JobMysqlBulkFile.Error.Label") + e.getMessage());
                result.setNrErrors(1);
              }

            } else {
              // Of course, the table should have been created already before the bulk load
              // operation
              db.disconnect();
              result.setNrErrors(1);
              if (log.isDetailed())
                logDetailed(
                    BaseMessages.getString(PKG, "JobMysqlBulkFile.TableNotExists1.Label")
                        + realTablename
                        + BaseMessages.getString(PKG, "JobMysqlBulkFile.TableNotExists2.Label"));
            }

          } catch (KettleDatabaseException dbe) {
            db.disconnect();
            result.setNrErrors(1);
            logError(
                BaseMessages.getString(PKG, "JobMysqlBulkFile.Error.Label")
                    + " "
                    + dbe.getMessage());
          }

        } else {
          // No database connection is defined
          result.setNrErrors(1);
          logError(BaseMessages.getString(PKG, "JobMysqlBulkFile.Nodatabase.Label"));
        }
      }

    } else {
      // No file was specified
      result.setNrErrors(1);
      logError(BaseMessages.getString(PKG, "JobMysqlBulkFile.Nofilename.Label"));
    }

    return result;
  }
  @Override
  public Result execute(Result previousResult, int nr) {
    Result result = previousResult;
    result.setResult(false);
    result.setNrErrors(1);
    String realCustomSQL = null;
    String realTablename = environmentSubstitute(tablename);
    String realSchemaname = environmentSubstitute(schemaname);

    if (connection == null) {
      logError(BaseMessages.getString(PKG, "JobEntryWaitForSQL.NoDbConnection"));
      return result;
    }

    if (iscustomSQL) {
      // clear result list rows
      if (isClearResultList) {
        result.getRows().clear();
      }

      realCustomSQL = customSQL;
      if (isUseVars) {
        realCustomSQL = environmentSubstitute(realCustomSQL);
      }
      if (log.isDebug()) {
        logDebug(
            BaseMessages.getString(PKG, "JobEntryWaitForSQL.Log.EnteredCustomSQL", realCustomSQL));
      }

      if (Const.isEmpty(realCustomSQL)) {
        logError(BaseMessages.getString(PKG, "JobEntryWaitForSQL.Error.NoCustomSQL"));
        return result;
      }

    } else {
      if (Const.isEmpty(realTablename)) {
        logError(BaseMessages.getString(PKG, "JobEntryWaitForSQL.Error.NoTableName"));
        return result;
      }
    }

    try {
      // check connection
      // connect and disconnect
      checkConnection();

      // starttime (in seconds)
      long timeStart = System.currentTimeMillis() / 1000;

      int nrRowsLimit = Const.toInt(environmentSubstitute(rowsCountValue), 0);
      if (log.isDetailed()) {
        logDetailed(
            BaseMessages.getString(PKG, "JobEntryWaitForSQL.Log.nrRowsLimit", "" + nrRowsLimit));
      }

      long iMaximumTimeout =
          Const.toInt(
              environmentSubstitute(maximumTimeout), Const.toInt(DEFAULT_MAXIMUM_TIMEOUT, 0));
      long iCycleTime =
          Const.toInt(
              environmentSubstitute(checkCycleTime), Const.toInt(DEFAULT_CHECK_CYCLE_TIME, 0));

      //
      // Sanity check on some values, and complain on insanity
      //
      if (iMaximumTimeout < 0) {
        iMaximumTimeout = Const.toInt(DEFAULT_MAXIMUM_TIMEOUT, 0);
        logBasic("Maximum timeout invalid, reset to " + iMaximumTimeout);
      }

      if (iCycleTime < 1) {
        // If lower than 1 set to the default
        iCycleTime = Const.toInt(DEFAULT_CHECK_CYCLE_TIME, 1);
        logBasic("Check cycle time invalid, reset to " + iCycleTime);
      }

      if (iMaximumTimeout == 0) {
        logBasic("Waiting indefinitely for SQL data");
      } else {
        logBasic("Waiting " + iMaximumTimeout + " seconds for SQL data");
      }

      boolean continueLoop = true;
      while (continueLoop && !parentJob.isStopped()) {
        if (SQLDataOK(result, nrRowsLimit, realSchemaname, realTablename, realCustomSQL)) {
          // SQL data exists, we're happy to exit
          logBasic("Detected SQL data within timeout");
          result.setResult(true);
          continueLoop = false;
        } else {
          long now = System.currentTimeMillis() / 1000;

          if ((iMaximumTimeout > 0) && (now > (timeStart + iMaximumTimeout))) {
            continueLoop = false;

            // SQL data doesn't exist after timeout, either true or false
            if (isSuccessOnTimeout()) {
              logBasic("Didn't detect SQL data before timeout, success");
              result.setResult(true);
            } else {
              logBasic("Didn't detect SQL data before timeout, failure");
              result.setResult(false);
            }
          }
          // sleep algorithm
          long sleepTime = 0;

          if (iMaximumTimeout == 0) {
            sleepTime = iCycleTime;
          } else {
            if ((now + iCycleTime) < (timeStart + iMaximumTimeout)) {
              sleepTime = iCycleTime;
            } else {
              sleepTime = iCycleTime - ((now + iCycleTime) - (timeStart + iMaximumTimeout));
            }
          }
          try {
            if (sleepTime > 0) {
              if (log.isDetailed()) {
                logDetailed("Sleeping " + sleepTime + " seconds before next check for SQL data");
              }
              Thread.sleep(sleepTime * 1000);
            }
          } catch (InterruptedException e) {
            // something strange happened
            result.setResult(false);
            continueLoop = false;
          }
        }
      }
    } catch (Exception e) {
      logBasic("Exception while waiting for SQL data: " + e.getMessage());
    }

    if (result.getResult()) {
      // Remove error count set at the beginning of the method
      // PDI-15437
      result.setNrErrors(0);
    }

    return result;
  }
Ejemplo n.º 11
0
  private void executeShell(Result result, List<RowMetaAndData> cmdRows, String[] args) {
    FileObject fileObject = null;
    String realScript = null;
    FileObject tempFile = null;

    try {
      // What's the exact command?
      String[] base = null;
      List<String> cmds = new ArrayList<String>();

      if (log.isBasic()) {
        logBasic(BaseMessages.getString(PKG, "JobShell.RunningOn", Const.getOS()));
      }

      if (insertScript) {
        realScript = environmentSubstitute(script);
      } else {
        String realFilename = environmentSubstitute(getFilename());
        fileObject = KettleVFS.getFileObject(realFilename, this);
      }

      if (Const.getOS().equals("Windows 95")) {
        base = new String[] {"command.com", "/C"};
        if (insertScript) {
          tempFile =
              KettleVFS.createTempFile(
                  "kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this);
          fileObject = createTemporaryShellFile(tempFile, realScript);
        }
      } else if (Const.getOS().startsWith("Windows")) {
        base = new String[] {"cmd.exe", "/C"};
        if (insertScript) {
          tempFile =
              KettleVFS.createTempFile(
                  "kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this);
          fileObject = createTemporaryShellFile(tempFile, realScript);
        }
      } else {
        if (insertScript) {
          tempFile =
              KettleVFS.createTempFile(
                  "kettle", "shell", System.getProperty("java.io.tmpdir"), this);
          fileObject = createTemporaryShellFile(tempFile, realScript);
        }
        base = new String[] {KettleVFS.getFilename(fileObject)};
      }

      // Construct the arguments...
      if (argFromPrevious && cmdRows != null) {
        // Add the base command...
        for (int i = 0; i < base.length; i++) {
          cmds.add(base[i]);
        }

        if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) {
          // for windows all arguments including the command itself
          // need to be
          // included in 1 argument to cmd/command.

          StringBuffer cmdline = new StringBuffer(300);

          cmdline.append('"');
          cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject)));
          // Add the arguments from previous results...
          for (int i = 0; i < cmdRows.size(); i++) {
            // Normally just one row, but once in a while to remain compatible we have multiple.

            RowMetaAndData r = cmdRows.get(i);
            for (int j = 0; j < r.size(); j++) {
              cmdline.append(' ');
              cmdline.append(Const.optionallyQuoteStringByOS(r.getString(j, null)));
            }
          }
          cmdline.append('"');
          cmds.add(cmdline.toString());
        } else {
          // Add the arguments from previous results...
          for (int i = 0; i < cmdRows.size(); i++) {
            // Normally just one row, but once in a while to remain compatible we have multiple.

            RowMetaAndData r = cmdRows.get(i);
            for (int j = 0; j < r.size(); j++) {
              cmds.add(Const.optionallyQuoteStringByOS(r.getString(j, null)));
            }
          }
        }
      } else if (args != null) {
        // Add the base command...
        for (int i = 0; i < base.length; i++) {
          cmds.add(base[i]);
        }

        if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) {
          // for windows all arguments including the command itself
          // need to be
          // included in 1 argument to cmd/command.

          StringBuffer cmdline = new StringBuffer(300);

          cmdline.append('"');
          cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject)));

          for (int i = 0; i < args.length; i++) {
            cmdline.append(' ');
            cmdline.append(Const.optionallyQuoteStringByOS(args[i]));
          }
          cmdline.append('"');
          cmds.add(cmdline.toString());
        } else {
          for (int i = 0; i < args.length; i++) {
            cmds.add(args[i]);
          }
        }
      }

      StringBuffer command = new StringBuffer();

      Iterator<String> it = cmds.iterator();
      boolean first = true;
      while (it.hasNext()) {
        if (!first) {
          command.append(' ');
        } else {
          first = false;
        }
        command.append(it.next());
      }
      if (log.isBasic()) {
        logBasic(BaseMessages.getString(PKG, "JobShell.ExecCommand", command.toString()));
      }

      // Build the environment variable list...
      ProcessBuilder procBuilder = new ProcessBuilder(cmds);
      Map<String, String> env = procBuilder.environment();
      String[] variables = listVariables();
      for (int i = 0; i < variables.length; i++) {
        env.put(variables[i], getVariable(variables[i]));
      }

      if (getWorkDirectory() != null && !Const.isEmpty(Const.rtrim(getWorkDirectory()))) {
        String vfsFilename = environmentSubstitute(getWorkDirectory());
        File file = new File(KettleVFS.getFilename(KettleVFS.getFileObject(vfsFilename, this)));
        procBuilder.directory(file);
      }
      Process proc = procBuilder.start();

      // any error message?
      StreamLogger errorLogger = new StreamLogger(log, proc.getErrorStream(), "(stderr)", true);

      // any output?
      StreamLogger outputLogger = new StreamLogger(log, proc.getInputStream(), "(stdout)");

      // kick them off
      Thread errorLoggerThread = new Thread(errorLogger);
      errorLoggerThread.start();
      Thread outputLoggerThread = new Thread(outputLogger);
      outputLoggerThread.start();

      proc.waitFor();
      if (log.isDetailed()) {
        logDetailed(BaseMessages.getString(PKG, "JobShell.CommandFinished", command.toString()));
      }

      // What's the exit status?
      result.setExitStatus(proc.exitValue());
      if (result.getExitStatus() != 0) {
        if (log.isDetailed()) {
          logDetailed(
              BaseMessages.getString(
                  PKG,
                  "JobShell.ExitStatus",
                  environmentSubstitute(getFilename()),
                  "" + result.getExitStatus()));
        }

        result.setNrErrors(1);
      }

      // wait until loggers read all data from stdout and stderr
      errorLoggerThread.join();
      outputLoggerThread.join();

      // close the streams
      // otherwise you get "Too many open files, java.io.IOException" after a lot of iterations
      proc.getErrorStream().close();
      proc.getOutputStream().close();

    } catch (IOException ioe) {
      logError(
          BaseMessages.getString(
              PKG,
              "JobShell.ErrorRunningShell",
              environmentSubstitute(getFilename()),
              ioe.toString()),
          ioe);
      result.setNrErrors(1);
    } catch (InterruptedException ie) {
      logError(
          BaseMessages.getString(
              PKG, "JobShell.Shellinterupted", environmentSubstitute(getFilename()), ie.toString()),
          ie);
      result.setNrErrors(1);
    } catch (Exception e) {
      logError(
          BaseMessages.getString(
              PKG, "JobShell.UnexpectedError", environmentSubstitute(getFilename()), e.toString()),
          e);
      result.setNrErrors(1);
    } finally {
      // If we created a temporary file, remove it...
      //
      if (tempFile != null) {
        try {
          tempFile.delete();
        } catch (Exception e) {
          BaseMessages.getString(
              PKG, "JobShell.UnexpectedError", tempFile.toString(), e.toString());
        }
      }
    }

    if (result.getNrErrors() > 0) {
      result.setResult(false);
    } else {
      result.setResult(true);
    }
  }
Ejemplo n.º 12
0
  public Result execute(Result result, int nr) throws KettleException {
    FileLoggingEventListener loggingEventListener = null;
    LogLevel shellLogLevel = parentJob.getLogLevel();
    if (setLogfile) {
      String realLogFilename = environmentSubstitute(getLogFilename());
      // We need to check here the log filename
      // if we do not have one, we must fail
      if (Const.isEmpty(realLogFilename)) {
        logError(BaseMessages.getString(PKG, "JobEntryShell.Exception.LogFilenameMissing"));
        result.setNrErrors(1);
        result.setResult(false);
        return result;
      }

      try {
        loggingEventListener =
            new FileLoggingEventListener(getLogChannelId(), realLogFilename, setAppendLogfile);
        KettleLogStore.getAppender().addLoggingEventListener(loggingEventListener);
      } catch (KettleException e) {
        logError(
            BaseMessages.getString(
                PKG, "JobEntryShell.Error.UnableopenAppenderFile", getLogFilename(), e.toString()));
        logError(Const.getStackTracker(e));
        result.setNrErrors(1);
        result.setResult(false);
        return result;
      }
      shellLogLevel = logFileLevel;
    }

    log.setLogLevel(shellLogLevel);

    result.setEntryNr(nr);

    // "Translate" the arguments for later
    String[] substArgs = null;
    if (arguments != null) {
      substArgs = new String[arguments.length];
      for (int idx = 0; idx < arguments.length; idx++) {
        substArgs[idx] = environmentSubstitute(arguments[idx]);
      }
    }

    int iteration = 0;
    String[] args = substArgs;
    RowMetaAndData resultRow = null;
    boolean first = true;
    List<RowMetaAndData> rows = result.getRows();

    if (log.isDetailed()) {
      logDetailed(
          BaseMessages.getString(
              PKG, "JobEntryShell.Log.FoundPreviousRows", "" + (rows != null ? rows.size() : 0)));
    }

    while ((first && !execPerRow)
        || (execPerRow && rows != null && iteration < rows.size() && result.getNrErrors() == 0)) {
      first = false;
      if (rows != null && execPerRow) {
        resultRow = rows.get(iteration);
      } else {
        resultRow = null;
      }

      List<RowMetaAndData> cmdRows = null;

      if (execPerRow) {
        // Execute for each input row

        if (argFromPrevious) {
          // Copy the input row to the (command line) arguments

          if (resultRow != null) {
            args = new String[resultRow.size()];
            for (int i = 0; i < resultRow.size(); i++) {
              args[i] = resultRow.getString(i, null);
            }
          }
        } else {
          // Just pass a single row
          List<RowMetaAndData> newList = new ArrayList<RowMetaAndData>();
          newList.add(resultRow);
          cmdRows = newList;
        }
      } else {
        if (argFromPrevious) {
          // Only put the first Row on the arguments
          args = null;
          if (resultRow != null) {
            args = new String[resultRow.size()];
            for (int i = 0; i < resultRow.size(); i++) {
              args[i] = resultRow.getString(i, null);
            }
          } else {
            cmdRows = rows;
          }
        } else {
          // Keep it as it was...
          cmdRows = rows;
        }
      }

      executeShell(result, cmdRows, args);

      iteration++;
    }

    if (setLogfile) {
      if (loggingEventListener != null) {
        KettleLogStore.getAppender().removeLoggingEventListener(loggingEventListener);
        loggingEventListener.close();

        ResultFile resultFile =
            new ResultFile(
                ResultFile.FILE_TYPE_LOG,
                loggingEventListener.getFile(),
                parentJob.getJobname(),
                getName());
        result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
      }
    }

    return result;
  }
  public Result execute(Result result, int nr) throws KettleException {
    result.setResult(true);
    result.setNrErrors(0);
    try {

      List<String> variables = new ArrayList<String>();
      List<String> variableValues = new ArrayList<String>();
      List<Integer> variableTypes = new ArrayList<Integer>();

      String realFilename = environmentSubstitute(filename);
      try {
        if (!Const.isEmpty(realFilename)) {
          Properties properties = new Properties();
          properties.load(KettleVFS.getInputStream(realFilename));
          for (Object key : properties.keySet()) {
            variables.add((String) key);
            variableValues.add((String) properties.get(key));
            variableTypes.add(fileVariableType);
          }
        }
      } catch (Exception e) {
        throw new KettleException(
            BaseMessages.getString(
                PKG, "JobEntrySetVariables.Error.UnableReadPropertiesFile", realFilename));
      }

      for (int i = 0; i < variableName.length; i++) {
        variables.add(variableName[i]);
        variableValues.add(variableValue[i]);
        variableTypes.add(variableType[i]);
      }

      for (int i = 0; i < variables.size(); i++) {
        String varname = variables.get(i);
        String value = variableValues.get(i);
        int type = variableTypes.get(i);

        if (replaceVars) {
          varname = environmentSubstitute(varname);
          value = environmentSubstitute(value);
        }

        // OK, where do we set this value...
        switch (type) {
          case VARIABLE_TYPE_JVM:
            {
              System.setProperty(varname, value);
              setVariable(varname, value);
              Job parentJobTraverse = parentJob;
              while (parentJobTraverse != null) {
                parentJobTraverse.setVariable(varname, value);
                parentJobTraverse = parentJobTraverse.getParentJob();
              }
            }
            break;
          case VARIABLE_TYPE_ROOT_JOB:
            {
              // set variable in this job entry
              setVariable(varname, value);
              Job rootJob = parentJob;
              while (rootJob != null) {
                rootJob.setVariable(varname, value);
                rootJob = rootJob.getParentJob();
              }
            }
            break;
          case VARIABLE_TYPE_CURRENT_JOB:
            {
              setVariable(varname, value);
              if (parentJob != null) {
                parentJob.setVariable(varname, value);
              } else {
                throw new KettleJobException(
                    BaseMessages.getString(
                        PKG, "JobEntrySetVariables.Error.UnableSetVariableCurrentJob", varname));
              }
            }
            break;
          case VARIABLE_TYPE_PARENT_JOB:
            {
              setVariable(varname, value);

              if (parentJob != null) {
                parentJob.setVariable(varname, value);
                Job gpJob = parentJob.getParentJob();
                if (gpJob != null) {
                  gpJob.setVariable(varname, value);
                } else {
                  throw new KettleJobException(
                      BaseMessages.getString(
                          PKG, "JobEntrySetVariables.Error.UnableSetVariableParentJob", varname));
                }
              } else {
                throw new KettleJobException(
                    BaseMessages.getString(
                        PKG, "JobEntrySetVariables.Error.UnableSetVariableCurrentJob", varname));
              }
            }
            break;
          default:
            break;
        }

        // ok we can process this line
        if (log.isDetailed()) {
          logDetailed(
              BaseMessages.getString(
                  PKG, "JobEntrySetVariables.Log.SetVariableToValue", varname, value));
        }
      }
    } catch (Exception e) {
      result.setResult(false);
      result.setNrErrors(1);

      logError(
          BaseMessages.getString(PKG, "JobEntrySetVariables.UnExcpectedError", e.getMessage()));
    }

    return result;
  }