コード例 #1
0
 /**
  * Finds the tab for the transformation that matches the metadata provided (either the file must
  * be the same or the repository id).
  *
  * @param trans Transformation metadata to look for
  * @return Tab with transformation open whose metadata matches {@code trans} or {@code null} if no
  *     tab exists.
  * @throws KettleFileException If there is a problem loading the file object for an open
  *     transformation with an invalid a filename.
  */
 public TabMapEntry findTabForTransformation(TransMeta trans) throws KettleFileException {
   // File for the transformation we're looking for. It will be loaded upon first request.
   FileObject transFile = null;
   for (TabMapEntry entry : tabMap) {
     if (entry == null || entry.getTabItem().isDisposed()) {
       continue;
     }
     if (trans.getFilename() != null && entry.getFilename() != null) {
       // If the entry has a file name it is the same as trans iff. they originated from the same
       // files
       FileObject entryFile = KettleVFS.getFileObject(entry.getFilename());
       if (transFile == null) {
         transFile = KettleVFS.getFileObject(trans.getFilename());
       }
       if (entryFile.equals(transFile)) {
         return entry;
       }
     } else if (trans.getObjectId() != null && entry.getObject() != null) {
       EngineMetaInterface meta = entry.getObject().getMeta();
       if (meta != null && trans.getObjectId().equals(meta.getObjectId())) {
         // If the transformation has an object id and the entry shares the same id they are the
         // same
         return entry;
       }
     }
   }
   // No tabs for the transformation exist and are not disposed
   return null;
 }
コード例 #2
0
ファイル: GPLoad.java プロジェクト: hoverzheng/kettle
  /**
   * Create the command line for a psql process depending on the meta information supplied.
   *
   * @param meta The meta data to create the command line from
   * @param password Use the real password or not
   * @return The string to execute.
   * @throws KettleException Upon any exception
   */
  public String createCommandLine(GPLoadMeta meta, boolean password) throws KettleException {
    StringBuffer sb = new StringBuffer(300);

    if (meta.getGploadPath() != null) {
      try {
        FileObject fileObject =
            KettleVFS.getFileObject(environmentSubstitute(meta.getGploadPath()), getTransMeta());
        String psqlexec = KettleVFS.getFilename(fileObject);
        // sb.append('\'').append(psqlexec).append('\'');
        sb.append(psqlexec);
      } catch (Exception ex) {
        throw new KettleException("Error retrieving sqlldr string", ex);
      }
    } else {
      throw new KettleException("No psql application specified");
    }

    if (meta.getControlFile() != null) {
      try {
        FileObject fileObject =
            KettleVFS.getFileObject(environmentSubstitute(meta.getControlFile()), getTransMeta());

        sb.append(" -f ");
        // sb.append('\'').append(KettleVFS.getFilename(fileObject)).append('\'');
        sb.append(KettleVFS.getFilename(fileObject));
      } catch (Exception ex) {
        throw new KettleException("Error retrieving controlfile string", ex);
      }
    } else {
      throw new KettleException("No control file specified");
    }

    if (meta.getLogFile() != null) {
      try {
        FileObject fileObject =
            KettleVFS.getFileObject(environmentSubstitute(meta.getLogFile()), getTransMeta());

        sb.append(" -l ");
        sb.append('\'').append(KettleVFS.getFilename(fileObject)).append('\'');
      } catch (Exception ex) {
        throw new KettleException("Error retrieving logfile string", ex);
      }
    }

    // hostname, port and so on are passed through the control file
    //

    return sb.toString();
  }
コード例 #3
0
  private boolean CreateFolder(String filefolder) {
    FileObject folder = null;
    try {
      folder = KettleVFS.getFileObject(filefolder, this);

      if (!folder.exists()) {
        if (createtargetfolder) {
          folder.createFolder();
          if (log.isDetailed()) {
            logDetailed(
                BaseMessages.getString(PKG, "JobSSH2GET.Log.FolderCreated", folder.toString()));
          }
        } else {
          return false;
        }
      }
      return true;
    } catch (Exception e) {
      logError(
          BaseMessages.getString(PKG, "JobSSH2GET.Log.CanNotCreateFolder", folder.toString()), e);

    } finally {
      if (folder != null) {
        try {
          folder.close();
        } catch (Exception ex) {
          /* Ignore */
        }
      }
    }
    return false;
  }
コード例 #4
0
  private boolean OpenFile() throws Exception {
    data.oneFileOpened = true;
    String realFilename = environmentSubstitute(meta.getFilename());
    if (log.isBasic()) {
      logBasic(BaseMessages.getString(PKG, "AccessOutput.log.WritingToFile", realFilename));
    }
    FileObject fileObject = KettleVFS.getFileObject(realFilename, getTransMeta());
    File file = FileUtils.toFile(fileObject.getURL());

    // First open or create the access file
    if (!file.exists()) {
      if (meta.isFileCreated()) {
        data.db = Database.create(file);
      } else {
        logError(
            BaseMessages.getString(PKG, "AccessOutput.InitError.FileDoesNotExist", realFilename));
        return false;
      }
    } else {
      data.db = Database.open(file);
    }

    // Add the filename to the result object...
    //
    if (meta.isAddToResultFiles()) {
      ResultFile resultFile =
          new ResultFile(
              ResultFile.FILE_TYPE_GENERAL, fileObject, getTransMeta().getName(), toString());
      resultFile.setComment("This file was created with an access output step");
      addResultFile(resultFile);
    }

    return true;
  }
コード例 #5
0
  /**
   * Since the exported transformation that runs this will reside in a ZIP file, we can't reference
   * files relatively. So what this does is turn the name of the base path into an absolute path.
   *
   * @param space the variable space to use
   * @param definitions
   * @param resourceNamingInterface
   * @param repository The repository to optionally load other resources from (to be converted to
   *     XML)
   * @param metaStore the metaStore in which non-kettle metadata could reside.
   * @return the filename of the exported resource
   */
  public String exportResources(
      VariableSpace space,
      Map<String, ResourceDefinition> definitions,
      ResourceNamingInterface resourceNamingInterface,
      Repository repository,
      IMetaStore metaStore)
      throws KettleException {
    try {
      // The object that we're modifying here is a copy of the original!
      // So let's change the filename from relative to absolute by grabbing the file object...
      // In case the name of the file comes from previous steps, forget about this!
      //
      if (!fileNameInField) {

        if (!Const.isEmpty(fileName)) {
          FileObject fileObject =
              KettleVFS.getFileObject(space.environmentSubstitute(fileName), space);
          fileName = resourceNamingInterface.nameResource(fileObject, space, true);
        }
      }

      return null;
    } catch (Exception e) {
      throw new KettleException(e);
    }
  }
コード例 #6
0
  @Test
  public void testZeroSizeFile() throws Exception {
    ByteArrayOutputStream log = new ByteArrayOutputStream();
    helper.redirectLog(log, LogLevel.BASIC);
    try (FileObject fileObj = KettleVFS.getFileObject(BASE_RAM_DIR + "test.json");
        LocaleChange enUs = new LocaleChange(Locale.US); ) {
      fileObj.createFile();
      JsonInputField price = new JsonInputField();
      price.setName("price");
      price.setType(ValueMetaInterface.TYPE_NUMBER);
      price.setPath("$..book[*].price");

      JsonInputMeta meta = createSimpleMeta("in file", price);
      meta.setIsAFile(true);
      meta.setRemoveSourceField(true);
      meta.setIgnoreEmptyFile(false);
      JsonInput jsonInput =
          createJsonInput(
              "in file", meta, new Object[][] {new Object[] {BASE_RAM_DIR + "test.json"}});
      processRows(jsonInput, 1);
      String logMsgs = log.toString();
      assertTrue(logMsgs, logMsgs.contains("is empty!"));
    } finally {
      deleteFiles();
    }
  }
コード例 #7
0
  private boolean deleteOrMoveFiles(FileObject file, String destinationFolder)
      throws KettleException {
    try {
      boolean retval = false;

      // Delete the file if this is needed!
      //
      if (afterFtpPut.equals("delete_file")) {
        file.delete();
        retval = true;
        if (log.isDetailed()) {
          logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.DeletedFile", file.toString()));
        }
      } else if (afterFtpPut.equals("move_file")) {
        // Move File
        FileObject destination = null;
        FileObject source = null;
        try {
          destination =
              KettleVFS.getFileObject(
                  destinationFolder + Const.FILE_SEPARATOR + file.getName().getBaseName(), this);
          file.moveTo(destination);
          retval = true;
        } catch (Exception e) {
          logError(
              BaseMessages.getString(
                  PKG,
                  "JobSSH2PUT.Cant_Move_File.Label",
                  file.toString(),
                  destinationFolder,
                  e.getMessage()));
        } finally {
          if (destination != null) {
            try {
              destination.close();
            } catch (Exception ex) {
              /* Ignore */
            }
          }
          if (source != null) {
            try {
              source.close();
            } catch (Exception ex) {
              /* Ignore */
            }
          }
        }
        if (log.isDetailed()) {
          logDetailed(
              BaseMessages.getString(
                  PKG, "JobSSH2PUT.Log.MovedFile", file.toString(), ftpDirectory));
        }
      }
      return retval;
    } catch (Exception e) {
      throw new KettleException(e);
    }
  }
コード例 #8
0
  private void ProcessFile(String filename, String wildcard) {

    FileObject filefolder = null;
    String realFilefoldername = environmentSubstitute(filename);
    String realwilcard = environmentSubstitute(wildcard);

    try {
      filefolder = KettleVFS.getFileObject(realFilefoldername);
      FileObject[] files = new FileObject[] {filefolder};
      if (filefolder.exists()) {
        // the file or folder exists
        if (filefolder.getType() == FileType.FOLDER) {
          // It's a folder
          if (isDetailed()) {
            logDetailed(
                BaseMessages.getString(
                    PKG, "JobEntryCheckFilesLocked.ProcessingFolder", realFilefoldername));
          }
          // Retrieve all files
          files = filefolder.findFiles(new TextFileSelector(filefolder.toString(), realwilcard));

          if (isDetailed()) {
            logDetailed(
                BaseMessages.getString(
                    PKG,
                    "JobEntryCheckFilesLocked.TotalFilesToCheck",
                    String.valueOf(files.length)));
          }
        } else {
          // It's a file
          if (isDetailed()) {
            logDetailed(
                BaseMessages.getString(
                    PKG, "JobEntryCheckFilesLocked.ProcessingFile", realFilefoldername));
          }
        }
        // Check files locked
        checkFilesLocked(files);
      } else {
        // We can not find thsi file
        logBasic(
            BaseMessages.getString(
                PKG, "JobEntryCheckFilesLocked.FileNotExist", realFilefoldername));
      }
    } catch (Exception e) {
      logError(
          BaseMessages.getString(
              PKG, "JobEntryCheckFilesLocked.CouldNotProcess", realFilefoldername, e.getMessage()));
    } finally {
      if (filefolder != null) {
        try {
          filefolder.close();
        } catch (IOException ex) {
          // Ignore
        }
      }
    }
  }
コード例 #9
0
  @Test
  public void testFileList() throws Exception {
    ByteArrayOutputStream err = new ByteArrayOutputStream();
    helper.redirectLog(err, LogLevel.ERROR);

    final String input1 = getBasicTestJson();
    final String input2 = "{ \"store\": { \"book\": [ { \"price\": 9.99 } ] } }";
    try (FileObject fileObj1 = KettleVFS.getFileObject(BASE_RAM_DIR + "test1.json");
        FileObject fileObj2 = KettleVFS.getFileObject(BASE_RAM_DIR + "test2.json")) {
      try (OutputStream out = fileObj1.getContent().getOutputStream()) {
        out.write(input1.getBytes());
      }
      try (OutputStream out = fileObj2.getContent().getOutputStream()) {
        out.write(input2.getBytes());
      }
      JsonInputField price = new JsonInputField();
      price.setName("price");
      price.setType(ValueMetaInterface.TYPE_NUMBER);
      price.setPath("$..book[*].price");
      List<FileObject> fileList = Arrays.asList(fileObj1, fileObj2);
      JsonInputMeta meta = createFileListMeta(fileList);
      meta.setInputFields(new JsonInputField[] {price});

      meta.setIncludeRowNumber(true);
      meta.setRowNumberField("rownbr");

      meta.setShortFileNameField("fname");

      JsonInput jsonInput = createJsonInput(meta);
      RowComparatorListener rowComparator =
          new RowComparatorListener(
              new Object[] {8.95d, 1L, "test1.json"},
              new Object[] {12.99d, 2L, "test1.json"},
              new Object[] {8.99d, 3L, "test1.json"},
              new Object[] {22.99d, 4L, "test1.json"},
              new Object[] {9.99d, 5L, "test2.json"});
      jsonInput.addRowListener(rowComparator);

      processRows(jsonInput, 5);
      disposeJsonInput(jsonInput);
      assertEquals(err.toString(), 0, jsonInput.getErrors());
    } finally {
      deleteFiles();
    }
  }
コード例 #10
0
ファイル: GPLoad.java プロジェクト: hoverzheng/kettle
  public void dispose(StepMetaInterface smi, StepDataInterface sdi) {
    meta = (GPLoadMeta) smi;
    data = (GPLoadData) sdi;

    super.dispose(smi, sdi);

    if (!preview && meta.isEraseFiles()) {
      // Erase the created cfg/dat files if requested. We don't erase
      // the rest of the files because it would be "stupid" to erase them
      // right after creation. If you don't want them, don't fill them in.
      FileObject fileObject = null;

      String method = meta.getLoadMethod();
      if ( // GPLoadMeta.METHOD_AUTO_CONCURRENT.equals(method) ||
      GPLoadMeta.METHOD_AUTO_END.equals(method)) {
        /*
         if ( meta.getControlFile() != null )
        {
         try
         {
                fileObject = KettleVFS.getFileObject(environmentSubstitute(meta.getControlFile()), getTransMeta());
                fileObject.delete();
                fileObject.close();
              }
            catch ( Exception ex )
            {
                logError("Error deleting control file \'" + KettleVFS.getFilename(fileObject) + "\': " + ex.getMessage());
            }
        }*/
      }

      if (GPLoadMeta.METHOD_AUTO_END.equals(method)) {
        // In concurrent mode the data is written to the control file.
        if (meta.getDataFile() != null) {
          try {
            fileObject =
                KettleVFS.getFileObject(environmentSubstitute(meta.getDataFile()), getTransMeta());
            fileObject.delete();
            fileObject.close();
          } catch (Exception ex) {
            logError(
                "Error deleting data file \'"
                    + KettleVFS.getFilename(fileObject)
                    + "\': "
                    + ex.getMessage(),
                ex);
          }
        }
      }

      if (GPLoadMeta.METHOD_MANUAL.equals(method)) {
        logBasic("Deletion of files is not compatible with \'manual load method\'");
      }
    }
  }
コード例 #11
0
  /**
   * Log only lines belonging to the specified log channel ID or one of it's children
   * (grandchildren) to the specified file.
   *
   * @param logChannelId
   * @param filename
   * @param append
   * @throws KettleException
   */
  public FileLoggingEventListener(String logChannelId, String filename, boolean append)
      throws KettleException {
    this.logChannelId = logChannelId;
    this.filename = filename;
    this.layout = new KettleLogLayout(true);
    this.exception = null;

    file = KettleVFS.getFileObject(filename);
    outputStream = null;
    try {
      outputStream = KettleVFS.getOutputStream(file, append);
    } catch (Exception e) {
      throw new KettleException(
          "Unable to create a logging event listener to write to file '" + filename + "'", e);
    }
  }
コード例 #12
0
  /**
   * Check existence of a local file
   *
   * @param filename
   * @return true, if file exists
   */
  public boolean FileExists(String filename) {

    FileObject file = null;
    try {
      file = KettleVFS.getFileObject(filename, this);
      if (!file.exists()) {
        return false;
      } else {
        if (file.getType() == FileType.FILE) {
          return true;
        } else {
          return false;
        }
      }
    } catch (Exception e) {
      return false;
    }
  }
コード例 #13
0
  @Test
  public void testZipFileInput() throws Exception {
    ByteArrayOutputStream err = new ByteArrayOutputStream();
    helper.redirectLog(err, LogLevel.ERROR);

    final String input = getBasicTestJson();
    try (FileObject fileObj = KettleVFS.getFileObject(BASE_RAM_DIR + "test.zip")) {
      fileObj.createFile();
      try (OutputStream out = fileObj.getContent().getOutputStream()) {
        try (ZipOutputStream zipOut = new ZipOutputStream(out)) {
          ZipEntry jsonFile = new ZipEntry("test.json");
          zipOut.putNextEntry(jsonFile);
          zipOut.write(input.getBytes());
          zipOut.closeEntry();
          zipOut.flush();
        }
      }
      JsonInputField price = new JsonInputField();
      price.setName("price");
      price.setType(ValueMetaInterface.TYPE_NUMBER);
      price.setPath("$..book[*].price");

      JsonInputMeta meta = createSimpleMeta("in file", price);
      meta.setIsAFile(true);
      meta.setRemoveSourceField(true);
      JsonInput jsonInput =
          createJsonInput(
              "in file",
              meta,
              new Object[][] {new Object[] {"zip:" + BASE_RAM_DIR + "test.zip!/test.json"}});
      RowComparatorListener rowComparator =
          new RowComparatorListener(
              new Object[] {8.95d},
              new Object[] {12.99d},
              new Object[] {8.99d},
              new Object[] {22.99d});
      jsonInput.addRowListener(rowComparator);
      processRows(jsonInput, 5);
      Assert.assertEquals(err.toString(), 0, jsonInput.getErrors());
    } finally {
      deleteFiles();
    }
  }
コード例 #14
0
  /**
   * Init Helper Class with connection settings
   *
   * @param serverIP IP address of remote server
   * @param serverPort port of remote server
   * @param userName username of remote server
   * @param privateKeyFilename filename of private key
   * @param passPhrase passphrase
   * @throws KettleJobException
   */
  public SFTPClient(
      InetAddress serverIP,
      int serverPort,
      String userName,
      String privateKeyFilename,
      String passPhrase)
      throws KettleJobException {

    if (serverIP == null || serverPort < 0 || userName == null || userName.equals("")) {
      throw new KettleJobException(
          "For a SFTP connection server name and username must be set and server port must be greater than zero.");
    }

    this.serverIP = serverIP;
    this.serverPort = serverPort;
    this.userName = userName;

    JSch jsch = new JSch();
    try {
      if (!Const.isEmpty(privateKeyFilename)) {
        // We need to use private key authentication
        this.prvkey = privateKeyFilename;
        byte[] passphrasebytes = new byte[0];
        if (!Const.isEmpty(passPhrase)) {
          // Set passphrase
          this.passphrase = passPhrase;
          passphrasebytes = GetPrivateKeyPassPhrase().getBytes();
        }
        jsch.addIdentity(
            getUserName(),
            FileUtil.getContent(KettleVFS.getFileObject(prvkey)), // byte[] privateKey
            null, // byte[] publicKey
            passphrasebytes); // byte[] passPhrase
      }
      s = jsch.getSession(userName, serverIP.getHostAddress(), serverPort);
    } catch (IOException e) {
      throw new KettleJobException(e);
    } catch (KettleFileException e) {
      throw new KettleJobException(e);
    } catch (JSchException e) {
      throw new KettleJobException(e);
    }
  }
コード例 #15
0
  private List<FileObject> getFiles(String localfolder) throws KettleFileException {
    try {
      List<FileObject> myFileList = new ArrayList<FileObject>();

      // Get all the files in the local directory...

      FileObject localFiles = KettleVFS.getFileObject(localfolder, this);
      FileObject[] children = localFiles.getChildren();
      if (children != null) {
        for (int i = 0; i < children.length; i++) {
          // Get filename of file or directory
          if (children[i].getType().equals(FileType.FILE)) {
            myFileList.add(children[i]);
          }
        } // end for
      }

      return myFileList;
    } catch (IOException e) {
      throw new KettleFileException(e);
    }
  }
コード例 #16
0
  /**
   * Since the exported transformation that runs this will reside in a ZIP file, we can't reference
   * files relatively. So what this does is turn the name of files into absolute paths OR it simply
   * includes the resource in the ZIP file. For now, we'll simply turn it into an absolute path and
   * pray that the file is on a shared drive or something like that.
   *
   * <p>TODO: create options to configure this behavior
   */
  public String exportResources(
      VariableSpace space,
      Map<String, ResourceDefinition> definitions,
      ResourceNamingInterface resourceNamingInterface,
      Repository repository)
      throws KettleException {
    try {
      // The object that we're modifying here is a copy of the original!
      // So let's change the filename from relative to absolute by grabbing the file object...
      // In case the name of the file comes from previous steps, forget about this!
      //
      if (Const.isEmpty(filenameField)) {
        // From : ${Internal.Transformation.Filename.Directory}/../foo/bar.csv
        // To   : /home/matt/test/files/foo/bar.csv
        //
        FileObject fileObject = KettleVFS.getFileObject(space.environmentSubstitute(filename));

        // If the file doesn't exist, forget about this effort too!
        //
        if (fileObject.exists()) {
          // Convert to an absolute path...
          //
          filename =
              resourceNamingInterface.nameResource(
                  fileObject.getName().getBaseName(),
                  fileObject.getParent().getName().getPath(),
                  space.toString(),
                  FileNamingType.DATA_FILE);

          return filename;
        }
      }
      return null;
    } catch (Exception e) {
      throw new KettleException(e); // $NON-NLS-1$
    }
  }
コード例 #17
0
  public Result execute(Result previousResult, int nr, Repository rep, Job parentJob) {
    LogWriter log = LogWriter.getInstance();
    Result result = previousResult;
    result.setResult(false);

    if (filename != null) {
      String realFilename = getRealFilename();
      try {
        FileObject file = KettleVFS.getFileObject(realFilename);
        if (file.exists() && file.isReadable()) {
          log.logDetailed(
              toString(),
              Messages.getString("JobEntryFileExists.File_Exists", realFilename)); // $NON-NLS-1$
          result.setResult(true);
        } else {
          log.logDetailed(
              toString(),
              Messages.getString(
                  "JobEntryFileExists.File_Does_Not_Exist", realFilename)); // $NON-NLS-1$
        }
      } catch (IOException e) {
        result.setNrErrors(1);
        log.logError(
            toString(),
            Messages.getString(
                "JobEntryFileExists.ERROR_0004_IO_Exception", e.toString())); // $NON-NLS-1$
      }
    } else {
      result.setNrErrors(1);
      log.logError(
          toString(),
          Messages.getString("JobEntryFileExists.ERROR_0005_No_Filename_Defined")); // $NON-NLS-1$
    }

    return result;
  }
コード例 #18
0
  public void openNewFile(String baseFilename) throws KettleException {
    data.writer = null;

    ResultFile resultFile = null;

    String filename = buildFilename(environmentSubstitute(baseFilename), true);

    try {
      if (meta.isFileAsCommand()) {
        if (log.isDebug()) logDebug("Spawning external process");
        if (data.cmdProc != null) {
          logError("Previous command not correctly terminated");
          setErrors(1);
        }
        String cmdstr = environmentSubstitute(meta.getFileName());
        if (Const.getOS().equals("Windows 95")) {
          cmdstr = "command.com /C " + cmdstr;
        } else {
          if (Const.getOS().startsWith("Windows")) {
            cmdstr = "cmd.exe /C " + cmdstr;
          }
        }
        if (log.isDetailed()) logDetailed("Starting: " + cmdstr);
        Runtime r = Runtime.getRuntime();
        data.cmdProc = r.exec(cmdstr, EnvUtil.getEnvironmentVariablesForRuntimeExec());
        data.writer = data.cmdProc.getOutputStream();
        StreamLogger stdoutLogger = new StreamLogger(data.cmdProc.getInputStream(), "(stdout)");
        StreamLogger stderrLogger = new StreamLogger(data.cmdProc.getErrorStream(), "(stderr)");
        new Thread(stdoutLogger).start();
        new Thread(stderrLogger).start();
      } else {
        // Add this to the result file names...
        resultFile =
            new ResultFile(
                ResultFile.FILE_TYPE_GENERAL,
                KettleVFS.getFileObject(filename),
                getTransMeta().getName(),
                getStepname());
        resultFile.setComment("This file was created with a text file output step");
        addResultFile(resultFile);

        OutputStream outputStream;

        if (!Const.isEmpty(meta.getFileCompression())
            && !meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_NONE)) {
          if (meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_ZIP)) {
            if (log.isDetailed())
              log.logDetailed(toString(), "Opening output stream in zipped mode");

            if (checkPreviouslyOpened(filename)) {
              data.fos = KettleVFS.getOutputStream(filename, true);
            } else {
              data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended());
            }
            data.zip = new ZipOutputStream(data.fos);
            File entry = new File(filename);
            ZipEntry zipentry = new ZipEntry(entry.getName());
            zipentry.setComment("Compressed by Kettle");
            data.zip.putNextEntry(zipentry);
            outputStream = data.zip;
          } else if (meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_GZIP)) {
            if (log.isDetailed())
              log.logDetailed(toString(), "Opening output stream in gzipped mode");
            if (checkPreviouslyOpened(filename)) {
              data.fos = KettleVFS.getOutputStream(filename, true);
            } else {
              data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended());
            }
            data.gzip = new GZIPOutputStream(data.fos);
            outputStream = data.gzip;
          } else {
            throw new KettleFileException("No compression method specified!");
          }
        } else {
          if (log.isDetailed())
            log.logDetailed(toString(), "Opening output stream in nocompress mode");
          if (checkPreviouslyOpened(filename)) {
            data.fos = KettleVFS.getOutputStream(filename, true);
          } else {
            data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended());
          }
          outputStream = data.fos;
        }

        if (!Const.isEmpty(meta.getEncoding())) {
          if (log.isDetailed())
            log.logDetailed(toString(), "Opening output stream in encoding: " + meta.getEncoding());
          data.writer = new BufferedOutputStream(outputStream, 5000);
        } else {
          if (log.isDetailed())
            log.logDetailed(toString(), "Opening output stream in default encoding");
          data.writer = new BufferedOutputStream(outputStream, 5000);
        }

        if (log.isDetailed()) logDetailed("Opened new file with name [" + filename + "]");
      }
    } catch (Exception e) {
      throw new KettleException("Error opening new file : " + e.toString());
    }
    // System.out.println("end of newFile(), splitnr="+splitnr);

    data.splitnr++;

    if (resultFile != null && meta.isAddToResultFiles()) {
      // Add this to the result file names...
      addResultFile(resultFile);
    }
  }
コード例 #19
0
ファイル: JsonInput.java プロジェクト: cnopens/BA
  private boolean ReadNextString() {

    try {
      data.readrow = getRow(); // Grab another row ...

      if (data.readrow == null) {
        // finished processing!
        if (log.isDetailed())
          logDetailed(BaseMessages.getString(PKG, "JsonInput.Log.FinishedProcessing"));
        return false;
      }

      if (first) {
        first = false;

        data.inputRowMeta = getInputRowMeta();
        data.outputRowMeta = data.inputRowMeta.clone();
        meta.getFields(data.outputRowMeta, getStepname(), null, null, this);

        // Get total previous fields
        data.totalpreviousfields = data.inputRowMeta.size();

        // Create convert meta-data objects that will contain Date & Number formatters
        data.convertRowMeta = data.outputRowMeta.clone();
        for (int i = 0; i < data.convertRowMeta.size(); i++)
          data.convertRowMeta.getValueMeta(i).setType(ValueMetaInterface.TYPE_STRING);

        // For String to <type> conversions, we allocate a conversion meta data row as well...
        //
        data.convertRowMeta = data.outputRowMeta.clone();
        for (int i = 0; i < data.convertRowMeta.size(); i++) {
          data.convertRowMeta.getValueMeta(i).setType(ValueMetaInterface.TYPE_STRING);
        }

        // Check if source field is provided
        if (Const.isEmpty(meta.getFieldValue())) {
          logError(BaseMessages.getString(PKG, "JsonInput.Log.NoField"));
          throw new KettleException(BaseMessages.getString(PKG, "JsonInput.Log.NoField"));
        }

        // cache the position of the field
        if (data.indexSourceField < 0) {
          data.indexSourceField = getInputRowMeta().indexOfValue(meta.getFieldValue());
          if (data.indexSourceField < 0) {
            // The field is unreachable !
            logError(
                BaseMessages.getString(
                    PKG,
                    "JsonInput.Log.ErrorFindingField",
                    meta.getFieldValue())); // $NON-NLS-1$ //$NON-NLS-2$
            throw new KettleException(
                BaseMessages.getString(
                    PKG,
                    "JsonInput.Exception.CouldnotFindField",
                    meta.getFieldValue())); // $NON-NLS-1$ //$NON-NLS-2$
          }
        }
      }

      // get source field value
      String fieldValue = getInputRowMeta().getString(data.readrow, data.indexSourceField);

      if (log.isDetailed())
        logDetailed(
            BaseMessages.getString(
                PKG, "JsonInput.Log.SourceValue", meta.getFieldValue(), fieldValue));

      if (meta.getIsAFile()) {

        // source is a file.
        data.file = KettleVFS.getFileObject(fieldValue, getTransMeta());
        if (meta.isIgnoreEmptyFile() && data.file.getContent().getSize() == 0) {
          // log only basic as a warning (was before logError)
          logBasic(
              BaseMessages.getString(PKG, "JsonInput.Error.FileSizeZero", data.file.getName()));
          ReadNextString();
        }
      } else {
        // read string
        data.stringToParse = fieldValue;
      }

      readFileOrString();
    } catch (Exception e) {
      logError(BaseMessages.getString(PKG, "JsonInput.Log.UnexpectedError", e.toString()));
      stopAll();
      logError(Const.getStackTracker(e));
      setErrors(1);
      return false;
    }
    return true;
  }
コード例 #20
0
ファイル: XMLOutput.java プロジェクト: hoverzheng/kettle
  public boolean openNewFile() {
    boolean retval = false;
    data.writer = null;

    try {

      FileObject file = KettleVFS.getFileObject(buildFilename(true), getTransMeta());

      if (meta.isAddToResultFiles()) {
        // Add this to the result file names...
        ResultFile resultFile =
            new ResultFile(
                ResultFile.FILE_TYPE_GENERAL, file, getTransMeta().getName(), getStepname());
        resultFile.setComment("This file was created with a xml output step"); // $NON-NLS-1$
        addResultFile(resultFile);
      }

      OutputStream outputStream;
      if (meta.isZipped()) {
        OutputStream fos = KettleVFS.getOutputStream(file, false);
        data.zip = new ZipOutputStream(fos);
        File entry = new File(buildFilename(false));
        ZipEntry zipentry = new ZipEntry(entry.getName());
        zipentry.setComment("Compressed by Kettle"); // $NON-NLS-1$
        data.zip.putNextEntry(zipentry);
        outputStream = data.zip;
      } else {
        OutputStream fos = KettleVFS.getOutputStream(file, false);
        outputStream = fos;
      }
      if (meta.getEncoding() != null && meta.getEncoding().length() > 0) {
        logBasic("Opening output stream in encoding: " + meta.getEncoding()); // $NON-NLS-1$
        data.writer = new OutputStreamWriter(outputStream, meta.getEncoding());
        data.writer.write(XMLHandler.getXMLHeader(meta.getEncoding()).toCharArray());
      } else {
        logBasic(
            "Opening output stream in default encoding : " + Const.XML_ENCODING); // $NON-NLS-1$
        data.writer = new OutputStreamWriter(outputStream);
        data.writer.write(XMLHandler.getXMLHeader(Const.XML_ENCODING).toCharArray());
      }

      // Add the name space if defined
      StringBuffer nameSpace = new StringBuffer();
      if ((meta.getNameSpace() != null) && (!"".equals(meta.getNameSpace()))) { // $NON-NLS-1$
        nameSpace.append(" xmlns=\""); // $NON-NLS-1$
        nameSpace.append(meta.getNameSpace());
        nameSpace.append("\""); // $NON-NLS-1$
      }

      // OK, write the header & the parent element:
      data.writer.write(
          ("<" + meta.getMainElement() + nameSpace.toString() + ">" + Const.CR)
              .toCharArray()); //$NON-NLS-1$//$NON-NLS-2$

      retval = true;
    } catch (Exception e) {
      logError("Error opening new file : " + e.toString()); // $NON-NLS-1$
    }
    // System.out.println("end of newFile(), splitnr="+splitnr);

    data.splitnr++;

    return retval;
  }
コード例 #21
0
  public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (AutoDocMeta) smi;
    data = (AutoDocData) sdi;

    Object[] row = getRow();
    if (row == null) {

      if (data.filenames.isEmpty()) {
        // Nothing to see here, move along!
        //
        setOutputDone();
        return false;
      }

      // End of the line, create the documentation...
      //
      FileObject targetFile =
          KettleVFS.getFileObject(environmentSubstitute(meta.getTargetFilename()));
      String targetFilename = KettleVFS.getFilename(targetFile);

      // Create the report builder
      //
      KettleReportBuilder kettleReportBuilder =
          new KettleReportBuilder(this, data.filenames, KettleVFS.getFilename(targetFile), meta);

      try {
        // Try to get the Classic Reporting Engine to boot inside of the plugin class loader...
        //
        if (ClassicEngineBoot.getInstance().isBootDone() == false) {

          ObjectUtilities.setClassLoader(getClass().getClassLoader());
          ObjectUtilities.setClassLoaderSource(ObjectUtilities.CLASS_CONTEXT);

          LibLoaderBoot.getInstance().start();
          LibFontBoot.getInstance().start();
          ClassicEngineBoot.getInstance().start();
        }

        // Do the reporting thing...
        //
        kettleReportBuilder.createReport();
        kettleReportBuilder.render();

        Object[] outputRowData = RowDataUtil.allocateRowData(data.outputRowMeta.size());
        int outputIndex = 0;
        outputRowData[outputIndex++] = targetFilename;

        // Pass along the data to the next steps...
        //
        putRow(data.outputRowMeta, outputRowData);

        // Add the target file to the result file list
        //
        ResultFile resultFile =
            new ResultFile(
                ResultFile.FILE_TYPE_GENERAL, targetFile, getTransMeta().getName(), toString());
        resultFile.setComment("This file was generated by the 'Auto Documentation Output' step");
        addResultFile(resultFile);
      } catch (Exception e) {
        throw new KettleException(
            BaseMessages.getString(PKG, "AutoDoc.Exception.UnableToRenderReport"), e);
      }

      setOutputDone();
      return false;
    }

    if (first) {
      first = false;

      data.outputRowMeta = getInputRowMeta().clone();
      meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);

      // Get the filename field index...
      //
      String filenameField = environmentSubstitute(meta.getFilenameField());
      data.fileNameFieldIndex = getInputRowMeta().indexOfValue(filenameField);
      if (data.fileNameFieldIndex < 0) {
        throw new KettleException(
            BaseMessages.getString(PKG, "AutoDoc.Exception.FilenameFieldNotFound", filenameField));
      }

      // Get the file type field index...
      //
      String fileTypeField = environmentSubstitute(meta.getFileTypeField());
      data.fileTypeFieldIndex = getInputRowMeta().indexOfValue(fileTypeField);
      if (data.fileTypeFieldIndex < 0) {
        throw new KettleException(
            BaseMessages.getString(PKG, "AutoDoc.Exception.FileTypeFieldNotFound", fileTypeField));
      }

      data.repository = getTrans().getRepository();
      if (data.repository != null) {
        data.tree = data.repository.loadRepositoryDirectoryTree();
      }

      // Initialize the repository information handlers (images, metadata, loading, etc)
      //
      TransformationInformation.init(getTrans().getRepository());
      JobInformation.init(getTrans().getRepository());
    }

    // One more transformation or job to place in the documentation.
    //
    String fileName = getInputRowMeta().getString(row, data.fileNameFieldIndex);
    String fileType = getInputRowMeta().getString(row, data.fileTypeFieldIndex);

    RepositoryObjectType objectType;
    if ("Transformation".equalsIgnoreCase(fileType)) {
      objectType = RepositoryObjectType.TRANSFORMATION;
    } else if ("Job".equalsIgnoreCase(fileType)) {
      objectType = RepositoryObjectType.JOB;
    } else {
      throw new KettleException(
          BaseMessages.getString(PKG, "AutoDoc.Exception.UnknownFileTypeValue", fileType));
    }

    ReportSubjectLocation location = null;
    if (getTrans().getRepository() == null) {
      switch (objectType) {
        case TRANSFORMATION:
          location = new ReportSubjectLocation(fileName, null, null, objectType);
          break;
        case JOB:
          location = new ReportSubjectLocation(fileName, null, null, objectType);
          break;
        default:
          break;
      }
    } else {
      int lastSlashIndex = fileName.lastIndexOf(RepositoryDirectory.DIRECTORY_SEPARATOR);
      if (lastSlashIndex < 0) {
        fileName = RepositoryDirectory.DIRECTORY_SEPARATOR + fileName;
        lastSlashIndex = 0;
      }

      String directoryName = fileName.substring(0, lastSlashIndex + 1);
      String objectName = fileName.substring(lastSlashIndex + 1);

      RepositoryDirectoryInterface directory = data.tree.findDirectory(directoryName);
      if (directory == null) {
        throw new KettleException(
            BaseMessages.getString(
                PKG, "AutoDoc.Exception.RepositoryDirectoryNotFound", directoryName));
      }

      location = new ReportSubjectLocation(null, directory, objectName, objectType);
    }

    if (location == null) {
      throw new KettleException(
          BaseMessages.getString(
              PKG, "AutoDoc.Exception.UnableToDetermineLocation", fileName, fileType));
    }

    if (meta.getOutputType() != OutputType.METADATA) {
      // Add the file location to the list for later processing in one output report
      //
      data.filenames.add(location);
    } else {
      // Load the metadata from the transformation / job...
      // Output it in one row for each input row
      //
      Object[] outputRow = RowDataUtil.resizeArray(row, data.outputRowMeta.size());
      int outputIndex = getInputRowMeta().size();

      List<AreaOwner> imageAreaList = null;

      switch (location.getObjectType()) {
        case TRANSFORMATION:
          TransformationInformation ti = TransformationInformation.getInstance();
          TransMeta transMeta = ti.getTransMeta(location);
          imageAreaList = ti.getImageAreaList(location);

          // TransMeta
          outputRow[outputIndex++] = transMeta;
          break;

        case JOB:
          JobInformation ji = JobInformation.getInstance();
          JobMeta jobMeta = ji.getJobMeta(location);
          imageAreaList = ji.getImageAreaList(location);

          // TransMeta
          outputRow[outputIndex++] = jobMeta;
          break;
        default:
          break;
      }

      // Name
      if (meta.isIncludingName()) {
        outputRow[outputIndex++] = KettleFileTableModel.getName(location);
      }

      // Description
      if (meta.isIncludingDescription()) {
        outputRow[outputIndex++] = KettleFileTableModel.getDescription(location);
      }

      // Extended Description
      if (meta.isIncludingExtendedDescription()) {
        outputRow[outputIndex++] = KettleFileTableModel.getExtendedDescription(location);
      }

      // created
      if (meta.isIncludingCreated()) {
        outputRow[outputIndex++] = KettleFileTableModel.getCreation(location);
      }

      // modified
      if (meta.isIncludingModified()) {
        outputRow[outputIndex++] = KettleFileTableModel.getModification(location);
      }

      // image
      if (meta.isIncludingImage()) {
        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
        try {
          BufferedImage image = KettleFileTableModel.getImage(location);
          ImageIO.write(image, "png", outputStream);

          outputRow[outputIndex++] = outputStream.toByteArray();
        } catch (Exception e) {
          throw new KettleException("Unable to serialize image to PNG", e);
        } finally {
          try {
            outputStream.close();
          } catch (IOException e) {
            throw new KettleException("Unable to serialize image to PNG", e);
          }
        }
      }

      if (meta.isIncludingLoggingConfiguration()) {
        outputRow[outputIndex++] = KettleFileTableModel.getLogging(location);
      }

      if (meta.isIncludingLastExecutionResult()) {
        outputRow[outputIndex++] = KettleFileTableModel.getLogging(location);
      }

      if (meta.isIncludingImageAreaList()) {
        outputRow[outputIndex++] = imageAreaList;
      }

      putRow(data.outputRowMeta, outputRow);
    }

    return true;
  }
コード例 #22
0
  public Result execute(Result previousResult, int nr) {

    String LimitNbrLignes = "";
    String ListOfColumn = "*";
    String strHighPriority = "";
    String OutDumpText = "";
    String OptionEnclosed = "";
    String FieldSeparator = "";
    String LinesTerminated = "";

    Result result = previousResult;
    result.setResult(false);

    // Let's check  the filename ...
    if (filename != null) {
      // User has specified a file, We can continue ...
      String realFilename = getRealFilename();
      File file = new File(realFilename);

      if (file.exists() && iffileexists == 2) {
        // the file exists and user want to Fail
        result.setResult(false);
        result.setNrErrors(1);
        logError(
            BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists1.Label")
                + realFilename
                + BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists2.Label"));

      } else if (file.exists() && iffileexists == 1) {
        // the file exists and user want to do nothing
        result.setResult(true);
        if (log.isDetailed())
          logDetailed(
              BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists1.Label")
                  + realFilename
                  + BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists2.Label"));

      } else {

        if (file.exists() && iffileexists == 0) {
          // File exists and user want to renamme it with unique name

          // Format Date

          // Try to clean filename (without wildcard)
          String wildcard =
              realFilename.substring(realFilename.length() - 4, realFilename.length());
          if (wildcard.substring(0, 1).equals(".")) {
            // Find wildcard
            realFilename =
                realFilename.substring(0, realFilename.length() - 4)
                    + "_"
                    + StringUtil.getFormattedDateTimeNow(true)
                    + wildcard;
          } else {
            // did not find wildcard
            realFilename = realFilename + "_" + StringUtil.getFormattedDateTimeNow(true);
          }

          logDebug(
              BaseMessages.getString(PKG, "JobMysqlBulkFile.FileNameChange1.Label")
                  + realFilename
                  + BaseMessages.getString(PKG, "JobMysqlBulkFile.FileNameChange1.Label"));
        }

        // User has specified an existing file, We can continue ...
        if (log.isDetailed())
          logDetailed(
              BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists1.Label")
                  + realFilename
                  + BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists2.Label"));

        if (connection != null) {
          // User has specified a connection, We can continue ...
          Database db = new Database(this, connection);
          db.shareVariablesWith(this);
          try {
            db.connect(parentJob.getTransactionId(), null);
            // Get schemaname
            String realSchemaname = environmentSubstitute(schemaname);
            // Get tablename
            String realTablename = environmentSubstitute(tablename);

            if (db.checkTableExists(realTablename)) {
              // The table existe, We can continue ...
              if (log.isDetailed())
                logDetailed(
                    BaseMessages.getString(PKG, "JobMysqlBulkFile.TableExists1.Label")
                        + realTablename
                        + BaseMessages.getString(PKG, "JobMysqlBulkFile.TableExists2.Label"));

              // Add schemaname (Most the time Schemaname.Tablename)
              if (schemaname != null) {
                realTablename = realSchemaname + "." + realTablename;
              }

              // Set the Limit lines
              if (Const.toInt(getRealLimitlines(), 0) > 0) {
                LimitNbrLignes = "LIMIT " + getRealLimitlines();
              }

              // Set list of Column, if null get all columns (*)
              if (getRealListColumn() != null) {
                ListOfColumn = MysqlString(getRealListColumn());
              }

              // Fields separator
              if (getRealSeparator() != null && outdumpvalue == 0) {
                FieldSeparator =
                    "FIELDS TERMINATED BY '" + Const.replace(getRealSeparator(), "'", "''") + "'";
              }

              // Lines Terminated by
              if (getRealLineterminated() != null && outdumpvalue == 0) {
                LinesTerminated =
                    "LINES TERMINATED BY '"
                        + Const.replace(getRealLineterminated(), "'", "''")
                        + "'";
              }

              // High Priority ?
              if (isHighPriority()) {
                strHighPriority = "HIGH_PRIORITY";
              }

              if (getRealEnclosed() != null && outdumpvalue == 0) {
                if (isOptionEnclosed()) {
                  OptionEnclosed = "OPTIONALLY ";
                }
                OptionEnclosed =
                    OptionEnclosed
                        + "ENCLOSED BY '"
                        + Const.replace(getRealEnclosed(), "'", "''")
                        + "'";
              }

              // OutFile or Dumpfile
              if (outdumpvalue == 0) {
                OutDumpText = "INTO OUTFILE";
              } else {
                OutDumpText = "INTO DUMPFILE";
              }

              String FILEBulkFile =
                  "SELECT "
                      + strHighPriority
                      + " "
                      + ListOfColumn
                      + " "
                      + OutDumpText
                      + " '"
                      + realFilename
                      + "' "
                      + FieldSeparator
                      + " "
                      + OptionEnclosed
                      + " "
                      + LinesTerminated
                      + " FROM "
                      + realTablename
                      + " "
                      + LimitNbrLignes
                      + " LOCK IN SHARE MODE";

              try {
                if (log.isDetailed()) logDetailed(FILEBulkFile);
                // Run the SQL
                PreparedStatement ps = db.prepareSQL(FILEBulkFile);
                ps.execute();

                // Everything is OK...we can disconnect now
                db.disconnect();

                if (isAddFileToResult()) {
                  // Add filename to output files
                  ResultFile resultFile =
                      new ResultFile(
                          ResultFile.FILE_TYPE_GENERAL,
                          KettleVFS.getFileObject(realFilename, this),
                          parentJob.getJobname(),
                          toString());
                  result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
                }

                result.setResult(true);

              } catch (SQLException je) {
                db.disconnect();
                result.setNrErrors(1);
                logError(
                    BaseMessages.getString(PKG, "JobMysqlBulkFile.Error.Label")
                        + " "
                        + je.getMessage());
              } catch (KettleFileException e) {
                logError(
                    BaseMessages.getString(PKG, "JobMysqlBulkFile.Error.Label") + e.getMessage());
                result.setNrErrors(1);
              }

            } else {
              // Of course, the table should have been created already before the bulk load
              // operation
              db.disconnect();
              result.setNrErrors(1);
              if (log.isDetailed())
                logDetailed(
                    BaseMessages.getString(PKG, "JobMysqlBulkFile.TableNotExists1.Label")
                        + realTablename
                        + BaseMessages.getString(PKG, "JobMysqlBulkFile.TableNotExists2.Label"));
            }

          } catch (KettleDatabaseException dbe) {
            db.disconnect();
            result.setNrErrors(1);
            logError(
                BaseMessages.getString(PKG, "JobMysqlBulkFile.Error.Label")
                    + " "
                    + dbe.getMessage());
          }

        } else {
          // No database connection is defined
          result.setNrErrors(1);
          logError(BaseMessages.getString(PKG, "JobMysqlBulkFile.Nodatabase.Label"));
        }
      }

    } else {
      // No file was specified
      result.setNrErrors(1);
      logError(BaseMessages.getString(PKG, "JobMysqlBulkFile.Nofilename.Label"));
    }

    return result;
  }
コード例 #23
0
  protected void pickFileVFS() {

    FileDialog dialog = new FileDialog(shell, SWT.OPEN);
    dialog.setFilterExtensions(Const.STRING_TRANS_FILTER_EXT);
    dialog.setFilterNames(Const.getTransformationFilterNames());
    String prevName = jobMeta.environmentSubstitute(wPath.getText());
    String parentFolder = null;
    try {
      parentFolder =
          KettleVFS.getFilename(
              KettleVFS.getFileObject(jobMeta.environmentSubstitute(jobMeta.getFilename()))
                  .getParent());
    } catch (Exception e) {
      // not that important
    }
    if (!Utils.isEmpty(prevName)) {
      try {
        if (KettleVFS.fileExists(prevName)) {
          dialog.setFilterPath(
              KettleVFS.getFilename(KettleVFS.getFileObject(prevName).getParent()));
        } else {

          if (!prevName.endsWith(".ktr")) {
            prevName = getEntryName(Const.trim(wPath.getText()) + ".ktr");
          }
          if (KettleVFS.fileExists(prevName)) {
            specificationMethod = ObjectLocationSpecificationMethod.FILENAME;
            wPath.setText(prevName);
            return;
          } else {
            // File specified doesn't exist. Ask if we should create the file...
            //
            MessageBox mb = new MessageBox(shell, SWT.YES | SWT.NO | SWT.ICON_QUESTION);
            mb.setMessage(
                BaseMessages.getString(
                    PKG, "JobTrans.Dialog.CreateTransformationQuestion.Message"));
            mb.setText(
                BaseMessages.getString(
                    PKG, "JobTrans.Dialog.CreateTransformationQuestion.Title")); // Sorry!
            int answer = mb.open();
            if (answer == SWT.YES) {

              Spoon spoon = Spoon.getInstance();
              spoon.newTransFile();
              TransMeta transMeta = spoon.getActiveTransformation();
              transMeta.initializeVariablesFrom(jobEntry);
              transMeta.setFilename(jobMeta.environmentSubstitute(prevName));
              wPath.setText(prevName);
              specificationMethod = ObjectLocationSpecificationMethod.FILENAME;
              spoon.saveFile();
              return;
            }
          }
        }
      } catch (Exception e) {
        dialog.setFilterPath(parentFolder);
      }
    } else if (!Utils.isEmpty(parentFolder)) {
      dialog.setFilterPath(parentFolder);
    }

    String fname = dialog.open();
    if (fname != null) {
      File file = new File(fname);
      String name = file.getName();
      String parentFolderSelection = file.getParentFile().toString();

      if (!Utils.isEmpty(parentFolder) && parentFolder.equals(parentFolderSelection)) {
        wPath.setText(getEntryName(name));
      } else {
        wPath.setText(fname);
      }
    }
  }
コード例 #24
0
  public Result execute(Result previousResult, int nr) {
    Result result = previousResult;
    result.setResult(false);

    // starttime (in seconds)
    long timeStart = System.currentTimeMillis() / 1000;

    if (filename != null) {
      FileObject fileObject = null;
      String realFilename = getRealFilename();
      try {
        fileObject = KettleVFS.getFileObject(realFilename, this);

        long iMaximumTimeout =
            Const.toInt(getRealMaximumTimeout(), Const.toInt(DEFAULT_MAXIMUM_TIMEOUT, 0));
        long iCycleTime =
            Const.toInt(getRealCheckCycleTime(), Const.toInt(DEFAULT_CHECK_CYCLE_TIME, 0));

        //
        // Sanity check on some values, and complain on insanity
        //
        if (iMaximumTimeout < 0) {
          iMaximumTimeout = Const.toInt(DEFAULT_MAXIMUM_TIMEOUT, 0);
          if (log.isBasic()) logBasic("Maximum timeout invalid, reset to " + iMaximumTimeout);
        }

        if (iCycleTime < 1) {
          // If lower than 1 set to the default
          iCycleTime = Const.toInt(DEFAULT_CHECK_CYCLE_TIME, 1);
          if (log.isBasic()) logBasic("Check cycle time invalid, reset to " + iCycleTime);
        }

        if (iMaximumTimeout == 0) {
          if (log.isBasic()) logBasic("Waiting indefinitely for file [" + realFilename + "]");
        } else {
          if (log.isBasic())
            logBasic("Waiting " + iMaximumTimeout + " seconds for file [" + realFilename + "]");
        }

        boolean continueLoop = true;
        while (continueLoop && !parentJob.isStopped()) {
          fileObject = KettleVFS.getFileObject(realFilename, this);

          if (fileObject.exists()) {
            // file exists, we're happy to exit
            if (log.isBasic()) logBasic("Detected file [" + realFilename + "] within timeout");
            result.setResult(true);
            continueLoop = false;

            // add filename to result filenames
            if (addFilenameToResult && fileObject.getType() == FileType.FILE) {
              ResultFile resultFile =
                  new ResultFile(
                      ResultFile.FILE_TYPE_GENERAL, fileObject, parentJob.getJobname(), toString());
              resultFile.setComment(BaseMessages.getString(PKG, "JobWaitForFile.FilenameAdded"));
              result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
            }
          } else {
            long now = System.currentTimeMillis() / 1000;

            if ((iMaximumTimeout > 0) && (now > (timeStart + iMaximumTimeout))) {
              continueLoop = false;

              // file doesn't exist after timeout, either true or false
              if (isSuccessOnTimeout()) {
                if (log.isBasic())
                  logBasic("Didn't detect file [" + realFilename + "] before timeout, success");
                result.setResult(true);
              } else {
                if (log.isBasic())
                  logBasic("Didn't detect file [" + realFilename + "] before timeout, failure");
                result.setResult(false);
              }
            }

            // sleep algorithm
            long sleepTime = 0;

            if (iMaximumTimeout == 0) {
              sleepTime = iCycleTime;
            } else {
              if ((now + iCycleTime) < (timeStart + iMaximumTimeout)) {
                sleepTime = iCycleTime;
              } else {
                sleepTime = iCycleTime - ((now + iCycleTime) - (timeStart + iMaximumTimeout));
              }
            }

            try {
              if (sleepTime > 0) {
                if (log.isDetailed()) {
                  logDetailed(
                      "Sleeping "
                          + sleepTime
                          + " seconds before next check for file ["
                          + realFilename
                          + "]");
                }
                Thread.sleep(sleepTime * 1000);
              }
            } catch (InterruptedException e) {
              // something strange happened
              result.setResult(false);
              continueLoop = false;
            }
          }
        }

        if (!parentJob.isStopped() && fileObject.exists() && isFileSizeCheck()) {
          long oldSize = -1;
          long newSize = fileObject.getContent().getSize();

          if (log.isDetailed())
            logDetailed("File [" + realFilename + "] is " + newSize + " bytes long");
          if (log.isBasic())
            logBasic(
                "Waiting until file ["
                    + realFilename
                    + "] stops growing for "
                    + iCycleTime
                    + " seconds");
          while (oldSize != newSize && !parentJob.isStopped()) {
            try {
              if (log.isDetailed()) {
                logDetailed(
                    "Sleeping "
                        + iCycleTime
                        + " seconds, waiting for file ["
                        + realFilename
                        + "] to stop growing");
              }
              Thread.sleep(iCycleTime * 1000);
            } catch (InterruptedException e) {
              // something strange happened
              result.setResult(false);
              continueLoop = false;
            }
            oldSize = newSize;
            newSize = fileObject.getContent().getSize();
            if (log.isDetailed()) {
              logDetailed("File [" + realFilename + "] is " + newSize + " bytes long");
            }
          }
          if (log.isBasic())
            logBasic("Stopped waiting for file [" + realFilename + "] to stop growing");
        }

        if (parentJob.isStopped()) {
          result.setResult(false);
        }
      } catch (Exception e) {
        logBasic("Exception while waiting for file [" + realFilename + "] to stop growing", e);
      } finally {
        if (fileObject != null) {
          try {
            fileObject.close();
          } catch (Exception e) {
          }
        }
      }
    } else {
      logError("No filename is defined.");
    }

    return result;
  }
コード例 #25
0
 /**
  * Resolve file name.
  *
  * @param fileName the filename to resolve. may contain Kettle Environment variables.
  * @return the data file name.
  * @throws KettleException the kettle exception
  */
 @SuppressWarnings("unused")
 private String resolveFileName(final String fileName) throws KettleException {
   final FileObject fileObject = KettleVFS.getFileObject(parent.environmentSubstitute(fileName));
   return KettleVFS.getFilename(fileObject);
 }
コード例 #26
0
  private boolean processOneXMLFile(
      String xmlfilename, String xslfilename, String outputfilename, Result result, Job parentJob) {
    boolean retval = false;
    FileObject xmlfile = null;
    FileObject xslfile = null;
    FileObject outputfile = null;

    try {
      xmlfile = KettleVFS.getFileObject(xmlfilename, this);
      xslfile = KettleVFS.getFileObject(xslfilename, this);
      outputfile = KettleVFS.getFileObject(outputfilename, this);

      if (xmlfile.exists() && xslfile.exists()) {
        if (outputfile.exists() && iffileexists == 2) {
          // Output file exists
          // User want to fail
          logError(
              BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists1.Label")
                  + outputfilename
                  + BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists2.Label"));
          return retval;

        } else if (outputfile.exists() && iffileexists == 1) {
          // Do nothing
          if (log.isDebug()) {
            logDebug(
                BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists1.Label")
                    + outputfilename
                    + BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists2.Label"));
          }
          retval = true;
          return retval;

        } else {
          if (outputfile.exists() && iffileexists == 0) {
            // the output file exists and user want to create new one with unique name
            // Format Date

            // Try to clean filename (without wildcard)
            String wildcard =
                outputfilename.substring(outputfilename.length() - 4, outputfilename.length());
            if (wildcard.substring(0, 1).equals(".")) {
              // Find wildcard
              outputfilename =
                  outputfilename.substring(0, outputfilename.length() - 4)
                      + "_"
                      + StringUtil.getFormattedDateTimeNow(true)
                      + wildcard;
            } else {
              // did not find wildcard
              outputfilename = outputfilename + "_" + StringUtil.getFormattedDateTimeNow(true);
            }
            if (log.isDebug()) {
              logDebug(
                  BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists1.Label")
                      + outputfilename
                      + BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists2.Label"));
              logDebug(
                  BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileNameChange1.Label")
                      + outputfilename
                      + BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileNameChange2.Label"));
            }
          }

          // Create transformer factory
          TransformerFactory factory = TransformerFactory.newInstance();

          if (xsltfactory.equals(FACTORY_SAXON)) {
            // Set the TransformerFactory to the SAXON implementation.
            factory = new net.sf.saxon.TransformerFactoryImpl();
          }

          if (log.isDetailed()) {
            log.logDetailed(
                BaseMessages.getString(PKG, "JobEntryXSL.Log.TransformerFactoryInfos"),
                BaseMessages.getString(
                    PKG, "JobEntryXSL.Log.TransformerFactory", factory.getClass().getName()));
          }

          InputStream xslInputStream = KettleVFS.getInputStream(xslfile);
          InputStream xmlInputStream = KettleVFS.getInputStream(xmlfile);
          OutputStream os = null;
          try {
            // Use the factory to create a template containing the xsl file
            Templates template = factory.newTemplates(new StreamSource(xslInputStream));

            // Use the template to create a transformer
            Transformer xformer = template.newTransformer();

            if (log.isDetailed()) {
              log.logDetailed(
                  BaseMessages.getString(PKG, "JobEntryXSL.Log.TransformerClassInfos"),
                  BaseMessages.getString(
                      PKG, "JobEntryXSL.Log.TransformerClass", xformer.getClass().getName()));
            }

            // Do we need to set output properties?
            if (setOutputProperties) {
              xformer.setOutputProperties(outputProperties);
            }

            // Do we need to pass parameters?
            if (useParameters) {
              for (int i = 0; i < nrParams; i++) {
                xformer.setParameter(nameOfParams[i], valueOfParams[i]);
              }
            }

            // Prepare the input and output files
            Source source = new StreamSource(xmlInputStream);
            os = KettleVFS.getOutputStream(outputfile, false);
            StreamResult resultat = new StreamResult(os);

            // Apply the xsl file to the source file and write the result to the output file
            xformer.transform(source, resultat);

            if (isAddFileToResult()) {
              // Add output filename to output files
              ResultFile resultFile =
                  new ResultFile(
                      ResultFile.FILE_TYPE_GENERAL,
                      KettleVFS.getFileObject(outputfilename, this),
                      parentJob.getJobname(),
                      toString());
              result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
            }

            // Everything is OK
            retval = true;
          } finally {
            try {
              xslInputStream.close();
            } catch (IOException ignored) {
              // ignore IO Exception on close
            }
            try {
              xmlInputStream.close();
            } catch (IOException ignored) {
              // ignore IO Exception on close
            }
            try {
              if (os != null) {
                os.close();
              }
            } catch (IOException ignored) {
              // ignore IO Exception on close
            }
          }
        }
      } else {

        if (!xmlfile.exists()) {
          logError(
              BaseMessages.getString(PKG, "JobEntryXSLT.FileDoesNotExist1.Label")
                  + xmlfilename
                  + BaseMessages.getString(PKG, "JobEntryXSLT.FileDoesNotExist2.Label"));
        }
        if (!xslfile.exists()) {
          logError(
              BaseMessages.getString(PKG, "JobEntryXSLT.FileDoesNotExist1.Label")
                  + xmlfilename
                  + BaseMessages.getString(PKG, "JobEntryXSLT.FileDoesNotExist2.Label"));
        }
      }
    } catch (Exception e) {
      logError(
          BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLST.Label")
              + BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLSTXML1.Label")
              + xmlfilename
              + BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLSTXML2.Label")
              + BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLSTXSL1.Label")
              + xslfilename
              + BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLSTXSL2.Label")
              + e.getMessage());
    } finally {
      try {
        if (xmlfile != null) {
          xmlfile.close();
        }

        if (xslfile != null) {
          xslfile.close();
        }
        if (outputfile != null) {
          outputfile.close();
        }
      } catch (IOException e) {
        logError("Unable to close file", e);
      }
    }

    return retval;
  }
コード例 #27
0
  public Result execute(Result previousResult, int nr) {
    Result result = previousResult;
    result.setResult(false);

    try {

      // Get real variable value
      String realServerName = environmentSubstitute(serverName);
      int realServerPort = Const.toInt(environmentSubstitute(serverPort), 22);
      String realUserName = environmentSubstitute(userName);
      String realServerPassword =
          Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(password));
      // Proxy Host
      String realProxyHost = environmentSubstitute(httpproxyhost);
      int realProxyPort = Const.toInt(environmentSubstitute(httpproxyport), 22);
      String realproxyUserName = environmentSubstitute(httpproxyusername);
      String realProxyPassword =
          Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(httpProxyPassword));
      // Key file
      String realKeyFilename = environmentSubstitute(keyFilename);
      String relKeyFilepass = environmentSubstitute(keyFilePass);
      // Source files
      String realLocalDirectory = environmentSubstitute(localDirectory);
      String realwildcard = environmentSubstitute(wildcard);
      // Remote destination
      String realftpDirectory = environmentSubstitute(ftpDirectory);
      // Destination folder (Move to)
      String realDestinationFolder = environmentSubstitute(destinationfolder);

      try {
        // Remote source
        realftpDirectory = FTPUtils.normalizePath(realftpDirectory);
        // Destination folder (Move to)
        realDestinationFolder = FTPUtils.normalizePath(realDestinationFolder);
      } catch (Exception e) {
        logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.CanNotNormalizePath", e.getMessage()));
        result.setNrErrors(1);
        return result;
      }

      // Check for mandatory fields
      boolean mandatoryok = true;
      if (Const.isEmpty(realServerName)) {
        mandatoryok = false;
        logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.ServernameMissing"));
      }
      if (usehttpproxy) {
        if (Const.isEmpty(realProxyHost)) {
          mandatoryok = false;
          logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.HttpProxyhostMissing"));
        }
      }
      if (publicpublickey) {
        if (Const.isEmpty(realKeyFilename)) {
          mandatoryok = false;
          logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.KeyFileMissing"));
        } else {
          // Let's check if folder exists...
          if (!KettleVFS.fileExists(realKeyFilename, this)) {
            mandatoryok = false;
            logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.KeyFileNotExist"));
          }
        }
      }

      if (Const.isEmpty(realLocalDirectory)) {
        mandatoryok = false;
        logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.LocalFolderMissing"));
      }
      if (afterFtpPut.equals("move_file")) {
        if (Const.isEmpty(realDestinationFolder)) {
          mandatoryok = false;
          logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.DestinatFolderMissing"));
        } else {
          FileObject folder = null;
          try {
            folder = KettleVFS.getFileObject(realDestinationFolder, this);
            // Let's check if folder exists...
            if (!folder.exists()) {
              // Do we need to create it?
              if (createDestinationFolder) {
                folder.createFolder();
              } else {
                logError(
                    BaseMessages.getString(
                        PKG, "JobSSH2PUT.Log.DestinatFolderNotExist", realDestinationFolder));
              }
            }
          } catch (Exception e) {
            throw new KettleException(e);
          } finally {
            if (folder != null) {
              try {
                folder.close();
                folder = null;
              } catch (Exception e) {
                /* Ignore */
              }
            }
          }
        }
      }

      if (mandatoryok) {
        Connection conn = null;
        SFTPv3Client client = null;
        boolean good = true;

        int nbfilestoput = 0;
        int nbput = 0;
        int nbrerror = 0;

        try {
          // Create a connection instance
          conn =
              getConnection(
                  realServerName,
                  realServerPort,
                  realProxyHost,
                  realProxyPort,
                  realproxyUserName,
                  realProxyPassword);

          if (timeout > 0) {
            // Use timeout
            // Cache Host Key
            if (cachehostkey) {
              conn.connect(new SimpleVerifier(database), 0, timeout * 1000);
            } else {
              conn.connect(null, 0, timeout * 1000);
            }

          } else {
            // Cache Host Key
            if (cachehostkey) {
              conn.connect(new SimpleVerifier(database));
            } else {
              conn.connect();
            }
          }

          // Authenticate

          boolean isAuthenticated = false;
          if (publicpublickey) {
            String keyContent =
                KettleVFS.getTextFileContent(realKeyFilename, this, Const.XML_ENCODING);
            isAuthenticated =
                conn.authenticateWithPublicKey(
                    realUserName, keyContent.toCharArray(), relKeyFilepass);
          } else {
            isAuthenticated = conn.authenticateWithPassword(realUserName, realServerPassword);
          }

          // LET'S CHECK AUTHENTICATION ...
          if (isAuthenticated == false) {
            logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.AuthenticationFailed"));
          } else {
            if (log.isBasic()) {
              logBasic(
                  BaseMessages.getString(PKG, "JobSSH2PUT.Log.Connected", serverName, userName));
            }

            client = new SFTPv3Client(conn);

            if (log.isDetailed()) {
              logDetailed(
                  BaseMessages.getString(
                      PKG, "JobSSH2PUT.Log.ProtocolVersion", "" + client.getProtocolVersion()));
            }

            // Check if remote directory exists
            if (!Const.isEmpty(realftpDirectory)) {
              if (!sshDirectoryExists(client, realftpDirectory)) {
                good = false;
                if (createRemoteFolder) {
                  good = CreateRemoteFolder(client, realftpDirectory);
                  if (good) {
                    logBasic(BaseMessages.getString(PKG, "JobSSH2PUT.Log.RemoteDirectoryCreated"));
                  }

                } else {
                  logError(
                      BaseMessages.getString(
                          PKG, "JobSSH2PUT.Log.RemoteDirectoryNotExist", realftpDirectory));
                }
              } else if (log.isDetailed()) {
                logDetailed(
                    BaseMessages.getString(
                        PKG, "JobSSH2PUT.Log.RemoteDirectoryExist", realftpDirectory));
              }
            }

            if (good) {
              // Get files list from local folder (source)
              List<FileObject> myFileList = getFiles(realLocalDirectory);

              // Prepare Pattern for wildcard
              Pattern pattern = null;
              if (!Const.isEmpty(realwildcard)) {
                pattern = Pattern.compile(realwildcard);
              }

              // Let's put files now ...
              // Get the files in the list
              for (int i = 0; i < myFileList.size() && !parentJob.isStopped(); i++) {
                FileObject myFile = myFileList.get(i);
                String localFilename = myFile.toString();
                String remoteFilename = myFile.getName().getBaseName();

                boolean getIt = true;

                // First see if the file matches the regular expression!
                if (pattern != null) {
                  Matcher matcher = pattern.matcher(remoteFilename);
                  getIt = matcher.matches();
                }

                // do we have a target directory?
                if (!Const.isEmpty(realftpDirectory)) {
                  remoteFilename = realftpDirectory + FTPUtils.FILE_SEPARATOR + remoteFilename;
                }

                if (onlyGettingNewFiles) {
                  // We get only new files
                  // ie not exist on the remote server
                  getIt = !sshFileExists(client, remoteFilename);
                }

                if (getIt) {
                  nbfilestoput++;

                  boolean putok = putFile(myFile, remoteFilename, client);
                  if (!putok) {
                    nbrerror++;
                    logError(
                        BaseMessages.getString(
                            PKG, "JobSSH2PUT.Log.Error.CanNotPutFile", localFilename));
                  } else {
                    nbput++;
                  }

                  if (putok && !afterFtpPut.equals("do_nothing")) {
                    deleteOrMoveFiles(myFile, realDestinationFolder);
                  }
                }
              }
              /** ****************************** RESULT ******************* */
              if (log.isDetailed()) {
                logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.JobEntryEnd1"));
                logDetailed(
                    BaseMessages.getString(
                        PKG, "JobSSH2PUT.Log.Result.TotalFiles", "" + nbfilestoput));
                logDetailed(
                    BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.TotalFilesPut", "" + nbput));
                logDetailed(
                    BaseMessages.getString(
                        PKG, "JobSSH2PUT.Log.Result.TotalFilesError", "" + nbrerror));
                logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.JobEntryEnd2"));
              }
              if (nbrerror == 0) {
                result.setResult(true);
                /** ****************************** RESULT ******************* */
              }
            }
          }

        } catch (Exception e) {
          result.setNrErrors(nbrerror);
          logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Error.ErrorFTP", e.getMessage()));
        } finally {
          if (conn != null) {
            conn.close();
          }
          if (client != null) {
            client.close();
          }
        }
      }
    } catch (Exception e) {
      result.setResult(false);
      result.setNrErrors(1L);
      logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Error.UnexpectedError"), e);
    }

    return result;
  }
コード例 #28
0
ファイル: Xslt.java プロジェクト: hzy001/pentaho-kettle
  public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (XsltMeta) smi;
    data = (XsltData) sdi;

    Object[] row = getRow();

    if (row == null) { // no more input to be expected...
      setOutputDone();
      return false;
    }
    if (first) {
      first = false;
      data.outputRowMeta = getInputRowMeta().clone();
      meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);

      // Check if The result field is given
      if (Const.isEmpty(meta.getResultfieldname())) {
        // Result Field is missing !
        logError(BaseMessages.getString(PKG, "Xslt.Log.ErrorResultFieldMissing"));
        throw new KettleStepException(
            BaseMessages.getString(PKG, "Xslt.Exception.ErrorResultFieldMissing"));
      }

      // Check if The XML field is given
      if (Const.isEmpty(meta.getFieldname())) {
        // Result Field is missing !
        logError(BaseMessages.getString(PKG, "Xslt.Exception.ErrorXMLFieldMissing"));
        throw new KettleStepException(
            BaseMessages.getString(PKG, "Xslt.Exception.ErrorXMLFieldMissing"));
      }

      // Try to get XML Field index
      data.fieldposition = getInputRowMeta().indexOfValue(meta.getFieldname());
      // Let's check the Field
      if (data.fieldposition < 0) {
        // The field is unreachable !
        logError(
            BaseMessages.getString(PKG, "Xslt.Log.ErrorFindingField")
                + "["
                + meta.getFieldname()
                + "]");
        throw new KettleStepException(
            BaseMessages.getString(PKG, "Xslt.Exception.CouldnotFindField", meta.getFieldname()));
      }

      // Check if the XSL Filename is contained in a column
      if (meta.useXSLField()) {
        if (Const.isEmpty(meta.getXSLFileField())) {
          // The field is missing
          // Result field is missing !
          logError(BaseMessages.getString(PKG, "Xslt.Log.ErrorXSLFileFieldMissing"));
          throw new KettleStepException(
              BaseMessages.getString(PKG, "Xslt.Exception.ErrorXSLFileFieldMissing"));
        }

        // Try to get Field index
        data.fielxslfiledposition = getInputRowMeta().indexOfValue(meta.getXSLFileField());

        // Let's check the Field
        if (data.fielxslfiledposition < 0) {
          // The field is unreachable !
          logError(
              BaseMessages.getString(PKG, "Xslt.Log.ErrorXSLFileFieldFinding")
                  + "["
                  + meta.getXSLFileField()
                  + "]");
          throw new KettleStepException(
              BaseMessages.getString(
                  PKG, "Xslt.Exception.ErrorXSLFileFieldFinding", meta.getXSLFileField()));
        }

      } else {
        if (Const.isEmpty(meta.getXslFilename())) {
          logError(BaseMessages.getString(PKG, "Xslt.Log.ErrorXSLFile"));
          throw new KettleStepException(BaseMessages.getString(PKG, "Xslt.Exception.ErrorXSLFile"));
        }

        // Check if XSL File exists!
        data.xslfilename = environmentSubstitute(meta.getXslFilename());
        FileObject file = null;
        try {
          file = KettleVFS.getFileObject(data.xslfilename);
          if (!file.exists()) {
            logError(
                BaseMessages.getString(PKG, "Xslt.Log.ErrorXSLFileNotExists", data.xslfilename));
            throw new KettleStepException(
                BaseMessages.getString(
                    PKG, "Xslt.Exception.ErrorXSLFileNotExists", data.xslfilename));
          }
          if (file.getType() != FileType.FILE) {
            logError(BaseMessages.getString(PKG, "Xslt.Log.ErrorXSLNotAFile", data.xslfilename));
            throw new KettleStepException(
                BaseMessages.getString(PKG, "Xslt.Exception.ErrorXSLNotAFile", data.xslfilename));
          }
        } catch (Exception e) {
          throw new KettleStepException(e);
        } finally {
          try {
            if (file != null) {
              file.close();
            }
          } catch (Exception e) {
            /* Ignore */
          }
        }
      }

      // Check output parameters
      int nrOutputProps =
          meta.getOutputPropertyName() == null ? 0 : meta.getOutputPropertyName().length;
      if (nrOutputProps > 0) {
        data.outputProperties = new Properties();
        for (int i = 0; i < nrOutputProps; i++) {
          data.outputProperties.put(
              meta.getOutputPropertyName()[i],
              environmentSubstitute(meta.getOutputPropertyValue()[i]));
        }
        data.setOutputProperties = true;
      }

      // Check parameters
      data.nrParams = meta.getParameterField() == null ? 0 : meta.getParameterField().length;
      if (data.nrParams > 0) {
        data.indexOfParams = new int[data.nrParams];
        data.nameOfParams = new String[data.nrParams];
        for (int i = 0; i < data.nrParams; i++) {
          String name = environmentSubstitute(meta.getParameterName()[i]);
          String field = environmentSubstitute(meta.getParameterField()[i]);
          if (Const.isEmpty(field)) {
            throw new KettleStepException(
                BaseMessages.getString(PKG, "Xslt.Exception.ParameterFieldMissing", name, i));
          }
          data.indexOfParams[i] = getInputRowMeta().indexOfValue(field);
          if (data.indexOfParams[i] < 0) {
            throw new KettleStepException(
                BaseMessages.getString(PKG, "Xslt.Exception.ParameterFieldNotFound", name));
          }
          data.nameOfParams[i] = name;
        }
        data.useParameters = true;
      }

      data.factory = TransformerFactory.newInstance();

      if (meta.getXSLFactory().equals("SAXON")) {
        // Set the TransformerFactory to the SAXON implementation.
        data.factory = new net.sf.saxon.TransformerFactoryImpl();
      }
    } // end if first

    // Get the field value
    String xmlValue = getInputRowMeta().getString(row, data.fieldposition);

    if (meta.useXSLField()) {
      // Get the value
      data.xslfilename = getInputRowMeta().getString(row, data.fielxslfiledposition);
      if (log.isDetailed()) {
        logDetailed(
            BaseMessages.getString(
                PKG, "Xslt.Log.XslfileNameFromFied", data.xslfilename, meta.getXSLFileField()));
      }
    }

    try {

      if (log.isDetailed()) {
        if (meta.isXSLFieldIsAFile()) {
          logDetailed(BaseMessages.getString(PKG, "Xslt.Log.Filexsl") + data.xslfilename);
        } else {
          logDetailed(BaseMessages.getString(PKG, "Xslt.Log.XslStream", data.xslfilename));
        }
      }

      // Get the template from the cache
      Transformer transformer = data.getTemplate(data.xslfilename, data.xslIsAfile);

      // Do we need to set output properties?
      if (data.setOutputProperties) {
        transformer.setOutputProperties(data.outputProperties);
      }

      // Do we need to pass parameters?
      if (data.useParameters) {
        for (int i = 0; i < data.nrParams; i++) {
          transformer.setParameter(data.nameOfParams[i], row[data.indexOfParams[i]]);
        }
      }

      Source source = new StreamSource(new StringReader(xmlValue));
      // Prepare output stream
      StreamResult result = new StreamResult(new StringWriter());
      // transform xml source
      transformer.transform(source, result);

      String xmlString = result.getWriter().toString();
      if (log.isDetailed()) {
        logDetailed(BaseMessages.getString(PKG, "Xslt.Log.FileResult"));
        logDetailed(xmlString);
      }
      Object[] outputRowData = RowDataUtil.addValueData(row, getInputRowMeta().size(), xmlString);

      if (log.isRowLevel()) {
        logRowlevel(
            BaseMessages.getString(PKG, "Xslt.Log.ReadRow")
                + " "
                + getInputRowMeta().getString(row));
      }

      // add new values to the row.
      putRow(data.outputRowMeta, outputRowData); // copy row to output rowset(s);

    } catch (Exception e) {
      String errorMessage = e.getMessage();
      StringWriter sw = new StringWriter();
      PrintWriter pw = new PrintWriter(sw);
      DefaultErrorHandler.printLocation(pw, e);
      pw.close();
      errorMessage = sw.toString() + "\n" + errorMessage;

      if (getStepMeta().isDoingErrorHandling()) {
        // Simply add this row to the error row
        putError(getInputRowMeta(), row, 1, errorMessage, meta.getResultfieldname(), "XSLT01");
      } else {
        logError(BaseMessages.getString(PKG, "Xslt.ErrorProcesing" + " : " + errorMessage));
        throw new KettleStepException(BaseMessages.getString(PKG, "Xslt.ErrorProcesing"), e);
      }
    }

    return true;
  }
コード例 #29
0
ファイル: LoadFileInput.java プロジェクト: jjeb/kettle-trunk
  private boolean openNextFile() {
    try {
      if (meta.getIsInFields()) {
        data.readrow = getRow(); // Grab another row ...

        if (data.readrow == null) // finished processing!
        {
          if (isDetailed())
            logDetailed(BaseMessages.getString(PKG, "LoadFileInput.Log.FinishedProcessing"));
          return false;
        }

        if (first) {
          first = false;

          data.inputRowMeta = getInputRowMeta();
          data.outputRowMeta = data.inputRowMeta.clone();
          meta.getFields(data.outputRowMeta, getStepname(), null, null, this);

          // Create convert meta-data objects that will contain Date & Number formatters
          //
          data.convertRowMeta = data.outputRowMeta.cloneToType(ValueMetaInterface.TYPE_STRING);

          if (meta.getIsInFields()) {
            // Check is filename field is provided
            if (Const.isEmpty(meta.getDynamicFilenameField())) {
              logError(BaseMessages.getString(PKG, "LoadFileInput.Log.NoField"));
              throw new KettleException(BaseMessages.getString(PKG, "LoadFileInput.Log.NoField"));
            }

            // cache the position of the field
            if (data.indexOfFilenameField < 0) {
              data.indexOfFilenameField =
                  data.inputRowMeta.indexOfValue(meta.getDynamicFilenameField());
              if (data.indexOfFilenameField < 0) {
                // The field is unreachable !
                logError(
                    BaseMessages.getString(PKG, "LoadFileInput.Log.ErrorFindingField")
                        + "["
                        + meta.getDynamicFilenameField()
                        + "]"); //$NON-NLS-1$ //$NON-NLS-2$
                throw new KettleException(
                    BaseMessages.getString(
                        PKG,
                        "LoadFileInput.Exception.CouldnotFindField",
                        meta.getDynamicFilenameField())); // $NON-NLS-1$ //$NON-NLS-2$
              }
            }
            // Get the number of previous fields
            data.totalpreviousfields = data.inputRowMeta.size();
          }
        } // end if first

        // get field value
        String Fieldvalue = data.inputRowMeta.getString(data.readrow, data.indexOfFilenameField);

        if (isDetailed())
          logDetailed(
              BaseMessages.getString(
                  PKG, "LoadFileInput.Log.Stream", meta.getDynamicFilenameField(), Fieldvalue));

        FileObject file = null;
        try {
          // Source is a file.
          data.file = KettleVFS.getFileObject(Fieldvalue);
        } catch (Exception e) {
          throw new KettleException(e);
        } finally {
          try {
            if (file != null) file.close();
          } catch (Exception e) {
          }
        }
      } else {
        if (data.filenr >= data.files.nrOfFiles()) // finished processing!
        {
          if (isDetailed())
            logDetailed(BaseMessages.getString(PKG, "LoadFileInput.Log.FinishedProcessing"));
          return false;
        }

        // Is this the last file?
        data.last_file = (data.filenr == data.files.nrOfFiles() - 1);
        data.file = (FileObject) data.files.getFile(data.filenr);
      }

      // Check if file is empty
      data.fileSize = data.file.getContent().getSize();
      // Move file pointer ahead!
      data.filenr++;

      if (meta.isIgnoreEmptyFile() && data.fileSize == 0) {
        logError(
            BaseMessages.getString(
                PKG, "LoadFileInput.Error.FileSizeZero", "" + data.file.getName()));
        openNextFile();

      } else {
        if (isDetailed())
          logDetailed(
              BaseMessages.getString(PKG, "LoadFileInput.Log.OpeningFile", data.file.toString()));
        data.filename = KettleVFS.getFilename(data.file);
        // Add additional fields?
        if (meta.getShortFileNameField() != null && meta.getShortFileNameField().length() > 0) {
          data.shortFilename = data.file.getName().getBaseName();
        }
        if (meta.getPathField() != null && meta.getPathField().length() > 0) {
          data.path = KettleVFS.getFilename(data.file.getParent());
        }
        if (meta.isHiddenField() != null && meta.isHiddenField().length() > 0) {
          data.hidden = data.file.isHidden();
        }
        if (meta.getExtensionField() != null && meta.getExtensionField().length() > 0) {
          data.extension = data.file.getName().getExtension();
        }
        if (meta.getLastModificationDateField() != null
            && meta.getLastModificationDateField().length() > 0) {
          data.lastModificationDateTime = new Date(data.file.getContent().getLastModifiedTime());
        }
        if (meta.getUriField() != null && meta.getUriField().length() > 0) {
          data.uriName = data.file.getName().getURI();
        }
        if (meta.getRootUriField() != null && meta.getRootUriField().length() > 0) {
          data.rootUriName = data.file.getName().getRootURI();
        }
        // get File content
        getFileContent();

        addFileToResultFilesname(data.file);

        if (isDetailed()) {
          logDetailed(
              BaseMessages.getString(PKG, "LoadFileInput.Log.FileOpened", data.file.toString()));
        }
      }

    } catch (Exception e) {
      logError(
          BaseMessages.getString(
              PKG,
              "LoadFileInput.Log.UnableToOpenFile",
              "" + data.filenr,
              data.file.toString(),
              e.toString()));
      stopAll();
      setErrors(1);
      return false;
    }
    return true;
  }
コード例 #30
0
  public boolean validate() {

    boolean retval = false;

    FileObject xmlfile = null;
    FileObject DTDfile = null;

    ByteArrayInputStream ba = null;
    try {
      if (xmlfilename != null
          && ((getDTDFilename() != null && !isInternDTD()) || (isInternDTD()))) {
        xmlfile = KettleVFS.getFileObject(getXMLFilename());

        if (xmlfile.exists()) {

          URL xmlFile = new File(KettleVFS.getFilename(xmlfile)).toURI().toURL();
          StringBuffer xmlStringbuffer = new StringBuffer("");

          BufferedReader xmlBufferedReader = null;
          InputStreamReader is = null;
          try {
            // open XML File
            is = new InputStreamReader(xmlFile.openStream());
            xmlBufferedReader = new BufferedReader(is);

            char[] buffertXML = new char[1024];
            int LenXML = -1;
            while ((LenXML = xmlBufferedReader.read(buffertXML)) != -1) {
              xmlStringbuffer.append(buffertXML, 0, LenXML);
            }
          } finally {
            if (is != null) {
              is.close();
            }
            if (xmlBufferedReader != null) {
              xmlBufferedReader.close();
            }
          }

          // Prepare parsing ...
          DocumentBuilderFactory DocBuilderFactory = DocumentBuilderFactory.newInstance();
          DocumentBuilder DocBuilder = DocBuilderFactory.newDocumentBuilder();

          // Let's try to get XML document encoding

          DocBuilderFactory.setValidating(false);
          ba = new ByteArrayInputStream(xmlStringbuffer.toString().getBytes("UTF-8"));
          Document xmlDocDTD = DocBuilder.parse(ba);
          if (ba != null) {
            ba.close();
          }

          String encoding = null;
          if (xmlDocDTD.getXmlEncoding() == null) {
            encoding = "UTF-8";
          } else {
            encoding = xmlDocDTD.getXmlEncoding();
          }

          int xmlStartDTD = xmlStringbuffer.indexOf("<!DOCTYPE");

          if (isInternDTD()) {
            // DTD find in the XML document
            if (xmlStartDTD != -1) {
              log.logBasic(
                  BaseMessages.getString(
                      PKG, "JobEntryDTDValidator.ERRORDTDFound.Label", getXMLFilename()));
            } else {
              setErrorMessage(
                  BaseMessages.getString(
                      PKG, "JobEntryDTDValidator.ERRORDTDNotFound.Label", getXMLFilename()));
            }

          } else {
            // DTD in external document
            // If we find an intern declaration, we remove it
            DTDfile = KettleVFS.getFileObject(getDTDFilename());

            if (DTDfile.exists()) {
              if (xmlStartDTD != -1) {
                int EndDTD = xmlStringbuffer.indexOf(">", xmlStartDTD);
                // String DocTypeDTD = xmlStringbuffer.substring(xmlStartDTD, EndDTD + 1);
                xmlStringbuffer.replace(xmlStartDTD, EndDTD + 1, "");
              }

              String xmlRootnodeDTD = xmlDocDTD.getDocumentElement().getNodeName();

              String RefDTD =
                  "<?xml version='"
                      + xmlDocDTD.getXmlVersion()
                      + "' encoding='"
                      + encoding
                      + "'?>\n<!DOCTYPE "
                      + xmlRootnodeDTD
                      + " SYSTEM '"
                      + KettleVFS.getFilename(DTDfile)
                      + "'>\n";

              int xmloffsetDTD = xmlStringbuffer.indexOf("<" + xmlRootnodeDTD);
              xmlStringbuffer.replace(0, xmloffsetDTD, RefDTD);
            } else {
              log.logError(
                  BaseMessages.getString(PKG, "JobEntryDTDValidator.ERRORDTDFileNotExists.Subject"),
                  BaseMessages.getString(
                      PKG, "JobEntryDTDValidator.ERRORDTDFileNotExists.Msg", getDTDFilename()));
            }
          }

          if (!(isInternDTD() && xmlStartDTD == -1 || (!isInternDTD() && !DTDfile.exists()))) {

            // Let's parse now ...
            MyErrorHandler error = new MyErrorHandler();
            DocBuilderFactory.setValidating(true);
            DocBuilder = DocBuilderFactory.newDocumentBuilder();
            DocBuilder.setErrorHandler(error);

            ba = new ByteArrayInputStream(xmlStringbuffer.toString().getBytes(encoding));
            xmlDocDTD = DocBuilder.parse(ba);

            if (error.errorMessage == null) {
              log.logBasic(
                  BaseMessages.getString(PKG, "JobEntryDTDValidator.DTDValidatorOK.Subject"),
                  BaseMessages.getString(
                      PKG, "JobEntryDTDValidator.DTDValidatorOK.Label", getXMLFilename()));

              // Everything is OK
              retval = true;
            } else {
              // Invalid DTD
              setNrErrors(error.nrErrors);
              setErrorMessage(
                  BaseMessages.getString(
                      PKG,
                      "JobEntryDTDValidator.DTDValidatorKO",
                      getXMLFilename(),
                      error.nrErrors,
                      error.errorMessage));
            }
          }

        } else {
          if (!xmlfile.exists()) {
            setErrorMessage(
                BaseMessages.getString(
                    PKG, "JobEntryDTDValidator.FileDoesNotExist.Label", getXMLFilename()));
          }
        }
      } else {
        setErrorMessage(BaseMessages.getString(PKG, "JobEntryDTDValidator.AllFilesNotNull.Label"));
      }
    } catch (Exception e) {
      setErrorMessage(
          BaseMessages.getString(
              PKG,
              "JobEntryDTDValidator.ErrorDTDValidator.Label",
              getXMLFilename(),
              getDTDFilename(),
              e.getMessage()));
    } finally {
      try {
        if (xmlfile != null) {
          xmlfile.close();
        }
        if (DTDfile != null) {
          DTDfile.close();
        }
        if (ba != null) {
          ba.close();
        }
      } catch (IOException e) {
        // Ignore close errors
      }
    }
    return retval;
  }