Exemplo n.º 1
0
  public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
    meta = (LoadFileInputMeta) smi;
    data = (LoadFileInputData) sdi;

    if (super.init(smi, sdi)) {
      if (!meta.getIsInFields()) {
        try {
          data.files = meta.getFiles(this);
          handleMissingFiles();
          // Create the output row meta-data
          data.outputRowMeta = new RowMeta();
          meta.getFields(
              data.outputRowMeta, getStepname(), null, null, this); // get the metadata populated

          // Create convert meta-data objects that will contain Date & Number formatters
          //
          data.convertRowMeta = data.outputRowMeta.cloneToType(ValueMetaInterface.TYPE_STRING);
        } catch (Exception e) {
          logError("Error at step initialization: " + e.toString());
          logError(Const.getStackTracker(e));
          return false;
        }
      }
      data.rownr = 1L;
      data.nrInputFields = meta.getInputFields().length;

      return true;
    }
    return false;
  }
Exemplo n.º 2
0
  private boolean openNextFile() {
    try {
      if (meta.getIsInFields()) {
        data.readrow = getRow(); // Grab another row ...

        if (data.readrow == null) // finished processing!
        {
          if (isDetailed())
            logDetailed(BaseMessages.getString(PKG, "LoadFileInput.Log.FinishedProcessing"));
          return false;
        }

        if (first) {
          first = false;

          data.inputRowMeta = getInputRowMeta();
          data.outputRowMeta = data.inputRowMeta.clone();
          meta.getFields(data.outputRowMeta, getStepname(), null, null, this);

          // Create convert meta-data objects that will contain Date & Number formatters
          //
          data.convertRowMeta = data.outputRowMeta.cloneToType(ValueMetaInterface.TYPE_STRING);

          if (meta.getIsInFields()) {
            // Check is filename field is provided
            if (Const.isEmpty(meta.getDynamicFilenameField())) {
              logError(BaseMessages.getString(PKG, "LoadFileInput.Log.NoField"));
              throw new KettleException(BaseMessages.getString(PKG, "LoadFileInput.Log.NoField"));
            }

            // cache the position of the field
            if (data.indexOfFilenameField < 0) {
              data.indexOfFilenameField =
                  data.inputRowMeta.indexOfValue(meta.getDynamicFilenameField());
              if (data.indexOfFilenameField < 0) {
                // The field is unreachable !
                logError(
                    BaseMessages.getString(PKG, "LoadFileInput.Log.ErrorFindingField")
                        + "["
                        + meta.getDynamicFilenameField()
                        + "]"); //$NON-NLS-1$ //$NON-NLS-2$
                throw new KettleException(
                    BaseMessages.getString(
                        PKG,
                        "LoadFileInput.Exception.CouldnotFindField",
                        meta.getDynamicFilenameField())); // $NON-NLS-1$ //$NON-NLS-2$
              }
            }
            // Get the number of previous fields
            data.totalpreviousfields = data.inputRowMeta.size();
          }
        } // end if first

        // get field value
        String Fieldvalue = data.inputRowMeta.getString(data.readrow, data.indexOfFilenameField);

        if (isDetailed())
          logDetailed(
              BaseMessages.getString(
                  PKG, "LoadFileInput.Log.Stream", meta.getDynamicFilenameField(), Fieldvalue));

        FileObject file = null;
        try {
          // Source is a file.
          data.file = KettleVFS.getFileObject(Fieldvalue);
        } catch (Exception e) {
          throw new KettleException(e);
        } finally {
          try {
            if (file != null) file.close();
          } catch (Exception e) {
          }
        }
      } else {
        if (data.filenr >= data.files.nrOfFiles()) // finished processing!
        {
          if (isDetailed())
            logDetailed(BaseMessages.getString(PKG, "LoadFileInput.Log.FinishedProcessing"));
          return false;
        }

        // Is this the last file?
        data.last_file = (data.filenr == data.files.nrOfFiles() - 1);
        data.file = (FileObject) data.files.getFile(data.filenr);
      }

      // Check if file is empty
      data.fileSize = data.file.getContent().getSize();
      // Move file pointer ahead!
      data.filenr++;

      if (meta.isIgnoreEmptyFile() && data.fileSize == 0) {
        logError(
            BaseMessages.getString(
                PKG, "LoadFileInput.Error.FileSizeZero", "" + data.file.getName()));
        openNextFile();

      } else {
        if (isDetailed())
          logDetailed(
              BaseMessages.getString(PKG, "LoadFileInput.Log.OpeningFile", data.file.toString()));
        data.filename = KettleVFS.getFilename(data.file);
        // Add additional fields?
        if (meta.getShortFileNameField() != null && meta.getShortFileNameField().length() > 0) {
          data.shortFilename = data.file.getName().getBaseName();
        }
        if (meta.getPathField() != null && meta.getPathField().length() > 0) {
          data.path = KettleVFS.getFilename(data.file.getParent());
        }
        if (meta.isHiddenField() != null && meta.isHiddenField().length() > 0) {
          data.hidden = data.file.isHidden();
        }
        if (meta.getExtensionField() != null && meta.getExtensionField().length() > 0) {
          data.extension = data.file.getName().getExtension();
        }
        if (meta.getLastModificationDateField() != null
            && meta.getLastModificationDateField().length() > 0) {
          data.lastModificationDateTime = new Date(data.file.getContent().getLastModifiedTime());
        }
        if (meta.getUriField() != null && meta.getUriField().length() > 0) {
          data.uriName = data.file.getName().getURI();
        }
        if (meta.getRootUriField() != null && meta.getRootUriField().length() > 0) {
          data.rootUriName = data.file.getName().getRootURI();
        }
        // get File content
        getFileContent();

        addFileToResultFilesname(data.file);

        if (isDetailed()) {
          logDetailed(
              BaseMessages.getString(PKG, "LoadFileInput.Log.FileOpened", data.file.toString()));
        }
      }

    } catch (Exception e) {
      logError(
          BaseMessages.getString(
              PKG,
              "LoadFileInput.Log.UnableToOpenFile",
              "" + data.filenr,
              data.file.toString(),
              e.toString()));
      stopAll();
      setErrors(1);
      return false;
    }
    return true;
  }
Exemplo n.º 3
0
  private Object[] getOneRow() throws KettleException {
    if (!openNextFile()) {
      return null;
    }

    // Build an empty row based on the meta-data
    Object[] outputRowData = buildEmptyRow();

    try {
      // Create new row	or clone
      if (meta.getIsInFields())
        System.arraycopy(data.readrow, 0, outputRowData, 0, data.readrow.length);

      // Read fields...
      for (int i = 0; i < data.nrInputFields; i++) {
        // Get field
        LoadFileInputField loadFileInputField = meta.getInputFields()[i];

        String o = null;
        switch (loadFileInputField.getElementType()) {
          case LoadFileInputField.ELEMENT_TYPE_FILECONTENT:

            // DO Trimming!
            switch (loadFileInputField.getTrimType()) {
              case LoadFileInputField.TYPE_TRIM_LEFT:
                data.filecontent = Const.ltrim(data.filecontent);
                break;
              case LoadFileInputField.TYPE_TRIM_RIGHT:
                data.filecontent = Const.rtrim(data.filecontent);
                break;
              case LoadFileInputField.TYPE_TRIM_BOTH:
                data.filecontent = Const.trim(data.filecontent);
                break;
              default:
                break;
            }
            o = data.filecontent;
            break;
          case LoadFileInputField.ELEMENT_TYPE_FILESIZE:
            o = String.valueOf(data.fileSize);
            break;
          default:
            break;
        }

        int indexField = data.totalpreviousfields + i;
        // Do conversions
        //
        ValueMetaInterface targetValueMeta = data.outputRowMeta.getValueMeta(indexField);
        ValueMetaInterface sourceValueMeta = data.convertRowMeta.getValueMeta(indexField);
        outputRowData[indexField] = targetValueMeta.convertData(sourceValueMeta, o);

        // Do we need to repeat this field if it is null?
        if (loadFileInputField.isRepeated()) {
          if (data.previousRow != null && o == null) {
            outputRowData[indexField] = data.previousRow[indexField];
          }
        }
      } // End of loop over fields...
      int rowIndex = data.totalpreviousfields + data.nrInputFields;

      // See if we need to add the filename to the row...
      if (meta.includeFilename()
          && meta.getFilenameField() != null
          && meta.getFilenameField().length() > 0) {
        outputRowData[rowIndex++] = data.filename;
      }

      // See if we need to add the row number to the row...
      if (meta.includeRowNumber()
          && meta.getRowNumberField() != null
          && meta.getRowNumberField().length() > 0) {
        outputRowData[rowIndex++] = new Long(data.rownr);
      }
      // Possibly add short filename...
      if (meta.getShortFileNameField() != null && meta.getShortFileNameField().length() > 0) {
        outputRowData[rowIndex++] = data.shortFilename;
      }
      // Add Extension
      if (meta.getExtensionField() != null && meta.getExtensionField().length() > 0) {
        outputRowData[rowIndex++] = data.extension;
      }
      // add path
      if (meta.getPathField() != null && meta.getPathField().length() > 0) {
        outputRowData[rowIndex++] = data.path;
      }

      // add Hidden
      if (meta.isHiddenField() != null && meta.isHiddenField().length() > 0) {
        outputRowData[rowIndex++] = new Boolean(data.hidden);
      }
      // Add modification date
      if (meta.getLastModificationDateField() != null
          && meta.getLastModificationDateField().length() > 0) {
        outputRowData[rowIndex++] = data.lastModificationDateTime;
      }
      // Add Uri
      if (meta.getUriField() != null && meta.getUriField().length() > 0) {
        outputRowData[rowIndex++] = data.uriName;
      }
      // Add RootUri
      if (meta.getRootUriField() != null && meta.getRootUriField().length() > 0) {
        outputRowData[rowIndex++] = data.rootUriName;
      }
      RowMetaInterface irow = getInputRowMeta();

      data.previousRow =
          irow == null ? outputRowData : (Object[]) irow.cloneRow(outputRowData); // copy it to make
      // surely the next step doesn't change it in between...

      incrementLinesInput();
      data.rownr++;

    } catch (Exception e) {
      throw new KettleException("Impossible de charger le fichier", e);
    }

    return outputRowData;
  }