コード例 #1
0
 /*
  * (non-Javadoc)
  * @see com.salesforce.dataloader.dao.csv.Writer#writeRow(java.util.Map)
  */
 public boolean writeRow(Map<String, Object> columnValues) throws DataAccessObjectException {
   CSVColumnVisitor visitor = new CSVColumnVisitor(fileOut);
   try {
     visitColumns(columnNames, columnValues, visitor);
     fileOut.newLine();
     visitor.newRow();
     currentRowNumber++;
     return true; // success unless there's an exception
   } catch (IOException e) {
     logger.error(Messages.getString("CSVWriter.errorWriting"), e); // $NON-NLS-1$
     throw new DataAccessObjectException(
         Messages.getString("CSVWriter.errorWriting"), e); // $NON-NLS-1$
   }
 }
コード例 #2
0
ファイル: DAORowUtil.java プロジェクト: rericka/dataloader
 /**
  * Validate column names and return a string if there's a validation warning.
  *
  * @param dao Data access object to validate columns for
  * @return A validation warning or null
  */
 public static String validateColumns(DataAccessObject dao) {
   HashSet<String> uniqueHeaders = new HashSet<String>();
   String warning = null;
   for (String header : dao.getColumnNames()) {
     if (header == null || header.length() == 0) {
       warning = Messages.getString("RowUtil.warningEmptyColumn"); // $NON-NLS-1$
       break;
     } else if (uniqueHeaders.contains(header)) {
       warning =
           Messages.getFormattedString("RowUtil.warningDuplicateColumn", header); // $NON-NLS-1$
       break;
     }
     uniqueHeaders.add(header);
   }
   if (warning != null) {
     logger.warn(warning);
   }
   return warning;
 }
コード例 #3
0
 private void writeHeaderRow() throws DataAccessObjectInitializationException {
   CSVColumnVisitor visitor = new CSVColumnVisitor(fileOut);
   try {
     visitHeaderColumns(this.columnNames, visitor);
     fileOut.newLine();
     visitor.newRow();
   } catch (IOException e) {
     String errMsg = Messages.getString("CSVWriter.errorWriting");
     logger.error(errMsg, e); // $NON-NLS-1$
     throw new DataAccessObjectInitializationException(errMsg, e);
   }
 }
コード例 #4
0
ファイル: CSVFileReader.java プロジェクト: rericka/dataloader
 private void readHeaderRow() throws DataAccessObjectInitializationException {
   try {
     synchronized (lock) {
       headerRow = csvReader.nextRecord();
     }
     if (headerRow == null) {
       LOGGER.error(Messages.getString("CSVFileDAO.errorHeaderRow"));
       throw new DataAccessObjectInitializationException(
           Messages.getString("CSVFileDAO.errorHeaderRow"));
     }
   } catch (IOException e) {
     String errMsg = Messages.getString("CSVFileDAO.errorHeaderRow");
     LOGGER.error(errMsg, e);
     throw new DataAccessObjectInitializationException(errMsg, e);
   } finally {
     // if there's a problem getting header row, the stream needs to be closed
     if (headerRow == null) {
       IOUtils.closeQuietly(input);
     }
   }
 }
コード例 #5
0
  /*
   * (non-Javadoc)
   * @see com.salesforce.dataloader.dao.DataWriter#setColumnNames(java.util.List)
   */
  public synchronized void setColumnNames(List<String> columnNames)
      throws DataAccessObjectInitializationException {
    if (columnNames == null || columnNames.isEmpty()) {
      String errMsg = Messages.getString("CSVFileDAO.errorOpenNoHeaderRow");
      logger.error(errMsg);
      throw new DataAccessObjectInitializationException(errMsg);
    }
    // save column names
    this.columnNames = columnNames;

    writeHeaderRow();
  }
コード例 #6
0
  /*
   * (non-Javadoc)
   * @see com.salesforce.dataloader.dao.csv.Writer#close()
   */
  public void close() {

    if (fileOut != null) {
      try {
        fileOut.close();
      } catch (IOException e) {
        logger.error(Messages.getString("CSVWriter.errorClosing"), e); // $NON-NLS-1$
      }
      if (!columnNames.isEmpty()) {
        columnNames.clear();
      }
    }
  }
コード例 #7
0
 private void processResult(
     Map<String, Object> dataRow, boolean isSuccess, String id, Error[] errors)
     throws DataAccessObjectException {
   // process success vs. error
   // extract error message from error result
   if (isSuccess) {
     writeSuccess(dataRow, id, null);
   } else {
     writeError(
         dataRow,
         errors == null
             ? Messages.getString("Visitor.noErrorReceivedMsg")
             : errors[0].getMessage());
   }
 }
コード例 #8
0
ファイル: CSVFileReader.java プロジェクト: rericka/dataloader
 private void initalizeInput() throws DataAccessObjectInitializationException {
   try {
     input = new FileInputStream(file);
     if (forceUTF8 || isUTF8File(file)) {
       csvReader = new CSVReader(input, "UTF-8", new char[] {',', '\t'});
     } else {
       csvReader = new CSVReader(input, new char[] {',', '\t'});
     }
     csvReader.setMaxRowsInFile(Integer.MAX_VALUE);
     csvReader.setMaxCharsInFile(Integer.MAX_VALUE);
   } catch (FileNotFoundException e) {
     String errMsg = Messages.getFormattedString("CSVFileDAO.errorOpen", file.getAbsolutePath());
     LOGGER.error(errMsg, e);
     throw new DataAccessObjectInitializationException(errMsg, e);
   } catch (UnsupportedEncodingException e) {
     String errMsg = Messages.getString("CSVFileDAO.errorUnsupportedEncoding");
     LOGGER.error(errMsg, e);
     throw new DataAccessObjectInitializationException(errMsg, e);
   } finally {
     if (csvReader == null) {
       IOUtils.closeQuietly(input);
     }
   }
 }
コード例 #9
0
 /**
  * Create database writer based on configuration
  *
  * @param config
  * @param dbConfigName
  * @throws DataAccessObjectInitializationException
  */
 DatabaseWriter(Config config, String dbConfigName)
     throws DataAccessObjectInitializationException {
   this.config = config;
   String dbConfigFilename =
       config.constructConfigFilePath(DatabaseContext.DEFAULT_CONFIG_FILENAME);
   if (!(new File(dbConfigFilename).exists())) {
     throw new DataAccessObjectInitializationException(
         Messages.getFormattedString(
             "DatabaseDAO.errorConfigFileExists", dbConfigFilename)); // $NON-NLS-1$
   }
   DatabaseConfig dbConfig = DatabaseConfig.getInstance(dbConfigFilename, dbConfigName);
   dataSource = dbConfig.getDataSource();
   sqlConfig = dbConfig.getSqlConfig();
   dbContext = new DatabaseContext(dbConfigName);
 }
コード例 #10
0
 /*
  * (non-Javadoc)
  * @see com.salesforce.dataloader.dao.DataAccessObject#open()
  */
 public void open() throws DataAccessObjectInitializationException {
   try {
     if (!useDefaultEncoding) {
       fileOut =
           new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), encoding));
     } else {
       fileOut = new BufferedWriter(new FileWriter(fileName));
     }
     currentRowNumber = 0;
     setOpen(true);
   } catch (IOException e) {
     String errMsg = Messages.getFormattedString("CSVWriter.errorOpening", fileName);
     logger.error(errMsg, e); // $NON-NLS-1$
     throw new DataAccessObjectInitializationException(errMsg, e); // $NON-NLS-1$
   }
 }
コード例 #11
0
 /** @param sqe */
 private void endException(int batchSize) {
   // Rollback if dbContext.getAutoCommit() is false
   try {
     dbContext.getDataConnection().rollback();
   } catch (SQLException sqe) {
     logger.error(
         Messages.getFormattedString(
             "DatabaseDAO.sqlExceptionRollback",
             new String[] {
               String.valueOf(currentRowNumber + 1 - batchSize),
               String.valueOf(currentRowNumber + 1),
               dbContext.getDbConfigName(),
               sqe.getMessage()
             }),
         sqe);
   }
 }
コード例 #12
0
ファイル: DAORowUtil.java プロジェクト: rericka/dataloader
  /**
   * Utility function for calculating the total number of rows available to current DAO instance
   *
   * @throws DataAccessObjectException
   */
  public static int calculateTotalRows(DataReader dataReader) throws DataAccessObjectException {
    try {
      // visit the rows
      DAOSizeVisitor visitor = new DAOSizeVisitor();
      for (Row row = dataReader.readRow(); isValidRow(row); row = dataReader.readRow()) {
        visitor.visit(row);
      }

      return visitor.getNumberOfRows();
    } catch (DataAccessObjectException daoe) {
      logger.error(Messages.getString("RowUtil.error"), daoe); // $NON-NLS-1$
      throw daoe;
    } finally {
      // since we've read all the rows, reopen the reader to reset the input
      dataReader.close();
      dataReader.open();
    }
  }
コード例 #13
0
ファイル: CSVFileReader.java プロジェクト: rericka/dataloader
  /**
   * Gets the next row from the current data access object data source. <i>Side effect:</i> Updates
   * the current record number
   */
  @Override
  public Row readRow() throws DataAccessObjectException {
    if (!isOpen) {
      open();
    }

    List<String> record;
    synchronized (lock) {
      try {
        record = csvReader.nextRecord();
      } catch (IOException e) {
        throw new DataAccessObjectException(e);
      }
    }

    if (!DAORowUtil.isValidRow(record)) {
      return null;
    }

    if (record.size() > headerRow.size()) {
      String errMsg =
          Messages.getFormattedString(
              "CSVFileDAO.errorRowTooLarge",
              new String[] {
                String.valueOf(currentRowNumber),
                String.valueOf(record.size()),
                String.valueOf(headerRow.size())
              });
      throw new DataAccessRowException(errMsg);
    }

    Row row = new Row(record.size());

    for (int i = 0; i < headerRow.size(); i++) {
      String value = record.get(i);
      if (value == null) {
        value = "";
      }
      row.put(headerRow.get(i), value);
    }
    currentRowNumber++;
    return row;
  }
コード例 #14
0
  private void writeOutputToWriter(Object[] results, List<Map<String, Object>> dataArray)
      throws DataAccessObjectException, LoadException {

    if (results.length != dataArray.size()) {
      getLogger().fatal(Messages.getString("Visitor.errorResultsLength")); // $NON-NLS-1$
      throw new LoadException(Messages.getString("Visitor.errorResultsLength"));
    }

    // have to do this because although saveResult and deleteResult
    // are a) not the same class yet b) not subclassed
    for (int i = 0; i < results.length; i++) {
      Map<String, Object> dataRow = dataArray.get(i);
      String statusMsg = null;
      if (results instanceof SaveResult[]) {
        SaveResult saveRes = (SaveResult) results[i];
        if (saveRes.getSuccess()) {
          if (OperationInfo.insert == getConfig().getOperationInfo()) {
            statusMsg = Messages.getString("DAOLoadVisitor.statusItemCreated");
          } else {
            statusMsg = Messages.getString("DAOLoadVisitor.statusItemUpdated");
          }
        }
        dataRow.put(Config.STATUS_COLUMN_NAME, statusMsg);
        processResult(dataRow, saveRes.getSuccess(), saveRes.getId(), saveRes.getErrors());
      } else if (results instanceof DeleteResult[]) {
        DeleteResult deleteRes = (DeleteResult) results[i];
        if (deleteRes.getSuccess()) {
          statusMsg = Messages.getString("DAOLoadVisitor.statusItemDeleted");
        }
        dataRow.put(Config.STATUS_COLUMN_NAME, statusMsg);
        processResult(dataRow, deleteRes.getSuccess(), deleteRes.getId(), deleteRes.getErrors());
      } else if (results instanceof UpsertResult[]) {
        UpsertResult upsertRes = (UpsertResult) results[i];
        if (upsertRes.getSuccess()) {
          statusMsg =
              upsertRes.getCreated()
                  ? Messages.getString("DAOLoadVisitor.statusItemCreated")
                  : Messages.getString("DAOLoadVisitor.statusItemUpdated");
        }
        dataRow.put(Config.STATUS_COLUMN_NAME, statusMsg);
        processResult(dataRow, upsertRes.getSuccess(), upsertRes.getId(), upsertRes.getErrors());
      }
    }
  }
コード例 #15
0
  @Override
  protected void loadBatch() throws DataAccessObjectException, LoadException {
    Object[] results = null;
    try {
      results = executeClientAction(getController().getPartnerClient(), dynaArray);
    } catch (ApiFault e) {
      handleException(e);
    } catch (ConnectionException e) {
      handleException(e);
    }

    // set the current processed
    int currentProcessed;
    try {
      currentProcessed = getConfig().getInt(LastRun.LAST_LOAD_BATCH_ROW);
    } catch (ParameterLoadException e) {
      // if there's a problem getting last batch row, start at the beginning
      currentProcessed = 0;
    }
    currentProcessed += results.length;
    getConfig().setValue(LastRun.LAST_LOAD_BATCH_ROW, currentProcessed);
    try {
      getConfig().saveLastRun();
    } catch (IOException e) {
      String errMsg = Messages.getString("LoadAction.errorLastRun");
      getLogger().error(errMsg, e);
      handleException(errMsg, e);
    }

    writeOutputToWriter(results, dataArray);

    // update Monitor
    getProgressMonitor().worked(results.length);
    getProgressMonitor()
        .setSubTask(getRateCalculator().calculateSubTask(getNumberOfRows(), getNumberErrors()));

    // now clear the arrays
    clearArrays();
  }
コード例 #16
0
  /*
   * (non-Javadoc)
   * @see com.salesforce.dataloader.dao.DataWriter#writeRowList(java.util.List)
   */
  @Override
  public boolean writeRowList(List<Row> inputRowList) throws DataAccessObjectException {

    // make sure that the update is setup and ready to go, otherwise stop
    if (!dbContext.isOpen()) {
      throw new DataAccessObjectInitializationException(
          Messages.getString("DatabaseDAO.errorUpdateNotOpen"));
    }

    boolean success = true;
    int startingRowNumber = currentRowNumber;

    try {
      // for batchsize = 1, don't do batching, this provides much better error output
      if (inputRowList.size() == 1) {
        dbContext.setSqlParamValues(sqlConfig, config, inputRowList.get(0));
        currentRowNumber++;
      } else {
        // for each row set the Sql params in the prepared statement
        dbContext.getDataStatement().clearBatch();
        for (Row inputRow : inputRowList) {
          dbContext.setSqlParamValues(sqlConfig, config, inputRow);
          dbContext.getDataStatement().addBatch();
          currentRowNumber++;
        }
      }
    } catch (ParameterLoadException e) {
      throw new DataAccessObjectException(e.getMessage(), e);
    } catch (SQLException sqe) {
      String errMsg =
          Messages.getFormattedString(
              "DatabaseDAO.sqlExceptionPrepareRow",
              new String[] {
                String.valueOf(currentRowNumber + 1), String.valueOf(startingRowNumber + 1),
                String.valueOf(startingRowNumber + inputRowList.size() + 1),
                    dbContext.getDbConfigName(),
                sqe.getMessage()
              });
      logger.error(errMsg, sqe);
      // batch failed: set current row number to the end of the batch
      currentRowNumber = startingRowNumber + inputRowList.size();
      throw new DataAccessObjectException(errMsg, sqe);
    } catch (Exception e) {
      String errMsg =
          Messages.getFormattedString(
              "DatabaseDAO.exceptionPrepareRow",
              new String[] {
                String.valueOf(currentRowNumber + 1), String.valueOf(startingRowNumber + 1),
                String.valueOf(startingRowNumber + inputRowList.size() + 1),
                    dbContext.getDbConfigName(),
                e.getMessage()
              });
      logger.error(errMsg, e);
      // batch failed: set current row number to the end of the batch
      currentRowNumber = startingRowNumber + inputRowList.size();
      throw new DataAccessObjectException(errMsg, e);
    }

    try {
      // for batchsize = 1, don't do batching, this provides much better error output
      int totalSuccessRows = 0;
      if (inputRowList.size() == 1) {
        // non-batch update returns exception, so it's always success unless exception is returned
        dbContext.getDataStatement().executeUpdate();
        success = true;
        totalSuccessRows = 1;
      } else {
        // execute the update SQL in batch
        int[] rowsUpdatedArray = dbContext.getDataStatement().executeBatch();
        for (int rowsUpdated : rowsUpdatedArray) {
          if (rowsUpdated == PreparedStatement.SUCCESS_NO_INFO) {
            totalSuccessRows = rowsUpdatedArray.length;
            success = true;
            break;
          }
        }
      }
      logger.debug(
          Messages.getFormattedString(
              "DatabaseDAO.updatedStatus",
              new String[] {String.valueOf(totalSuccessRows), String.valueOf(currentRowNumber)}));

      // commit the change
      dbContext.getDataConnection().commit();

    } catch (SQLException sqe) {
      if (sqe instanceof BatchUpdateException) {
        int[] updateCountArray = ((BatchUpdateException) sqe).getUpdateCounts();
        for (int i = 0; i < updateCountArray.length; i++) {
          if (updateCountArray[i] == PreparedStatement.EXECUTE_FAILED) {
            // FIXME all results are the same, return
            success = false;
            break;
          }
        }
      }
      String errMsg =
          Messages.getFormattedString(
              "DatabaseDAO.sqlExceptionWriteRow",
              new String[] {
                String.valueOf(currentRowNumber + 1 - inputRowList.size()),
                    String.valueOf(currentRowNumber + 1),
                dbContext.getDbConfigName(), sqe.getMessage()
              });
      logger.error(errMsg, sqe);

      endException(inputRowList.size());
      throw new DataAccessObjectException(errMsg, sqe);
    } catch (Exception e) {
      String errMsg =
          Messages.getFormattedString(
              "DatabaseDAO.exceptionWriteRow",
              new String[] {
                String.valueOf(currentRowNumber + 1 - inputRowList.size()),
                    String.valueOf(currentRowNumber + 1),
                dbContext.getDbConfigName(), e.getMessage()
              });
      logger.error(errMsg, e);

      endException(inputRowList.size());
      throw new DataAccessObjectException(errMsg, e);
    }

    return success;
  }
コード例 #17
0
ファイル: DAORowUtil.java プロジェクト: rericka/dataloader
 private void handleError(final Exception e, String msgKey) throws LoadException {
   final String errMsg = Messages.getMessage(getClass(), msgKey);
   logger.error(errMsg, e);
   throw new LoadException(errMsg, e);
 }