/*
* (non-Javadoc)
* @see com.salesforce.dataloader.dao.csv.Writer#writeRow(java.util.Map)
*/
public boolean writeRow(Map<String, Object> columnValues) throws DataAccessObjectException {
CSVColumnVisitor visitor = new CSVColumnVisitor(fileOut);
try {
visitColumns(columnNames, columnValues, visitor);
fileOut.newLine();
visitor.newRow();
currentRowNumber++;
return true; // success unless there's an exception
} catch (IOException e) {
logger.error(Messages.getString("CSVWriter.errorWriting"), e); // $NON-NLS-1$
throw new DataAccessObjectException(
Messages.getString("CSVWriter.errorWriting"), e); // $NON-NLS-1$
}
}
/**
* Validate column names and return a string if there's a validation warning.
*
* @param dao Data access object to validate columns for
* @return A validation warning or null
*/
public static String validateColumns(DataAccessObject dao) {
HashSet<String> uniqueHeaders = new HashSet<String>();
String warning = null;
for (String header : dao.getColumnNames()) {
if (header == null || header.length() == 0) {
warning = Messages.getString("RowUtil.warningEmptyColumn"); // $NON-NLS-1$
break;
} else if (uniqueHeaders.contains(header)) {
warning =
Messages.getFormattedString("RowUtil.warningDuplicateColumn", header); // $NON-NLS-1$
break;
}
uniqueHeaders.add(header);
}
if (warning != null) {
logger.warn(warning);
}
return warning;
}
private void writeHeaderRow() throws DataAccessObjectInitializationException {
CSVColumnVisitor visitor = new CSVColumnVisitor(fileOut);
try {
visitHeaderColumns(this.columnNames, visitor);
fileOut.newLine();
visitor.newRow();
} catch (IOException e) {
String errMsg = Messages.getString("CSVWriter.errorWriting");
logger.error(errMsg, e); // $NON-NLS-1$
throw new DataAccessObjectInitializationException(errMsg, e);
}
}
private void readHeaderRow() throws DataAccessObjectInitializationException {
try {
synchronized (lock) {
headerRow = csvReader.nextRecord();
}
if (headerRow == null) {
LOGGER.error(Messages.getString("CSVFileDAO.errorHeaderRow"));
throw new DataAccessObjectInitializationException(
Messages.getString("CSVFileDAO.errorHeaderRow"));
}
} catch (IOException e) {
String errMsg = Messages.getString("CSVFileDAO.errorHeaderRow");
LOGGER.error(errMsg, e);
throw new DataAccessObjectInitializationException(errMsg, e);
} finally {
// if there's a problem getting header row, the stream needs to be closed
if (headerRow == null) {
IOUtils.closeQuietly(input);
}
}
}
/*
* (non-Javadoc)
* @see com.salesforce.dataloader.dao.DataWriter#setColumnNames(java.util.List)
*/
public synchronized void setColumnNames(List<String> columnNames)
throws DataAccessObjectInitializationException {
if (columnNames == null || columnNames.isEmpty()) {
String errMsg = Messages.getString("CSVFileDAO.errorOpenNoHeaderRow");
logger.error(errMsg);
throw new DataAccessObjectInitializationException(errMsg);
}
// save column names
this.columnNames = columnNames;
writeHeaderRow();
}
/*
* (non-Javadoc)
* @see com.salesforce.dataloader.dao.csv.Writer#close()
*/
public void close() {
if (fileOut != null) {
try {
fileOut.close();
} catch (IOException e) {
logger.error(Messages.getString("CSVWriter.errorClosing"), e); // $NON-NLS-1$
}
if (!columnNames.isEmpty()) {
columnNames.clear();
}
}
}
private void processResult(
Map<String, Object> dataRow, boolean isSuccess, String id, Error[] errors)
throws DataAccessObjectException {
// process success vs. error
// extract error message from error result
if (isSuccess) {
writeSuccess(dataRow, id, null);
} else {
writeError(
dataRow,
errors == null
? Messages.getString("Visitor.noErrorReceivedMsg")
: errors[0].getMessage());
}
}
private void initalizeInput() throws DataAccessObjectInitializationException {
try {
input = new FileInputStream(file);
if (forceUTF8 || isUTF8File(file)) {
csvReader = new CSVReader(input, "UTF-8", new char[] {',', '\t'});
} else {
csvReader = new CSVReader(input, new char[] {',', '\t'});
}
csvReader.setMaxRowsInFile(Integer.MAX_VALUE);
csvReader.setMaxCharsInFile(Integer.MAX_VALUE);
} catch (FileNotFoundException e) {
String errMsg = Messages.getFormattedString("CSVFileDAO.errorOpen", file.getAbsolutePath());
LOGGER.error(errMsg, e);
throw new DataAccessObjectInitializationException(errMsg, e);
} catch (UnsupportedEncodingException e) {
String errMsg = Messages.getString("CSVFileDAO.errorUnsupportedEncoding");
LOGGER.error(errMsg, e);
throw new DataAccessObjectInitializationException(errMsg, e);
} finally {
if (csvReader == null) {
IOUtils.closeQuietly(input);
}
}
}
/**
* Create database writer based on configuration
*
* @param config
* @param dbConfigName
* @throws DataAccessObjectInitializationException
*/
DatabaseWriter(Config config, String dbConfigName)
throws DataAccessObjectInitializationException {
this.config = config;
String dbConfigFilename =
config.constructConfigFilePath(DatabaseContext.DEFAULT_CONFIG_FILENAME);
if (!(new File(dbConfigFilename).exists())) {
throw new DataAccessObjectInitializationException(
Messages.getFormattedString(
"DatabaseDAO.errorConfigFileExists", dbConfigFilename)); // $NON-NLS-1$
}
DatabaseConfig dbConfig = DatabaseConfig.getInstance(dbConfigFilename, dbConfigName);
dataSource = dbConfig.getDataSource();
sqlConfig = dbConfig.getSqlConfig();
dbContext = new DatabaseContext(dbConfigName);
}
/*
* (non-Javadoc)
* @see com.salesforce.dataloader.dao.DataAccessObject#open()
*/
public void open() throws DataAccessObjectInitializationException {
try {
if (!useDefaultEncoding) {
fileOut =
new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), encoding));
} else {
fileOut = new BufferedWriter(new FileWriter(fileName));
}
currentRowNumber = 0;
setOpen(true);
} catch (IOException e) {
String errMsg = Messages.getFormattedString("CSVWriter.errorOpening", fileName);
logger.error(errMsg, e); // $NON-NLS-1$
throw new DataAccessObjectInitializationException(errMsg, e); // $NON-NLS-1$
}
}
/** @param sqe */
private void endException(int batchSize) {
// Rollback if dbContext.getAutoCommit() is false
try {
dbContext.getDataConnection().rollback();
} catch (SQLException sqe) {
logger.error(
Messages.getFormattedString(
"DatabaseDAO.sqlExceptionRollback",
new String[] {
String.valueOf(currentRowNumber + 1 - batchSize),
String.valueOf(currentRowNumber + 1),
dbContext.getDbConfigName(),
sqe.getMessage()
}),
sqe);
}
}
/**
* Utility function for calculating the total number of rows available to current DAO instance
*
* @throws DataAccessObjectException
*/
public static int calculateTotalRows(DataReader dataReader) throws DataAccessObjectException {
try {
// visit the rows
DAOSizeVisitor visitor = new DAOSizeVisitor();
for (Row row = dataReader.readRow(); isValidRow(row); row = dataReader.readRow()) {
visitor.visit(row);
}
return visitor.getNumberOfRows();
} catch (DataAccessObjectException daoe) {
logger.error(Messages.getString("RowUtil.error"), daoe); // $NON-NLS-1$
throw daoe;
} finally {
// since we've read all the rows, reopen the reader to reset the input
dataReader.close();
dataReader.open();
}
}
/**
* Gets the next row from the current data access object data source. <i>Side effect:</i> Updates
* the current record number
*/
@Override
public Row readRow() throws DataAccessObjectException {
if (!isOpen) {
open();
}
List<String> record;
synchronized (lock) {
try {
record = csvReader.nextRecord();
} catch (IOException e) {
throw new DataAccessObjectException(e);
}
}
if (!DAORowUtil.isValidRow(record)) {
return null;
}
if (record.size() > headerRow.size()) {
String errMsg =
Messages.getFormattedString(
"CSVFileDAO.errorRowTooLarge",
new String[] {
String.valueOf(currentRowNumber),
String.valueOf(record.size()),
String.valueOf(headerRow.size())
});
throw new DataAccessRowException(errMsg);
}
Row row = new Row(record.size());
for (int i = 0; i < headerRow.size(); i++) {
String value = record.get(i);
if (value == null) {
value = "";
}
row.put(headerRow.get(i), value);
}
currentRowNumber++;
return row;
}
private void writeOutputToWriter(Object[] results, List<Map<String, Object>> dataArray)
throws DataAccessObjectException, LoadException {
if (results.length != dataArray.size()) {
getLogger().fatal(Messages.getString("Visitor.errorResultsLength")); // $NON-NLS-1$
throw new LoadException(Messages.getString("Visitor.errorResultsLength"));
}
// have to do this because although saveResult and deleteResult
// are a) not the same class yet b) not subclassed
for (int i = 0; i < results.length; i++) {
Map<String, Object> dataRow = dataArray.get(i);
String statusMsg = null;
if (results instanceof SaveResult[]) {
SaveResult saveRes = (SaveResult) results[i];
if (saveRes.getSuccess()) {
if (OperationInfo.insert == getConfig().getOperationInfo()) {
statusMsg = Messages.getString("DAOLoadVisitor.statusItemCreated");
} else {
statusMsg = Messages.getString("DAOLoadVisitor.statusItemUpdated");
}
}
dataRow.put(Config.STATUS_COLUMN_NAME, statusMsg);
processResult(dataRow, saveRes.getSuccess(), saveRes.getId(), saveRes.getErrors());
} else if (results instanceof DeleteResult[]) {
DeleteResult deleteRes = (DeleteResult) results[i];
if (deleteRes.getSuccess()) {
statusMsg = Messages.getString("DAOLoadVisitor.statusItemDeleted");
}
dataRow.put(Config.STATUS_COLUMN_NAME, statusMsg);
processResult(dataRow, deleteRes.getSuccess(), deleteRes.getId(), deleteRes.getErrors());
} else if (results instanceof UpsertResult[]) {
UpsertResult upsertRes = (UpsertResult) results[i];
if (upsertRes.getSuccess()) {
statusMsg =
upsertRes.getCreated()
? Messages.getString("DAOLoadVisitor.statusItemCreated")
: Messages.getString("DAOLoadVisitor.statusItemUpdated");
}
dataRow.put(Config.STATUS_COLUMN_NAME, statusMsg);
processResult(dataRow, upsertRes.getSuccess(), upsertRes.getId(), upsertRes.getErrors());
}
}
}
@Override
protected void loadBatch() throws DataAccessObjectException, LoadException {
Object[] results = null;
try {
results = executeClientAction(getController().getPartnerClient(), dynaArray);
} catch (ApiFault e) {
handleException(e);
} catch (ConnectionException e) {
handleException(e);
}
// set the current processed
int currentProcessed;
try {
currentProcessed = getConfig().getInt(LastRun.LAST_LOAD_BATCH_ROW);
} catch (ParameterLoadException e) {
// if there's a problem getting last batch row, start at the beginning
currentProcessed = 0;
}
currentProcessed += results.length;
getConfig().setValue(LastRun.LAST_LOAD_BATCH_ROW, currentProcessed);
try {
getConfig().saveLastRun();
} catch (IOException e) {
String errMsg = Messages.getString("LoadAction.errorLastRun");
getLogger().error(errMsg, e);
handleException(errMsg, e);
}
writeOutputToWriter(results, dataArray);
// update Monitor
getProgressMonitor().worked(results.length);
getProgressMonitor()
.setSubTask(getRateCalculator().calculateSubTask(getNumberOfRows(), getNumberErrors()));
// now clear the arrays
clearArrays();
}
/*
* (non-Javadoc)
* @see com.salesforce.dataloader.dao.DataWriter#writeRowList(java.util.List)
*/
@Override
public boolean writeRowList(List<Row> inputRowList) throws DataAccessObjectException {
// make sure that the update is setup and ready to go, otherwise stop
if (!dbContext.isOpen()) {
throw new DataAccessObjectInitializationException(
Messages.getString("DatabaseDAO.errorUpdateNotOpen"));
}
boolean success = true;
int startingRowNumber = currentRowNumber;
try {
// for batchsize = 1, don't do batching, this provides much better error output
if (inputRowList.size() == 1) {
dbContext.setSqlParamValues(sqlConfig, config, inputRowList.get(0));
currentRowNumber++;
} else {
// for each row set the Sql params in the prepared statement
dbContext.getDataStatement().clearBatch();
for (Row inputRow : inputRowList) {
dbContext.setSqlParamValues(sqlConfig, config, inputRow);
dbContext.getDataStatement().addBatch();
currentRowNumber++;
}
}
} catch (ParameterLoadException e) {
throw new DataAccessObjectException(e.getMessage(), e);
} catch (SQLException sqe) {
String errMsg =
Messages.getFormattedString(
"DatabaseDAO.sqlExceptionPrepareRow",
new String[] {
String.valueOf(currentRowNumber + 1), String.valueOf(startingRowNumber + 1),
String.valueOf(startingRowNumber + inputRowList.size() + 1),
dbContext.getDbConfigName(),
sqe.getMessage()
});
logger.error(errMsg, sqe);
// batch failed: set current row number to the end of the batch
currentRowNumber = startingRowNumber + inputRowList.size();
throw new DataAccessObjectException(errMsg, sqe);
} catch (Exception e) {
String errMsg =
Messages.getFormattedString(
"DatabaseDAO.exceptionPrepareRow",
new String[] {
String.valueOf(currentRowNumber + 1), String.valueOf(startingRowNumber + 1),
String.valueOf(startingRowNumber + inputRowList.size() + 1),
dbContext.getDbConfigName(),
e.getMessage()
});
logger.error(errMsg, e);
// batch failed: set current row number to the end of the batch
currentRowNumber = startingRowNumber + inputRowList.size();
throw new DataAccessObjectException(errMsg, e);
}
try {
// for batchsize = 1, don't do batching, this provides much better error output
int totalSuccessRows = 0;
if (inputRowList.size() == 1) {
// non-batch update returns exception, so it's always success unless exception is returned
dbContext.getDataStatement().executeUpdate();
success = true;
totalSuccessRows = 1;
} else {
// execute the update SQL in batch
int[] rowsUpdatedArray = dbContext.getDataStatement().executeBatch();
for (int rowsUpdated : rowsUpdatedArray) {
if (rowsUpdated == PreparedStatement.SUCCESS_NO_INFO) {
totalSuccessRows = rowsUpdatedArray.length;
success = true;
break;
}
}
}
logger.debug(
Messages.getFormattedString(
"DatabaseDAO.updatedStatus",
new String[] {String.valueOf(totalSuccessRows), String.valueOf(currentRowNumber)}));
// commit the change
dbContext.getDataConnection().commit();
} catch (SQLException sqe) {
if (sqe instanceof BatchUpdateException) {
int[] updateCountArray = ((BatchUpdateException) sqe).getUpdateCounts();
for (int i = 0; i < updateCountArray.length; i++) {
if (updateCountArray[i] == PreparedStatement.EXECUTE_FAILED) {
// FIXME all results are the same, return
success = false;
break;
}
}
}
String errMsg =
Messages.getFormattedString(
"DatabaseDAO.sqlExceptionWriteRow",
new String[] {
String.valueOf(currentRowNumber + 1 - inputRowList.size()),
String.valueOf(currentRowNumber + 1),
dbContext.getDbConfigName(), sqe.getMessage()
});
logger.error(errMsg, sqe);
endException(inputRowList.size());
throw new DataAccessObjectException(errMsg, sqe);
} catch (Exception e) {
String errMsg =
Messages.getFormattedString(
"DatabaseDAO.exceptionWriteRow",
new String[] {
String.valueOf(currentRowNumber + 1 - inputRowList.size()),
String.valueOf(currentRowNumber + 1),
dbContext.getDbConfigName(), e.getMessage()
});
logger.error(errMsg, e);
endException(inputRowList.size());
throw new DataAccessObjectException(errMsg, e);
}
return success;
}
private void handleError(final Exception e, String msgKey) throws LoadException {
final String errMsg = Messages.getMessage(getClass(), msgKey);
logger.error(errMsg, e);
throw new LoadException(errMsg, e);
}