private void doPreparedStatement(
PreparedStatement ps, DbDialect dbDialect, LobCreator lobCreator, EventData data)
throws SQLException {
EventType type = data.getEventType();
// 注意insert/update语句对应的字段数序都是将主键排在后面
List<EventColumn> columns = new ArrayList<EventColumn>();
if (type.isInsert()) {
columns.addAll(data.getColumns()); // insert为所有字段
columns.addAll(data.getKeys());
} else if (type.isDelete()) {
columns.addAll(data.getKeys());
} else if (type.isUpdate()) {
boolean existOldKeys = !CollectionUtils.isEmpty(data.getOldKeys());
columns.addAll(data.getUpdatedColumns()); // 只更新带有isUpdate=true的字段
columns.addAll(data.getKeys());
if (existOldKeys) {
columns.addAll(data.getOldKeys());
}
}
for (int i = 0; i < columns.size(); i++) {
int paramIndex = i + 1;
EventColumn column = columns.get(i);
int sqlType = column.getColumnType();
// 获取一下当前字段名的数据是否必填
Table table = dbDialect.findTable(data.getSchemaName(), data.getTableName());
Map<String, Boolean> isRequiredMap = new HashMap<String, Boolean>();
for (Column tableColumn : table.getColumns()) {
isRequiredMap.put(StringUtils.lowerCase(tableColumn.getName()), tableColumn.isRequired());
}
Boolean isRequired = isRequiredMap.get(StringUtils.lowerCase(column.getColumnName()));
if (isRequired == null) {
throw new ClaveException(
String.format(
"column name %s is not found in Table[%s]",
column.getColumnName(), table.toString()));
}
Object param =
SqlUtils.stringToSqlValue(
column.getColumnValue(), sqlType, isRequired, dbDialect.isEmptyStringNulled());
try {
switch (sqlType) {
case Types.CLOB:
lobCreator.setClobAsString(ps, paramIndex, (String) param);
break;
case Types.BLOB:
lobCreator.setBlobAsBytes(ps, paramIndex, (byte[]) param);
break;
default:
StatementCreatorUtils.setParameterValue(ps, paramIndex, sqlType, null, param);
break;
}
} catch (SQLException ex) {
logger.error(
"## SetParam error , [table={}, sqltype={}, value={}]",
new Object[] {data.getSchemaName() + "." + data.getTableName(), sqlType, param});
throw ex;
}
}
}
private Exception doCall() {
RuntimeException error = null;
ExecuteResult exeResult = null;
int index = 0; // 记录下处理成功的记录下标
for (; index < datas.size(); ) {
// 处理数据切分
final List<EventData> splitDatas = new ArrayList<EventData>();
if (useBatch && canBatch) {
int end = (index + batchSize > datas.size()) ? datas.size() : (index + batchSize);
splitDatas.addAll(datas.subList(index, end));
index = end; // 移动到下一批次
} else {
splitDatas.add(datas.get(index));
index = index + 1; // 移动到下一条
}
int retryCount = 0;
while (true) {
try {
if (CollectionUtils.isEmpty(failedDatas) == false) {
splitDatas.clear();
splitDatas.addAll(failedDatas); // 下次重试时，只处理错误的记录
} else {
failedDatas.addAll(splitDatas); // 先添加为出错记录，可能获取lob,datasource会出错
}
final LobCreator lobCreator = dbDialect.getLobHandler().getLobCreator();
if (useBatch && canBatch) {
// 处理batch
final String sql = splitDatas.get(0).getSql();
int[] affects = new int[splitDatas.size()];
affects =
(int[])
dbDialect
.getTransactionTemplate()
.execute(
new TransactionCallback() {
public Object doInTransaction(TransactionStatus status) {
// 初始化一下内容
try {
failedDatas.clear(); // 先清理
processedDatas.clear();
interceptor.transactionBegin(context, splitDatas, dbDialect);
JdbcTemplate template = dbDialect.getJdbcTemplate();
int[] affects =
template.batchUpdate(
sql,
new BatchPreparedStatementSetter() {
public void setValues(PreparedStatement ps, int idx)
throws SQLException {
doPreparedStatement(
ps, dbDialect, lobCreator, splitDatas.get(idx));
}
public int getBatchSize() {
return splitDatas.size();
}
});
interceptor.transactionEnd(context, splitDatas, dbDialect);
return affects;
} finally {
lobCreator.close();
}
}
});
// 更新统计信息
for (int i = 0; i < splitDatas.size(); i++) {
processStat(splitDatas.get(i), affects[i], true);
}
} else {
final EventData data = splitDatas.get(0); // 直接取第一条
int affect = 0;
affect =
(Integer)
dbDialect
.getTransactionTemplate()
.execute(
new TransactionCallback() {
public Object doInTransaction(TransactionStatus status) {
try {
failedDatas.clear(); // 先清理
processedDatas.clear();
interceptor.transactionBegin(
context, Arrays.asList(data), dbDialect);
JdbcTemplate template = dbDialect.getJdbcTemplate();
int affect =
template.update(
data.getSql(),
new PreparedStatementSetter() {
public void setValues(PreparedStatement ps)
throws SQLException {
doPreparedStatement(
ps, dbDialect, lobCreator, data);
}
});
interceptor.transactionEnd(
context, Arrays.asList(data), dbDialect);
return affect;
} finally {
lobCreator.close();
}
}
});
// 更新统计信息
processStat(data, affect, false);
}
error = null;
exeResult = ExecuteResult.SUCCESS;
} catch (DeadlockLoserDataAccessException ex) {
error =
new ClaveException(
ExceptionUtils.getFullStackTrace(ex), DbLoadDumper.dumpEventDatas(splitDatas));
exeResult = ExecuteResult.RETRY;
} catch (DataIntegrityViolationException ex) {
error =
new ClaveException(
ExceptionUtils.getFullStackTrace(ex), DbLoadDumper.dumpEventDatas(splitDatas));
// if (StringUtils.contains(ex.getMessage(), "ORA-00001")) {
// exeResult = ExecuteResult.RETRY;
// } else {
// exeResult = ExecuteResult.ERROR;
// }
exeResult = ExecuteResult.ERROR;
} catch (RuntimeException ex) {
error =
new ClaveException(
ExceptionUtils.getFullStackTrace(ex), DbLoadDumper.dumpEventDatas(splitDatas));
exeResult = ExecuteResult.ERROR;
} catch (Throwable ex) {
error =
new ClaveException(
ExceptionUtils.getFullStackTrace(ex), DbLoadDumper.dumpEventDatas(splitDatas));
exeResult = ExecuteResult.ERROR;
}
if (ExecuteResult.SUCCESS == exeResult) {
allFailedDatas.addAll(failedDatas); // 记录一下异常到all记录中
allProcesedDatas.addAll(processedDatas);
failedDatas.clear(); // 清空上一轮的处理
processedDatas.clear();
break; // do next eventData
} else if (ExecuteResult.RETRY == exeResult) {
retryCount = retryCount + 1; // 计数一次
// 出现异常，理论上当前的批次都会失败
processedDatas.clear();
failedDatas.clear();
failedDatas.addAll(splitDatas);
if (retryCount >= retry) {
processFailedDatas(index); // 重试已结束，添加出错记录并退出
throw new ClaveException(
String.format("execute retry %s times failed", retryCount), error);
} else {
try {
int wait = retryCount * retryWait;
wait = (wait < retryWait) ? retryWait : wait;
Thread.sleep(wait);
} catch (InterruptedException ex) {
Thread.interrupted();
processFailedDatas(index); // 局部处理出错了
throw new ClaveException(ex);
}
}
} else {
// 出现异常，理论上当前的批次都会失败
processedDatas.clear();
failedDatas.clear();
failedDatas.addAll(splitDatas);
processFailedDatas(index); // 局部处理出错了
throw error;
}
}
}
// 记录一下当前处理过程中失败的记录,affect = 0的记录
context.getFailedDatas().addAll(allFailedDatas);
context.getProcessedDatas().addAll(allProcesedDatas);
return null;
}
