Пример #1
0
    private void doPreparedStatement(
        PreparedStatement ps, DbDialect dbDialect, LobCreator lobCreator, EventData data)
        throws SQLException {
      EventType type = data.getEventType();
      // 注意insert/update语句对应的字段数序都是将主键排在后面
      List<EventColumn> columns = new ArrayList<EventColumn>();
      if (type.isInsert()) {
        columns.addAll(data.getColumns()); // insert为所有字段
        columns.addAll(data.getKeys());
      } else if (type.isDelete()) {
        columns.addAll(data.getKeys());
      } else if (type.isUpdate()) {
        boolean existOldKeys = !CollectionUtils.isEmpty(data.getOldKeys());
        columns.addAll(data.getUpdatedColumns()); // 只更新带有isUpdate=true的字段
        columns.addAll(data.getKeys());
        if (existOldKeys) {
          columns.addAll(data.getOldKeys());
        }
      }
      for (int i = 0; i < columns.size(); i++) {
        int paramIndex = i + 1;
        EventColumn column = columns.get(i);
        int sqlType = column.getColumnType();

        // 获取一下当前字段名的数据是否必填
        Table table = dbDialect.findTable(data.getSchemaName(), data.getTableName());
        Map<String, Boolean> isRequiredMap = new HashMap<String, Boolean>();
        for (Column tableColumn : table.getColumns()) {
          isRequiredMap.put(StringUtils.lowerCase(tableColumn.getName()), tableColumn.isRequired());
        }

        Boolean isRequired = isRequiredMap.get(StringUtils.lowerCase(column.getColumnName()));
        if (isRequired == null) {
          throw new ClaveException(
              String.format(
                  "column name %s is not found in Table[%s]",
                  column.getColumnName(), table.toString()));
        }

        Object param =
            SqlUtils.stringToSqlValue(
                column.getColumnValue(), sqlType, isRequired, dbDialect.isEmptyStringNulled());
        try {
          switch (sqlType) {
            case Types.CLOB:
              lobCreator.setClobAsString(ps, paramIndex, (String) param);
              break;

            case Types.BLOB:
              lobCreator.setBlobAsBytes(ps, paramIndex, (byte[]) param);
              break;

            default:
              StatementCreatorUtils.setParameterValue(ps, paramIndex, sqlType, null, param);
              break;
          }
        } catch (SQLException ex) {
          logger.error(
              "## SetParam error , [table={}, sqltype={}, value={}]",
              new Object[] {data.getSchemaName() + "." + data.getTableName(), sqlType, param});
          throw ex;
        }
      }
    }
Пример #2
0
    private Exception doCall() {
      RuntimeException error = null;
      ExecuteResult exeResult = null;
      int index = 0; // 记录下处理成功的记录下标
      for (; index < datas.size(); ) {
        // 处理数据切分
        final List<EventData> splitDatas = new ArrayList<EventData>();
        if (useBatch && canBatch) {
          int end = (index + batchSize > datas.size()) ? datas.size() : (index + batchSize);
          splitDatas.addAll(datas.subList(index, end));
          index = end; // 移动到下一批次
        } else {
          splitDatas.add(datas.get(index));
          index = index + 1; // 移动到下一条
        }

        int retryCount = 0;
        while (true) {
          try {
            if (CollectionUtils.isEmpty(failedDatas) == false) {
              splitDatas.clear();
              splitDatas.addAll(failedDatas); // 下次重试时,只处理错误的记录
            } else {
              failedDatas.addAll(splitDatas); // 先添加为出错记录,可能获取lob,datasource会出错
            }

            final LobCreator lobCreator = dbDialect.getLobHandler().getLobCreator();
            if (useBatch && canBatch) {
              // 处理batch
              final String sql = splitDatas.get(0).getSql();
              int[] affects = new int[splitDatas.size()];
              affects =
                  (int[])
                      dbDialect
                          .getTransactionTemplate()
                          .execute(
                              new TransactionCallback() {

                                public Object doInTransaction(TransactionStatus status) {
                                  // 初始化一下内容
                                  try {
                                    failedDatas.clear(); // 先清理
                                    processedDatas.clear();
                                    interceptor.transactionBegin(context, splitDatas, dbDialect);
                                    JdbcTemplate template = dbDialect.getJdbcTemplate();
                                    int[] affects =
                                        template.batchUpdate(
                                            sql,
                                            new BatchPreparedStatementSetter() {

                                              public void setValues(PreparedStatement ps, int idx)
                                                  throws SQLException {
                                                doPreparedStatement(
                                                    ps, dbDialect, lobCreator, splitDatas.get(idx));
                                              }

                                              public int getBatchSize() {
                                                return splitDatas.size();
                                              }
                                            });
                                    interceptor.transactionEnd(context, splitDatas, dbDialect);
                                    return affects;
                                  } finally {
                                    lobCreator.close();
                                  }
                                }
                              });

              // 更新统计信息
              for (int i = 0; i < splitDatas.size(); i++) {
                processStat(splitDatas.get(i), affects[i], true);
              }
            } else {
              final EventData data = splitDatas.get(0); // 直接取第一条
              int affect = 0;
              affect =
                  (Integer)
                      dbDialect
                          .getTransactionTemplate()
                          .execute(
                              new TransactionCallback() {

                                public Object doInTransaction(TransactionStatus status) {
                                  try {
                                    failedDatas.clear(); // 先清理
                                    processedDatas.clear();
                                    interceptor.transactionBegin(
                                        context, Arrays.asList(data), dbDialect);
                                    JdbcTemplate template = dbDialect.getJdbcTemplate();
                                    int affect =
                                        template.update(
                                            data.getSql(),
                                            new PreparedStatementSetter() {

                                              public void setValues(PreparedStatement ps)
                                                  throws SQLException {
                                                doPreparedStatement(
                                                    ps, dbDialect, lobCreator, data);
                                              }
                                            });
                                    interceptor.transactionEnd(
                                        context, Arrays.asList(data), dbDialect);
                                    return affect;
                                  } finally {
                                    lobCreator.close();
                                  }
                                }
                              });
              // 更新统计信息
              processStat(data, affect, false);
            }

            error = null;
            exeResult = ExecuteResult.SUCCESS;
          } catch (DeadlockLoserDataAccessException ex) {
            error =
                new ClaveException(
                    ExceptionUtils.getFullStackTrace(ex), DbLoadDumper.dumpEventDatas(splitDatas));
            exeResult = ExecuteResult.RETRY;
          } catch (DataIntegrityViolationException ex) {
            error =
                new ClaveException(
                    ExceptionUtils.getFullStackTrace(ex), DbLoadDumper.dumpEventDatas(splitDatas));
            // if (StringUtils.contains(ex.getMessage(), "ORA-00001")) {
            // exeResult = ExecuteResult.RETRY;
            // } else {
            // exeResult = ExecuteResult.ERROR;
            // }
            exeResult = ExecuteResult.ERROR;
          } catch (RuntimeException ex) {
            error =
                new ClaveException(
                    ExceptionUtils.getFullStackTrace(ex), DbLoadDumper.dumpEventDatas(splitDatas));
            exeResult = ExecuteResult.ERROR;
          } catch (Throwable ex) {
            error =
                new ClaveException(
                    ExceptionUtils.getFullStackTrace(ex), DbLoadDumper.dumpEventDatas(splitDatas));
            exeResult = ExecuteResult.ERROR;
          }

          if (ExecuteResult.SUCCESS == exeResult) {
            allFailedDatas.addAll(failedDatas); // 记录一下异常到all记录中
            allProcesedDatas.addAll(processedDatas);
            failedDatas.clear(); // 清空上一轮的处理
            processedDatas.clear();
            break; // do next eventData
          } else if (ExecuteResult.RETRY == exeResult) {
            retryCount = retryCount + 1; // 计数一次
            // 出现异常,理论上当前的批次都会失败
            processedDatas.clear();
            failedDatas.clear();
            failedDatas.addAll(splitDatas);
            if (retryCount >= retry) {
              processFailedDatas(index); // 重试已结束,添加出错记录并退出
              throw new ClaveException(
                  String.format("execute retry %s times failed", retryCount), error);
            } else {
              try {
                int wait = retryCount * retryWait;
                wait = (wait < retryWait) ? retryWait : wait;
                Thread.sleep(wait);
              } catch (InterruptedException ex) {
                Thread.interrupted();
                processFailedDatas(index); // 局部处理出错了
                throw new ClaveException(ex);
              }
            }
          } else {
            // 出现异常,理论上当前的批次都会失败
            processedDatas.clear();
            failedDatas.clear();
            failedDatas.addAll(splitDatas);
            processFailedDatas(index); // 局部处理出错了
            throw error;
          }
        }
      }

      // 记录一下当前处理过程中失败的记录,affect = 0的记录
      context.getFailedDatas().addAll(allFailedDatas);
      context.getProcessedDatas().addAll(allProcesedDatas);
      return null;
    }