Exemplo n.º 1
0
    /** 根据视图同步定义的columnPair,获取需要反查的字段列表,不包括主键 */
    private List<String> buildMaxColumnsFromColumnPairs(
        List<DataMediaPair> mediaPairs, List<EventColumn> pks) {
      Set<String> allColumns = new HashSet<String>();
      Map<String, EventColumn> pkMap = new HashMap<String, EventColumn>(pks.size(), 1f);
      for (EventColumn pk : pks) {
        pkMap.put(StringUtils.lowerCase(pk.getColumnName()), pk);
      }

      for (DataMediaPair mediaPair : mediaPairs) { // 一个源库可以对应多个目标,多路复制
        List<ColumnPair> columnPairs = mediaPair.getColumnPairs();

        if (CollectionUtils.isEmpty(columnPairs) || mediaPair.getColumnPairMode().isExclude()) {
          // 1. 如果有一个没有视图定义,说明需要所有字段
          // 2. 如果有一个表存在exclude模式,简单处理,直接反查所有字段,到后面进行过滤
          return new ArrayList<String>(); // 返回空集合,代表没有view
          // filter,需要所有字段
        } else {
          for (ColumnPair columnPair : columnPairs) {
            String columnName = columnPair.getSourceColumn().getName();
            if (!pkMap.containsKey(StringUtils.lowerCase(columnName))) {
              allColumns.add(columnPair.getSourceColumn().getName()); // 加入的为非主键
            }
          }
        }
      }

      return new ArrayList<String>(allColumns);
    }
Exemplo n.º 2
0
 private String dumpEventColumn(List<EventColumn> columns) {
   StringBuilder builder = new StringBuilder(event_default_capacity);
   int size = columns.size();
   for (int i = 0; i < size; i++) {
     EventColumn column = columns.get(i);
     builder.append("\t").append(column.toString());
     if (i < columns.size() - 1) {
       builder.append(SEP);
     }
   }
   return builder.toString();
 }
Exemplo n.º 3
0
 private EventColumn buildColumn(BatchProto.Column columnProto) {
   EventColumn column = new EventColumn();
   column.setColumnName(columnProto.getName());
   column.setNull(columnProto.getIsNull());
   column.setColumnType(columnProto.getType());
   column.setColumnValue(columnProto.getValue());
   column.setKey(columnProto.getIsPrimaryKey());
   column.setIndex(columnProto.getIndex());
   column.setUpdate(columnProto.getIsUpdate()); // add by ljh 2012-08-30,标记变更字段
   return column;
 }
Exemplo n.º 4
0
 private BatchProto.Column buildColumn(EventColumn keyColumn) {
   BatchProto.Column.Builder columnBuilder = BatchProto.Column.newBuilder();
   columnBuilder.setName(keyColumn.getColumnName());
   columnBuilder.setType(keyColumn.getColumnType());
   columnBuilder.setIsNull(keyColumn.isNull());
   columnBuilder.setIsPrimaryKey(keyColumn.isKey());
   columnBuilder.setIndex(keyColumn.getIndex());
   if (keyColumn.getColumnValue() != null) {
     columnBuilder.setValue(keyColumn.getColumnValue());
   }
   columnBuilder.setIsUpdate(keyColumn.isUpdate()); // add by ljh 2012-08-30,标记变更字段
   return columnBuilder.build();
 }
Exemplo n.º 5
0
    /** 构建数据库主键字段的信息 */
    private TableData buildTableData(Table table, List<EventColumn> keys) {
      Column[] tableColumns = table.getColumns();

      TableData data = new TableData();
      data.indexs = new int[keys.size()];
      data.columnNames = new String[keys.size()];
      data.columnTypes = new int[keys.size()];
      data.columnValues = new Object[keys.size()];

      int i = 0;
      int index = 0;
      for (EventColumn keyColumn : keys) {
        for (Column tableColumn : tableColumns) {
          if (StringUtils.equalsIgnoreCase(keyColumn.getColumnName(), tableColumn.getName())) {
            data.indexs[i] = index;
            data.columnNames[i] = tableColumn.getName();
            data.columnTypes[i] = tableColumn.getTypeCode();
            data.columnValues[i] =
                SqlUtils.stringToSqlValue(
                    keyColumn.getColumnValue(),
                    tableColumn.getTypeCode(),
                    tableColumn.isRequired(),
                    false);

            i++;
            break;
          }
          index++;
        }
      }

      if (i != keys.size()) {
        throw new ExtractException(
            "keys is not found in table " + table.toString() + " keys : " + dumpEventColumn(keys));
      }
      return data;
    }
Exemplo n.º 6
0
    @SuppressWarnings("unused")
    private boolean checkNeedDbForRowMode(
        Table table, List<String> viewColumns, EventData eventData) {
      if (viewColumns.size() != 0) { // 说明有视图
        if (viewColumns.size() != eventData.getColumns().size()) {
          return true;
        }

        // 检查一下当前是否所有字段都在view字段列表里
        for (EventColumn column : eventData.getColumns()) {
          if (!viewColumns.contains(column.getColumnName())) {
            return true;
          }
        }

        return false;
      } else {
        if (table.getColumnCount() == eventData.getColumns().size() + eventData.getKeys().size()) {
          return false;
        } else {
          return true;
        }
      }
    }
Exemplo n.º 7
0
 private EventColumn buildColumn(
     String name, int type, String value, boolean isKey, boolean isNull) {
   EventColumn column = new EventColumn();
   column.setColumnName(name);
   column.setColumnType(type);
   column.setColumnValue(value);
   column.setKey(isKey);
   column.setNull(isNull);
   return column;
 }
Exemplo n.º 8
0
    /** 构建数据库非主键字段的信息 */
    private TableData buildTableData(
        Table table, List<EventColumn> columns, boolean needAll, List<String> viewColumnNames) {
      Column[] tableColumns = table.getColumns();
      List<Column> noPkcolumns = new ArrayList<Column>();
      for (Column tableColumn : tableColumns) {
        if (!tableColumn.isPrimaryKey()) {
          noPkcolumns.add(tableColumn);
        }
      }

      TableData data = new TableData();
      int size = columns.size();
      if (needAll) {
        size =
            viewColumnNames.size() != 0
                ? viewColumnNames.size()
                : noPkcolumns.size(); // 如果view不为空就使用view作为反查字段
      }

      data.indexs = new int[size];
      data.columnNames = new String[size];
      data.columnTypes = new int[size];
      data.columnValues = new Object[size];

      int i = 0;
      if (needAll) {
        int index = 0;
        if (viewColumnNames.size() != 0) { // 存在视图定义
          for (Column tableColumn : tableColumns) {
            if (viewColumnNames.contains(tableColumn.getName())) { // 只放入在view中定义的
              data.indexs[i] = index; // 计算下下标
              data.columnNames[i] = tableColumn.getName();
              data.columnTypes[i] = tableColumn.getTypeCode();
              i++;
            }

            index++;
          }
        } else {
          for (Column tableColumn : tableColumns) {
            if (!tableColumn.isPrimaryKey()) {
              data.indexs[i] = index; // 计算下下标
              data.columnNames[i] = tableColumn.getName();
              data.columnTypes[i] = tableColumn.getTypeCode();
              i++;
            }
            index++;
          }
        }
      } else {
        for (EventColumn column : columns) {
          int index = 0;
          for (Column tableColumn : tableColumns) {
            if (StringUtils.equalsIgnoreCase(column.getColumnName(), tableColumn.getName())) {
              data.indexs[i] = index; // 计算下下标
              data.columnNames[i] = tableColumn.getName();
              data.columnTypes[i] = tableColumn.getTypeCode();

              i++;
              break;
            }
            index++;
          }
        }

        if (i != columns.size()) {
          throw new ExtractException(
              "columns is not found in table "
                  + table.toString()
                  + " columns : "
                  + dumpEventColumn(columns));
        }
      }

      return data;
    }
Exemplo n.º 9
0
    public void run() {
      try {
        MDC.put(OtterConstants.splitPipelineLogFileKey, String.valueOf(pipeline.getId()));
        Thread.currentThread()
            .setName(String.format(WORKER_NAME_FORMAT, pipeline.getId(), pipeline.getName()));
        // 获取数据表信息
        DataMedia dataMedia = ConfigHelper.findDataMedia(pipeline, eventData.getTableId());
        DbDialect dbDialect =
            dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) dataMedia.getSource());
        Table table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());
        TableData keyTableData = buildTableData(table, eventData.getKeys());

        // oracle类型特殊处理下
        if (dbDialect instanceof OracleDialect) {
          keyTableData.columnTypes = getOraclePkTypes(table, keyTableData.columnNames);
        }

        boolean needAll =
            pipeline.getParameters().getSyncMode().isRow()
                || (eventData.getSyncMode() != null && eventData.getSyncMode().isRow());

        // 增加一种case, 针对oracle erosa有时侯结果记录只有主键,没有变更字段,需要做一次反查,获取所有字段
        needAll |=
            CollectionUtils.isEmpty(eventData.getUpdatedColumns())
                && dataMedia.getSource().getType().isOracle();

        List<DataMediaPair> mediaParis =
            ConfigHelper.findDataMediaPairByMediaId(pipeline, dataMedia.getId());
        List<String> viewColumnNames =
            buildMaxColumnsFromColumnPairs(mediaParis, eventData.getKeys());

        // TODO 后续版本测试下
        // if (needAll) {
        // boolean needDb = checkNeedDbForRowMode(table,
        // viewColumnNames, eventData);
        // if (needAll && !needDb) {// 不需要进行反查
        // item.setFilter(false);
        // return;
        // }
        // }

        // modified by ljh at 2012-11-04
        // 反查数据时只反查带update=true标识的数据,因为update=false的记录可能只是进行filter需要用到的数据,不需要反查
        TableData columnTableData =
            buildTableData(table, eventData.getUpdatedColumns(), needAll, viewColumnNames);

        if (columnTableData.columnNames.length == 0) {
          // 全主键,不需要进行反查
        } else {
          List<String> newColumnValues =
              select(
                  dbDialect,
                  eventData.getSchemaName(),
                  eventData.getTableName(),
                  keyTableData,
                  columnTableData);

          if (newColumnValues == null) {
            // miss from db
            // 设置为filter=true,可能存在丢数据的风险.
            // 比如针对源库发生主备切换,otter反查的是备库,查询不到对应的记录
            // item.setFilter(true);

            // 针对需要自定义反查数据库的,允许忽略
            // a. 自由门触发的数据,不存在时可以忽略
            // b. 回环补救算法触发的数据,不存在时可以忽略
            boolean needFilter = eventData.isRemedy() || pipeline.getParameters().getSkipNoRow();
            item.setFilter(needFilter);

            // 判断主键是否有变更,如果变更了,就原样返回item
            int index = 0;
            for (EventColumn oldKey : eventData.getOldKeys()) {
              if (!oldKey.equals(eventData.getKeys().get(index))) {
                item.setFilter(false);
                break;
              }
            }
          } else {
            // 构造反查的返回结果
            List<EventColumn> newEventColumns = new ArrayList<EventColumn>();
            for (int i = 0; i < newColumnValues.size(); i++) {
              EventColumn column = new EventColumn();
              column.setIndex(columnTableData.indexs[i]);
              column.setColumnName(columnTableData.columnNames[i]);
              column.setColumnType(columnTableData.columnTypes[i]);
              column.setNull(newColumnValues.get(i) == null);
              column.setColumnValue(newColumnValues.get(i));
              column.setUpdate(true);
              newEventColumns.add(column);
            }

            // 处理下columns中不在反查字段内的字段列表
            for (EventColumn column : eventData.getColumns()) {
              boolean override = false;
              for (EventColumn newEventColumn : newEventColumns) {
                if (StringUtils.equalsIgnoreCase(
                    newEventColumn.getColumnName(), column.getColumnName())) {
                  override = true;
                  break;
                }
              }

              if (!override) { // 针对newcolumns不存在的记录进行添加
                newEventColumns.add(column);
              }
            }

            Collections.sort(newEventColumns, new EventColumnIndexComparable()); // 重新排个序
            eventData.setColumns(newEventColumns);
          }
        }
      } catch (InterruptedException e) {
        // ignore
      } finally {
        Thread.currentThread().setName(WORKER_NAME);
        MDC.remove(OtterConstants.splitPipelineLogFileKey);
      }
    }