Beispiel #1
0
  private void parseOneRow(
      RowData.Builder rowDataBuilder,
      RowsLogEvent event,
      RowsLogBuffer buffer,
      BitSet cols,
      boolean isAfter,
      TableMeta tableMeta)
      throws UnsupportedEncodingException {
    final int columnCnt = event.getTable().getColumnCnt();
    final ColumnInfo[] columnInfo = event.getTable().getColumnInfo();

    // check table fileds count,只能处理加字段
    if (tableMeta != null && columnInfo.length > tableMeta.getFileds().size()) {
      throw new CanalParseException(
          "column size is not match for table:" + tableMeta.getFullName());
    }

    for (int i = 0; i < columnCnt; i++) {
      ColumnInfo info = columnInfo[i];
      buffer.nextValue(info.type, info.meta);

      FieldMeta fieldMeta = null;
      Column.Builder columnBuilder = Column.newBuilder();
      columnBuilder.setIndex(i);
      columnBuilder.setIsNull(false);
      if (tableMeta != null) {
        // 处理file meta
        fieldMeta = tableMeta.getFileds().get(i);
        columnBuilder.setName(fieldMeta.getColumnName());
        columnBuilder.setIsKey(fieldMeta.isKey());
      }
      final int javaType = buffer.getJavaType();
      columnBuilder.setSqlType(javaType);
      if (buffer.isNull()) {
        columnBuilder.setIsNull(true);
      } else {
        final Serializable value = buffer.getValue();
        // 处理各种类型
        switch (javaType) {
          case Types.INTEGER:
          case Types.TINYINT:
          case Types.SMALLINT:
          case Types.BIGINT:
            // 处理unsigned类型
            Number number = (Number) value;
            if (fieldMeta != null && fieldMeta.isUnsigned() && number.longValue() < 0) {
              switch (buffer.getLength()) {
                case 1: /* MYSQL_TYPE_TINY */
                  columnBuilder.setValue(
                      String.valueOf(Integer.valueOf(TINYINT_MAX_VALUE + number.intValue())));

                case 2: /* MYSQL_TYPE_SHORT */
                  columnBuilder.setValue(
                      String.valueOf(Integer.valueOf(SMALLINT_MAX_VALUE + number.intValue())));

                case 3: /* MYSQL_TYPE_INT24 */
                  columnBuilder.setValue(
                      String.valueOf(Integer.valueOf(MEDIUMINT_MAX_VALUE + number.intValue())));

                case 4: /* MYSQL_TYPE_LONG */
                  columnBuilder.setValue(
                      String.valueOf(Long.valueOf(INTEGER_MAX_VALUE + number.longValue())));

                case 8: /* MYSQL_TYPE_LONGLONG */
                  columnBuilder.setValue(
                      BIGINT_MAX_VALUE.add(BigInteger.valueOf(number.longValue())).toString());
              }
            } else {
              // 对象为number类型,直接valueof即可
              columnBuilder.setValue(String.valueOf(value));
            }
            break;
          case Types.REAL: // float
          case Types.DOUBLE: // double
          case Types.BIT: // bit
            // 对象为number类型,直接valueof即可
            columnBuilder.setValue(String.valueOf(value));
            break;
          case Types.DECIMAL:
            columnBuilder.setValue(((BigDecimal) value).toPlainString());
            break;
          case Types.TIMESTAMP:
            String v = value.toString();
            v = v.substring(0, v.length() - 2);
            columnBuilder.setValue(v);
            break;
          case Types.TIME:
          case Types.DATE:
            // 需要处理year
            columnBuilder.setValue(value.toString());
            break;
          case Types.BINARY:
          case Types.VARBINARY:
          case Types.LONGVARBINARY:
            // fixed text encoding https://github.com/AlibabaTech/canal/issues/18
            // mysql binlog中blob/text都处理为blob类型,需要反查table meta,按编码解析text
            if (isText(fieldMeta.getColumnType())) {
              columnBuilder.setValue(new String((byte[]) value, charset));
            } else {
              // byte数组,直接使用iso-8859-1保留对应编码,浪费内存
              columnBuilder.setValue(new String((byte[]) value, ISO_8859_1));
            }
            break;
          case Types.CHAR:
          case Types.VARCHAR:
            // 本身对象为string
            columnBuilder.setValue(value.toString());
            break;
          default:
            columnBuilder.setValue(value.toString());
        }
      }

      // 设置是否update的标记位
      columnBuilder.setUpdated(
          isAfter
              && isUpdate(
                  rowDataBuilder.getBeforeColumnsList(),
                  columnBuilder.getIsNull() ? null : columnBuilder.getValue(),
                  i));
      if (isAfter) {
        rowDataBuilder.addAfterColumns(columnBuilder.build());
      } else {
        rowDataBuilder.addBeforeColumns(columnBuilder.build());
      }
    }
  }
Beispiel #2
0
  private Entry parseRowsEvent(RowsLogEvent event) {
    try {
      TableMapLogEvent table = event.getTable();
      if (table == null) {
        // tableId对应的记录不存在
        throw new TableIdNotFoundException("not found tableId:" + event.getTableId());
      }

      String fullname = getSchemaNameAndTableName(table);
      if (nameFilter != null && !nameFilter.filter(fullname)) { // check name filter
        return null;
      }

      Header header =
          createHeader(binlogFileName, event.getHeader(), table.getDbName(), table.getTableName());
      RowChange.Builder rowChangeBuider = RowChange.newBuilder();
      rowChangeBuider.setTableId(event.getTableId());
      rowChangeBuider.setIsDdl(false);
      EventType eventType = null;
      int type = event.getHeader().getType();
      if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
        eventType = EventType.INSERT;
      } else if (LogEvent.UPDATE_ROWS_EVENT_V1 == type || LogEvent.UPDATE_ROWS_EVENT == type) {
        eventType = EventType.UPDATE;
      } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
        eventType = EventType.DELETE;
      } else {
        throw new CanalParseException("unsupport event type :" + event.getHeader().getType());
      }
      rowChangeBuider.setEventType(eventType);

      RowsLogBuffer buffer = event.getRowsBuf(charset.name());
      BitSet columns = event.getColumns();
      BitSet changeColumns = event.getColumns();
      TableMeta tableMeta = null;
      if (tableMetaCache != null) { // 入错存在table meta cache
        tableMeta = tableMetaCache.getTableMeta(fullname);
        if (tableMeta == null) {
          throw new CanalParseException("not found [" + fullname + "] in db , pls check!");
        }
      }

      while (buffer.nextOneRow(columns)) {
        // 处理row记录
        RowData.Builder rowDataBuilder = RowData.newBuilder();
        if (EventType.INSERT == eventType) {
          // insert的记录放在before字段中
          parseOneRow(rowDataBuilder, event, buffer, columns, true, tableMeta);
        } else if (EventType.DELETE == eventType) {
          // delete的记录放在before字段中
          parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
        } else {
          // update需要处理before/after
          parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
          if (!buffer.nextOneRow(changeColumns)) {
            rowChangeBuider.addRowDatas(rowDataBuilder.build());
            break;
          }

          parseOneRow(rowDataBuilder, event, buffer, event.getChangeColumns(), true, tableMeta);
        }

        rowChangeBuider.addRowDatas(rowDataBuilder.build());
      }
      return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
    } catch (Exception e) {
      throw new CanalParseException("parse row data failed.", e);
    }
  }