private void parseOneRow(
RowData.Builder rowDataBuilder,
RowsLogEvent event,
RowsLogBuffer buffer,
BitSet cols,
boolean isAfter,
TableMeta tableMeta)
throws UnsupportedEncodingException {
final int columnCnt = event.getTable().getColumnCnt();
final ColumnInfo[] columnInfo = event.getTable().getColumnInfo();
// check table fileds count，只能处理加字段
if (tableMeta != null && columnInfo.length > tableMeta.getFileds().size()) {
throw new CanalParseException(
"column size is not match for table:" + tableMeta.getFullName());
}
for (int i = 0; i < columnCnt; i++) {
ColumnInfo info = columnInfo[i];
buffer.nextValue(info.type, info.meta);
FieldMeta fieldMeta = null;
Column.Builder columnBuilder = Column.newBuilder();
columnBuilder.setIndex(i);
columnBuilder.setIsNull(false);
if (tableMeta != null) {
// 处理file meta
fieldMeta = tableMeta.getFileds().get(i);
columnBuilder.setName(fieldMeta.getColumnName());
columnBuilder.setIsKey(fieldMeta.isKey());
}
final int javaType = buffer.getJavaType();
columnBuilder.setSqlType(javaType);
if (buffer.isNull()) {
columnBuilder.setIsNull(true);
} else {
final Serializable value = buffer.getValue();
// 处理各种类型
switch (javaType) {
case Types.INTEGER:
case Types.TINYINT:
case Types.SMALLINT:
case Types.BIGINT:
// 处理unsigned类型
Number number = (Number) value;
if (fieldMeta != null && fieldMeta.isUnsigned() && number.longValue() < 0) {
switch (buffer.getLength()) {
case 1: /* MYSQL_TYPE_TINY */
columnBuilder.setValue(
String.valueOf(Integer.valueOf(TINYINT_MAX_VALUE + number.intValue())));
case 2: /* MYSQL_TYPE_SHORT */
columnBuilder.setValue(
String.valueOf(Integer.valueOf(SMALLINT_MAX_VALUE + number.intValue())));
case 3: /* MYSQL_TYPE_INT24 */
columnBuilder.setValue(
String.valueOf(Integer.valueOf(MEDIUMINT_MAX_VALUE + number.intValue())));
case 4: /* MYSQL_TYPE_LONG */
columnBuilder.setValue(
String.valueOf(Long.valueOf(INTEGER_MAX_VALUE + number.longValue())));
case 8: /* MYSQL_TYPE_LONGLONG */
columnBuilder.setValue(
BIGINT_MAX_VALUE.add(BigInteger.valueOf(number.longValue())).toString());
}
} else {
// 对象为number类型，直接valueof即可
columnBuilder.setValue(String.valueOf(value));
}
break;
case Types.REAL: // float
case Types.DOUBLE: // double
case Types.BIT: // bit
// 对象为number类型，直接valueof即可
columnBuilder.setValue(String.valueOf(value));
break;
case Types.DECIMAL:
columnBuilder.setValue(((BigDecimal) value).toPlainString());
break;
case Types.TIMESTAMP:
String v = value.toString();
v = v.substring(0, v.length() - 2);
columnBuilder.setValue(v);
break;
case Types.TIME:
case Types.DATE:
// 需要处理year
columnBuilder.setValue(value.toString());
break;
case Types.BINARY:
case Types.VARBINARY:
case Types.LONGVARBINARY:
// fixed text encoding https://github.com/AlibabaTech/canal/issues/18
// mysql binlog中blob/text都处理为blob类型，需要反查table meta，按编码解析text
if (isText(fieldMeta.getColumnType())) {
columnBuilder.setValue(new String((byte[]) value, charset));
} else {
// byte数组，直接使用iso-8859-1保留对应编码，浪费内存
columnBuilder.setValue(new String((byte[]) value, ISO_8859_1));
}
break;
case Types.CHAR:
case Types.VARCHAR:
// 本身对象为string
columnBuilder.setValue(value.toString());
break;
default:
columnBuilder.setValue(value.toString());
}
}
// 设置是否update的标记位
columnBuilder.setUpdated(
isAfter
&& isUpdate(
rowDataBuilder.getBeforeColumnsList(),
columnBuilder.getIsNull() ? null : columnBuilder.getValue(),
i));
if (isAfter) {
rowDataBuilder.addAfterColumns(columnBuilder.build());
} else {
rowDataBuilder.addBeforeColumns(columnBuilder.build());
}
}
}
private Entry parseRowsEvent(RowsLogEvent event) {
try {
TableMapLogEvent table = event.getTable();
if (table == null) {
// tableId对应的记录不存在
throw new TableIdNotFoundException("not found tableId:" + event.getTableId());
}
String fullname = getSchemaNameAndTableName(table);
if (nameFilter != null && !nameFilter.filter(fullname)) { // check name filter
return null;
}
Header header =
createHeader(binlogFileName, event.getHeader(), table.getDbName(), table.getTableName());
RowChange.Builder rowChangeBuider = RowChange.newBuilder();
rowChangeBuider.setTableId(event.getTableId());
rowChangeBuider.setIsDdl(false);
EventType eventType = null;
int type = event.getHeader().getType();
if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
eventType = EventType.INSERT;
} else if (LogEvent.UPDATE_ROWS_EVENT_V1 == type || LogEvent.UPDATE_ROWS_EVENT == type) {
eventType = EventType.UPDATE;
} else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
eventType = EventType.DELETE;
} else {
throw new CanalParseException("unsupport event type :" + event.getHeader().getType());
}
rowChangeBuider.setEventType(eventType);
RowsLogBuffer buffer = event.getRowsBuf(charset.name());
BitSet columns = event.getColumns();
BitSet changeColumns = event.getColumns();
TableMeta tableMeta = null;
if (tableMetaCache != null) { // 入错存在table meta cache
tableMeta = tableMetaCache.getTableMeta(fullname);
if (tableMeta == null) {
throw new CanalParseException("not found [" + fullname + "] in db , pls check!");
}
}
while (buffer.nextOneRow(columns)) {
// 处理row记录
RowData.Builder rowDataBuilder = RowData.newBuilder();
if (EventType.INSERT == eventType) {
// insert的记录放在before字段中
parseOneRow(rowDataBuilder, event, buffer, columns, true, tableMeta);
} else if (EventType.DELETE == eventType) {
// delete的记录放在before字段中
parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
} else {
// update需要处理before/after
parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
if (!buffer.nextOneRow(changeColumns)) {
rowChangeBuider.addRowDatas(rowDataBuilder.build());
break;
}
parseOneRow(rowDataBuilder, event, buffer, event.getChangeColumns(), true, tableMeta);
}
rowChangeBuider.addRowDatas(rowDataBuilder.build());
}
return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
} catch (Exception e) {
throw new CanalParseException("parse row data failed.", e);
}
}
