protected void preDump(ErosaConnection connection) { if (!(connection instanceof MysqlConnection)) { throw new CanalParseException( "Unsupported connection type : " + connection.getClass().getSimpleName()); } // 开始启动心跳包 if (detectingEnable && StringUtils.isNotBlank(detectingSQL)) { // fixed issue #56,避免重复创建heartbeat线程 if (mysqlHeartBeatTimeTask == null) { logger.info("start heart beat.... "); startHeartbeat((MysqlConnection) connection.fork()); } } if (binlogParser != null && binlogParser instanceof LogEventConvert) { metaConnection = (MysqlConnection) connection.fork(); try { metaConnection.connect(); } catch (IOException e) { throw new CanalParseException(e); } tableMetaCache = new TableMetaCache(metaConnection); ((LogEventConvert) binlogParser).setTableMetaCache(tableMetaCache); } }
protected void afterDump(ErosaConnection connection) { super.afterDump(connection); if (!(connection instanceof MysqlConnection)) { throw new CanalParseException( "Unsupported connection type : " + connection.getClass().getSimpleName()); } if (metaConnection != null) { try { metaConnection.disconnect(); } catch (IOException e) { logger.error( "ERROR # disconnect meta connection for address:{}", metaConnection.getConnector().getAddress(), e); } } }
// 根据想要的position,可能这个position对应的记录为rowdata,需要找到事务头,避免丢数据 // 主要考虑一个事务执行时间可能会几秒种,如果仅仅按照timestamp相同,则可能会丢失事务的前半部分数据 private Long findTransactionBeginPosition( ErosaConnection mysqlConnection, final EntryPosition entryPosition) throws IOException { // 尝试找到一个合适的位置 final AtomicBoolean reDump = new AtomicBoolean(false); mysqlConnection.reconnect(); mysqlConnection.seek( entryPosition.getJournalName(), entryPosition.getPosition(), new SinkFunction<LogEvent>() { private LogPosition lastPosition; public boolean sink(LogEvent event) { try { CanalEntry.Entry entry = parseAndProfilingIfNecessary(event); if (entry == null) { return true; } // 直接查询第一条业务数据,确认是否为事务Begin/End if (CanalEntry.EntryType.TRANSACTIONBEGIN == entry.getEntryType() || CanalEntry.EntryType.TRANSACTIONEND == entry.getEntryType()) { lastPosition = buildLastPosition(entry); return false; } else { reDump.set(true); lastPosition = buildLastPosition(entry); return false; } } catch (Exception e) { // 上一次记录的poistion可能为一条update/insert/delete变更事件,直接进行dump的话,会缺少tableMap事件,导致tableId未进行解析 processError( e, lastPosition, entryPosition.getJournalName(), entryPosition.getPosition()); reDump.set(true); return false; } } }); // 针对开始的第一条为非Begin记录,需要从该binlog扫描 if (reDump.get()) { final AtomicLong preTransactionStartPosition = new AtomicLong(0L); mysqlConnection.reconnect(); mysqlConnection.seek( entryPosition.getJournalName(), 4L, new SinkFunction<LogEvent>() { private LogPosition lastPosition; public boolean sink(LogEvent event) { try { CanalEntry.Entry entry = parseAndProfilingIfNecessary(event); if (entry == null) { return true; } // 直接查询第一条业务数据,确认是否为事务Begin // 记录一下transaction begin position if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN && entry.getHeader().getLogfileOffset() < entryPosition.getPosition()) { preTransactionStartPosition.set(entry.getHeader().getLogfileOffset()); } if (entry.getHeader().getLogfileOffset() >= entryPosition.getPosition()) { return false; // 退出 } lastPosition = buildLastPosition(entry); } catch (Exception e) { processError( e, lastPosition, entryPosition.getJournalName(), entryPosition.getPosition()); return false; } return running; } }); // 判断一下找到的最接近position的事务头的位置 if (preTransactionStartPosition.get() > entryPosition.getPosition()) { logger.error( "preTransactionEndPosition greater than startPosition from zk or localconf, maybe lost data"); throw new CanalParseException( "preTransactionStartPosition greater than startPosition from zk or localconf, maybe lost data"); } return preTransactionStartPosition.get(); } else { return entryPosition.getPosition(); } }