Esempio n. 1
0
  protected void preDump(ErosaConnection connection) {
    if (!(connection instanceof MysqlConnection)) {
      throw new CanalParseException(
          "Unsupported connection type : " + connection.getClass().getSimpleName());
    }

    // 开始启动心跳包
    if (detectingEnable && StringUtils.isNotBlank(detectingSQL)) {
      // fixed issue #56,避免重复创建heartbeat线程
      if (mysqlHeartBeatTimeTask == null) {
        logger.info("start heart beat.... ");
        startHeartbeat((MysqlConnection) connection.fork());
      }
    }

    if (binlogParser != null && binlogParser instanceof LogEventConvert) {
      metaConnection = (MysqlConnection) connection.fork();
      try {
        metaConnection.connect();
      } catch (IOException e) {
        throw new CanalParseException(e);
      }

      tableMetaCache = new TableMetaCache(metaConnection);
      ((LogEventConvert) binlogParser).setTableMetaCache(tableMetaCache);
    }
  }
Esempio n. 2
0
  protected void afterDump(ErosaConnection connection) {
    super.afterDump(connection);

    if (!(connection instanceof MysqlConnection)) {
      throw new CanalParseException(
          "Unsupported connection type : " + connection.getClass().getSimpleName());
    }

    if (metaConnection != null) {
      try {
        metaConnection.disconnect();
      } catch (IOException e) {
        logger.error(
            "ERROR # disconnect meta connection for address:{}",
            metaConnection.getConnector().getAddress(),
            e);
      }
    }
  }
Esempio n. 3
0
  // 根据想要的position,可能这个position对应的记录为rowdata,需要找到事务头,避免丢数据
  // 主要考虑一个事务执行时间可能会几秒种,如果仅仅按照timestamp相同,则可能会丢失事务的前半部分数据
  private Long findTransactionBeginPosition(
      ErosaConnection mysqlConnection, final EntryPosition entryPosition) throws IOException {
    // 尝试找到一个合适的位置
    final AtomicBoolean reDump = new AtomicBoolean(false);
    mysqlConnection.reconnect();
    mysqlConnection.seek(
        entryPosition.getJournalName(),
        entryPosition.getPosition(),
        new SinkFunction<LogEvent>() {

          private LogPosition lastPosition;

          public boolean sink(LogEvent event) {
            try {
              CanalEntry.Entry entry = parseAndProfilingIfNecessary(event);
              if (entry == null) {
                return true;
              }

              // 直接查询第一条业务数据,确认是否为事务Begin/End
              if (CanalEntry.EntryType.TRANSACTIONBEGIN == entry.getEntryType()
                  || CanalEntry.EntryType.TRANSACTIONEND == entry.getEntryType()) {
                lastPosition = buildLastPosition(entry);
                return false;
              } else {
                reDump.set(true);
                lastPosition = buildLastPosition(entry);
                return false;
              }
            } catch (Exception e) {
              // 上一次记录的poistion可能为一条update/insert/delete变更事件,直接进行dump的话,会缺少tableMap事件,导致tableId未进行解析
              processError(
                  e, lastPosition, entryPosition.getJournalName(), entryPosition.getPosition());
              reDump.set(true);
              return false;
            }
          }
        });
    // 针对开始的第一条为非Begin记录,需要从该binlog扫描
    if (reDump.get()) {
      final AtomicLong preTransactionStartPosition = new AtomicLong(0L);
      mysqlConnection.reconnect();
      mysqlConnection.seek(
          entryPosition.getJournalName(),
          4L,
          new SinkFunction<LogEvent>() {

            private LogPosition lastPosition;

            public boolean sink(LogEvent event) {
              try {
                CanalEntry.Entry entry = parseAndProfilingIfNecessary(event);
                if (entry == null) {
                  return true;
                }

                // 直接查询第一条业务数据,确认是否为事务Begin
                // 记录一下transaction begin position
                if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
                    && entry.getHeader().getLogfileOffset() < entryPosition.getPosition()) {
                  preTransactionStartPosition.set(entry.getHeader().getLogfileOffset());
                }

                if (entry.getHeader().getLogfileOffset() >= entryPosition.getPosition()) {
                  return false; // 退出
                }

                lastPosition = buildLastPosition(entry);
              } catch (Exception e) {
                processError(
                    e, lastPosition, entryPosition.getJournalName(), entryPosition.getPosition());
                return false;
              }

              return running;
            }
          });

      // 判断一下找到的最接近position的事务头的位置
      if (preTransactionStartPosition.get() > entryPosition.getPosition()) {
        logger.error(
            "preTransactionEndPosition greater than startPosition from zk or localconf, maybe lost data");
        throw new CanalParseException(
            "preTransactionStartPosition greater than startPosition from zk or localconf, maybe lost data");
      }
      return preTransactionStartPosition.get();
    } else {
      return entryPosition.getPosition();
    }
  }