Esempio n. 1
0
  @Override
  protected EntryPosition findStartPosition(ErosaConnection connection) {
    // 处理逻辑
    // 1. 首先查询上一次解析成功的最后一条记录
    // 2. 存在最后一条记录,判断一下当前记录是否发生过主备切换
    // // a. 无机器切换,直接返回
    // // b. 存在机器切换,按最后一条记录的stamptime进行查找
    // 3. 不存在最后一条记录,则从默认的位置开始启动
    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    if (logPosition == null) { // 找不到历史成功记录
      EntryPosition entryPosition = masterPosition;

      // 判断一下是否需要按时间订阅
      if (StringUtils.isEmpty(entryPosition.getJournalName())) {
        // 如果没有指定binlogName,尝试按照timestamp进行查找
        if (entryPosition.getTimestamp() != null) {
          return new EntryPosition(entryPosition.getTimestamp());
        }
      } else {
        if (entryPosition.getPosition() != null) {
          // 如果指定binlogName + offest,直接返回
          return entryPosition;
        } else {
          return new EntryPosition(entryPosition.getTimestamp());
        }
      }
    } else {
      return logPosition.getPostion();
    }

    return null;
  }
  /** 检查是否存在需要get的数据,并且数量>=batchSize */
  private boolean checkUnGetSlotAt(LogPosition startPosition, int batchSize) {
    if (batchMode.isItemSize()) {
      long current = getSequence.get();
      long maxAbleSequence = putSequence.get();
      long next = current;
      if (startPosition == null
          || !startPosition.getPostion().isIncluded()) { // 第一次订阅之后,需要包含一下start位置,防止丢失第一条记录
        next = next + 1; // 少一条数据
      }

      if (current < maxAbleSequence && next + batchSize - 1 <= maxAbleSequence) {
        return true;
      } else {
        return false;
      }
    } else {
      // 处理内存大小判断
      long currentSize = getMemSize.get();
      long maxAbleSize = putMemSize.get();

      if (maxAbleSize - currentSize >= batchSize * bufferMemUnit) {
        return true;
      } else {
        return false;
      }
    }
  }
Esempio n. 3
0
  /** 根据给定的时间戳,在指定的binlog中找到最接近于该时间戳(必须是小于时间戳)的一个事务起始位置。针对最后一个binlog会给定endPosition,避免无尽的查询 */
  private EntryPosition findAsPerTimestampInSpecificLogFile(
      MysqlConnection mysqlConnection,
      final Long startTimestamp,
      final EntryPosition endPosition,
      final String searchBinlogFile) {

    final LogPosition logPosition = new LogPosition();
    try {
      mysqlConnection.reconnect();
      // 开始遍历文件
      mysqlConnection.seek(
          searchBinlogFile,
          4L,
          new SinkFunction<LogEvent>() {

            private LogPosition lastPosition;

            public boolean sink(LogEvent event) {
              EntryPosition entryPosition = null;
              try {
                CanalEntry.Entry entry = parseAndProfilingIfNecessary(event);
                if (entry == null) {
                  return true;
                }

                String logfilename = entry.getHeader().getLogfileName();
                Long logfileoffset = entry.getHeader().getLogfileOffset();
                Long logposTimestamp = entry.getHeader().getExecuteTime();

                if (CanalEntry.EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType())) {
                  logger.debug(
                      "compare exit condition:{},{},{}, startTimestamp={}...",
                      new Object[] {logfilename, logfileoffset, logposTimestamp, startTimestamp});
                  // 寻找第一条记录时间戳,如果最小的一条记录都不满足条件,可直接退出
                  if (logposTimestamp >= startTimestamp) {
                    return false;
                  }
                }

                if (StringUtils.equals(endPosition.getJournalName(), logfilename)
                    && endPosition.getPosition() <= (logfileoffset + event.getEventLen())) {
                  return false;
                }

                // 记录一下上一个事务结束的位置,即下一个事务的position
                // position = current + data.length,代表该事务的下一条offest,避免多余的事务重复
                if (CanalEntry.EntryType.TRANSACTIONEND.equals(entry.getEntryType())) {
                  entryPosition =
                      new EntryPosition(
                          logfilename, logfileoffset + event.getEventLen(), logposTimestamp);
                  logger.debug(
                      "set {} to be pending start position before finding another proper one...",
                      entryPosition);
                  logPosition.setPostion(entryPosition);
                }

                lastPosition = buildLastPosition(entry);
              } catch (Exception e) {
                processError(e, lastPosition, searchBinlogFile, 4L);
              }

              return running;
            }
          });

    } catch (IOException e) {
      logger.error("ERROR ## findAsPerTimestampInSpecificLogFile has an error", e);
    }

    if (logPosition.getPostion() != null) {
      return logPosition.getPostion();
    } else {
      return null;
    }
  }
Esempio n. 4
0
  protected EntryPosition findStartPositionInternal(ErosaConnection connection) {
    MysqlConnection mysqlConnection = (MysqlConnection) connection;
    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    if (logPosition == null) { // 找不到历史成功记录
      EntryPosition entryPosition = null;
      if (masterInfo != null
          && mysqlConnection.getConnector().getAddress().equals(masterInfo.getAddress())) {
        entryPosition = masterPosition;
      } else if (standbyInfo != null
          && mysqlConnection.getConnector().getAddress().equals(standbyInfo.getAddress())) {
        entryPosition = standbyPosition;
      }

      if (entryPosition == null) {
        entryPosition = findEndPosition(mysqlConnection); // 默认从当前最后一个位置进行消费
      }

      // 判断一下是否需要按时间订阅
      if (StringUtils.isEmpty(entryPosition.getJournalName())) {
        // 如果没有指定binlogName,尝试按照timestamp进行查找
        if (entryPosition.getTimestamp() != null && entryPosition.getTimestamp() > 0L) {
          logger.warn(
              "prepare to find start position {}:{}:{}",
              new Object[] {"", "", entryPosition.getTimestamp()});
          return findByStartTimeStamp(mysqlConnection, entryPosition.getTimestamp());
        } else {
          logger.warn("prepare to find start position just show master status");
          return findEndPosition(mysqlConnection); // 默认从当前最后一个位置进行消费
        }
      } else {
        if (entryPosition.getPosition() != null && entryPosition.getPosition() > 0L) {
          // 如果指定binlogName + offest,直接返回
          logger.warn(
              "prepare to find start position {}:{}:{}",
              new Object[] {entryPosition.getJournalName(), entryPosition.getPosition(), ""});
          return entryPosition;
        } else {
          EntryPosition specificLogFilePosition = null;
          if (entryPosition.getTimestamp() != null && entryPosition.getTimestamp() > 0L) {
            // 如果指定binlogName + timestamp,但没有指定对应的offest,尝试根据时间找一下offest
            EntryPosition endPosition = findEndPosition(mysqlConnection);
            if (endPosition != null) {
              logger.warn(
                  "prepare to find start position {}:{}:{}",
                  new Object[] {entryPosition.getJournalName(), "", entryPosition.getTimestamp()});
              specificLogFilePosition =
                  findAsPerTimestampInSpecificLogFile(
                      mysqlConnection,
                      entryPosition.getTimestamp(),
                      endPosition,
                      entryPosition.getJournalName());
            }
          }

          if (specificLogFilePosition == null) {
            // position不存在,从文件头开始
            entryPosition.setPosition(BINLOG_START_OFFEST);
            return entryPosition;
          } else {
            return specificLogFilePosition;
          }
        }
      }
    } else {
      if (logPosition
          .getIdentity()
          .getSourceAddress()
          .equals(mysqlConnection.getConnector().getAddress())) {
        logger.warn("prepare to find start position just last position");
        return logPosition.getPostion();
      } else {
        // 针对切换的情况,考虑回退时间
        long newStartTimestamp =
            logPosition.getPostion().getTimestamp() - fallbackIntervalInSeconds * 1000;
        logger.warn(
            "prepare to find start position by switch {}:{}:{}",
            new Object[] {"", "", logPosition.getPostion().getTimestamp()});
        return findByStartTimeStamp(mysqlConnection, newStartTimestamp);
      }
    }
  }
  private Events<Event> doGet(Position start, int batchSize) throws CanalStoreException {
    LogPosition startPosition = (LogPosition) start;

    long current = getSequence.get();
    long maxAbleSequence = putSequence.get();
    long next = current;
    long end = current;
    // 如果startPosition为null,说明是第一次,默认+1处理
    if (startPosition == null
        || !startPosition.getPostion().isIncluded()) { // 第一次订阅之后,需要包含一下start位置,防止丢失第一条记录
      next = next + 1;
    }

    if (current >= maxAbleSequence) {
      return new Events<Event>();
    }

    Events<Event> result = new Events<Event>();
    List<Event> entrys = result.getEvents();
    long memsize = 0;
    if (batchMode.isItemSize()) {
      end = (next + batchSize - 1) < maxAbleSequence ? (next + batchSize - 1) : maxAbleSequence;
      // 提取数据并返回
      for (; next <= end; next++) {
        Event event = entries[getIndex(next)];
        if (ddlIsolation && isDdl(event.getEntry().getHeader().getEventType())) {
          // 如果是ddl隔离,直接返回
          if (entrys.size() == 0) {
            entrys.add(event); // 如果没有DML事件,加入当前的DDL事件
            end = next; // 更新end为当前
          } else {
            // 如果之前已经有DML事件,直接返回了,因为不包含当前next这记录,需要回退一个位置
            end = next - 1; // next-1一定大于current,不需要判断
          }
          break;
        } else {
          entrys.add(event);
        }
      }
    } else {
      long maxMemSize = batchSize * bufferMemUnit;
      for (; memsize <= maxMemSize && next <= maxAbleSequence; next++) {
        // 永远保证可以取出第一条的记录,避免死锁
        Event event = entries[getIndex(next)];
        if (ddlIsolation && isDdl(event.getEntry().getHeader().getEventType())) {
          // 如果是ddl隔离,直接返回
          if (entrys.size() == 0) {
            entrys.add(event); // 如果没有DML事件,加入当前的DDL事件
            end = next; // 更新end为当前
          } else {
            // 如果之前已经有DML事件,直接返回了,因为不包含当前next这记录,需要回退一个位置
            end = next - 1; // next-1一定大于current,不需要判断
          }
          break;
        } else {
          entrys.add(event);
          memsize += calculateSize(event);
          end = next; // 记录end位点
        }
      }
    }

    PositionRange<LogPosition> range = new PositionRange<LogPosition>();
    result.setPositionRange(range);

    range.setStart(CanalEventUtils.createPosition(entrys.get(0)));
    range.setEnd(CanalEventUtils.createPosition(entrys.get(result.getEvents().size() - 1)));
    // 记录一下是否存在可以被ack的点

    for (int i = entrys.size() - 1; i >= 0; i--) {
      Event event = entrys.get(i);
      if (CanalEntry.EntryType.TRANSACTIONBEGIN == event.getEntry().getEntryType()
          || CanalEntry.EntryType.TRANSACTIONEND == event.getEntry().getEntryType()
          || isDdl(event.getEntry().getHeader().getEventType())) {
        // 将事务头/尾设置可被为ack的点
        range.setAck(CanalEventUtils.createPosition(event));
        break;
      }
    }

    if (getSequence.compareAndSet(current, end)) {
      getMemSize.addAndGet(memsize);
      notFull.signal();
      return result;
    } else {
      return new Events<Event>();
    }
  }