Exemplo n.º 1
0
  @Override
  public FetchResult fetchAll(final FetchRequest fetchRequest, long timeout, TimeUnit timeUnit)
      throws MetaClientException, InterruptedException {
    if (timeout <= 0 || timeUnit == null) {
      timeout = this.consumerConfig.getFetchTimeoutInMills();
      timeUnit = TimeUnit.MILLISECONDS;
    }
    final long start = System.currentTimeMillis();
    boolean success = false;
    final long currentOffset = fetchRequest.getOffset();
    try {
      GetCommand getCmd = null;
      SubscriberInfo subInfo = this.topicSubcriberRegistry.get(fetchRequest.getTopic());
      Set<String> messageTypeList = null;
      if (subInfo != null) {
        messageTypeList = subInfo.getMessageTypes();
      }
      if (this.consumerConfig.isVersion2() && messageTypeList != null) { // 用户使用新版本的接口才使用新的协议
        getCmd =
            new FetchCommand(
                this.consumerConfig.getVersion(),
                fetchRequest.getTopic(),
                this.consumerConfig.getGroup(),
                fetchRequest.getPartition(),
                currentOffset,
                fetchRequest.getMaxSize(),
                OpaqueGenerator.getNextOpaque(),
                MetaMessageSessionFactory.startTime);
      } else {
        getCmd =
            new GetCommand(
                fetchRequest.getTopic(),
                this.consumerConfig.getGroup(),
                fetchRequest.getPartition(),
                currentOffset,
                fetchRequest.getMaxSize(),
                OpaqueGenerator.getNextOpaque());
      }

      final String serverUrl = fetchRequest.getBroker().getZKString();
      final ResponseCommand response =
          this.remotingClient.invokeToGroup(serverUrl, getCmd, timeout, timeUnit);
      if (response instanceof DataCommand) {
        final DataCommand dataCmd = (DataCommand) response;
        final byte[] data = dataCmd.getData();
        if (data.length < (MetaMessageDecoder.MessageFlagPostion + 4)) {
          log.fatal("fetch a invalid message " + data.length);
          return null;
        }

        // 识别服务器版本
        java.nio.ByteBuffer byteBuffer = java.nio.ByteBuffer.wrap(data);
        int messageFlag = byteBuffer.getInt(MetaMessageDecoder.MessageFlagPostion);
        // 2.0版本
        if ((messageFlag & MetaMessageDecoder.NewServerFlag) == MetaMessageDecoder.NewServerFlag) {
          server14 = false;
          success = true;
          List<Message> msgList = new ArrayList<Message>(100);
          List<MetaMessageWrapper> wrapperList = MetaMessageDecoder.decodes(byteBuffer);
          if (!wrapperList.isEmpty()) {
            for (MetaMessageWrapper wrapper : wrapperList) {
              String type = wrapper.getMetaMessage().getType();
              if (messageTypeList != null
                  && !messageTypeList.contains("*")
                  && !messageTypeList.contains(type)) {
                continue;
              }
              Message msg =
                  new Message(
                      fetchRequest.getTopic(),
                      wrapper.getMetaMessage().getBody(),
                      wrapper.getMetaMessage().getAttribute());
              msg.setOffset(wrapper.getMetaMessageAnnotation().getQueueOffset());
              msg.setId(wrapper.getMetaMessageAnnotation().getPhysicOffset());
              msg.setMsgNewId(wrapper.getMetaMessageAnnotation().getMsgId());
              msgList.add(msg);
            }
          }

          if (msgList.isEmpty()) {
            if (wrapperList != null && !wrapperList.isEmpty()) {
              MetaMessageWrapper wrapper = wrapperList.get(wrapperList.size() - 1);
              fetchRequest.setOffset(
                  wrapper.getMetaMessageAnnotation().getQueueOffset() + 1,
                  wrapper.getMetaMessageAnnotation().getPhysicOffset(),
                  true);
            }
            return null;
          }
          return new FetchResult(true, msgList, null);
        }
        // 1.4版本
        else {
          server14 = true;
          // 获取的数据严重不足的时候,缩减maxSize
          if (data.length < fetchRequest.getMaxSize() / 2) {
            fetchRequest.decreaseMaxSize();
          }
          success = true;
          return new FetchResult(false, null, new MessageIterator(fetchRequest.getTopic(), data));
        }
      } else {
        final BooleanCommand booleanCmd = (BooleanCommand) response;
        switch (booleanCmd.getCode()) {
          case HttpStatus.NotFound:
            success = true;
            if (log.isDebugEnabled()) {
              log.debug(booleanCmd.getErrorMsg());
            }
            return null;
          case HttpStatus.Forbidden:
            success = true;
            return null;
          case HttpStatus.Moved:
            success = true;
            fetchRequest.resetRetries();
            long serverPushedOffset = Long.parseLong(booleanCmd.getErrorMsg());
            fetchRequest.setOffset(serverPushedOffset, 100, true);
            if (!server14) {
              log.warn(
                  "consumer request offset: "
                      + currentOffset
                      + " invalid or not matched, server pushed new offset: "
                      + serverPushedOffset);
            }
            return null;
          case HttpStatus.Continue:
            success = true;
            SubscriberInfo info = registeMessageType(fetchRequest);
            if (info == null) {
              log.error("consumer report message types failed.");
            } else {
              log.info(
                  "consumer report message types success : " + info.getMessageTypes().toString());
            }
            return null;
          default:
            throw new MetaClientException(((BooleanCommand) response).getErrorMsg());
        }
      }

    } catch (final TimeoutException e) {
      throw new MetaOpeartionTimeoutException(
          "pull message timeout in "
              + this.consumerConfig.getFetchTimeoutInMills()
              + " mills, requestOffset "
              + currentOffset);
    } catch (final MetaClientException e) {
      throw e;
    } catch (final InterruptedException e) {
      throw e;
    } catch (final Exception e) {
      throw new MetaClientException(
          "get message failed,topic="
              + fetchRequest.getTopic()
              + ",partition="
              + fetchRequest.getPartition()
              + ",offset="
              + fetchRequest.getOffset(),
          e);
    } finally {
      final long duration = System.currentTimeMillis() - start;
      if (duration > 200) {
        MetaStatLog.addStatValue2(
            null, StatConstants.GET_TIME_STAT, fetchRequest.getTopic(), duration);
      }
      if (!success) {
        MetaStatLog.addStat(null, StatConstants.GET_FAILED_STAT, fetchRequest.getTopic());
      }
    }
  }
Exemplo n.º 2
0
  @Override
  public DequeueResult fetchSync(final FetchRequest fetchRequest, long timeout, TimeUnit timeUnit)
      throws MetaClientException, InterruptedException {
    if (timeout <= 0 || timeUnit == null) {
      timeout = this.consumerConfig.getFetchTimeoutInMills();
      timeUnit = TimeUnit.MILLISECONDS;
    }
    final long start = System.currentTimeMillis();
    boolean success = false;
    final long currentOffset = fetchRequest.getOffset();
    try {
      final GetCommand getCmd =
          new GetCommand(
              fetchRequest.getTopic(),
              this.consumerConfig.getGroup(),
              fetchRequest.getPartition(),
              currentOffset,
              fetchRequest.getMaxSize(),
              OpaqueGenerator.getNextOpaque());
      final String serverUrl = fetchRequest.getBroker().getZKString();
      final ResponseCommand response =
          this.remotingClient.invokeToGroup(serverUrl, getCmd, timeout, timeUnit);
      if (response instanceof DataCommand) {
        final DataCommand dataCmd = (DataCommand) response;
        final byte[] data = dataCmd.getData();
        if (data.length < (MetaMessageDecoder.MessageFlagPostion + 4)) {
          log.fatal("fetch a invalid message " + data.length);
          return new DequeueResult(DequeueStatus.STATUS_OTHER_ERROR, null, 0);
        }

        java.nio.ByteBuffer byteBuffer = java.nio.ByteBuffer.wrap(data);
        int messageFlag = byteBuffer.getInt(MetaMessageDecoder.MessageFlagPostion);
        // 2.0
        if ((messageFlag & MetaMessageDecoder.NewServerFlag) == MetaMessageDecoder.NewServerFlag) {
          success = true;
          List<Message> msgList = new ArrayList<Message>(100);
          List<MetaMessageWrapper> wrapperList = MetaMessageDecoder.decodes(byteBuffer);
          if (!wrapperList.isEmpty()) {
            for (MetaMessageWrapper wrapper : wrapperList) {
              Message msg =
                  new Message(
                      fetchRequest.getTopic(),
                      wrapper.getMetaMessage().getBody(),
                      wrapper.getMetaMessage().getAttribute());
              msg.setOffset(wrapper.getMetaMessageAnnotation().getQueueOffset());
              msg.setId(wrapper.getMetaMessageAnnotation().getPhysicOffset());
              msg.setMsgNewId(wrapper.getMetaMessageAnnotation().getMsgId());
              msgList.add(msg);
            }
          }

          if (msgList.isEmpty()) {
            log.error("fetch sync OK, but no message");
            return new DequeueResult(DequeueStatus.STATUS_OTHER_ERROR, null, 0);
          }

          return new DequeueResult(DequeueStatus.STATUS_OK, msgList, 0);
        }
        // 1.4
        else {
          if (data.length < fetchRequest.getMaxSize() / 2) {
            fetchRequest.decreaseMaxSize();
          }
          success = true;
          log.info("server is not 2.0");
          return new DequeueResult(DequeueStatus.STATUS_OTHER_ERROR, null, 0);
        }
      } else {
        final BooleanCommand booleanCmd = (BooleanCommand) response;
        switch (booleanCmd.getCode()) {
          case HttpStatus.NotFound:
            success = true;
            if (log.isDebugEnabled()) {
              log.debug(booleanCmd.getErrorMsg());
            }
            return new DequeueResult(DequeueStatus.STATUS_NOT_FOUND, null, 0);
          case HttpStatus.Forbidden:
            success = true;
            return new DequeueResult(DequeueStatus.STATUS_OTHER_ERROR, null, 0);
          case HttpStatus.Moved:
            success = true;
            fetchRequest.resetRetries();
            long serverPushedOffset = Long.parseLong(booleanCmd.getErrorMsg());
            fetchRequest.setOffset(serverPushedOffset, 100, true);
            log.warn(
                "consumer request offset: "
                    + currentOffset
                    + " invalid or not matched, server pushed new offset: "
                    + serverPushedOffset);
            return new DequeueResult(DequeueStatus.STATUS_MOVED, null, serverPushedOffset);
          default:
            throw new MetaClientException(((BooleanCommand) response).getErrorMsg());
        }
      }

    } catch (final Exception e) {
      log.error(
          "fetchSync message failed,topic="
              + fetchRequest.getTopic()
              + ",partition="
              + fetchRequest.getPartition()
              + ",offset="
              + fetchRequest.getOffset(),
          e);

      return new DequeueResult(DequeueStatus.STATUS_OTHER_ERROR, null, 0);
    }
  }
Exemplo n.º 3
0
  @Override
  public MessageIterator fetch(final FetchRequest fetchRequest, long timeout, TimeUnit timeUnit)
      throws MetaClientException, InterruptedException {
    if (timeout <= 0 || timeUnit == null) {
      timeout = this.consumerConfig.getFetchTimeoutInMills();
      timeUnit = TimeUnit.MILLISECONDS;
    }
    final long start = System.currentTimeMillis();
    boolean success = false;
    try {
      final long currentOffset = fetchRequest.getOffset();
      final GetCommand getCmd =
          new GetCommand(
              fetchRequest.getTopic(),
              this.consumerConfig.getGroup(),
              fetchRequest.getPartition(),
              currentOffset,
              fetchRequest.getMaxSize(),
              OpaqueGenerator.getNextOpaque());
      final String serverUrl = fetchRequest.getBroker().getZKString();
      final ResponseCommand response =
          this.remotingClient.invokeToGroup(serverUrl, getCmd, timeout, timeUnit);
      if (response instanceof DataCommand) {
        final DataCommand dataCmd = (DataCommand) response;
        final byte[] data = dataCmd.getData();
        // 获取的数据严重不足的时候,缩减maxSize
        if (data.length < fetchRequest.getMaxSize() / 2) {
          fetchRequest.decreaseMaxSize();
        }
        success = true;
        return new MessageIterator(fetchRequest.getTopic(), data);
      } else {
        final BooleanCommand booleanCmd = (BooleanCommand) response;
        switch (booleanCmd.getCode()) {
          case HttpStatus.NotFound:
            success = true;
            return null;
          case HttpStatus.Forbidden:
            success = true;
            return null;
          case HttpStatus.Moved:
            success = true;
            fetchRequest.resetRetries();
            fetchRequest.setOffset(Long.parseLong(booleanCmd.getErrorMsg()), -1, true);
            return null;
          default:
            throw new MetaClientException(((BooleanCommand) response).getErrorMsg());
        }
      }

    } catch (final TimeoutException e) {
      throw new MetaOpeartionTimeoutException(
          "Send message timeout in " + this.consumerConfig.getFetchTimeoutInMills() + " mills");
    } catch (final MetaClientException e) {
      throw e;
    } catch (final InterruptedException e) {
      throw e;
    } catch (final Exception e) {
      throw new MetaClientException(
          "get message failed,topic="
              + fetchRequest.getTopic()
              + ",partition="
              + fetchRequest.getPartition()
              + ",offset="
              + fetchRequest.getOffset(),
          e);
    } finally {
      final long duration = System.currentTimeMillis() - start;
      if (duration > 200) {
        MetaStatLog.addStatValue2(
            null, StatConstants.GET_TIME_STAT, fetchRequest.getTopic(), duration);
      }
      if (!success) {
        MetaStatLog.addStat(null, StatConstants.GET_FAILED_STAT, fetchRequest.getTopic());
      }
    }
  }