private void assertMsg(Message msg) {
   assertNotNull(msg);
   assertEquals(9999L, msg.getId());
   assertEquals("test", msg.getTopic());
   assertFalse(msg.hasAttribute());
   assertEquals(0, MessageAccessor.getFlag(msg));
   assertEquals("hello", new String(msg.getData()));
 }
Esempio n. 2
0
 @Override
 public void appendCouldNotProcessMessage(final Message message) throws IOException {
   // 目前的处理是交给本地存储管理并重试
   log.warn(
       "Message could not process,save to local.MessageId="
           + message.getId()
           + ",Topic="
           + message.getTopic()
           + ",Partition="
           + message.getPartition());
   this.recoverStorageManager.append(this.consumerConfig.getGroup(), message);
 }
Esempio n. 3
0
  /**
   * 将消息属性和消息payload打包,结构如下:</br></br> 0或者1个定长attribute + payload
   *
   * @param message
   * @return
   */
  public static final byte[] encodePayload(final Message message) {
    final byte[] payload = message.getData();
    final String attribute = message.getAttribute();
    byte[] attrData = null;
    if (attribute != null) {
      attrData = ByteUtils.getBytes(attribute);
    } else {
      return payload;
    }
    final int attrLen = attrData == null ? 0 : attrData.length;
    final ByteBuffer buffer = ByteBuffer.allocate(4 + attrLen + payload.length);
    if (attribute != null) {
      buffer.putInt(attrLen);
      if (attrData != null) {
        buffer.put(attrData);
      }
    }

    buffer.put(payload);
    return buffer.array();
  }
Esempio n. 4
0
 /**
  * 从binary数据中解出消息
  *
  * @param topic
  * @param data
  * @param offset
  * @return
  * @throws InvalidMessageException
  */
 public static final DecodedMessage decodeMessage(
     final String topic, final byte[] data, final int offset) throws InvalidMessageException {
   final ByteBuffer buf = ByteBuffer.wrap(data, offset, HEADER_LEN);
   final int msgLen = buf.getInt();
   final int checksum = buf.getInt();
   vailidateMessage(offset + HEADER_LEN, msgLen, checksum, data);
   final long id = buf.getLong();
   // 取flag
   final int flag = buf.getInt();
   String attribute = null;
   int payLoadOffset = offset + HEADER_LEN;
   int payLoadLen = msgLen;
   if (payLoadLen > MAX_READ_BUFFER_SIZE) {
     throw new InvalidMessageException("Too much long payload length:" + payLoadLen);
   }
   // 如果有属性,需要解析属性
   if (MessageFlagUtils.hasAttribute(flag)) {
     // 取4个字节的属性长度
     final int attrLen = getInt(offset + HEADER_LEN, data);
     // 取消息属性
     final byte[] attrData = new byte[attrLen];
     System.arraycopy(data, offset + HEADER_LEN + 4, attrData, 0, attrLen);
     attribute = ByteUtils.getString(attrData);
     // 递增payloadOffset,加上4个字节的消息长度和消息长度本身
     payLoadOffset = offset + HEADER_LEN + 4 + attrLen;
     // payload长度递减,减去4个字节的消息长度和消息长度本身
     payLoadLen = msgLen - 4 - attrLen;
   }
   // 获取payload
   final byte[] payload = new byte[payLoadLen];
   System.arraycopy(data, payLoadOffset, payload, 0, payLoadLen);
   final Message msg = new Message(topic, payload);
   MessageAccessor.setFlag(msg, flag);
   msg.setAttribute(attribute);
   MessageAccessor.setId(msg, id);
   return new DecodedMessage(
       payLoadOffset + payLoadLen,
       msg,
       ByteBuffer.wrap(data, offset, payLoadOffset + payLoadLen - offset));
 }
 public void persistent(Message message) {
   List<Span> spanList;
   try {
     spanList = (List) PB.parsePBBytes(message.getData());
   } catch (Exception e) {
     log.error(e.getMessage());
     return;
   }
   try {
     if (spanList != null) {
       for (Span s : spanList) {
         insertService.addSpan(s);
         insertService.addAnnotation(s);
         insertService.addTrace(s);
       }
     }
   } catch (Exception e) {
     e.printStackTrace();
     log.error(e.getMessage());
   }
 }
Esempio n. 6
0
  @Override
  public DequeueResult fetchSync(final FetchRequest fetchRequest, long timeout, TimeUnit timeUnit)
      throws MetaClientException, InterruptedException {
    if (timeout <= 0 || timeUnit == null) {
      timeout = this.consumerConfig.getFetchTimeoutInMills();
      timeUnit = TimeUnit.MILLISECONDS;
    }
    final long start = System.currentTimeMillis();
    boolean success = false;
    final long currentOffset = fetchRequest.getOffset();
    try {
      final GetCommand getCmd =
          new GetCommand(
              fetchRequest.getTopic(),
              this.consumerConfig.getGroup(),
              fetchRequest.getPartition(),
              currentOffset,
              fetchRequest.getMaxSize(),
              OpaqueGenerator.getNextOpaque());
      final String serverUrl = fetchRequest.getBroker().getZKString();
      final ResponseCommand response =
          this.remotingClient.invokeToGroup(serverUrl, getCmd, timeout, timeUnit);
      if (response instanceof DataCommand) {
        final DataCommand dataCmd = (DataCommand) response;
        final byte[] data = dataCmd.getData();
        if (data.length < (MetaMessageDecoder.MessageFlagPostion + 4)) {
          log.fatal("fetch a invalid message " + data.length);
          return new DequeueResult(DequeueStatus.STATUS_OTHER_ERROR, null, 0);
        }

        java.nio.ByteBuffer byteBuffer = java.nio.ByteBuffer.wrap(data);
        int messageFlag = byteBuffer.getInt(MetaMessageDecoder.MessageFlagPostion);
        // 2.0
        if ((messageFlag & MetaMessageDecoder.NewServerFlag) == MetaMessageDecoder.NewServerFlag) {
          success = true;
          List<Message> msgList = new ArrayList<Message>(100);
          List<MetaMessageWrapper> wrapperList = MetaMessageDecoder.decodes(byteBuffer);
          if (!wrapperList.isEmpty()) {
            for (MetaMessageWrapper wrapper : wrapperList) {
              Message msg =
                  new Message(
                      fetchRequest.getTopic(),
                      wrapper.getMetaMessage().getBody(),
                      wrapper.getMetaMessage().getAttribute());
              msg.setOffset(wrapper.getMetaMessageAnnotation().getQueueOffset());
              msg.setId(wrapper.getMetaMessageAnnotation().getPhysicOffset());
              msg.setMsgNewId(wrapper.getMetaMessageAnnotation().getMsgId());
              msgList.add(msg);
            }
          }

          if (msgList.isEmpty()) {
            log.error("fetch sync OK, but no message");
            return new DequeueResult(DequeueStatus.STATUS_OTHER_ERROR, null, 0);
          }

          return new DequeueResult(DequeueStatus.STATUS_OK, msgList, 0);
        }
        // 1.4
        else {
          if (data.length < fetchRequest.getMaxSize() / 2) {
            fetchRequest.decreaseMaxSize();
          }
          success = true;
          log.info("server is not 2.0");
          return new DequeueResult(DequeueStatus.STATUS_OTHER_ERROR, null, 0);
        }
      } else {
        final BooleanCommand booleanCmd = (BooleanCommand) response;
        switch (booleanCmd.getCode()) {
          case HttpStatus.NotFound:
            success = true;
            if (log.isDebugEnabled()) {
              log.debug(booleanCmd.getErrorMsg());
            }
            return new DequeueResult(DequeueStatus.STATUS_NOT_FOUND, null, 0);
          case HttpStatus.Forbidden:
            success = true;
            return new DequeueResult(DequeueStatus.STATUS_OTHER_ERROR, null, 0);
          case HttpStatus.Moved:
            success = true;
            fetchRequest.resetRetries();
            long serverPushedOffset = Long.parseLong(booleanCmd.getErrorMsg());
            fetchRequest.setOffset(serverPushedOffset, 100, true);
            log.warn(
                "consumer request offset: "
                    + currentOffset
                    + " invalid or not matched, server pushed new offset: "
                    + serverPushedOffset);
            return new DequeueResult(DequeueStatus.STATUS_MOVED, null, serverPushedOffset);
          default:
            throw new MetaClientException(((BooleanCommand) response).getErrorMsg());
        }
      }

    } catch (final Exception e) {
      log.error(
          "fetchSync message failed,topic="
              + fetchRequest.getTopic()
              + ",partition="
              + fetchRequest.getPartition()
              + ",offset="
              + fetchRequest.getOffset(),
          e);

      return new DequeueResult(DequeueStatus.STATUS_OTHER_ERROR, null, 0);
    }
  }
Esempio n. 7
0
  @Override
  public FetchResult fetchAll(final FetchRequest fetchRequest, long timeout, TimeUnit timeUnit)
      throws MetaClientException, InterruptedException {
    if (timeout <= 0 || timeUnit == null) {
      timeout = this.consumerConfig.getFetchTimeoutInMills();
      timeUnit = TimeUnit.MILLISECONDS;
    }
    final long start = System.currentTimeMillis();
    boolean success = false;
    final long currentOffset = fetchRequest.getOffset();
    try {
      GetCommand getCmd = null;
      SubscriberInfo subInfo = this.topicSubcriberRegistry.get(fetchRequest.getTopic());
      Set<String> messageTypeList = null;
      if (subInfo != null) {
        messageTypeList = subInfo.getMessageTypes();
      }
      if (this.consumerConfig.isVersion2() && messageTypeList != null) { // 用户使用新版本的接口才使用新的协议
        getCmd =
            new FetchCommand(
                this.consumerConfig.getVersion(),
                fetchRequest.getTopic(),
                this.consumerConfig.getGroup(),
                fetchRequest.getPartition(),
                currentOffset,
                fetchRequest.getMaxSize(),
                OpaqueGenerator.getNextOpaque(),
                MetaMessageSessionFactory.startTime);
      } else {
        getCmd =
            new GetCommand(
                fetchRequest.getTopic(),
                this.consumerConfig.getGroup(),
                fetchRequest.getPartition(),
                currentOffset,
                fetchRequest.getMaxSize(),
                OpaqueGenerator.getNextOpaque());
      }

      final String serverUrl = fetchRequest.getBroker().getZKString();
      final ResponseCommand response =
          this.remotingClient.invokeToGroup(serverUrl, getCmd, timeout, timeUnit);
      if (response instanceof DataCommand) {
        final DataCommand dataCmd = (DataCommand) response;
        final byte[] data = dataCmd.getData();
        if (data.length < (MetaMessageDecoder.MessageFlagPostion + 4)) {
          log.fatal("fetch a invalid message " + data.length);
          return null;
        }

        // 识别服务器版本
        java.nio.ByteBuffer byteBuffer = java.nio.ByteBuffer.wrap(data);
        int messageFlag = byteBuffer.getInt(MetaMessageDecoder.MessageFlagPostion);
        // 2.0版本
        if ((messageFlag & MetaMessageDecoder.NewServerFlag) == MetaMessageDecoder.NewServerFlag) {
          server14 = false;
          success = true;
          List<Message> msgList = new ArrayList<Message>(100);
          List<MetaMessageWrapper> wrapperList = MetaMessageDecoder.decodes(byteBuffer);
          if (!wrapperList.isEmpty()) {
            for (MetaMessageWrapper wrapper : wrapperList) {
              String type = wrapper.getMetaMessage().getType();
              if (messageTypeList != null
                  && !messageTypeList.contains("*")
                  && !messageTypeList.contains(type)) {
                continue;
              }
              Message msg =
                  new Message(
                      fetchRequest.getTopic(),
                      wrapper.getMetaMessage().getBody(),
                      wrapper.getMetaMessage().getAttribute());
              msg.setOffset(wrapper.getMetaMessageAnnotation().getQueueOffset());
              msg.setId(wrapper.getMetaMessageAnnotation().getPhysicOffset());
              msg.setMsgNewId(wrapper.getMetaMessageAnnotation().getMsgId());
              msgList.add(msg);
            }
          }

          if (msgList.isEmpty()) {
            if (wrapperList != null && !wrapperList.isEmpty()) {
              MetaMessageWrapper wrapper = wrapperList.get(wrapperList.size() - 1);
              fetchRequest.setOffset(
                  wrapper.getMetaMessageAnnotation().getQueueOffset() + 1,
                  wrapper.getMetaMessageAnnotation().getPhysicOffset(),
                  true);
            }
            return null;
          }
          return new FetchResult(true, msgList, null);
        }
        // 1.4版本
        else {
          server14 = true;
          // 获取的数据严重不足的时候,缩减maxSize
          if (data.length < fetchRequest.getMaxSize() / 2) {
            fetchRequest.decreaseMaxSize();
          }
          success = true;
          return new FetchResult(false, null, new MessageIterator(fetchRequest.getTopic(), data));
        }
      } else {
        final BooleanCommand booleanCmd = (BooleanCommand) response;
        switch (booleanCmd.getCode()) {
          case HttpStatus.NotFound:
            success = true;
            if (log.isDebugEnabled()) {
              log.debug(booleanCmd.getErrorMsg());
            }
            return null;
          case HttpStatus.Forbidden:
            success = true;
            return null;
          case HttpStatus.Moved:
            success = true;
            fetchRequest.resetRetries();
            long serverPushedOffset = Long.parseLong(booleanCmd.getErrorMsg());
            fetchRequest.setOffset(serverPushedOffset, 100, true);
            if (!server14) {
              log.warn(
                  "consumer request offset: "
                      + currentOffset
                      + " invalid or not matched, server pushed new offset: "
                      + serverPushedOffset);
            }
            return null;
          case HttpStatus.Continue:
            success = true;
            SubscriberInfo info = registeMessageType(fetchRequest);
            if (info == null) {
              log.error("consumer report message types failed.");
            } else {
              log.info(
                  "consumer report message types success : " + info.getMessageTypes().toString());
            }
            return null;
          default:
            throw new MetaClientException(((BooleanCommand) response).getErrorMsg());
        }
      }

    } catch (final TimeoutException e) {
      throw new MetaOpeartionTimeoutException(
          "pull message timeout in "
              + this.consumerConfig.getFetchTimeoutInMills()
              + " mills, requestOffset "
              + currentOffset);
    } catch (final MetaClientException e) {
      throw e;
    } catch (final InterruptedException e) {
      throw e;
    } catch (final Exception e) {
      throw new MetaClientException(
          "get message failed,topic="
              + fetchRequest.getTopic()
              + ",partition="
              + fetchRequest.getPartition()
              + ",offset="
              + fetchRequest.getOffset(),
          e);
    } finally {
      final long duration = System.currentTimeMillis() - start;
      if (duration > 200) {
        MetaStatLog.addStatValue2(
            null, StatConstants.GET_TIME_STAT, fetchRequest.getTopic(), duration);
      }
      if (!success) {
        MetaStatLog.addStat(null, StatConstants.GET_FAILED_STAT, fetchRequest.getTopic());
      }
    }
  }