コード例 #1
0
 @Override
 public long offset(final FetchRequest fetchRequest) throws MetaClientException {
   final long start = System.currentTimeMillis();
   boolean success = false;
   try {
     final long currentOffset = fetchRequest.getOffset();
     final OffsetCommand offsetCmd =
         new OffsetCommand(
             fetchRequest.getTopic(),
             this.consumerConfig.getGroup(),
             fetchRequest.getPartition(),
             currentOffset,
             OpaqueGenerator.getNextOpaque());
     final String serverUrl = fetchRequest.getBroker().getZKString();
     final BooleanCommand booleanCmd =
         (BooleanCommand)
             this.remotingClient.invokeToGroup(
                 serverUrl,
                 offsetCmd,
                 this.consumerConfig.getFetchTimeoutInMills(),
                 TimeUnit.MILLISECONDS);
     switch (booleanCmd.getCode()) {
       case HttpStatus.Success:
         success = true;
         return Long.parseLong(booleanCmd.getErrorMsg());
       default:
         throw new MetaClientException(booleanCmd.getErrorMsg());
     }
   } catch (final MetaClientException e) {
     throw e;
   } catch (final TimeoutException e) {
     throw new MetaOpeartionTimeoutException(
         "Send message timeout in " + this.consumerConfig.getFetchTimeoutInMills() + " mills");
   } catch (final Exception e) {
     throw new MetaClientException(
         "get offset failed,topic="
             + fetchRequest.getTopic()
             + ",partition="
             + fetchRequest.getPartition()
             + ",current offset="
             + fetchRequest.getOffset(),
         e);
   } finally {
     final long duration = System.currentTimeMillis() - start;
     if (duration > 200) {
       MetaStatLog.addStatValue2(
           null, StatConstants.OFFSET_TIME_STAT, fetchRequest.getTopic(), duration);
     }
     if (!success) {
       MetaStatLog.addStat(null, StatConstants.OFFSET_FAILED_STAT, fetchRequest.getTopic());
     }
   }
 }
コード例 #2
0
 private SubscriberInfo registeMessageType(FetchRequest req) {
   SubscriberInfo info = this.topicSubcriberRegistry.get(req.getTopic());
   if (info == null) {
     log.warn("query topic's[" + req.getTopic() + "] subscriberInfo is null.");
     return null;
   }
   Set<String> messageTypeList = info.getMessageTypes();
   MessageTypeCommand mtCmd =
       new MessageTypeCommand(
           this.consumerConfig.getVersion(),
           this.consumerConfig.getGroup(),
           req.getTopic(),
           OpaqueGenerator.getNextOpaque(),
           messageTypeList,
           MetaMessageSessionFactory.startTime);
   try {
     ResponseCommand response =
         this.remotingClient.invokeToGroup(
             req.getBroker().getZKString(),
             mtCmd,
             this.consumerConfig.getFetchTimeoutInMills(),
             TimeUnit.MILLISECONDS);
     if (response instanceof BooleanCommand) {
       BooleanCommand bc = (BooleanCommand) response;
       if (bc.getCode() == HttpStatus.Success) {
         return info;
       } else {
         return null;
       }
     }
   } catch (InterruptedException e) {
     log.error("registe message type interrupted," + e.getMessage(), e.getCause());
   } catch (TimeoutException e) {
     log.error("registe message type timeout," + e.getMessage(), e.getCause());
   } catch (NotifyRemotingException e) {
     log.error("registe message type failed, " + e.getMessage(), e.getCause());
   }
   return null;
 }
コード例 #3
0
 private Map<InetSocketAddress, StatsResult> getStats0(InetSocketAddress target, String item)
     throws InterruptedException {
   Set<String> groups = remotingClient.getGroupSet();
   if (groups == null || groups.size() <= 1) {
     return Collections.emptyMap();
   }
   Map<InetSocketAddress, StatsResult> rt = new HashMap<InetSocketAddress, StatsResult>();
   try {
     for (String group : groups) {
       if (!group.equals(Constants.DEFAULT_GROUP)) {
         URI uri = new URI(group);
         InetSocketAddress sockAddr = new InetSocketAddress(uri.getHost(), uri.getPort());
         if (target == null || target.equals(sockAddr)) {
           BooleanCommand resp =
               (BooleanCommand)
                   remotingClient.invokeToGroup(
                       group,
                       new StatsCommand(OpaqueGenerator.getNextOpaque(), item),
                       STATS_OPTIMEOUT,
                       TimeUnit.MILLISECONDS);
           if (resp.getResponseStatus() == ResponseStatus.NO_ERROR) {
             String body = resp.getErrorMsg();
             if (body != null) {
               parseStatsValues(sockAddr, rt, group, body);
             }
           }
         }
       }
     }
     return rt;
   } catch (InterruptedException e) {
     throw e;
   } catch (Exception e) {
     throw new IllegalStateException("Get statistics from brokers failed", e);
   }
 }
コード例 #4
0
 public NotifyDummyRequestCommand(final String dummy) {
   this.opaque = OpaqueGenerator.getNextOpaque();
   this.opCode = OpCode.DUMMY;
   this.dummy = dummy;
 }
コード例 #5
0
  @Override
  public DequeueResult fetchSync(final FetchRequest fetchRequest, long timeout, TimeUnit timeUnit)
      throws MetaClientException, InterruptedException {
    if (timeout <= 0 || timeUnit == null) {
      timeout = this.consumerConfig.getFetchTimeoutInMills();
      timeUnit = TimeUnit.MILLISECONDS;
    }
    final long start = System.currentTimeMillis();
    boolean success = false;
    final long currentOffset = fetchRequest.getOffset();
    try {
      final GetCommand getCmd =
          new GetCommand(
              fetchRequest.getTopic(),
              this.consumerConfig.getGroup(),
              fetchRequest.getPartition(),
              currentOffset,
              fetchRequest.getMaxSize(),
              OpaqueGenerator.getNextOpaque());
      final String serverUrl = fetchRequest.getBroker().getZKString();
      final ResponseCommand response =
          this.remotingClient.invokeToGroup(serverUrl, getCmd, timeout, timeUnit);
      if (response instanceof DataCommand) {
        final DataCommand dataCmd = (DataCommand) response;
        final byte[] data = dataCmd.getData();
        if (data.length < (MetaMessageDecoder.MessageFlagPostion + 4)) {
          log.fatal("fetch a invalid message " + data.length);
          return new DequeueResult(DequeueStatus.STATUS_OTHER_ERROR, null, 0);
        }

        java.nio.ByteBuffer byteBuffer = java.nio.ByteBuffer.wrap(data);
        int messageFlag = byteBuffer.getInt(MetaMessageDecoder.MessageFlagPostion);
        // 2.0
        if ((messageFlag & MetaMessageDecoder.NewServerFlag) == MetaMessageDecoder.NewServerFlag) {
          success = true;
          List<Message> msgList = new ArrayList<Message>(100);
          List<MetaMessageWrapper> wrapperList = MetaMessageDecoder.decodes(byteBuffer);
          if (!wrapperList.isEmpty()) {
            for (MetaMessageWrapper wrapper : wrapperList) {
              Message msg =
                  new Message(
                      fetchRequest.getTopic(),
                      wrapper.getMetaMessage().getBody(),
                      wrapper.getMetaMessage().getAttribute());
              msg.setOffset(wrapper.getMetaMessageAnnotation().getQueueOffset());
              msg.setId(wrapper.getMetaMessageAnnotation().getPhysicOffset());
              msg.setMsgNewId(wrapper.getMetaMessageAnnotation().getMsgId());
              msgList.add(msg);
            }
          }

          if (msgList.isEmpty()) {
            log.error("fetch sync OK, but no message");
            return new DequeueResult(DequeueStatus.STATUS_OTHER_ERROR, null, 0);
          }

          return new DequeueResult(DequeueStatus.STATUS_OK, msgList, 0);
        }
        // 1.4
        else {
          if (data.length < fetchRequest.getMaxSize() / 2) {
            fetchRequest.decreaseMaxSize();
          }
          success = true;
          log.info("server is not 2.0");
          return new DequeueResult(DequeueStatus.STATUS_OTHER_ERROR, null, 0);
        }
      } else {
        final BooleanCommand booleanCmd = (BooleanCommand) response;
        switch (booleanCmd.getCode()) {
          case HttpStatus.NotFound:
            success = true;
            if (log.isDebugEnabled()) {
              log.debug(booleanCmd.getErrorMsg());
            }
            return new DequeueResult(DequeueStatus.STATUS_NOT_FOUND, null, 0);
          case HttpStatus.Forbidden:
            success = true;
            return new DequeueResult(DequeueStatus.STATUS_OTHER_ERROR, null, 0);
          case HttpStatus.Moved:
            success = true;
            fetchRequest.resetRetries();
            long serverPushedOffset = Long.parseLong(booleanCmd.getErrorMsg());
            fetchRequest.setOffset(serverPushedOffset, 100, true);
            log.warn(
                "consumer request offset: "
                    + currentOffset
                    + " invalid or not matched, server pushed new offset: "
                    + serverPushedOffset);
            return new DequeueResult(DequeueStatus.STATUS_MOVED, null, serverPushedOffset);
          default:
            throw new MetaClientException(((BooleanCommand) response).getErrorMsg());
        }
      }

    } catch (final Exception e) {
      log.error(
          "fetchSync message failed,topic="
              + fetchRequest.getTopic()
              + ",partition="
              + fetchRequest.getPartition()
              + ",offset="
              + fetchRequest.getOffset(),
          e);

      return new DequeueResult(DequeueStatus.STATUS_OTHER_ERROR, null, 0);
    }
  }
コード例 #6
0
  @Override
  public FetchResult fetchAll(final FetchRequest fetchRequest, long timeout, TimeUnit timeUnit)
      throws MetaClientException, InterruptedException {
    if (timeout <= 0 || timeUnit == null) {
      timeout = this.consumerConfig.getFetchTimeoutInMills();
      timeUnit = TimeUnit.MILLISECONDS;
    }
    final long start = System.currentTimeMillis();
    boolean success = false;
    final long currentOffset = fetchRequest.getOffset();
    try {
      GetCommand getCmd = null;
      SubscriberInfo subInfo = this.topicSubcriberRegistry.get(fetchRequest.getTopic());
      Set<String> messageTypeList = null;
      if (subInfo != null) {
        messageTypeList = subInfo.getMessageTypes();
      }
      if (this.consumerConfig.isVersion2() && messageTypeList != null) { // 用户使用新版本的接口才使用新的协议
        getCmd =
            new FetchCommand(
                this.consumerConfig.getVersion(),
                fetchRequest.getTopic(),
                this.consumerConfig.getGroup(),
                fetchRequest.getPartition(),
                currentOffset,
                fetchRequest.getMaxSize(),
                OpaqueGenerator.getNextOpaque(),
                MetaMessageSessionFactory.startTime);
      } else {
        getCmd =
            new GetCommand(
                fetchRequest.getTopic(),
                this.consumerConfig.getGroup(),
                fetchRequest.getPartition(),
                currentOffset,
                fetchRequest.getMaxSize(),
                OpaqueGenerator.getNextOpaque());
      }

      final String serverUrl = fetchRequest.getBroker().getZKString();
      final ResponseCommand response =
          this.remotingClient.invokeToGroup(serverUrl, getCmd, timeout, timeUnit);
      if (response instanceof DataCommand) {
        final DataCommand dataCmd = (DataCommand) response;
        final byte[] data = dataCmd.getData();
        if (data.length < (MetaMessageDecoder.MessageFlagPostion + 4)) {
          log.fatal("fetch a invalid message " + data.length);
          return null;
        }

        // 识别服务器版本
        java.nio.ByteBuffer byteBuffer = java.nio.ByteBuffer.wrap(data);
        int messageFlag = byteBuffer.getInt(MetaMessageDecoder.MessageFlagPostion);
        // 2.0版本
        if ((messageFlag & MetaMessageDecoder.NewServerFlag) == MetaMessageDecoder.NewServerFlag) {
          server14 = false;
          success = true;
          List<Message> msgList = new ArrayList<Message>(100);
          List<MetaMessageWrapper> wrapperList = MetaMessageDecoder.decodes(byteBuffer);
          if (!wrapperList.isEmpty()) {
            for (MetaMessageWrapper wrapper : wrapperList) {
              String type = wrapper.getMetaMessage().getType();
              if (messageTypeList != null
                  && !messageTypeList.contains("*")
                  && !messageTypeList.contains(type)) {
                continue;
              }
              Message msg =
                  new Message(
                      fetchRequest.getTopic(),
                      wrapper.getMetaMessage().getBody(),
                      wrapper.getMetaMessage().getAttribute());
              msg.setOffset(wrapper.getMetaMessageAnnotation().getQueueOffset());
              msg.setId(wrapper.getMetaMessageAnnotation().getPhysicOffset());
              msg.setMsgNewId(wrapper.getMetaMessageAnnotation().getMsgId());
              msgList.add(msg);
            }
          }

          if (msgList.isEmpty()) {
            if (wrapperList != null && !wrapperList.isEmpty()) {
              MetaMessageWrapper wrapper = wrapperList.get(wrapperList.size() - 1);
              fetchRequest.setOffset(
                  wrapper.getMetaMessageAnnotation().getQueueOffset() + 1,
                  wrapper.getMetaMessageAnnotation().getPhysicOffset(),
                  true);
            }
            return null;
          }
          return new FetchResult(true, msgList, null);
        }
        // 1.4版本
        else {
          server14 = true;
          // 获取的数据严重不足的时候,缩减maxSize
          if (data.length < fetchRequest.getMaxSize() / 2) {
            fetchRequest.decreaseMaxSize();
          }
          success = true;
          return new FetchResult(false, null, new MessageIterator(fetchRequest.getTopic(), data));
        }
      } else {
        final BooleanCommand booleanCmd = (BooleanCommand) response;
        switch (booleanCmd.getCode()) {
          case HttpStatus.NotFound:
            success = true;
            if (log.isDebugEnabled()) {
              log.debug(booleanCmd.getErrorMsg());
            }
            return null;
          case HttpStatus.Forbidden:
            success = true;
            return null;
          case HttpStatus.Moved:
            success = true;
            fetchRequest.resetRetries();
            long serverPushedOffset = Long.parseLong(booleanCmd.getErrorMsg());
            fetchRequest.setOffset(serverPushedOffset, 100, true);
            if (!server14) {
              log.warn(
                  "consumer request offset: "
                      + currentOffset
                      + " invalid or not matched, server pushed new offset: "
                      + serverPushedOffset);
            }
            return null;
          case HttpStatus.Continue:
            success = true;
            SubscriberInfo info = registeMessageType(fetchRequest);
            if (info == null) {
              log.error("consumer report message types failed.");
            } else {
              log.info(
                  "consumer report message types success : " + info.getMessageTypes().toString());
            }
            return null;
          default:
            throw new MetaClientException(((BooleanCommand) response).getErrorMsg());
        }
      }

    } catch (final TimeoutException e) {
      throw new MetaOpeartionTimeoutException(
          "pull message timeout in "
              + this.consumerConfig.getFetchTimeoutInMills()
              + " mills, requestOffset "
              + currentOffset);
    } catch (final MetaClientException e) {
      throw e;
    } catch (final InterruptedException e) {
      throw e;
    } catch (final Exception e) {
      throw new MetaClientException(
          "get message failed,topic="
              + fetchRequest.getTopic()
              + ",partition="
              + fetchRequest.getPartition()
              + ",offset="
              + fetchRequest.getOffset(),
          e);
    } finally {
      final long duration = System.currentTimeMillis() - start;
      if (duration > 200) {
        MetaStatLog.addStatValue2(
            null, StatConstants.GET_TIME_STAT, fetchRequest.getTopic(), duration);
      }
      if (!success) {
        MetaStatLog.addStat(null, StatConstants.GET_FAILED_STAT, fetchRequest.getTopic());
      }
    }
  }
コード例 #7
0
  @Override
  public MessageIterator fetch(final FetchRequest fetchRequest, long timeout, TimeUnit timeUnit)
      throws MetaClientException, InterruptedException {
    if (timeout <= 0 || timeUnit == null) {
      timeout = this.consumerConfig.getFetchTimeoutInMills();
      timeUnit = TimeUnit.MILLISECONDS;
    }
    final long start = System.currentTimeMillis();
    boolean success = false;
    try {
      final long currentOffset = fetchRequest.getOffset();
      final GetCommand getCmd =
          new GetCommand(
              fetchRequest.getTopic(),
              this.consumerConfig.getGroup(),
              fetchRequest.getPartition(),
              currentOffset,
              fetchRequest.getMaxSize(),
              OpaqueGenerator.getNextOpaque());
      final String serverUrl = fetchRequest.getBroker().getZKString();
      final ResponseCommand response =
          this.remotingClient.invokeToGroup(serverUrl, getCmd, timeout, timeUnit);
      if (response instanceof DataCommand) {
        final DataCommand dataCmd = (DataCommand) response;
        final byte[] data = dataCmd.getData();
        // 获取的数据严重不足的时候,缩减maxSize
        if (data.length < fetchRequest.getMaxSize() / 2) {
          fetchRequest.decreaseMaxSize();
        }
        success = true;
        return new MessageIterator(fetchRequest.getTopic(), data);
      } else {
        final BooleanCommand booleanCmd = (BooleanCommand) response;
        switch (booleanCmd.getCode()) {
          case HttpStatus.NotFound:
            success = true;
            return null;
          case HttpStatus.Forbidden:
            success = true;
            return null;
          case HttpStatus.Moved:
            success = true;
            fetchRequest.resetRetries();
            fetchRequest.setOffset(Long.parseLong(booleanCmd.getErrorMsg()), -1, true);
            return null;
          default:
            throw new MetaClientException(((BooleanCommand) response).getErrorMsg());
        }
      }

    } catch (final TimeoutException e) {
      throw new MetaOpeartionTimeoutException(
          "Send message timeout in " + this.consumerConfig.getFetchTimeoutInMills() + " mills");
    } catch (final MetaClientException e) {
      throw e;
    } catch (final InterruptedException e) {
      throw e;
    } catch (final Exception e) {
      throw new MetaClientException(
          "get message failed,topic="
              + fetchRequest.getTopic()
              + ",partition="
              + fetchRequest.getPartition()
              + ",offset="
              + fetchRequest.getOffset(),
          e);
    } finally {
      final long duration = System.currentTimeMillis() - start;
      if (duration > 200) {
        MetaStatLog.addStatValue2(
            null, StatConstants.GET_TIME_STAT, fetchRequest.getTopic(), duration);
      }
      if (!success) {
        MetaStatLog.addStat(null, StatConstants.GET_FAILED_STAT, fetchRequest.getTopic());
      }
    }
  }