@Override
public long offset(final FetchRequest fetchRequest) throws MetaClientException {
final long start = System.currentTimeMillis();
boolean success = false;
try {
final long currentOffset = fetchRequest.getOffset();
final OffsetCommand offsetCmd =
new OffsetCommand(
fetchRequest.getTopic(),
this.consumerConfig.getGroup(),
fetchRequest.getPartition(),
currentOffset,
OpaqueGenerator.getNextOpaque());
final String serverUrl = fetchRequest.getBroker().getZKString();
final BooleanCommand booleanCmd =
(BooleanCommand)
this.remotingClient.invokeToGroup(
serverUrl,
offsetCmd,
this.consumerConfig.getFetchTimeoutInMills(),
TimeUnit.MILLISECONDS);
switch (booleanCmd.getCode()) {
case HttpStatus.Success:
success = true;
return Long.parseLong(booleanCmd.getErrorMsg());
default:
throw new MetaClientException(booleanCmd.getErrorMsg());
}
} catch (final MetaClientException e) {
throw e;
} catch (final TimeoutException e) {
throw new MetaOpeartionTimeoutException(
"Send message timeout in " + this.consumerConfig.getFetchTimeoutInMills() + " mills");
} catch (final Exception e) {
throw new MetaClientException(
"get offset failed,topic="
+ fetchRequest.getTopic()
+ ",partition="
+ fetchRequest.getPartition()
+ ",current offset="
+ fetchRequest.getOffset(),
e);
} finally {
final long duration = System.currentTimeMillis() - start;
if (duration > 200) {
MetaStatLog.addStatValue2(
null, StatConstants.OFFSET_TIME_STAT, fetchRequest.getTopic(), duration);
}
if (!success) {
MetaStatLog.addStat(null, StatConstants.OFFSET_FAILED_STAT, fetchRequest.getTopic());
}
}
}
private SubscriberInfo registeMessageType(FetchRequest req) {
SubscriberInfo info = this.topicSubcriberRegistry.get(req.getTopic());
if (info == null) {
log.warn("query topic's[" + req.getTopic() + "] subscriberInfo is null.");
return null;
}
Set<String> messageTypeList = info.getMessageTypes();
MessageTypeCommand mtCmd =
new MessageTypeCommand(
this.consumerConfig.getVersion(),
this.consumerConfig.getGroup(),
req.getTopic(),
OpaqueGenerator.getNextOpaque(),
messageTypeList,
MetaMessageSessionFactory.startTime);
try {
ResponseCommand response =
this.remotingClient.invokeToGroup(
req.getBroker().getZKString(),
mtCmd,
this.consumerConfig.getFetchTimeoutInMills(),
TimeUnit.MILLISECONDS);
if (response instanceof BooleanCommand) {
BooleanCommand bc = (BooleanCommand) response;
if (bc.getCode() == HttpStatus.Success) {
return info;
} else {
return null;
}
}
} catch (InterruptedException e) {
log.error("registe message type interrupted," + e.getMessage(), e.getCause());
} catch (TimeoutException e) {
log.error("registe message type timeout," + e.getMessage(), e.getCause());
} catch (NotifyRemotingException e) {
log.error("registe message type failed, " + e.getMessage(), e.getCause());
}
return null;
}
private Map<InetSocketAddress, StatsResult> getStats0(InetSocketAddress target, String item)
throws InterruptedException {
Set<String> groups = remotingClient.getGroupSet();
if (groups == null || groups.size() <= 1) {
return Collections.emptyMap();
}
Map<InetSocketAddress, StatsResult> rt = new HashMap<InetSocketAddress, StatsResult>();
try {
for (String group : groups) {
if (!group.equals(Constants.DEFAULT_GROUP)) {
URI uri = new URI(group);
InetSocketAddress sockAddr = new InetSocketAddress(uri.getHost(), uri.getPort());
if (target == null || target.equals(sockAddr)) {
BooleanCommand resp =
(BooleanCommand)
remotingClient.invokeToGroup(
group,
new StatsCommand(OpaqueGenerator.getNextOpaque(), item),
STATS_OPTIMEOUT,
TimeUnit.MILLISECONDS);
if (resp.getResponseStatus() == ResponseStatus.NO_ERROR) {
String body = resp.getErrorMsg();
if (body != null) {
parseStatsValues(sockAddr, rt, group, body);
}
}
}
}
}
return rt;
} catch (InterruptedException e) {
throw e;
} catch (Exception e) {
throw new IllegalStateException("Get statistics from brokers failed", e);
}
}
public NotifyDummyRequestCommand(final String dummy) {
this.opaque = OpaqueGenerator.getNextOpaque();
this.opCode = OpCode.DUMMY;
this.dummy = dummy;
}
@Override
public DequeueResult fetchSync(final FetchRequest fetchRequest, long timeout, TimeUnit timeUnit)
throws MetaClientException, InterruptedException {
if (timeout <= 0 || timeUnit == null) {
timeout = this.consumerConfig.getFetchTimeoutInMills();
timeUnit = TimeUnit.MILLISECONDS;
}
final long start = System.currentTimeMillis();
boolean success = false;
final long currentOffset = fetchRequest.getOffset();
try {
final GetCommand getCmd =
new GetCommand(
fetchRequest.getTopic(),
this.consumerConfig.getGroup(),
fetchRequest.getPartition(),
currentOffset,
fetchRequest.getMaxSize(),
OpaqueGenerator.getNextOpaque());
final String serverUrl = fetchRequest.getBroker().getZKString();
final ResponseCommand response =
this.remotingClient.invokeToGroup(serverUrl, getCmd, timeout, timeUnit);
if (response instanceof DataCommand) {
final DataCommand dataCmd = (DataCommand) response;
final byte[] data = dataCmd.getData();
if (data.length < (MetaMessageDecoder.MessageFlagPostion + 4)) {
log.fatal("fetch a invalid message " + data.length);
return new DequeueResult(DequeueStatus.STATUS_OTHER_ERROR, null, 0);
}
java.nio.ByteBuffer byteBuffer = java.nio.ByteBuffer.wrap(data);
int messageFlag = byteBuffer.getInt(MetaMessageDecoder.MessageFlagPostion);
// 2.0
if ((messageFlag & MetaMessageDecoder.NewServerFlag) == MetaMessageDecoder.NewServerFlag) {
success = true;
List<Message> msgList = new ArrayList<Message>(100);
List<MetaMessageWrapper> wrapperList = MetaMessageDecoder.decodes(byteBuffer);
if (!wrapperList.isEmpty()) {
for (MetaMessageWrapper wrapper : wrapperList) {
Message msg =
new Message(
fetchRequest.getTopic(),
wrapper.getMetaMessage().getBody(),
wrapper.getMetaMessage().getAttribute());
msg.setOffset(wrapper.getMetaMessageAnnotation().getQueueOffset());
msg.setId(wrapper.getMetaMessageAnnotation().getPhysicOffset());
msg.setMsgNewId(wrapper.getMetaMessageAnnotation().getMsgId());
msgList.add(msg);
}
}
if (msgList.isEmpty()) {
log.error("fetch sync OK, but no message");
return new DequeueResult(DequeueStatus.STATUS_OTHER_ERROR, null, 0);
}
return new DequeueResult(DequeueStatus.STATUS_OK, msgList, 0);
}
// 1.4
else {
if (data.length < fetchRequest.getMaxSize() / 2) {
fetchRequest.decreaseMaxSize();
}
success = true;
log.info("server is not 2.0");
return new DequeueResult(DequeueStatus.STATUS_OTHER_ERROR, null, 0);
}
} else {
final BooleanCommand booleanCmd = (BooleanCommand) response;
switch (booleanCmd.getCode()) {
case HttpStatus.NotFound:
success = true;
if (log.isDebugEnabled()) {
log.debug(booleanCmd.getErrorMsg());
}
return new DequeueResult(DequeueStatus.STATUS_NOT_FOUND, null, 0);
case HttpStatus.Forbidden:
success = true;
return new DequeueResult(DequeueStatus.STATUS_OTHER_ERROR, null, 0);
case HttpStatus.Moved:
success = true;
fetchRequest.resetRetries();
long serverPushedOffset = Long.parseLong(booleanCmd.getErrorMsg());
fetchRequest.setOffset(serverPushedOffset, 100, true);
log.warn(
"consumer request offset: "
+ currentOffset
+ " invalid or not matched, server pushed new offset: "
+ serverPushedOffset);
return new DequeueResult(DequeueStatus.STATUS_MOVED, null, serverPushedOffset);
default:
throw new MetaClientException(((BooleanCommand) response).getErrorMsg());
}
}
} catch (final Exception e) {
log.error(
"fetchSync message failed,topic="
+ fetchRequest.getTopic()
+ ",partition="
+ fetchRequest.getPartition()
+ ",offset="
+ fetchRequest.getOffset(),
e);
return new DequeueResult(DequeueStatus.STATUS_OTHER_ERROR, null, 0);
}
}
@Override
public FetchResult fetchAll(final FetchRequest fetchRequest, long timeout, TimeUnit timeUnit)
throws MetaClientException, InterruptedException {
if (timeout <= 0 || timeUnit == null) {
timeout = this.consumerConfig.getFetchTimeoutInMills();
timeUnit = TimeUnit.MILLISECONDS;
}
final long start = System.currentTimeMillis();
boolean success = false;
final long currentOffset = fetchRequest.getOffset();
try {
GetCommand getCmd = null;
SubscriberInfo subInfo = this.topicSubcriberRegistry.get(fetchRequest.getTopic());
Set<String> messageTypeList = null;
if (subInfo != null) {
messageTypeList = subInfo.getMessageTypes();
}
if (this.consumerConfig.isVersion2() && messageTypeList != null) { // 用户使用新版本的接口才使用新的协议
getCmd =
new FetchCommand(
this.consumerConfig.getVersion(),
fetchRequest.getTopic(),
this.consumerConfig.getGroup(),
fetchRequest.getPartition(),
currentOffset,
fetchRequest.getMaxSize(),
OpaqueGenerator.getNextOpaque(),
MetaMessageSessionFactory.startTime);
} else {
getCmd =
new GetCommand(
fetchRequest.getTopic(),
this.consumerConfig.getGroup(),
fetchRequest.getPartition(),
currentOffset,
fetchRequest.getMaxSize(),
OpaqueGenerator.getNextOpaque());
}
final String serverUrl = fetchRequest.getBroker().getZKString();
final ResponseCommand response =
this.remotingClient.invokeToGroup(serverUrl, getCmd, timeout, timeUnit);
if (response instanceof DataCommand) {
final DataCommand dataCmd = (DataCommand) response;
final byte[] data = dataCmd.getData();
if (data.length < (MetaMessageDecoder.MessageFlagPostion + 4)) {
log.fatal("fetch a invalid message " + data.length);
return null;
}
// 识别服务器版本
java.nio.ByteBuffer byteBuffer = java.nio.ByteBuffer.wrap(data);
int messageFlag = byteBuffer.getInt(MetaMessageDecoder.MessageFlagPostion);
// 2.0版本
if ((messageFlag & MetaMessageDecoder.NewServerFlag) == MetaMessageDecoder.NewServerFlag) {
server14 = false;
success = true;
List<Message> msgList = new ArrayList<Message>(100);
List<MetaMessageWrapper> wrapperList = MetaMessageDecoder.decodes(byteBuffer);
if (!wrapperList.isEmpty()) {
for (MetaMessageWrapper wrapper : wrapperList) {
String type = wrapper.getMetaMessage().getType();
if (messageTypeList != null
&& !messageTypeList.contains("*")
&& !messageTypeList.contains(type)) {
continue;
}
Message msg =
new Message(
fetchRequest.getTopic(),
wrapper.getMetaMessage().getBody(),
wrapper.getMetaMessage().getAttribute());
msg.setOffset(wrapper.getMetaMessageAnnotation().getQueueOffset());
msg.setId(wrapper.getMetaMessageAnnotation().getPhysicOffset());
msg.setMsgNewId(wrapper.getMetaMessageAnnotation().getMsgId());
msgList.add(msg);
}
}
if (msgList.isEmpty()) {
if (wrapperList != null && !wrapperList.isEmpty()) {
MetaMessageWrapper wrapper = wrapperList.get(wrapperList.size() - 1);
fetchRequest.setOffset(
wrapper.getMetaMessageAnnotation().getQueueOffset() + 1,
wrapper.getMetaMessageAnnotation().getPhysicOffset(),
true);
}
return null;
}
return new FetchResult(true, msgList, null);
}
// 1.4版本
else {
server14 = true;
// 获取的数据严重不足的时候，缩减maxSize
if (data.length < fetchRequest.getMaxSize() / 2) {
fetchRequest.decreaseMaxSize();
}
success = true;
return new FetchResult(false, null, new MessageIterator(fetchRequest.getTopic(), data));
}
} else {
final BooleanCommand booleanCmd = (BooleanCommand) response;
switch (booleanCmd.getCode()) {
case HttpStatus.NotFound:
success = true;
if (log.isDebugEnabled()) {
log.debug(booleanCmd.getErrorMsg());
}
return null;
case HttpStatus.Forbidden:
success = true;
return null;
case HttpStatus.Moved:
success = true;
fetchRequest.resetRetries();
long serverPushedOffset = Long.parseLong(booleanCmd.getErrorMsg());
fetchRequest.setOffset(serverPushedOffset, 100, true);
if (!server14) {
log.warn(
"consumer request offset: "
+ currentOffset
+ " invalid or not matched, server pushed new offset: "
+ serverPushedOffset);
}
return null;
case HttpStatus.Continue:
success = true;
SubscriberInfo info = registeMessageType(fetchRequest);
if (info == null) {
log.error("consumer report message types failed.");
} else {
log.info(
"consumer report message types success : " + info.getMessageTypes().toString());
}
return null;
default:
throw new MetaClientException(((BooleanCommand) response).getErrorMsg());
}
}
} catch (final TimeoutException e) {
throw new MetaOpeartionTimeoutException(
"pull message timeout in "
+ this.consumerConfig.getFetchTimeoutInMills()
+ " mills, requestOffset "
+ currentOffset);
} catch (final MetaClientException e) {
throw e;
} catch (final InterruptedException e) {
throw e;
} catch (final Exception e) {
throw new MetaClientException(
"get message failed,topic="
+ fetchRequest.getTopic()
+ ",partition="
+ fetchRequest.getPartition()
+ ",offset="
+ fetchRequest.getOffset(),
e);
} finally {
final long duration = System.currentTimeMillis() - start;
if (duration > 200) {
MetaStatLog.addStatValue2(
null, StatConstants.GET_TIME_STAT, fetchRequest.getTopic(), duration);
}
if (!success) {
MetaStatLog.addStat(null, StatConstants.GET_FAILED_STAT, fetchRequest.getTopic());
}
}
}
@Override
public MessageIterator fetch(final FetchRequest fetchRequest, long timeout, TimeUnit timeUnit)
throws MetaClientException, InterruptedException {
if (timeout <= 0 || timeUnit == null) {
timeout = this.consumerConfig.getFetchTimeoutInMills();
timeUnit = TimeUnit.MILLISECONDS;
}
final long start = System.currentTimeMillis();
boolean success = false;
try {
final long currentOffset = fetchRequest.getOffset();
final GetCommand getCmd =
new GetCommand(
fetchRequest.getTopic(),
this.consumerConfig.getGroup(),
fetchRequest.getPartition(),
currentOffset,
fetchRequest.getMaxSize(),
OpaqueGenerator.getNextOpaque());
final String serverUrl = fetchRequest.getBroker().getZKString();
final ResponseCommand response =
this.remotingClient.invokeToGroup(serverUrl, getCmd, timeout, timeUnit);
if (response instanceof DataCommand) {
final DataCommand dataCmd = (DataCommand) response;
final byte[] data = dataCmd.getData();
// 获取的数据严重不足的时候，缩减maxSize
if (data.length < fetchRequest.getMaxSize() / 2) {
fetchRequest.decreaseMaxSize();
}
success = true;
return new MessageIterator(fetchRequest.getTopic(), data);
} else {
final BooleanCommand booleanCmd = (BooleanCommand) response;
switch (booleanCmd.getCode()) {
case HttpStatus.NotFound:
success = true;
return null;
case HttpStatus.Forbidden:
success = true;
return null;
case HttpStatus.Moved:
success = true;
fetchRequest.resetRetries();
fetchRequest.setOffset(Long.parseLong(booleanCmd.getErrorMsg()), -1, true);
return null;
default:
throw new MetaClientException(((BooleanCommand) response).getErrorMsg());
}
}
} catch (final TimeoutException e) {
throw new MetaOpeartionTimeoutException(
"Send message timeout in " + this.consumerConfig.getFetchTimeoutInMills() + " mills");
} catch (final MetaClientException e) {
throw e;
} catch (final InterruptedException e) {
throw e;
} catch (final Exception e) {
throw new MetaClientException(
"get message failed,topic="
+ fetchRequest.getTopic()
+ ",partition="
+ fetchRequest.getPartition()
+ ",offset="
+ fetchRequest.getOffset(),
e);
} finally {
final long duration = System.currentTimeMillis() - start;
if (duration > 200) {
MetaStatLog.addStatValue2(
null, StatConstants.GET_TIME_STAT, fetchRequest.getTopic(), duration);
}
if (!success) {
MetaStatLog.addStat(null, StatConstants.GET_FAILED_STAT, fetchRequest.getTopic());
}
}
}
