public long removeMessage(final List<MessageExt> msgs) {
long result = -1;
final long now = System.currentTimeMillis();
try {
this.lockTreeMap.writeLock().lockInterruptibly();
this.lastConsumeTimestamp = now;
try {
if (!msgTreeMap.isEmpty()) {
result = this.queueOffsetMax + 1;
int removedCnt = 0;
for (MessageExt msg : msgs) {
MessageExt prev = msgTreeMap.remove(msg.getQueueOffset());
if (prev != null) {
removedCnt--;
}
}
msgCount.addAndGet(removedCnt);
if (!msgTreeMap.isEmpty()) {
result = msgTreeMap.firstKey();
}
}
} finally {
this.lockTreeMap.writeLock().unlock();
}
} catch (Throwable t) {
log.error("removeMessage exception", t);
}
return result;
}
public void prepareMsg() {
while (true) {
BatchMessage msgTuple = null;
try {
msgTuple = super.getBatchQueue().poll(1, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
return;
}
if (msgTuple == null) {
return;
}
if (msgTuple.getMsgList().size() == 0) {
super.finish(msgTuple.getBatchId());
return;
}
BatchMsgsTag partTag = new BatchMsgsTag();
Set<String> msgIds = partTag.getMsgIds();
for (MessageExt msg : msgTuple.getMsgList()) {
String msgId = msg.getMsgId();
msgIds.add(msgId);
MessageCacheItem item =
new MessageCacheItem(msgTuple.getBatchId(), msg, msgTuple.getMessageStat());
msgCache.put(msgId, item);
msgQueue.offer(item);
}
batchMsgsMap.put(msgTuple.getBatchId(), partTag);
}
}
private MessageExtBrokerInner messageTimeup(MessageExt msgExt) {
MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
msgInner.setBody(msgExt.getBody());
msgInner.setFlag(msgExt.getFlag());
msgInner.setProperties(msgExt.getProperties());
TopicFilterType topicFilterType = MessageExt.parseTopicFilterType(msgInner.getSysFlag());
long tagsCodeValue =
MessageExtBrokerInner.tagsString2tagsCode(topicFilterType, msgInner.getTags());
msgInner.setTagsCode(tagsCodeValue);
msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties()));
msgInner.setSysFlag(msgExt.getSysFlag());
msgInner.setBornTimestamp(msgExt.getBornTimestamp());
msgInner.setBornHost(msgExt.getBornHost());
msgInner.setStoreHost(msgExt.getStoreHost());
msgInner.setReconsumeTimes(msgExt.getReconsumeTimes());
msgInner.setWaitStoreMsgOK(false);
msgInner.clearProperty(Message.PROPERTY_DELAY_TIME_LEVEL);
// 恢复Topic
msgInner.setTopic(msgInner.getProperty(Message.PROPERTY_REAL_TOPIC));
// 恢复QueueId
String queueIdStr = msgInner.getProperty(Message.PROPERTY_REAL_QUEUE_ID);
int queueId = Integer.parseInt(queueIdStr);
msgInner.setQueueId(queueId);
return msgInner;
}
void queryByKey(final DefaultMQAdminExt admin, final String topic, final String key)
throws MQClientException, InterruptedException {
admin.start();
QueryResult queryResult = admin.queryMessage(topic, key, 32, 0, Long.MAX_VALUE);
System.out.printf(
"%-50s %-4s %s\n", //
"#Message ID", //
"#QID", //
"#Offset");
for (MessageExt msg : queryResult.getMessageList()) {
System.out.printf("%-50s %-4d %d\n", msg.getMsgId(), msg.getQueueId(), msg.getQueueOffset());
}
}
public void makeMessageToCosumeAgain(List<MessageExt> msgs) {
try {
this.lockTreeMap.writeLock().lockInterruptibly();
try {
for (MessageExt msg : msgs) {
this.msgTreeMapTemp.remove(msg.getQueueOffset());
this.msgTreeMap.put(msg.getQueueOffset(), msg);
}
} finally {
this.lockTreeMap.writeLock().unlock();
}
} catch (InterruptedException e) {
log.error("makeMessageToCosumeAgain exception", e);
}
}
/**
* parse message body.
*
* @param msgs
* @return
*/
public List<String> getMessageBody(List<MessageExt> msgs) {
List<String> messageBodyList = Lists.newArrayList();
Iterator<MessageExt> it = msgs.iterator();
while (it.hasNext()) {
MessageExt msg = it.next();
try {
// System.out.println(Thread.currentThread().getName() + " Receive New
// Messages,topic:" + msg.getTopic()
// +",message body:"+new String(msg.getBody(),"utf-8"));
messageBodyList.add(new String(msg.getBody(), "utf-8"));
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
}
return messageBodyList;
}
private Serializable decodeMsg(MessageExt msg) {
if (msg == null) {
return null;
}
// 1.反序列化
try {
return HessianUtils.decode(msg.getBody());
} catch (IOException e) {
logger.error("反序列化出错!" + e.getMessage(), e);
return null;
}
}
public boolean putMessage(final List<MessageExt> msgs) {
boolean dispatchToConsume = false;
try {
this.lockTreeMap.writeLock().lockInterruptibly();
try {
int validMsgCnt = 0;
for (MessageExt msg : msgs) {
MessageExt old = msgTreeMap.put(msg.getQueueOffset(), msg);
if (null == old) {
validMsgCnt++;
this.queueOffsetMax = msg.getQueueOffset();
}
}
msgCount.addAndGet(validMsgCnt);
if (!msgTreeMap.isEmpty() && !this.consuming) {
dispatchToConsume = true;
this.consuming = true;
}
if (!msgs.isEmpty()) {
MessageExt messageExt = msgs.get(msgs.size() - 1);
String property = messageExt.getProperty(MessageConst.PROPERTY_MAX_OFFSET);
if (property != null) {
long accTotal = Long.parseLong(property) - messageExt.getQueueOffset();
if (accTotal > 0) {
this.msgAccCnt = accTotal;
}
}
}
} finally {
this.lockTreeMap.writeLock().unlock();
}
} catch (InterruptedException e) {
log.error("putMessage exception", e);
}
return dispatchToConsume;
}
/** 客户端使用，SLAVE也会使用 */
public static MessageExt decode(java.nio.ByteBuffer byteBuffer, final boolean readBody) {
try {
MessageExt msgExt = new MessageExt();
// 1 TOTALSIZE
int storeSize = byteBuffer.getInt();
msgExt.setStoreSize(storeSize);
// 2 MAGICCODE
byteBuffer.getInt();
// 3 BODYCRC
int bodyCRC = byteBuffer.getInt();
msgExt.setBodyCRC(bodyCRC);
// 4 QUEUEID
int queueId = byteBuffer.getInt();
msgExt.setQueueId(queueId);
// 5 FLAG
int flag = byteBuffer.getInt();
msgExt.setFlag(flag);
// 6 QUEUEOFFSET
long queueOffset = byteBuffer.getLong();
msgExt.setQueueOffset(queueOffset);
// 7 PHYSICALOFFSET
long physicOffset = byteBuffer.getLong();
msgExt.setCommitLogOffset(physicOffset);
// 8 SYSFLAG
int sysFlag = byteBuffer.getInt();
msgExt.setSysFlag(sysFlag);
// 9 BORNTIMESTAMP
long bornTimeStamp = byteBuffer.getLong();
msgExt.setBornTimestamp(bornTimeStamp);
// 10 BORNHOST
byte[] bornHost = new byte[4];
byteBuffer.get(bornHost, 0, 4);
int port = byteBuffer.getInt();
msgExt.setBornHost(new InetSocketAddress(InetAddress.getByAddress(bornHost), port));
// 11 STORETIMESTAMP
long storeTimestamp = byteBuffer.getLong();
msgExt.setStoreTimestamp(storeTimestamp);
// 12 STOREHOST
byte[] storeHost = new byte[4];
byteBuffer.get(storeHost, 0, 4);
port = byteBuffer.getInt();
msgExt.setStoreHost(new InetSocketAddress(InetAddress.getByAddress(storeHost), port));
// 13 RECONSUMETIMES
int reconsumeTimes = byteBuffer.getInt();
msgExt.setReconsumeTimes(reconsumeTimes);
// 14 Prepared Transaction Offset
long preparedTransactionOffset = byteBuffer.getLong();
msgExt.setPreparedTransactionOffset(preparedTransactionOffset);
// 15 BODY
int bodyLen = byteBuffer.getInt();
if (bodyLen > 0) {
if (readBody) {
byte[] body = new byte[bodyLen];
byteBuffer.get(body);
// uncompress body
if ((sysFlag & MessageSysFlag.CompressedFlag) == MessageSysFlag.CompressedFlag) {
body = UtilAll.uncompress(body);
}
msgExt.setBody(body);
} else {
byteBuffer.position(byteBuffer.position() + bodyLen);
}
}
// 16 TOPIC
byte topicLen = byteBuffer.get();
byte[] topic = new byte[(int) topicLen];
byteBuffer.get(topic);
msgExt.setTopic(new String(topic));
// 17 properties
short propertiesLength = byteBuffer.getShort();
if (propertiesLength > 0) {
byte[] properties = new byte[propertiesLength];
byteBuffer.get(properties);
String propertiesString = new String(properties);
Map<String, String> map = string2messageProperties(propertiesString);
msgExt.setProperties(map);
}
// 消息ID
ByteBuffer byteBufferMsgId = ByteBuffer.allocate(MSG_ID_LENGTH);
String msgId =
createMessageId(byteBufferMsgId, msgExt.getStoreHostBytes(), msgExt.getCommitLogOffset());
msgExt.setMsgId(msgId);
return msgExt;
} catch (UnknownHostException e) {
byteBuffer.position(byteBuffer.limit());
} catch (BufferUnderflowException e) {
byteBuffer.position(byteBuffer.limit());
} catch (Exception e) {
byteBuffer.position(byteBuffer.limit());
}
return null;
}
/**
* 服务端使用 检查消息并返回消息大小
*
* @return 0 表示走到文件末尾 >0 正常消息 -1 消息校验失败
*/
public DispatchRequest checkMessageAndReturnSize(
java.nio.ByteBuffer byteBuffer, final boolean checkCRC, final boolean readBody) {
try {
java.nio.ByteBuffer byteBufferMessage =
((DefaultAppendMessageCallback) this.appendMessageCallback).getMsgStoreItemMemory();
byte[] bytesContent = byteBufferMessage.array();
// 1 TOTALSIZE
int totalSize = byteBuffer.getInt();
// 2 MAGICCODE
int magicCode = byteBuffer.getInt();
switch (magicCode) {
case MessageMagicCode:
break;
case BlankMagicCode:
return new DispatchRequest(0);
default:
log.warn("found a illegal magic code 0x" + Integer.toHexString(magicCode));
return new DispatchRequest(-1);
}
// 3 BODYCRC
int bodyCRC = byteBuffer.getInt();
// 4 QUEUEID
int queueId = byteBuffer.getInt();
// 5 FLAG
int flag = byteBuffer.getInt();
flag = flag + 0;
// 6 QUEUEOFFSET
long queueOffset = byteBuffer.getLong();
// 7 PHYSICALOFFSET
long physicOffset = byteBuffer.getLong();
// 8 SYSFLAG
int sysFlag = byteBuffer.getInt();
// 9 BORNTIMESTAMP
long bornTimeStamp = byteBuffer.getLong();
bornTimeStamp = bornTimeStamp + 0;
// 10 BORNHOST（IP+PORT）
byteBuffer.get(bytesContent, 0, 8);
// 11 STORETIMESTAMP
long storeTimestamp = byteBuffer.getLong();
// 12 STOREHOST（IP+PORT）
byteBuffer.get(bytesContent, 0, 8);
// 13 RECONSUMETIMES
int reconsumeTimes = byteBuffer.getInt();
// 14 Prepared Transaction Offset
long preparedTransactionOffset = byteBuffer.getLong();
// 15 BODY
int bodyLen = byteBuffer.getInt();
if (bodyLen > 0) {
if (readBody) {
byteBuffer.get(bytesContent, 0, bodyLen);
// 校验CRC
if (checkCRC) {
int crc = UtilAll.crc32(bytesContent, 0, bodyLen);
if (crc != bodyCRC) {
log.warn("CRC check failed " + crc + " " + bodyCRC);
return new DispatchRequest(-1);
}
}
} else {
byteBuffer.position(byteBuffer.position() + bodyLen);
}
}
// 16 TOPIC
byte topicLen = byteBuffer.get();
byteBuffer.get(bytesContent, 0, topicLen);
String topic = new String(bytesContent, 0, topicLen);
long tagsCode = 0;
String keys = "";
// 17 properties
short propertiesLength = byteBuffer.getShort();
if (propertiesLength > 0) {
byteBuffer.get(bytesContent, 0, propertiesLength);
String properties = new String(bytesContent, 0, propertiesLength);
Map<String, String> propertiesMap = MessageDecoder.string2messageProperties(properties);
keys = propertiesMap.get(MessageConst.PROPERTY_KEYS);
String tags = propertiesMap.get(MessageConst.PROPERTY_TAGS);
if (tags != null && tags.length() > 0) {
tagsCode =
MessageExtBrokerInner.tagsString2tagsCode(
MessageExt.parseTopicFilterType(sysFlag), tags);
}
}
return new DispatchRequest( //
topic, // 1
queueId, // 2
physicOffset, // 3
totalSize, // 4
tagsCode, // 5
storeTimestamp, // 6
queueOffset, // 7
keys, // 8
sysFlag, // 9
0L, // 10
preparedTransactionOffset, // 11
null // 12
);
} catch (BufferUnderflowException e) {
byteBuffer.position(byteBuffer.limit());
} catch (Exception e) {
byteBuffer.position(byteBuffer.limit());
}
return new DispatchRequest(-1);
}
private ByteBuffer messageToByteBuffer(final MessageExt msg) throws IOException {
int sysFlag = MessageSysFlag.clearCompressedFlag(msg.getSysFlag());
if (msg.getBody() != null) {
if (msg.getBody().length
>= this.filtersrvController.getFiltersrvConfig().getCompressMsgBodyOverHowmuch()) {
byte[] data =
UtilAll.compress(
msg.getBody(), this.filtersrvController.getFiltersrvConfig().getZipCompressLevel());
if (data != null) {
msg.setBody(data);
sysFlag |= MessageSysFlag.CompressedFlag;
}
}
}
final int bodyLength = msg.getBody() != null ? msg.getBody().length : 0;
byte[] topicData = msg.getTopic().getBytes(MixAll.DEFAULT_CHARSET);
final int topicLength = topicData.length;
String properties = MessageDecoder.messageProperties2String(msg.getProperties());
byte[] propertiesData = properties.getBytes(MixAll.DEFAULT_CHARSET);
final int propertiesLength = propertiesData.length;
final int msgLen =
4 // 1 TOTALSIZE
+ 4 // 2 MAGICCODE
+ 4 // 3 BODYCRC
+ 4 // 4 QUEUEID
+ 4 // 5 FLAG
+ 8 // 6 QUEUEOFFSET
+ 8 // 7 PHYSICALOFFSET
+ 4 // 8 SYSFLAG
+ 8 // 9 BORNTIMESTAMP
+ 8 // 10 BORNHOST
+ 8 // 11 STORETIMESTAMP
+ 8 // 12 STOREHOSTADDRESS
+ 4 // 13 RECONSUMETIMES
+ 8 // 14 Prepared Transaction Offset
+ 4
+ bodyLength // 14 BODY
+ 1
+ topicLength // 15 TOPIC
+ 2
+ propertiesLength // 16 propertiesLength
+ 0;
ByteBuffer msgStoreItemMemory = ByteBuffer.allocate(msgLen);
final MessageExt msgInner = msg;
// 1 TOTALSIZE
msgStoreItemMemory.putInt(msgLen);
// 2 MAGICCODE
msgStoreItemMemory.putInt(CommitLog.MessageMagicCode);
// 3 BODYCRC
msgStoreItemMemory.putInt(UtilAll.crc32(msgInner.getBody()));
// 4 QUEUEID
msgStoreItemMemory.putInt(msgInner.getQueueId());
// 5 FLAG
msgStoreItemMemory.putInt(msgInner.getFlag());
// 6 QUEUEOFFSET
msgStoreItemMemory.putLong(msgInner.getQueueOffset());
// 7 PHYSICALOFFSET
msgStoreItemMemory.putLong(msgInner.getCommitLogOffset());
// 8 SYSFLAG
msgStoreItemMemory.putInt(sysFlag);
// 9 BORNTIMESTAMP
msgStoreItemMemory.putLong(msgInner.getBornTimestamp());
// 10 BORNHOST
msgStoreItemMemory.put(msgInner.getBornHostBytes());
// 11 STORETIMESTAMP
msgStoreItemMemory.putLong(msgInner.getStoreTimestamp());
// 12 STOREHOSTADDRESS
msgStoreItemMemory.put(msgInner.getStoreHostBytes());
// 13 RECONSUMETIMES
msgStoreItemMemory.putInt(msgInner.getReconsumeTimes());
// 14 Prepared Transaction Offset
msgStoreItemMemory.putLong(msgInner.getPreparedTransactionOffset());
// 15 BODY
msgStoreItemMemory.putInt(bodyLength);
if (bodyLength > 0) msgStoreItemMemory.put(msgInner.getBody());
// 16 TOPIC
msgStoreItemMemory.put((byte) topicLength);
msgStoreItemMemory.put(topicData);
// 17 PROPERTIES
msgStoreItemMemory.putShort((short) propertiesLength);
if (propertiesLength > 0) msgStoreItemMemory.put(propertiesData);
return msgStoreItemMemory;
}
public void executeOnTimeup() {
ConsumeQueue cq =
ScheduleMessageService.this.defaultMessageStore.findConsumeQueue(
SCHEDULE_TOPIC, delayLevel2QueueId(delayLevel));
if (cq != null) {
SelectMapedBufferResult bufferCQ = cq.getIndexBuffer(this.offset);
if (bufferCQ != null) {
try {
long nextOffset = offset;
int i = 0;
for (; i < bufferCQ.getSize(); i += ConsumeQueue.CQStoreUnitSize) {
long offsetPy = bufferCQ.getByteBuffer().getLong();
int sizePy = bufferCQ.getByteBuffer().getInt();
long tagsCode = bufferCQ.getByteBuffer().getLong();
// 队列里存储的tagsCode实际是一个时间点
long deliverTimestamp = tagsCode;
nextOffset = offset + (i / ConsumeQueue.CQStoreUnitSize);
long countdown = deliverTimestamp - System.currentTimeMillis();
// 时间到了，该投递
if (countdown <= 0) {
MessageExt msgExt =
ScheduleMessageService.this.defaultMessageStore.lookMessageByOffset(
offsetPy, sizePy);
if (msgExt != null) {
MessageExtBrokerInner msgInner = this.messageTimeup(msgExt);
PutMessageResult putMessageResult =
ScheduleMessageService.this.defaultMessageStore.putMessage(msgInner);
// 成功
if (putMessageResult != null
&& putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK) {
continue;
}
// 失败
else {
log.error(
"a message time up, but reput it failed, topic: {} msgId {}",
msgExt.getTopic(),
msgExt.getMsgId());
ScheduleMessageService.this.timer.schedule(
new DeliverDelayedMessageTimerTask(this.delayLevel, nextOffset),
DELAY_FOR_A_PERIOD);
ScheduleMessageService.this.updateOffset(this.delayLevel, nextOffset);
return;
}
}
}
// 时候未到，继续定时
else {
ScheduleMessageService.this.timer.schedule(
new DeliverDelayedMessageTimerTask(this.delayLevel, nextOffset), countdown);
ScheduleMessageService.this.updateOffset(this.delayLevel, nextOffset);
return;
}
} // end of for
nextOffset = offset + (i / ConsumeQueue.CQStoreUnitSize);
ScheduleMessageService.this.timer.schedule(
new DeliverDelayedMessageTimerTask(this.delayLevel, nextOffset), DELAY_FOR_A_WHILE);
ScheduleMessageService.this.updateOffset(this.delayLevel, nextOffset);
return;
} finally {
// 必须释放资源
bufferCQ.release();
}
} // end of if (bufferCQ != null)
} // end of if (cq != null)
ScheduleMessageService.this.timer.schedule(
new DeliverDelayedMessageTimerTask(this.delayLevel, this.offset), DELAY_FOR_A_WHILE);
}