private MessageExtBrokerInner messageTimeup(MessageExt msgExt) { MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); msgInner.setBody(msgExt.getBody()); msgInner.setFlag(msgExt.getFlag()); msgInner.setProperties(msgExt.getProperties()); TopicFilterType topicFilterType = MessageExt.parseTopicFilterType(msgInner.getSysFlag()); long tagsCodeValue = MessageExtBrokerInner.tagsString2tagsCode(topicFilterType, msgInner.getTags()); msgInner.setTagsCode(tagsCodeValue); msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties())); msgInner.setSysFlag(msgExt.getSysFlag()); msgInner.setBornTimestamp(msgExt.getBornTimestamp()); msgInner.setBornHost(msgExt.getBornHost()); msgInner.setStoreHost(msgExt.getStoreHost()); msgInner.setReconsumeTimes(msgExt.getReconsumeTimes()); msgInner.setWaitStoreMsgOK(false); msgInner.clearProperty(Message.PROPERTY_DELAY_TIME_LEVEL); // 恢复Topic msgInner.setTopic(msgInner.getProperty(Message.PROPERTY_REAL_TOPIC)); // 恢复QueueId String queueIdStr = msgInner.getProperty(Message.PROPERTY_REAL_QUEUE_ID); int queueId = Integer.parseInt(queueIdStr); msgInner.setQueueId(queueId); return msgInner; }
private SendResult sendKernelImpl( final Message msg, // final MessageQueue mq, // final CommunicationMode communicationMode, // final SendCallback sendCallback // ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); if (null == brokerAddr) { // TODO 此处可能对Name Server压力过大,需要调优 tryToFindTopicPublishInfo(mq.getTopic()); brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); } SendMessageContext context = null; if (brokerAddr != null) { byte[] prevBody = msg.getBody(); try { int sysFlag = 0; if (this.tryToCompressMessage(msg)) { sysFlag |= MessageSysFlag.CompressedFlag; } final String tranMsg = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); if (tranMsg != null && Boolean.parseBoolean(tranMsg)) { sysFlag |= MessageSysFlag.TransactionPreparedType; } // 执行hook if (this.hasHook()) { context = new SendMessageContext(); context.setProducerGroup(this.defaultMQProducer.getProducerGroup()); context.setCommunicationMode(communicationMode); context.setBrokerAddr(brokerAddr); context.setMessage(msg); context.setMq(mq); this.executeHookBefore(context); } SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); requestHeader.setProducerGroup(this.defaultMQProducer.getProducerGroup()); requestHeader.setTopic(msg.getTopic()); requestHeader.setDefaultTopic(this.defaultMQProducer.getCreateTopicKey()); requestHeader.setDefaultTopicQueueNums(this.defaultMQProducer.getDefaultTopicQueueNums()); requestHeader.setQueueId(mq.getQueueId()); requestHeader.setSysFlag(sysFlag); requestHeader.setBornTimestamp(System.currentTimeMillis()); requestHeader.setFlag(msg.getFlag()); requestHeader.setProperties(MessageDecoder.messageProperties2String(msg.getProperties())); SendResult sendResult = this.mQClientFactory .getMQClientAPIImpl() .sendMessage( // brokerAddr, // 1 mq.getBrokerName(), // 2 msg, // 3 requestHeader, // 4 this.defaultMQProducer.getSendMsgTimeout(), // 5 communicationMode, // 6 sendCallback // 7 ); // 执行hook if (this.hasHook()) { context.setSendResult(sendResult); this.executeHookAfter(context); } return sendResult; } catch (RemotingException e) { // 执行hook if (this.hasHook()) { context.setException(e); this.executeHookAfter(context); } throw e; } catch (MQBrokerException e) { // 执行hook if (this.hasHook()) { context.setException(e); this.executeHookAfter(context); } throw e; } catch (InterruptedException e) { // 执行hook if (this.hasHook()) { context.setException(e); this.executeHookAfter(context); } throw e; } finally { msg.setBody(prevBody); } } throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); }
public PutMessageResult putMessage(final MessageExtBrokerInner msg) { // 设置存储时间 msg.setStoreTimestamp(System.currentTimeMillis()); // 设置消息体BODY CRC(考虑在客户端设置最合适) msg.setBodyCRC(UtilAll.crc32(msg.getBody())); // 返回结果 AppendMessageResult result = null; StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService(); String topic = msg.getTopic(); int queueId = msg.getQueueId(); long tagsCode = msg.getTagsCode(); final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag()); if (tranType == MessageSysFlag.TransactionNotType // || tranType == MessageSysFlag.TransactionCommitType) { // 延时投递 if (msg.getDelayTimeLevel() > 0) { if (msg.getDelayTimeLevel() > this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel()) { msg.setDelayTimeLevel( this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel()); } topic = ScheduleMessageService.SCHEDULE_TOPIC; queueId = ScheduleMessageService.delayLevel2QueueId(msg.getDelayTimeLevel()); tagsCode = this.defaultMessageStore .getScheduleMessageService() .computeDeliverTimestamp(msg.getDelayTimeLevel(), msg.getStoreTimestamp()); /** 备份真实的topic,queueId */ msg.putProperty(MessageConst.PROPERTY_REAL_TOPIC, msg.getTopic()); msg.putProperty(MessageConst.PROPERTY_REAL_QUEUE_ID, String.valueOf(msg.getQueueId())); msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties())); msg.setTopic(topic); msg.setQueueId(queueId); } } // 写文件要加锁 synchronized (this) { long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now(); // 这里设置存储时间戳,才能保证全局有序 msg.setStoreTimestamp(beginLockTimestamp); // 尝试写入 MapedFile mapedFile = this.mapedFileQueue.getLastMapedFile(); if (null == mapedFile) { log.error( "create maped file1 error, topic: " + msg.getTopic() + " clientAddr: " + msg.getBornHostString()); return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, null); } result = mapedFile.appendMessage(msg, this.appendMessageCallback); switch (result.getStatus()) { // 成功追加消息 case PUT_OK: break; // 走到文件末尾 case END_OF_FILE: // 创建新文件,重新写消息 mapedFile = this.mapedFileQueue.getLastMapedFile(); if (null == mapedFile) { log.error( "create maped file2 error, topic: " + msg.getTopic() + " clientAddr: " + msg.getBornHostString()); return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, result); } result = mapedFile.appendMessage(msg, this.appendMessageCallback); break; // 消息大小超限 case MESSAGE_SIZE_EXCEEDED: return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result); // 未知错误 case UNKNOWN_ERROR: return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result); default: return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result); } DispatchRequest dispatchRequest = new DispatchRequest( // topic, // 1 queueId, // 2 result.getWroteOffset(), // 3 result.getWroteBytes(), // 4 tagsCode, // 5 msg.getStoreTimestamp(), // 6 result.getLogicsOffset(), // 7 msg.getKeys(), // 8 /** 事务部分 */ msg.getSysFlag(), // 9 msg.getQueueOffset(), // 10 msg.getPreparedTransactionOffset(), // 11 msg.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP) // 12 ); this.defaultMessageStore.putDispatchRequest(dispatchRequest); long eclipseTime = this.defaultMessageStore.getSystemClock().now() - beginLockTimestamp; if (eclipseTime > 1000) { log.warn("putMessage in lock eclipse time(ms) " + eclipseTime); } } // 返回结果 PutMessageResult putMessageResult = new PutMessageResult(PutMessageStatus.PUT_OK, result); // 统计消息SIZE storeStatsService.getSinglePutMessageTopicSizeTotal(topic).addAndGet(result.getWroteBytes()); GroupCommitRequest request = null; // 同步刷盘 if (FlushDiskType.SYNC_FLUSH == this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) { GroupCommitService service = (GroupCommitService) this.flushCommitLogService; if (msg.isWaitStoreMsgOK()) { request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes()); service.putRequest(request); boolean flushOK = request.waitForFlush( this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout()); if (!flushOK) { log.error( "do groupcommit, wait for flush failed, topic: " + msg.getTopic() + " tags: " + msg.getTags() + " client address: " + msg.getBornHostString()); putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_DISK_TIMEOUT); } } else { service.wakeup(); } } // 异步刷盘 else { this.flushCommitLogService.wakeup(); } // 同步双写 if (BrokerRole.SYNC_MASTER == this.defaultMessageStore.getMessageStoreConfig().getBrokerRole()) { HAService service = this.defaultMessageStore.getHaService(); if (msg.isWaitStoreMsgOK()) { // 判断是否要等待 if (service.isSlaveOK(result.getWroteOffset() + result.getWroteBytes())) { if (null == request) { request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes()); } service.putRequest(request); service.getWaitNotifyObject().wakeupAll(); boolean flushOK = // TODO 此处参数与刷盘公用是否合适 request.waitForFlush( this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout()); if (!flushOK) { log.error( "do sync transfer other node, wait return, but failed, topic: " + msg.getTopic() + " tags: " + msg.getTags() + " client address: " + msg.getBornHostString()); putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_SLAVE_TIMEOUT); } } // Slave异常 else { // 告诉发送方,Slave异常 putMessageResult.setPutMessageStatus(PutMessageStatus.SLAVE_NOT_AVAILABLE); } } } // 向发送方返回结果 return putMessageResult; }
private ByteBuffer messageToByteBuffer(final MessageExt msg) throws IOException { int sysFlag = MessageSysFlag.clearCompressedFlag(msg.getSysFlag()); if (msg.getBody() != null) { if (msg.getBody().length >= this.filtersrvController.getFiltersrvConfig().getCompressMsgBodyOverHowmuch()) { byte[] data = UtilAll.compress( msg.getBody(), this.filtersrvController.getFiltersrvConfig().getZipCompressLevel()); if (data != null) { msg.setBody(data); sysFlag |= MessageSysFlag.CompressedFlag; } } } final int bodyLength = msg.getBody() != null ? msg.getBody().length : 0; byte[] topicData = msg.getTopic().getBytes(MixAll.DEFAULT_CHARSET); final int topicLength = topicData.length; String properties = MessageDecoder.messageProperties2String(msg.getProperties()); byte[] propertiesData = properties.getBytes(MixAll.DEFAULT_CHARSET); final int propertiesLength = propertiesData.length; final int msgLen = 4 // 1 TOTALSIZE + 4 // 2 MAGICCODE + 4 // 3 BODYCRC + 4 // 4 QUEUEID + 4 // 5 FLAG + 8 // 6 QUEUEOFFSET + 8 // 7 PHYSICALOFFSET + 4 // 8 SYSFLAG + 8 // 9 BORNTIMESTAMP + 8 // 10 BORNHOST + 8 // 11 STORETIMESTAMP + 8 // 12 STOREHOSTADDRESS + 4 // 13 RECONSUMETIMES + 8 // 14 Prepared Transaction Offset + 4 + bodyLength // 14 BODY + 1 + topicLength // 15 TOPIC + 2 + propertiesLength // 16 propertiesLength + 0; ByteBuffer msgStoreItemMemory = ByteBuffer.allocate(msgLen); final MessageExt msgInner = msg; // 1 TOTALSIZE msgStoreItemMemory.putInt(msgLen); // 2 MAGICCODE msgStoreItemMemory.putInt(CommitLog.MessageMagicCode); // 3 BODYCRC msgStoreItemMemory.putInt(UtilAll.crc32(msgInner.getBody())); // 4 QUEUEID msgStoreItemMemory.putInt(msgInner.getQueueId()); // 5 FLAG msgStoreItemMemory.putInt(msgInner.getFlag()); // 6 QUEUEOFFSET msgStoreItemMemory.putLong(msgInner.getQueueOffset()); // 7 PHYSICALOFFSET msgStoreItemMemory.putLong(msgInner.getCommitLogOffset()); // 8 SYSFLAG msgStoreItemMemory.putInt(sysFlag); // 9 BORNTIMESTAMP msgStoreItemMemory.putLong(msgInner.getBornTimestamp()); // 10 BORNHOST msgStoreItemMemory.put(msgInner.getBornHostBytes()); // 11 STORETIMESTAMP msgStoreItemMemory.putLong(msgInner.getStoreTimestamp()); // 12 STOREHOSTADDRESS msgStoreItemMemory.put(msgInner.getStoreHostBytes()); // 13 RECONSUMETIMES msgStoreItemMemory.putInt(msgInner.getReconsumeTimes()); // 14 Prepared Transaction Offset msgStoreItemMemory.putLong(msgInner.getPreparedTransactionOffset()); // 15 BODY msgStoreItemMemory.putInt(bodyLength); if (bodyLength > 0) msgStoreItemMemory.put(msgInner.getBody()); // 16 TOPIC msgStoreItemMemory.put((byte) topicLength); msgStoreItemMemory.put(topicData); // 17 PROPERTIES msgStoreItemMemory.putShort((short) propertiesLength); if (propertiesLength > 0) msgStoreItemMemory.put(propertiesData); return msgStoreItemMemory; }