@Test public void test_findMapedFileByOffset() { final String fixedMsg = "abcd"; System.out.println("================================================================"); AllocateMapedFileService allocateMapedFileService = new AllocateMapedFileService(); allocateMapedFileService.start(); MapedFileQueue mapedFileQueue = new MapedFileQueue("./unit_test_store/b/", 1024, allocateMapedFileService); for (int i = 0; i < 1024; i++) { MapedFile mapedFile = mapedFileQueue.getLastMapedFile(); assertTrue(mapedFile != null); boolean result = mapedFile.appendMessage(fixedMsg.getBytes()); // System.out.println("appendMessage " + bytes); assertTrue(result); } MapedFile mapedFile = mapedFileQueue.findMapedFileByOffset(0); assertTrue(mapedFile != null); assertEquals(mapedFile.getFileFromOffset(), 0); System.out.println(mapedFile.getFileFromOffset()); mapedFile = mapedFileQueue.findMapedFileByOffset(100); assertTrue(mapedFile != null); assertEquals(mapedFile.getFileFromOffset(), 0); System.out.println(mapedFile.getFileFromOffset()); mapedFile = mapedFileQueue.findMapedFileByOffset(1024); assertTrue(mapedFile != null); assertEquals(mapedFile.getFileFromOffset(), 1024); System.out.println(mapedFile.getFileFromOffset()); mapedFile = mapedFileQueue.findMapedFileByOffset(1024 + 100); assertTrue(mapedFile != null); assertEquals(mapedFile.getFileFromOffset(), 1024); System.out.println(mapedFile.getFileFromOffset()); mapedFile = mapedFileQueue.findMapedFileByOffset(1024 * 2); assertTrue(mapedFile != null); assertEquals(mapedFile.getFileFromOffset(), 1024 * 2); System.out.println(mapedFile.getFileFromOffset()); mapedFile = mapedFileQueue.findMapedFileByOffset(1024 * 2 + 100); assertTrue(mapedFile != null); assertEquals(mapedFile.getFileFromOffset(), 1024 * 2); System.out.println(mapedFile.getFileFromOffset()); mapedFile = mapedFileQueue.findMapedFileByOffset(1024 * 4); assertTrue(mapedFile == null); mapedFile = mapedFileQueue.findMapedFileByOffset(1024 * 4 + 100); assertTrue(mapedFile == null); mapedFileQueue.shutdown(1000); mapedFileQueue.destroy(); allocateMapedFileService.shutdown(); System.out.println("MapedFileQueue.findMapedFileByOffset() OK"); }
@Test public void test_commit() { final String fixedMsg = "0123456789abcdef"; System.out.println("================================================================"); AllocateMapedFileService allocateMapedFileService = new AllocateMapedFileService(); allocateMapedFileService.start(); MapedFileQueue mapedFileQueue = new MapedFileQueue("./unit_test_store/c/", 1024, allocateMapedFileService); for (int i = 0; i < 1024; i++) { MapedFile mapedFile = mapedFileQueue.getLastMapedFile(); assertTrue(mapedFile != null); boolean result = mapedFile.appendMessage(fixedMsg.getBytes()); assertTrue(result); } // 不断尝试提交 boolean result = mapedFileQueue.commit(0); assertFalse(result); assertEquals(1024 * 1, mapedFileQueue.getCommittedWhere()); System.out.println("1 " + result + " " + mapedFileQueue.getCommittedWhere()); result = mapedFileQueue.commit(0); assertFalse(result); assertEquals(1024 * 2, mapedFileQueue.getCommittedWhere()); System.out.println("2 " + result + " " + mapedFileQueue.getCommittedWhere()); result = mapedFileQueue.commit(0); assertFalse(result); assertEquals(1024 * 3, mapedFileQueue.getCommittedWhere()); System.out.println("3 " + result + " " + mapedFileQueue.getCommittedWhere()); result = mapedFileQueue.commit(0); assertFalse(result); assertEquals(1024 * 4, mapedFileQueue.getCommittedWhere()); System.out.println("4 " + result + " " + mapedFileQueue.getCommittedWhere()); result = mapedFileQueue.commit(0); assertFalse(result); assertEquals(1024 * 5, mapedFileQueue.getCommittedWhere()); System.out.println("5 " + result + " " + mapedFileQueue.getCommittedWhere()); result = mapedFileQueue.commit(0); assertFalse(result); assertEquals(1024 * 6, mapedFileQueue.getCommittedWhere()); System.out.println("6 " + result + " " + mapedFileQueue.getCommittedWhere()); mapedFileQueue.shutdown(1000); mapedFileQueue.destroy(); allocateMapedFileService.shutdown(); System.out.println("MapedFileQueue.commit() OK"); }
public boolean appendData(long startOffset, byte[] data) { // 写文件要加锁 synchronized (this) { // 尝试写入 MapedFile mapedFile = this.mapedFileQueue.getLastMapedFile(startOffset); if (null == mapedFile) { log.error("appendData getLastMapedFile error " + startOffset); return false; } return mapedFile.appendMessage(data); } }
@Test public void test_getMapedMemorySize() { final String fixedMsg = "abcd"; System.out.println("================================================================"); AllocateMapedFileService allocateMapedFileService = new AllocateMapedFileService(); allocateMapedFileService.start(); MapedFileQueue mapedFileQueue = new MapedFileQueue("./unit_test_store/d/", 1024, allocateMapedFileService); for (int i = 0; i < 1024; i++) { MapedFile mapedFile = mapedFileQueue.getLastMapedFile(); assertTrue(mapedFile != null); boolean result = mapedFile.appendMessage(fixedMsg.getBytes()); assertTrue(result); } assertEquals(fixedMsg.length() * 1024, mapedFileQueue.getMapedMemorySize()); mapedFileQueue.shutdown(1000); mapedFileQueue.destroy(); allocateMapedFileService.shutdown(); System.out.println("MapedFileQueue.getMapedMemorySize() OK"); }
@Test public void test_getLastMapedFile() { final String fixedMsg = "0123456789abcdef"; System.out.println("================================================================"); AllocateMapedFileService allocateMapedFileService = new AllocateMapedFileService(); allocateMapedFileService.start(); MapedFileQueue mapedFileQueue = new MapedFileQueue("./unit_test_store/a/", 1024, allocateMapedFileService); for (int i = 0; i < 1024; i++) { MapedFile mapedFile = mapedFileQueue.getLastMapedFile(); assertTrue(mapedFile != null); boolean result = mapedFile.appendMessage(fixedMsg.getBytes()); if (!result) { System.out.println("appendMessage " + i); } assertTrue(result); } mapedFileQueue.shutdown(1000); mapedFileQueue.destroy(); allocateMapedFileService.shutdown(); System.out.println("MapedFileQueue.getLastMapedFile() OK"); }
public PutMessageResult putMessage(final MessageExtBrokerInner msg) { // 设置存储时间 msg.setStoreTimestamp(System.currentTimeMillis()); // 设置消息体BODY CRC(考虑在客户端设置最合适) msg.setBodyCRC(UtilAll.crc32(msg.getBody())); // 返回结果 AppendMessageResult result = null; StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService(); String topic = msg.getTopic(); int queueId = msg.getQueueId(); long tagsCode = msg.getTagsCode(); final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag()); if (tranType == MessageSysFlag.TransactionNotType // || tranType == MessageSysFlag.TransactionCommitType) { // 延时投递 if (msg.getDelayTimeLevel() > 0) { if (msg.getDelayTimeLevel() > this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel()) { msg.setDelayTimeLevel( this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel()); } topic = ScheduleMessageService.SCHEDULE_TOPIC; queueId = ScheduleMessageService.delayLevel2QueueId(msg.getDelayTimeLevel()); tagsCode = this.defaultMessageStore .getScheduleMessageService() .computeDeliverTimestamp(msg.getDelayTimeLevel(), msg.getStoreTimestamp()); /** 备份真实的topic,queueId */ msg.putProperty(MessageConst.PROPERTY_REAL_TOPIC, msg.getTopic()); msg.putProperty(MessageConst.PROPERTY_REAL_QUEUE_ID, String.valueOf(msg.getQueueId())); msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties())); msg.setTopic(topic); msg.setQueueId(queueId); } } // 写文件要加锁 synchronized (this) { long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now(); // 这里设置存储时间戳,才能保证全局有序 msg.setStoreTimestamp(beginLockTimestamp); // 尝试写入 MapedFile mapedFile = this.mapedFileQueue.getLastMapedFile(); if (null == mapedFile) { log.error( "create maped file1 error, topic: " + msg.getTopic() + " clientAddr: " + msg.getBornHostString()); return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, null); } result = mapedFile.appendMessage(msg, this.appendMessageCallback); switch (result.getStatus()) { // 成功追加消息 case PUT_OK: break; // 走到文件末尾 case END_OF_FILE: // 创建新文件,重新写消息 mapedFile = this.mapedFileQueue.getLastMapedFile(); if (null == mapedFile) { log.error( "create maped file2 error, topic: " + msg.getTopic() + " clientAddr: " + msg.getBornHostString()); return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, result); } result = mapedFile.appendMessage(msg, this.appendMessageCallback); break; // 消息大小超限 case MESSAGE_SIZE_EXCEEDED: return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result); // 未知错误 case UNKNOWN_ERROR: return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result); default: return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result); } DispatchRequest dispatchRequest = new DispatchRequest( // topic, // 1 queueId, // 2 result.getWroteOffset(), // 3 result.getWroteBytes(), // 4 tagsCode, // 5 msg.getStoreTimestamp(), // 6 result.getLogicsOffset(), // 7 msg.getKeys(), // 8 /** 事务部分 */ msg.getSysFlag(), // 9 msg.getQueueOffset(), // 10 msg.getPreparedTransactionOffset(), // 11 msg.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP) // 12 ); this.defaultMessageStore.putDispatchRequest(dispatchRequest); long eclipseTime = this.defaultMessageStore.getSystemClock().now() - beginLockTimestamp; if (eclipseTime > 1000) { log.warn("putMessage in lock eclipse time(ms) " + eclipseTime); } } // 返回结果 PutMessageResult putMessageResult = new PutMessageResult(PutMessageStatus.PUT_OK, result); // 统计消息SIZE storeStatsService.getSinglePutMessageTopicSizeTotal(topic).addAndGet(result.getWroteBytes()); GroupCommitRequest request = null; // 同步刷盘 if (FlushDiskType.SYNC_FLUSH == this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) { GroupCommitService service = (GroupCommitService) this.flushCommitLogService; if (msg.isWaitStoreMsgOK()) { request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes()); service.putRequest(request); boolean flushOK = request.waitForFlush( this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout()); if (!flushOK) { log.error( "do groupcommit, wait for flush failed, topic: " + msg.getTopic() + " tags: " + msg.getTags() + " client address: " + msg.getBornHostString()); putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_DISK_TIMEOUT); } } else { service.wakeup(); } } // 异步刷盘 else { this.flushCommitLogService.wakeup(); } // 同步双写 if (BrokerRole.SYNC_MASTER == this.defaultMessageStore.getMessageStoreConfig().getBrokerRole()) { HAService service = this.defaultMessageStore.getHaService(); if (msg.isWaitStoreMsgOK()) { // 判断是否要等待 if (service.isSlaveOK(result.getWroteOffset() + result.getWroteBytes())) { if (null == request) { request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes()); } service.putRequest(request); service.getWaitNotifyObject().wakeupAll(); boolean flushOK = // TODO 此处参数与刷盘公用是否合适 request.waitForFlush( this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout()); if (!flushOK) { log.error( "do sync transfer other node, wait return, but failed, topic: " + msg.getTopic() + " tags: " + msg.getTags() + " client address: " + msg.getBornHostString()); putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_SLAVE_TIMEOUT); } } // Slave异常 else { // 告诉发送方,Slave异常 putMessageResult.setPutMessageStatus(PutMessageStatus.SLAVE_NOT_AVAILABLE); } } } // 向发送方返回结果 return putMessageResult; }