public long getMinOffset() { MapedFile mapedFile = this.mapedFileQueue.getFirstMapedFileOnLock(); if (mapedFile != null) { if (mapedFile.isAvailable()) { return mapedFile.getFileFromOffset(); } else { return this.rollNextFile(mapedFile.getFileFromOffset()); } } return -1; }
public SelectMapedBufferResult getData(final long offset, final boolean returnFirstOnNotFound) { int mapedFileSize = this.defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog(); MapedFile mapedFile = this.mapedFileQueue.findMapedFileByOffset(offset, returnFirstOnNotFound); if (mapedFile != null) { int pos = (int) (offset % mapedFileSize); SelectMapedBufferResult result = mapedFile.selectMapedBuffer(pos); return result; } return null; }
@Test public void test_commit() { final String fixedMsg = "0123456789abcdef"; System.out.println("================================================================"); AllocateMapedFileService allocateMapedFileService = new AllocateMapedFileService(); allocateMapedFileService.start(); MapedFileQueue mapedFileQueue = new MapedFileQueue("./unit_test_store/c/", 1024, allocateMapedFileService); for (int i = 0; i < 1024; i++) { MapedFile mapedFile = mapedFileQueue.getLastMapedFile(); assertTrue(mapedFile != null); boolean result = mapedFile.appendMessage(fixedMsg.getBytes()); assertTrue(result); } // 不断尝试提交 boolean result = mapedFileQueue.commit(0); assertFalse(result); assertEquals(1024 * 1, mapedFileQueue.getCommittedWhere()); System.out.println("1 " + result + " " + mapedFileQueue.getCommittedWhere()); result = mapedFileQueue.commit(0); assertFalse(result); assertEquals(1024 * 2, mapedFileQueue.getCommittedWhere()); System.out.println("2 " + result + " " + mapedFileQueue.getCommittedWhere()); result = mapedFileQueue.commit(0); assertFalse(result); assertEquals(1024 * 3, mapedFileQueue.getCommittedWhere()); System.out.println("3 " + result + " " + mapedFileQueue.getCommittedWhere()); result = mapedFileQueue.commit(0); assertFalse(result); assertEquals(1024 * 4, mapedFileQueue.getCommittedWhere()); System.out.println("4 " + result + " " + mapedFileQueue.getCommittedWhere()); result = mapedFileQueue.commit(0); assertFalse(result); assertEquals(1024 * 5, mapedFileQueue.getCommittedWhere()); System.out.println("5 " + result + " " + mapedFileQueue.getCommittedWhere()); result = mapedFileQueue.commit(0); assertFalse(result); assertEquals(1024 * 6, mapedFileQueue.getCommittedWhere()); System.out.println("6 " + result + " " + mapedFileQueue.getCommittedWhere()); mapedFileQueue.shutdown(1000); mapedFileQueue.destroy(); allocateMapedFileService.shutdown(); System.out.println("MapedFileQueue.commit() OK"); }
public boolean appendData(long startOffset, byte[] data) { // 写文件要加锁 synchronized (this) { // 尝试写入 MapedFile mapedFile = this.mapedFileQueue.getLastMapedFile(startOffset); if (null == mapedFile) { log.error("appendData getLastMapedFile error " + startOffset); return false; } return mapedFile.appendMessage(data); } }
/** 读取消息 */ public SelectMapedBufferResult getMessage(final long offset, final int size) { int mapedFileSize = this.defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog(); MapedFile mapedFile = this.mapedFileQueue.findMapedFileByOffset(offset, (0 == offset ? true : false)); if (mapedFile != null) { int pos = (int) (offset % mapedFileSize); SelectMapedBufferResult result = mapedFile.selectMapedBuffer(pos, size); return result; } return null; }
private boolean isMapedFileMatchedRecover(final MapedFile mapedFile) { ByteBuffer byteBuffer = mapedFile.sliceByteBuffer(); int magicCode = byteBuffer.getInt(MessageDecoder.MessageMagicCodePostion); if (magicCode != MessageMagicCode) { return false; } long storeTimestamp = byteBuffer.getLong(MessageDecoder.MessageStoreTimestampPostion); if (0 == storeTimestamp) { return false; } if (this.defaultMessageStore.getMessageStoreConfig().isMessageIndexEnable() // && this.defaultMessageStore.getMessageStoreConfig().isMessageIndexSafe()) { if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestampIndex()) { log.info( "find check timestamp, {} {}", // storeTimestamp, // UtilAll.timeMillisToHumanString(storeTimestamp)); return true; } } else { if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestamp()) { log.info( "find check timestamp, {} {}", // storeTimestamp, // UtilAll.timeMillisToHumanString(storeTimestamp)); return true; } } return false; }
/** 正常退出时,数据恢复,所有内存数据都已经刷盘 */ public void recoverNormally() { boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover(); final List<MapedFile> mapedFiles = this.mapedFileQueue.getMapedFiles(); if (!mapedFiles.isEmpty()) { // 从倒数第三个文件开始恢复 int index = mapedFiles.size() - 3; if (index < 0) index = 0; MapedFile mapedFile = mapedFiles.get(index); ByteBuffer byteBuffer = mapedFile.sliceByteBuffer(); long processOffset = mapedFile.getFileFromOffset(); long mapedFileOffset = 0; while (true) { DispatchRequest dispatchRequest = this.checkMessageAndReturnSize(byteBuffer, checkCRCOnRecover); int size = dispatchRequest.getMsgSize(); // 正常数据 if (size > 0) { mapedFileOffset += size; } // 文件中间读到错误 else if (size == -1) { log.info("recover physics file end, " + mapedFile.getFileName()); break; } // 走到文件末尾,切换至下一个文件 // 由于返回0代表是遇到了最后的空洞,这个可以不计入truncate offset中 else if (size == 0) { index++; if (index >= mapedFiles.size()) { // 当前条件分支不可能发生 log.info( "recover last 3 physics file over, last maped file " + mapedFile.getFileName()); break; } else { mapedFile = mapedFiles.get(index); byteBuffer = mapedFile.sliceByteBuffer(); processOffset = mapedFile.getFileFromOffset(); mapedFileOffset = 0; log.info("recover next physics file, " + mapedFile.getFileName()); } } } processOffset += mapedFileOffset; this.mapedFileQueue.setCommittedWhere(processOffset); this.mapedFileQueue.truncateDirtyFiles(processOffset); } }
@Test public void test_getMapedMemorySize() { final String fixedMsg = "abcd"; System.out.println("================================================================"); AllocateMapedFileService allocateMapedFileService = new AllocateMapedFileService(); allocateMapedFileService.start(); MapedFileQueue mapedFileQueue = new MapedFileQueue("./unit_test_store/d/", 1024, allocateMapedFileService); for (int i = 0; i < 1024; i++) { MapedFile mapedFile = mapedFileQueue.getLastMapedFile(); assertTrue(mapedFile != null); boolean result = mapedFile.appendMessage(fixedMsg.getBytes()); assertTrue(result); } assertEquals(fixedMsg.length() * 1024, mapedFileQueue.getMapedMemorySize()); mapedFileQueue.shutdown(1000); mapedFileQueue.destroy(); allocateMapedFileService.shutdown(); System.out.println("MapedFileQueue.getMapedMemorySize() OK"); }
@Test public void test_getLastMapedFile() { final String fixedMsg = "0123456789abcdef"; System.out.println("================================================================"); AllocateMapedFileService allocateMapedFileService = new AllocateMapedFileService(); allocateMapedFileService.start(); MapedFileQueue mapedFileQueue = new MapedFileQueue("./unit_test_store/a/", 1024, allocateMapedFileService); for (int i = 0; i < 1024; i++) { MapedFile mapedFile = mapedFileQueue.getLastMapedFile(); assertTrue(mapedFile != null); boolean result = mapedFile.appendMessage(fixedMsg.getBytes()); if (!result) { System.out.println("appendMessage " + i); } assertTrue(result); } mapedFileQueue.shutdown(1000); mapedFileQueue.destroy(); allocateMapedFileService.shutdown(); System.out.println("MapedFileQueue.getLastMapedFile() OK"); }
public PutMessageResult putMessage(final MessageExtBrokerInner msg) { // 设置存储时间 msg.setStoreTimestamp(System.currentTimeMillis()); // 设置消息体BODY CRC(考虑在客户端设置最合适) msg.setBodyCRC(UtilAll.crc32(msg.getBody())); // 返回结果 AppendMessageResult result = null; StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService(); String topic = msg.getTopic(); int queueId = msg.getQueueId(); long tagsCode = msg.getTagsCode(); final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag()); if (tranType == MessageSysFlag.TransactionNotType // || tranType == MessageSysFlag.TransactionCommitType) { // 延时投递 if (msg.getDelayTimeLevel() > 0) { if (msg.getDelayTimeLevel() > this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel()) { msg.setDelayTimeLevel( this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel()); } topic = ScheduleMessageService.SCHEDULE_TOPIC; queueId = ScheduleMessageService.delayLevel2QueueId(msg.getDelayTimeLevel()); tagsCode = this.defaultMessageStore .getScheduleMessageService() .computeDeliverTimestamp(msg.getDelayTimeLevel(), msg.getStoreTimestamp()); /** 备份真实的topic,queueId */ msg.putProperty(MessageConst.PROPERTY_REAL_TOPIC, msg.getTopic()); msg.putProperty(MessageConst.PROPERTY_REAL_QUEUE_ID, String.valueOf(msg.getQueueId())); msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties())); msg.setTopic(topic); msg.setQueueId(queueId); } } // 写文件要加锁 synchronized (this) { long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now(); // 这里设置存储时间戳,才能保证全局有序 msg.setStoreTimestamp(beginLockTimestamp); // 尝试写入 MapedFile mapedFile = this.mapedFileQueue.getLastMapedFile(); if (null == mapedFile) { log.error( "create maped file1 error, topic: " + msg.getTopic() + " clientAddr: " + msg.getBornHostString()); return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, null); } result = mapedFile.appendMessage(msg, this.appendMessageCallback); switch (result.getStatus()) { // 成功追加消息 case PUT_OK: break; // 走到文件末尾 case END_OF_FILE: // 创建新文件,重新写消息 mapedFile = this.mapedFileQueue.getLastMapedFile(); if (null == mapedFile) { log.error( "create maped file2 error, topic: " + msg.getTopic() + " clientAddr: " + msg.getBornHostString()); return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, result); } result = mapedFile.appendMessage(msg, this.appendMessageCallback); break; // 消息大小超限 case MESSAGE_SIZE_EXCEEDED: return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result); // 未知错误 case UNKNOWN_ERROR: return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result); default: return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result); } DispatchRequest dispatchRequest = new DispatchRequest( // topic, // 1 queueId, // 2 result.getWroteOffset(), // 3 result.getWroteBytes(), // 4 tagsCode, // 5 msg.getStoreTimestamp(), // 6 result.getLogicsOffset(), // 7 msg.getKeys(), // 8 /** 事务部分 */ msg.getSysFlag(), // 9 msg.getQueueOffset(), // 10 msg.getPreparedTransactionOffset(), // 11 msg.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP) // 12 ); this.defaultMessageStore.putDispatchRequest(dispatchRequest); long eclipseTime = this.defaultMessageStore.getSystemClock().now() - beginLockTimestamp; if (eclipseTime > 1000) { log.warn("putMessage in lock eclipse time(ms) " + eclipseTime); } } // 返回结果 PutMessageResult putMessageResult = new PutMessageResult(PutMessageStatus.PUT_OK, result); // 统计消息SIZE storeStatsService.getSinglePutMessageTopicSizeTotal(topic).addAndGet(result.getWroteBytes()); GroupCommitRequest request = null; // 同步刷盘 if (FlushDiskType.SYNC_FLUSH == this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) { GroupCommitService service = (GroupCommitService) this.flushCommitLogService; if (msg.isWaitStoreMsgOK()) { request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes()); service.putRequest(request); boolean flushOK = request.waitForFlush( this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout()); if (!flushOK) { log.error( "do groupcommit, wait for flush failed, topic: " + msg.getTopic() + " tags: " + msg.getTags() + " client address: " + msg.getBornHostString()); putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_DISK_TIMEOUT); } } else { service.wakeup(); } } // 异步刷盘 else { this.flushCommitLogService.wakeup(); } // 同步双写 if (BrokerRole.SYNC_MASTER == this.defaultMessageStore.getMessageStoreConfig().getBrokerRole()) { HAService service = this.defaultMessageStore.getHaService(); if (msg.isWaitStoreMsgOK()) { // 判断是否要等待 if (service.isSlaveOK(result.getWroteOffset() + result.getWroteBytes())) { if (null == request) { request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes()); } service.putRequest(request); service.getWaitNotifyObject().wakeupAll(); boolean flushOK = // TODO 此处参数与刷盘公用是否合适 request.waitForFlush( this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout()); if (!flushOK) { log.error( "do sync transfer other node, wait return, but failed, topic: " + msg.getTopic() + " tags: " + msg.getTags() + " client address: " + msg.getBornHostString()); putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_SLAVE_TIMEOUT); } } // Slave异常 else { // 告诉发送方,Slave异常 putMessageResult.setPutMessageStatus(PutMessageStatus.SLAVE_NOT_AVAILABLE); } } } // 向发送方返回结果 return putMessageResult; }
public void recoverAbnormally() { // 根据最小时间戳来恢复 boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover(); final List<MapedFile> mapedFiles = this.mapedFileQueue.getMapedFiles(); if (!mapedFiles.isEmpty()) { // 寻找从哪个文件开始恢复 int index = mapedFiles.size() - 1; MapedFile mapedFile = null; for (; index >= 0; index--) { mapedFile = mapedFiles.get(index); if (this.isMapedFileMatchedRecover(mapedFile)) { log.info("recover from this maped file " + mapedFile.getFileName()); break; } } if (index < 0) { index = 0; mapedFile = mapedFiles.get(index); } ByteBuffer byteBuffer = mapedFile.sliceByteBuffer(); long processOffset = mapedFile.getFileFromOffset(); long mapedFileOffset = 0; while (true) { DispatchRequest dispatchRequest = this.checkMessageAndReturnSize(byteBuffer, checkCRCOnRecover); int size = dispatchRequest.getMsgSize(); // 正常数据 if (size > 0) { mapedFileOffset += size; this.defaultMessageStore.putDispatchRequest(dispatchRequest); } // 文件中间读到错误 else if (size == -1) { log.info("recover physics file end, " + mapedFile.getFileName()); break; } // 走到文件末尾,切换至下一个文件 // 由于返回0代表是遇到了最后的空洞,这个可以不计入truncate offset中 else if (size == 0) { index++; if (index >= mapedFiles.size()) { // 当前条件分支正常情况下不应该发生 log.info("recover physics file over, last maped file " + mapedFile.getFileName()); break; } else { mapedFile = mapedFiles.get(index); byteBuffer = mapedFile.sliceByteBuffer(); processOffset = mapedFile.getFileFromOffset(); mapedFileOffset = 0; log.info("recover next physics file, " + mapedFile.getFileName()); } } } processOffset += mapedFileOffset; this.mapedFileQueue.setCommittedWhere(processOffset); this.mapedFileQueue.truncateDirtyFiles(processOffset); // 清除ConsumeQueue的多余数据 this.defaultMessageStore.truncateDirtyLogicFiles(processOffset); } // 物理文件都被删除情况下 else { this.mapedFileQueue.setCommittedWhere(0); this.defaultMessageStore.destroyLogics(); } }
@Test public void test_findMapedFileByOffset() { final String fixedMsg = "abcd"; System.out.println("================================================================"); AllocateMapedFileService allocateMapedFileService = new AllocateMapedFileService(); allocateMapedFileService.start(); MapedFileQueue mapedFileQueue = new MapedFileQueue("./unit_test_store/b/", 1024, allocateMapedFileService); for (int i = 0; i < 1024; i++) { MapedFile mapedFile = mapedFileQueue.getLastMapedFile(); assertTrue(mapedFile != null); boolean result = mapedFile.appendMessage(fixedMsg.getBytes()); // System.out.println("appendMessage " + bytes); assertTrue(result); } MapedFile mapedFile = mapedFileQueue.findMapedFileByOffset(0); assertTrue(mapedFile != null); assertEquals(mapedFile.getFileFromOffset(), 0); System.out.println(mapedFile.getFileFromOffset()); mapedFile = mapedFileQueue.findMapedFileByOffset(100); assertTrue(mapedFile != null); assertEquals(mapedFile.getFileFromOffset(), 0); System.out.println(mapedFile.getFileFromOffset()); mapedFile = mapedFileQueue.findMapedFileByOffset(1024); assertTrue(mapedFile != null); assertEquals(mapedFile.getFileFromOffset(), 1024); System.out.println(mapedFile.getFileFromOffset()); mapedFile = mapedFileQueue.findMapedFileByOffset(1024 + 100); assertTrue(mapedFile != null); assertEquals(mapedFile.getFileFromOffset(), 1024); System.out.println(mapedFile.getFileFromOffset()); mapedFile = mapedFileQueue.findMapedFileByOffset(1024 * 2); assertTrue(mapedFile != null); assertEquals(mapedFile.getFileFromOffset(), 1024 * 2); System.out.println(mapedFile.getFileFromOffset()); mapedFile = mapedFileQueue.findMapedFileByOffset(1024 * 2 + 100); assertTrue(mapedFile != null); assertEquals(mapedFile.getFileFromOffset(), 1024 * 2); System.out.println(mapedFile.getFileFromOffset()); mapedFile = mapedFileQueue.findMapedFileByOffset(1024 * 4); assertTrue(mapedFile == null); mapedFile = mapedFileQueue.findMapedFileByOffset(1024 * 4 + 100); assertTrue(mapedFile == null); mapedFileQueue.shutdown(1000); mapedFileQueue.destroy(); allocateMapedFileService.shutdown(); System.out.println("MapedFileQueue.findMapedFileByOffset() OK"); }