public long getMinOffset() {
MapedFile mapedFile = this.mapedFileQueue.getFirstMapedFileOnLock();
if (mapedFile != null) {
if (mapedFile.isAvailable()) {
return mapedFile.getFileFromOffset();
} else {
return this.rollNextFile(mapedFile.getFileFromOffset());
}
}
return -1;
}
public SelectMapedBufferResult getData(final long offset, final boolean returnFirstOnNotFound) {
int mapedFileSize =
this.defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog();
MapedFile mapedFile = this.mapedFileQueue.findMapedFileByOffset(offset, returnFirstOnNotFound);
if (mapedFile != null) {
int pos = (int) (offset % mapedFileSize);
SelectMapedBufferResult result = mapedFile.selectMapedBuffer(pos);
return result;
}
return null;
}
@Test
public void test_commit() {
final String fixedMsg = "0123456789abcdef";
System.out.println("================================================================");
AllocateMapedFileService allocateMapedFileService = new AllocateMapedFileService();
allocateMapedFileService.start();
MapedFileQueue mapedFileQueue =
new MapedFileQueue("./unit_test_store/c/", 1024, allocateMapedFileService);
for (int i = 0; i < 1024; i++) {
MapedFile mapedFile = mapedFileQueue.getLastMapedFile();
assertTrue(mapedFile != null);
boolean result = mapedFile.appendMessage(fixedMsg.getBytes());
assertTrue(result);
}
// 不断尝试提交
boolean result = mapedFileQueue.commit(0);
assertFalse(result);
assertEquals(1024 * 1, mapedFileQueue.getCommittedWhere());
System.out.println("1 " + result + " " + mapedFileQueue.getCommittedWhere());
result = mapedFileQueue.commit(0);
assertFalse(result);
assertEquals(1024 * 2, mapedFileQueue.getCommittedWhere());
System.out.println("2 " + result + " " + mapedFileQueue.getCommittedWhere());
result = mapedFileQueue.commit(0);
assertFalse(result);
assertEquals(1024 * 3, mapedFileQueue.getCommittedWhere());
System.out.println("3 " + result + " " + mapedFileQueue.getCommittedWhere());
result = mapedFileQueue.commit(0);
assertFalse(result);
assertEquals(1024 * 4, mapedFileQueue.getCommittedWhere());
System.out.println("4 " + result + " " + mapedFileQueue.getCommittedWhere());
result = mapedFileQueue.commit(0);
assertFalse(result);
assertEquals(1024 * 5, mapedFileQueue.getCommittedWhere());
System.out.println("5 " + result + " " + mapedFileQueue.getCommittedWhere());
result = mapedFileQueue.commit(0);
assertFalse(result);
assertEquals(1024 * 6, mapedFileQueue.getCommittedWhere());
System.out.println("6 " + result + " " + mapedFileQueue.getCommittedWhere());
mapedFileQueue.shutdown(1000);
mapedFileQueue.destroy();
allocateMapedFileService.shutdown();
System.out.println("MapedFileQueue.commit() OK");
}
public boolean appendData(long startOffset, byte[] data) {
// 写文件要加锁
synchronized (this) {
// 尝试写入
MapedFile mapedFile = this.mapedFileQueue.getLastMapedFile(startOffset);
if (null == mapedFile) {
log.error("appendData getLastMapedFile error " + startOffset);
return false;
}
return mapedFile.appendMessage(data);
}
}
/** 读取消息 */
public SelectMapedBufferResult getMessage(final long offset, final int size) {
int mapedFileSize =
this.defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog();
MapedFile mapedFile =
this.mapedFileQueue.findMapedFileByOffset(offset, (0 == offset ? true : false));
if (mapedFile != null) {
int pos = (int) (offset % mapedFileSize);
SelectMapedBufferResult result = mapedFile.selectMapedBuffer(pos, size);
return result;
}
return null;
}
private boolean isMapedFileMatchedRecover(final MapedFile mapedFile) {
ByteBuffer byteBuffer = mapedFile.sliceByteBuffer();
int magicCode = byteBuffer.getInt(MessageDecoder.MessageMagicCodePostion);
if (magicCode != MessageMagicCode) {
return false;
}
long storeTimestamp = byteBuffer.getLong(MessageDecoder.MessageStoreTimestampPostion);
if (0 == storeTimestamp) {
return false;
}
if (this.defaultMessageStore.getMessageStoreConfig().isMessageIndexEnable() //
&& this.defaultMessageStore.getMessageStoreConfig().isMessageIndexSafe()) {
if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestampIndex()) {
log.info(
"find check timestamp, {} {}", //
storeTimestamp, //
UtilAll.timeMillisToHumanString(storeTimestamp));
return true;
}
} else {
if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestamp()) {
log.info(
"find check timestamp, {} {}", //
storeTimestamp, //
UtilAll.timeMillisToHumanString(storeTimestamp));
return true;
}
}
return false;
}
/** 正常退出时，数据恢复，所有内存数据都已经刷盘 */
public void recoverNormally() {
boolean checkCRCOnRecover =
this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover();
final List<MapedFile> mapedFiles = this.mapedFileQueue.getMapedFiles();
if (!mapedFiles.isEmpty()) {
// 从倒数第三个文件开始恢复
int index = mapedFiles.size() - 3;
if (index < 0) index = 0;
MapedFile mapedFile = mapedFiles.get(index);
ByteBuffer byteBuffer = mapedFile.sliceByteBuffer();
long processOffset = mapedFile.getFileFromOffset();
long mapedFileOffset = 0;
while (true) {
DispatchRequest dispatchRequest =
this.checkMessageAndReturnSize(byteBuffer, checkCRCOnRecover);
int size = dispatchRequest.getMsgSize();
// 正常数据
if (size > 0) {
mapedFileOffset += size;
}
// 文件中间读到错误
else if (size == -1) {
log.info("recover physics file end, " + mapedFile.getFileName());
break;
}
// 走到文件末尾，切换至下一个文件
// 由于返回0代表是遇到了最后的空洞，这个可以不计入truncate offset中
else if (size == 0) {
index++;
if (index >= mapedFiles.size()) {
// 当前条件分支不可能发生
log.info(
"recover last 3 physics file over, last maped file " + mapedFile.getFileName());
break;
} else {
mapedFile = mapedFiles.get(index);
byteBuffer = mapedFile.sliceByteBuffer();
processOffset = mapedFile.getFileFromOffset();
mapedFileOffset = 0;
log.info("recover next physics file, " + mapedFile.getFileName());
}
}
}
processOffset += mapedFileOffset;
this.mapedFileQueue.setCommittedWhere(processOffset);
this.mapedFileQueue.truncateDirtyFiles(processOffset);
}
}
@Test
public void test_getMapedMemorySize() {
final String fixedMsg = "abcd";
System.out.println("================================================================");
AllocateMapedFileService allocateMapedFileService = new AllocateMapedFileService();
allocateMapedFileService.start();
MapedFileQueue mapedFileQueue =
new MapedFileQueue("./unit_test_store/d/", 1024, allocateMapedFileService);
for (int i = 0; i < 1024; i++) {
MapedFile mapedFile = mapedFileQueue.getLastMapedFile();
assertTrue(mapedFile != null);
boolean result = mapedFile.appendMessage(fixedMsg.getBytes());
assertTrue(result);
}
assertEquals(fixedMsg.length() * 1024, mapedFileQueue.getMapedMemorySize());
mapedFileQueue.shutdown(1000);
mapedFileQueue.destroy();
allocateMapedFileService.shutdown();
System.out.println("MapedFileQueue.getMapedMemorySize() OK");
}
@Test
public void test_getLastMapedFile() {
final String fixedMsg = "0123456789abcdef";
System.out.println("================================================================");
AllocateMapedFileService allocateMapedFileService = new AllocateMapedFileService();
allocateMapedFileService.start();
MapedFileQueue mapedFileQueue =
new MapedFileQueue("./unit_test_store/a/", 1024, allocateMapedFileService);
for (int i = 0; i < 1024; i++) {
MapedFile mapedFile = mapedFileQueue.getLastMapedFile();
assertTrue(mapedFile != null);
boolean result = mapedFile.appendMessage(fixedMsg.getBytes());
if (!result) {
System.out.println("appendMessage " + i);
}
assertTrue(result);
}
mapedFileQueue.shutdown(1000);
mapedFileQueue.destroy();
allocateMapedFileService.shutdown();
System.out.println("MapedFileQueue.getLastMapedFile() OK");
}
public PutMessageResult putMessage(final MessageExtBrokerInner msg) {
// 设置存储时间
msg.setStoreTimestamp(System.currentTimeMillis());
// 设置消息体BODY CRC（考虑在客户端设置最合适）
msg.setBodyCRC(UtilAll.crc32(msg.getBody()));
// 返回结果
AppendMessageResult result = null;
StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService();
String topic = msg.getTopic();
int queueId = msg.getQueueId();
long tagsCode = msg.getTagsCode();
final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag());
if (tranType == MessageSysFlag.TransactionNotType //
|| tranType == MessageSysFlag.TransactionCommitType) {
// 延时投递
if (msg.getDelayTimeLevel() > 0) {
if (msg.getDelayTimeLevel()
> this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel()) {
msg.setDelayTimeLevel(
this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel());
}
topic = ScheduleMessageService.SCHEDULE_TOPIC;
queueId = ScheduleMessageService.delayLevel2QueueId(msg.getDelayTimeLevel());
tagsCode =
this.defaultMessageStore
.getScheduleMessageService()
.computeDeliverTimestamp(msg.getDelayTimeLevel(), msg.getStoreTimestamp());
/** 备份真实的topic，queueId */
msg.putProperty(MessageConst.PROPERTY_REAL_TOPIC, msg.getTopic());
msg.putProperty(MessageConst.PROPERTY_REAL_QUEUE_ID, String.valueOf(msg.getQueueId()));
msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));
msg.setTopic(topic);
msg.setQueueId(queueId);
}
}
// 写文件要加锁
synchronized (this) {
long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now();
// 这里设置存储时间戳，才能保证全局有序
msg.setStoreTimestamp(beginLockTimestamp);
// 尝试写入
MapedFile mapedFile = this.mapedFileQueue.getLastMapedFile();
if (null == mapedFile) {
log.error(
"create maped file1 error, topic: "
+ msg.getTopic()
+ " clientAddr: "
+ msg.getBornHostString());
return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, null);
}
result = mapedFile.appendMessage(msg, this.appendMessageCallback);
switch (result.getStatus()) {
// 成功追加消息
case PUT_OK:
break;
// 走到文件末尾
case END_OF_FILE:
// 创建新文件，重新写消息
mapedFile = this.mapedFileQueue.getLastMapedFile();
if (null == mapedFile) {
log.error(
"create maped file2 error, topic: "
+ msg.getTopic()
+ " clientAddr: "
+ msg.getBornHostString());
return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, result);
}
result = mapedFile.appendMessage(msg, this.appendMessageCallback);
break;
// 消息大小超限
case MESSAGE_SIZE_EXCEEDED:
return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result);
// 未知错误
case UNKNOWN_ERROR:
return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result);
default:
return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result);
}
DispatchRequest dispatchRequest =
new DispatchRequest( //
topic, // 1
queueId, // 2
result.getWroteOffset(), // 3
result.getWroteBytes(), // 4
tagsCode, // 5
msg.getStoreTimestamp(), // 6
result.getLogicsOffset(), // 7
msg.getKeys(), // 8
/** 事务部分 */
msg.getSysFlag(), // 9
msg.getQueueOffset(), // 10
msg.getPreparedTransactionOffset(), // 11
msg.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP) // 12
);
this.defaultMessageStore.putDispatchRequest(dispatchRequest);
long eclipseTime = this.defaultMessageStore.getSystemClock().now() - beginLockTimestamp;
if (eclipseTime > 1000) {
log.warn("putMessage in lock eclipse time(ms) " + eclipseTime);
}
}
// 返回结果
PutMessageResult putMessageResult = new PutMessageResult(PutMessageStatus.PUT_OK, result);
// 统计消息SIZE
storeStatsService.getSinglePutMessageTopicSizeTotal(topic).addAndGet(result.getWroteBytes());
GroupCommitRequest request = null;
// 同步刷盘
if (FlushDiskType.SYNC_FLUSH
== this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) {
GroupCommitService service = (GroupCommitService) this.flushCommitLogService;
if (msg.isWaitStoreMsgOK()) {
request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes());
service.putRequest(request);
boolean flushOK =
request.waitForFlush(
this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());
if (!flushOK) {
log.error(
"do groupcommit, wait for flush failed, topic: "
+ msg.getTopic()
+ " tags: "
+ msg.getTags()
+ " client address: "
+ msg.getBornHostString());
putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_DISK_TIMEOUT);
}
} else {
service.wakeup();
}
}
// 异步刷盘
else {
this.flushCommitLogService.wakeup();
}
// 同步双写
if (BrokerRole.SYNC_MASTER
== this.defaultMessageStore.getMessageStoreConfig().getBrokerRole()) {
HAService service = this.defaultMessageStore.getHaService();
if (msg.isWaitStoreMsgOK()) {
// 判断是否要等待
if (service.isSlaveOK(result.getWroteOffset() + result.getWroteBytes())) {
if (null == request) {
request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes());
}
service.putRequest(request);
service.getWaitNotifyObject().wakeupAll();
boolean flushOK =
// TODO 此处参数与刷盘公用是否合适
request.waitForFlush(
this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());
if (!flushOK) {
log.error(
"do sync transfer other node, wait return, but failed, topic: "
+ msg.getTopic()
+ " tags: "
+ msg.getTags()
+ " client address: "
+ msg.getBornHostString());
putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_SLAVE_TIMEOUT);
}
}
// Slave异常
else {
// 告诉发送方，Slave异常
putMessageResult.setPutMessageStatus(PutMessageStatus.SLAVE_NOT_AVAILABLE);
}
}
}
// 向发送方返回结果
return putMessageResult;
}
public void recoverAbnormally() {
// 根据最小时间戳来恢复
boolean checkCRCOnRecover =
this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover();
final List<MapedFile> mapedFiles = this.mapedFileQueue.getMapedFiles();
if (!mapedFiles.isEmpty()) {
// 寻找从哪个文件开始恢复
int index = mapedFiles.size() - 1;
MapedFile mapedFile = null;
for (; index >= 0; index--) {
mapedFile = mapedFiles.get(index);
if (this.isMapedFileMatchedRecover(mapedFile)) {
log.info("recover from this maped file " + mapedFile.getFileName());
break;
}
}
if (index < 0) {
index = 0;
mapedFile = mapedFiles.get(index);
}
ByteBuffer byteBuffer = mapedFile.sliceByteBuffer();
long processOffset = mapedFile.getFileFromOffset();
long mapedFileOffset = 0;
while (true) {
DispatchRequest dispatchRequest =
this.checkMessageAndReturnSize(byteBuffer, checkCRCOnRecover);
int size = dispatchRequest.getMsgSize();
// 正常数据
if (size > 0) {
mapedFileOffset += size;
this.defaultMessageStore.putDispatchRequest(dispatchRequest);
}
// 文件中间读到错误
else if (size == -1) {
log.info("recover physics file end, " + mapedFile.getFileName());
break;
}
// 走到文件末尾，切换至下一个文件
// 由于返回0代表是遇到了最后的空洞，这个可以不计入truncate offset中
else if (size == 0) {
index++;
if (index >= mapedFiles.size()) {
// 当前条件分支正常情况下不应该发生
log.info("recover physics file over, last maped file " + mapedFile.getFileName());
break;
} else {
mapedFile = mapedFiles.get(index);
byteBuffer = mapedFile.sliceByteBuffer();
processOffset = mapedFile.getFileFromOffset();
mapedFileOffset = 0;
log.info("recover next physics file, " + mapedFile.getFileName());
}
}
}
processOffset += mapedFileOffset;
this.mapedFileQueue.setCommittedWhere(processOffset);
this.mapedFileQueue.truncateDirtyFiles(processOffset);
// 清除ConsumeQueue的多余数据
this.defaultMessageStore.truncateDirtyLogicFiles(processOffset);
}
// 物理文件都被删除情况下
else {
this.mapedFileQueue.setCommittedWhere(0);
this.defaultMessageStore.destroyLogics();
}
}
@Test
public void test_findMapedFileByOffset() {
final String fixedMsg = "abcd";
System.out.println("================================================================");
AllocateMapedFileService allocateMapedFileService = new AllocateMapedFileService();
allocateMapedFileService.start();
MapedFileQueue mapedFileQueue =
new MapedFileQueue("./unit_test_store/b/", 1024, allocateMapedFileService);
for (int i = 0; i < 1024; i++) {
MapedFile mapedFile = mapedFileQueue.getLastMapedFile();
assertTrue(mapedFile != null);
boolean result = mapedFile.appendMessage(fixedMsg.getBytes());
// System.out.println("appendMessage " + bytes);
assertTrue(result);
}
MapedFile mapedFile = mapedFileQueue.findMapedFileByOffset(0);
assertTrue(mapedFile != null);
assertEquals(mapedFile.getFileFromOffset(), 0);
System.out.println(mapedFile.getFileFromOffset());
mapedFile = mapedFileQueue.findMapedFileByOffset(100);
assertTrue(mapedFile != null);
assertEquals(mapedFile.getFileFromOffset(), 0);
System.out.println(mapedFile.getFileFromOffset());
mapedFile = mapedFileQueue.findMapedFileByOffset(1024);
assertTrue(mapedFile != null);
assertEquals(mapedFile.getFileFromOffset(), 1024);
System.out.println(mapedFile.getFileFromOffset());
mapedFile = mapedFileQueue.findMapedFileByOffset(1024 + 100);
assertTrue(mapedFile != null);
assertEquals(mapedFile.getFileFromOffset(), 1024);
System.out.println(mapedFile.getFileFromOffset());
mapedFile = mapedFileQueue.findMapedFileByOffset(1024 * 2);
assertTrue(mapedFile != null);
assertEquals(mapedFile.getFileFromOffset(), 1024 * 2);
System.out.println(mapedFile.getFileFromOffset());
mapedFile = mapedFileQueue.findMapedFileByOffset(1024 * 2 + 100);
assertTrue(mapedFile != null);
assertEquals(mapedFile.getFileFromOffset(), 1024 * 2);
System.out.println(mapedFile.getFileFromOffset());
mapedFile = mapedFileQueue.findMapedFileByOffset(1024 * 4);
assertTrue(mapedFile == null);
mapedFile = mapedFileQueue.findMapedFileByOffset(1024 * 4 + 100);
assertTrue(mapedFile == null);
mapedFileQueue.shutdown(1000);
mapedFileQueue.destroy();
allocateMapedFileService.shutdown();
System.out.println("MapedFileQueue.findMapedFileByOffset() OK");
}
