private boolean processWhenRetryTooMany(final FetchRequest request, final MessageIterator it) {
if (SimpleFetchManager.this.isRetryTooMany(request)) {
try {
final Message couldNotProecssMsg = it.next();
MessageAccessor.setPartition(couldNotProecssMsg, request.getPartitionObject());
MetaStatLog.addStat(null, StatConstants.SKIP_MSG_COUNT, couldNotProecssMsg.getTopic());
SimpleFetchManager.this.consumer.appendCouldNotProcessMessage(couldNotProecssMsg);
} catch (final InvalidMessageException e) {
MetaStatLog.addStat(null, StatConstants.INVALID_MSG_STAT, request.getTopic());
// 消息体非法，获取有效offset，重新发起查询
this.getOffsetAddRequest(request, e);
return true;
} catch (final Throwable t) {
this.LogAddRequest(request, t);
return true;
}
request.resetRetries();
// 跳过这条不能处理的消息
request.setOffset(request.getOffset() + it.getOffset(), it.getPrevMessage().getId(), true);
// 强制设置延迟为0
request.setDelay(0);
SimpleFetchManager.this.addFetchRequest(request);
return true;
} else {
return false;
}
}
private void ackRequest(
final FetchRequest request, final MessageIterator it, final boolean ack) {
request.setOffset(
request.getOffset() + it.getOffset(),
it.getPrevMessage() != null ? it.getPrevMessage().getId() : -1,
ack);
this.addRequst(request);
}
private void postReceiveMessage(
final FetchRequest request, final MessageIterator it, final Partition partition) {
// 如果offset仍然没有前进，递增重试次数
if (it.getOffset() == 0) {
request.incrementRetriesAndGet();
} else {
request.resetRetries();
}
// 非自动ack模式
if (!partition.isAutoAck()) {
// 如果是回滚,则回滚offset，再次发起请求
if (partition.isRollback()) {
request.rollbackOffset();
partition.reset();
this.addRequst(request);
}
// 如果提交，则更新临时offset到存储
else if (partition.isAcked()) {
partition.reset();
this.ackRequest(request, it, true);
} else {
// 都不是，递增临时offset
this.ackRequest(request, it, false);
}
} else {
// 自动ack模式
this.ackRequest(request, it, true);
}
}
/**
* 返回是否需要跳过后续的处理
*
* @param request
* @param it
* @param listener
* @param partition
* @return
*/
private boolean processReceiveMessage(
final FetchRequest request,
final MessageIterator it,
final MessageListener listener,
final Partition partition) {
int count = 0;
while (it.hasNext()) {
final int prevOffset = it.getOffset();
try {
final Message msg = it.next();
MessageAccessor.setPartition(msg, partition);
if (((SimpleMessageConsumer) consumer).canConsumer(msg)) listener.recieveMessages(msg);
if (partition.isAutoAck()) {
count++;
} else {
// 提交或者回滚都必须跳出循环
if (partition.isAcked()) {
count++;
break;
} else if (partition.isRollback()) {
break;
} else {
// 不是提交也不是回滚，仅递增计数
count++;
}
}
} catch (final InvalidMessageException e) {
MetaStatLog.addStat(null, StatConstants.INVALID_MSG_STAT, request.getTopic());
// 消息体非法，获取有效offset，重新发起查询
this.getOffsetAddRequest(request, e);
return true;
} catch (final Throwable e) {
// 将指针移到上一条消息
it.setOffset(prevOffset);
log.error(
"MessageListener处理消息异常,topic="
+ request.getTopic()
+ ",partition="
+ request.getPartition(),
e);
// 跳出循环，处理消息异常，到此为止
break;
}
}
MetaStatLog.addStatValue2(null, StatConstants.GET_MSG_COUNT_STAT, request.getTopic(), count);
return false;
}
public Conversation next() {
index++;
if (files == null || files.length <= index) {
return null;
}
for (; index < files.length; index++) {
File file = files[index];
if (!file.isFile()) {
continue;
}
try {
String name = file.getName();
long uid = Long.parseLong(name);
RandomAccessFile f = new RandomAccessFile(file, "r");
MessageIterator iter = new MessageIterator(f);
IMessage msg = iter.next();
if (msg == null) {
continue;
}
Conversation conv = new Conversation();
conv.type = this.type;
conv.cid = uid;
conv.message = msg;
return conv;
} catch (NumberFormatException e) {
e.printStackTrace();
continue;
} catch (FileNotFoundException e) {
e.printStackTrace();
continue;
} catch (IOException e) {
e.printStackTrace();
continue;
}
}
return null;
}
/**
* 处理消息的整个流程：<br>
*
* <ul>
* <li>1.判断是否有消息可以处理，如果没有消息并且有数据递增重试次数，并判断是否需要递增maxSize
* <li>2.判断消息是否重试多次，如果超过设定次数，就跳过该消息继续往下走。跳过的消息可能在本地重试或者交给notify重投
* <li>3.进入消息处理流程，根据是否自动ack的情况进行处理:
* <ul>
* <li>(1)如果消息是自动ack，如果消费发生异常，则不修改offset，延迟消费等待重试
* <li>(2)如果消息是自动ack，如果消费正常，递增offset
* <li>(3)如果消息非自动ack，如果消费正常并ack，将offset修改为tmp offset，并重设tmp offset
* <li>(4)如果消息非自动ack，如果消费正常并rollback，不递增offset，重设tmp offset
* <li>(5)如果消息非自动ack，如果消费正常不ack也不rollback，不递增offset，递增tmp offset
* </ul>
* </ul>
*
* @param request
* @param it
* @param listener
*/
private void receiveMessages(
final FetchRequest request, final MessageIterator it, final MessageListener listener) {
if (it != null && it.hasNext()) {
if (this.processWhenRetryTooMany(request, it)) {
return;
}
final Partition partition = request.getPartitionObject();
if (this.processReceiveMessage(request, it, listener, partition)) {
return;
}
this.postReceiveMessage(request, it, partition);
} else {
// 尝试多次无法解析出获取的数据，可能需要增大maxSize
if (SimpleFetchManager.this.isRetryTooManyForIncrease(request)
&& it != null
&& it.getDataLength() > 0) {
request.increaseMaxSize();
log.warn(
"警告，第"
+ request.getRetries()
+ "次无法拉取topic="
+ request.getTopic()
+ ",partition="
+ request.getPartitionObject()
+ "的消息，递增maxSize="
+ request.getMaxSize()
+ " Bytes");
}
// 一定要判断it是否为null,否则正常的拉到结尾时(返回null)也将进行Retries记数,会导致以后再拉到消息时进入recover
if (it != null) {
request.incrementRetriesAndGet();
}
this.updateDelay(request);
SimpleFetchManager.this.addFetchRequest(request);
}
}
public boolean hasNext() {
try {
if (folderIter == null) {
folderIter = new FolderIterator(mailbox);
}
// get next message from the folder
// if folder is exhausted get next folder
// loop till a valid mail or all folders exhausted.
while (msgIter == null || !msgIter.hasNext()) {
Folder next = folderIter.hasNext() ? folderIter.next() : null;
if (next == null) {
return false;
}
msgIter = new MessageIterator(next, batchSize);
}
} catch (EmailFetchException e) {
LOG.error("Fetching email failed", e);
return false;
}
return true;
}
public String getFolder() {
if (msgIter == null) {
return null;
}
return msgIter.getFolder();
}
public IMAPMessage next() {
return (IMAPMessage) msgIter.next();
}
