@Override
public void run() {
while (true) {
RedoTask task;
try {
task = mQueue.take();
} catch (InterruptedException e) {
break;
}
if (task.isShutdownTask()) break;
if (hadError()) {
// If there was an error, keep consuming from the queue without executing anything.
// This thread must consume all tasks until shutdown task is received. If this
// thread stopped consuming, the producer may not be able to enqueue the shutdown
// task.
continue;
}
RedoableOp op = task.getOp();
try {
if (ZimbraLog.redolog.isDebugEnabled())
ZimbraLog.redolog.info("Executing: " + op.toString());
op.redo();
} catch (OutOfMemoryError oome) {
Zimbra.halt("Out of memory while executing redo op", oome);
} catch (Throwable e) {
ZimbraLog.redolog.error("Unable to execute redo op: " + op.toString(), e);
if (!ignoreReplayErrors()) raiseError(e);
}
}
}
@Override
protected void playOp(RedoableOp op) throws Exception {
checkError();
int mboxId = op.getMailboxId();
if (mboxId == RedoableOp.MAILBOX_ID_ALL || mboxId == RedoableOp.UNKNOWN_ID) {
// Multi-mailbox ops are executed by the main thread to prevent later ops
// that depend on this op's result aren't run out of order.
if (ZimbraLog.redolog.isDebugEnabled()) ZimbraLog.redolog.info("Executing: " + op.toString());
op.redo();
} else {
// Ops for the same mailbox must be played back in order. To ensure that,
// all ops for the same mailbox are sent to the same player thread. The
// ops are added to the thread's internal queue and played back in order.
// This assignment of ops to threads will result in uneven distribution.
int index = Math.abs(mboxId % mPlayerThreads.length);
PlayerThread player = mPlayerThreads[index];
RedoTask task = new RedoTask(op);
if (ZimbraLog.redolog.isDebugEnabled()) ZimbraLog.redolog.info("Enqueuing: " + op.toString());
try {
player.enqueue(task);
} catch (InterruptedException e) {
}
}
}
public synchronized RedoableOp getNextOp() throws IOException {
long pos = mRAF.getFilePointer();
if (pos == mFileSizeAtOpen) {
// EOF reached.
return null;
}
boolean first = true;
long currPos = pos;
while (true) {
try {
RedoableOp op = RedoableOp.deserializeOp(mIN);
mLastOpStartOffset = currPos;
if (!first) {
String msg =
String.format(
"Skipped bad bytes in redolog %s; resuming at offset 0x%08x after skipping %d bytes",
mFile.getAbsolutePath(), currPos, currPos - pos);
ZimbraLog.redolog.warn(msg);
}
return op;
} catch (IOException e) {
if (e instanceof EOFException) throw e;
if (first) {
String msg =
String.format(
"Error while parsing redolog %s, offset=0x%08x; bad bytes will be skipped",
mFile.getAbsolutePath(), pos);
ZimbraLog.redolog.warn(msg, e);
}
}
first = false;
// Skip over bad bytes by looking for the next occurrence of "ZMREDO" redo op marker.
mRAF.seek(currPos + 1);
if (searchInRAF(RedoableOp.REDO_MAGIC.getBytes())) {
currPos = mRAF.getFilePointer();
} else {
String msg =
String.format(
"Found %d junk bytes from offset 0x%08x to end of file, in redolog %s",
mFileSizeAtOpen - pos, pos, mFile.getAbsolutePath());
throw new IOException(msg);
}
}
}