private void saveState(byte[] snapshot, int lastCID) {
logLock.lock();
Logger.println("(TOMLayer.saveState) Saving state of CID " + lastCID);
log.newCheckpoint(snapshot, computeHash(snapshot), lastCID);
log.setLastCID(-1);
log.setLastCheckpointCID(lastCID);
logLock.unlock();
Logger.println("(TOMLayer.saveState) Finished saving state of CID " + lastCID);
}
@Override
public int setState(ApplicationState recvState) {
int lastCID = -1;
if (recvState instanceof CSTState) {
CSTState state = (CSTState) recvState;
int lastCheckpointCID = state.getCheckpointCID();
lastCID = state.getLastCID();
bftsmart.tom.util.Logger.println(
"(DurabilityCoordinator.setState) I'm going to update myself from CID "
+ lastCheckpointCID
+ " to CID "
+ lastCID);
stateLock.lock();
if (state.getSerializedState() != null) {
System.out.println("The state is not null. Will install it");
log.update(state);
installSnapshot(state.getSerializedState());
}
System.out.print("--- Installing log from " + (lastCheckpointCID + 1) + " to " + lastCID);
for (int cid = lastCheckpointCID + 1; cid <= lastCID; cid++) {
try {
bftsmart.tom.util.Logger.println(
"(DurabilityCoordinator.setState) interpreting and verifying batched requests for CID "
+ cid);
CommandsInfo cmdInfo = state.getMessageBatch(cid);
byte[][] commands = cmdInfo.commands;
MessageContext[] msgCtx = cmdInfo.msgCtx;
if (commands == null || msgCtx == null || msgCtx[0].isNoOp()) {
continue;
}
appExecuteBatch(commands, msgCtx);
} catch (Exception e) {
e.printStackTrace(System.err);
}
}
System.out.println("--- Installed");
stateLock.unlock();
}
return lastCID;
}
/**
* This is the method invoked by the client side communication system.
*
* @param reply The reply delivered by the client side communication system
*/
@Override
public void replyReceived(TOMMessage reply) {
canReceiveLock.lock();
if (reqId == -1) { // no message being expected
Logger.println(
"throwing out request: sender=" + reply.getSender() + " reqId=" + reply.getSequence());
canReceiveLock.unlock();
return;
}
int pos = getViewManager().getCurrentViewPos(reply.getSender());
if (pos < 0) { // ignore messages that don't come from replicas
canReceiveLock.unlock();
return;
}
if (reply.getSequence() == reqId && reply.getReqType() == requestType) {
if (replyListener != null) {
replyListener.replyReceived(reply);
canReceiveLock.unlock();
return;
}
Logger.println(
"Receiving reply from "
+ reply.getSender()
+ " with reqId:"
+ reply.getSequence()
+ ". Putting on pos="
+ pos);
if (replies[pos] == null) {
receivedReplies++;
}
replies[pos] = reply;
// Compare the reply just received, to the others
int sameContent = 1;
for (int i = 0; i < replies.length; i++) {
if (i != pos
&& replies[i] != null
&& (comparator.compare(replies[i].getContent(), reply.getContent()) == 0)) {
sameContent++;
if (sameContent >= replyQuorum) {
response = extractor.extractResponse(replies, sameContent, pos);
reqId = -1;
this.sm.release(); // resumes the thread that is executing the "invoke" method
break;
}
}
}
if (response == null) {
if (requestType.equals(TOMMessageType.ORDERED_REQUEST)) {
if (receivedReplies == getViewManager().getCurrentViewN()) {
reqId = -1;
this.sm.release(); // resumes the thread that is executing the "invoke" method
}
} else { // UNORDERED
if (receivedReplies != sameContent) {
reqId = -1;
this.sm.release(); // resumes the thread that is executing the "invoke" method
}
}
}
}
// Critical section ends here. The semaphore can be released
canReceiveLock.unlock();
}
// ******* EDUARDO BEGIN **************//
private void reconfigureTo(View v) {
Logger.println("Installing a most up-to-date view with id=" + v.getId());
getViewManager().reconfigureTo(v);
replies = new TOMMessage[getViewManager().getCurrentViewN()];
getCommunicationSystem().updateConnections();
}
/**
* This method sends a request to the replicas, and returns the related reply. If the servers take
* more than invokeTimeout seconds the method returns null. This method is thread-safe.
*
* @param request Request to be sent
* @param reqType TOM_NORMAL_REQUESTS for service requests, and other for reconfig requests.
* @return The reply from the replicas related to request
*/
public byte[] invoke(byte[] request, TOMMessageType reqType) {
canSendLock.lock();
// Clean all statefull data to prepare for receiving next replies
Arrays.fill(replies, null);
receivedReplies = 0;
response = null;
replyListener = null;
replyQuorum =
(int)
Math.ceil(
(getViewManager().getCurrentViewN() + getViewManager().getCurrentViewF()) / 2)
+ 1;
// Send the request to the replicas, and get its ID
reqId = generateRequestId(reqType);
requestType = reqType;
TOMulticast(request, reqId, reqType);
Logger.println("Sending request (" + reqType + ") with reqId=" + reqId);
Logger.println("Expected number of matching replies: " + replyQuorum);
// This instruction blocks the thread, until a response is obtained.
// The thread will be unblocked when the method replyReceived is invoked
// by the client side communication system
try {
if (!this.sm.tryAcquire(invokeTimeout, TimeUnit.SECONDS)) {
Logger.println("###################TIMEOUT#######################");
Logger.println("Reply timeout for reqId=" + reqId);
canSendLock.unlock();
System.out.print(getProcessId() + " // " + reqId + " // TIMEOUT // ");
System.out.println("Replies received: " + receivedReplies);
return null;
}
} catch (InterruptedException ex) {
}
Logger.println("Response extracted = " + response);
byte[] ret = null;
if (response == null) {
// the response can be null if n-f replies are received but there isn't
// a replyQuorum of matching replies
Logger.println("Received n-f replies and no response could be extracted.");
canSendLock.unlock();
if (reqType == TOMMessageType.UNORDERED_REQUEST) {
// invoke the operation again, whitout the read-only flag
Logger.println("###################RETRY#######################");
return invokeOrdered(request);
} else {
throw new RuntimeException("Received n-f replies without f+1 of them matching.");
}
} else {
// normal operation
// ******* EDUARDO BEGIN **************//
if (reqType == TOMMessageType.ORDERED_REQUEST) {
// Reply to a normal request!
if (response.getViewID() == getViewManager().getCurrentViewId()) {
ret = response.getContent(); // return the response
} else { // if(response.getViewID() > getViewManager().getCurrentViewId())
// updated view received
reconfigureTo((View) TOMUtil.getObject(response.getContent()));
canSendLock.unlock();
return invoke(request, reqType);
}
} else {
if (response.getViewID() > getViewManager().getCurrentViewId()) {
// Reply to a reconfigure request!
Logger.println("Reconfiguration request' reply received!");
Object r = TOMUtil.getObject(response.getContent());
if (r
instanceof
View) { // did not executed the request because it is using an outdated view
reconfigureTo((View) r);
canSendLock.unlock();
return invoke(request, reqType);
} else { // reconfiguration executed!
reconfigureTo(((ReconfigureReply) r).getView());
ret = response.getContent();
}
} else {
// Reply to readonly request
ret = response.getContent();
}
}
}
// ******* EDUARDO END **************//
canSendLock.unlock();
return ret;
}
private byte[][] executeBatch(byte[][] commands, MessageContext[] msgCtx, boolean noop) {
int cid = msgCtx[msgCtx.length - 1].getConsensusId();
int[] cids = consensusIds(msgCtx);
int checkpointIndex = findCheckpointPosition(cids);
byte[][] replies = new byte[commands.length][];
// During the consensus IDs contained in this batch of commands none of the
// replicas is supposed to take a checkpoint, so the replica will only execute
// the command and return the replies
if (checkpointIndex == -1) {
if (!noop) {
stateLock.lock();
replies = appExecuteBatch(commands, msgCtx);
stateLock.unlock();
}
Logger.println(
"(DurabilityCoordinator.executeBatch) Storing message batch in the state log for consensus "
+ cid);
saveCommands(commands, msgCtx);
} else {
// there is a replica supposed to take the checkpoint. In this case, the commands
// has to be executed in two steps. First the batch of commands containing commands
// until the checkpoint period is executed and the log saved or checkpoint taken
// if this replica is the one supposed to take the checkpoint. After the checkpoint
// or log, the pointer in the log is updated and then the remaining portion of the
// commands is executed
byte[][] firstHalf = new byte[checkpointIndex + 1][];
MessageContext[] firstHalfMsgCtx = new MessageContext[firstHalf.length];
byte[][] secondHalf = new byte[commands.length - (checkpointIndex + 1)][];
MessageContext[] secondHalfMsgCtx = new MessageContext[secondHalf.length];
System.arraycopy(commands, 0, firstHalf, 0, checkpointIndex + 1);
System.arraycopy(msgCtx, 0, firstHalfMsgCtx, 0, checkpointIndex + 1);
if (secondHalf.length > 0) {
System.arraycopy(
commands, checkpointIndex + 1, secondHalf, 0, commands.length - (checkpointIndex + 1));
System.arraycopy(
msgCtx,
checkpointIndex + 1,
secondHalfMsgCtx,
0,
commands.length - (checkpointIndex + 1));
} else firstHalfMsgCtx = msgCtx;
byte[][] firstHalfReplies = new byte[firstHalf.length][];
byte[][] secondHalfReplies = new byte[secondHalf.length][];
// execute the first half
cid = msgCtx[checkpointIndex].getConsensusId();
if (!noop) {
stateLock.lock();
firstHalfReplies = appExecuteBatch(firstHalf, firstHalfMsgCtx);
stateLock.unlock();
}
if (cid % globalCheckpointPeriod == replicaCkpIndex && lastCkpCID < cid) {
Logger.println(
"(DurabilityCoordinator.executeBatch) Performing checkpoint for consensus " + cid);
stateLock.lock();
byte[] snapshot = getSnapshot();
stateLock.unlock();
saveState(snapshot, cid);
lastCkpCID = cid;
} else {
Logger.println(
"(DurabilityCoordinator.executeBatch) Storing message batch in the state log for consensus "
+ cid);
saveCommands(firstHalf, firstHalfMsgCtx);
}
System.arraycopy(firstHalfReplies, 0, replies, 0, firstHalfReplies.length);
// execute the second half if it exists
if (secondHalf.length > 0) {
// System.out.println("----THERE IS A SECOND HALF----");
cid = msgCtx[msgCtx.length - 1].getConsensusId();
if (!noop) {
stateLock.lock();
secondHalfReplies = appExecuteBatch(secondHalf, secondHalfMsgCtx);
stateLock.unlock();
}
Logger.println(
"(DurabilityCoordinator.executeBatch) Storing message batch in the state log for consensus "
+ cid);
saveCommands(secondHalf, secondHalfMsgCtx);
System.arraycopy(
secondHalfReplies, 0, replies, firstHalfReplies.length, secondHalfReplies.length);
}
}
if (cids != null && cids.length > 0) getStateManager().setLastCID(cids[cids.length - 1]);
return replies;
}