private void saveState(byte[] snapshot, int lastCID) { logLock.lock(); Logger.println("(TOMLayer.saveState) Saving state of CID " + lastCID); log.newCheckpoint(snapshot, computeHash(snapshot), lastCID); log.setLastCID(-1); log.setLastCheckpointCID(lastCID); logLock.unlock(); Logger.println("(TOMLayer.saveState) Finished saving state of CID " + lastCID); }
@Override public int setState(ApplicationState recvState) { int lastCID = -1; if (recvState instanceof CSTState) { CSTState state = (CSTState) recvState; int lastCheckpointCID = state.getCheckpointCID(); lastCID = state.getLastCID(); bftsmart.tom.util.Logger.println( "(DurabilityCoordinator.setState) I'm going to update myself from CID " + lastCheckpointCID + " to CID " + lastCID); stateLock.lock(); if (state.getSerializedState() != null) { System.out.println("The state is not null. Will install it"); log.update(state); installSnapshot(state.getSerializedState()); } System.out.print("--- Installing log from " + (lastCheckpointCID + 1) + " to " + lastCID); for (int cid = lastCheckpointCID + 1; cid <= lastCID; cid++) { try { bftsmart.tom.util.Logger.println( "(DurabilityCoordinator.setState) interpreting and verifying batched requests for CID " + cid); CommandsInfo cmdInfo = state.getMessageBatch(cid); byte[][] commands = cmdInfo.commands; MessageContext[] msgCtx = cmdInfo.msgCtx; if (commands == null || msgCtx == null || msgCtx[0].isNoOp()) { continue; } appExecuteBatch(commands, msgCtx); } catch (Exception e) { e.printStackTrace(System.err); } } System.out.println("--- Installed"); stateLock.unlock(); } return lastCID; }
/** * This is the method invoked by the client side communication system. * * @param reply The reply delivered by the client side communication system */ @Override public void replyReceived(TOMMessage reply) { canReceiveLock.lock(); if (reqId == -1) { // no message being expected Logger.println( "throwing out request: sender=" + reply.getSender() + " reqId=" + reply.getSequence()); canReceiveLock.unlock(); return; } int pos = getViewManager().getCurrentViewPos(reply.getSender()); if (pos < 0) { // ignore messages that don't come from replicas canReceiveLock.unlock(); return; } if (reply.getSequence() == reqId && reply.getReqType() == requestType) { if (replyListener != null) { replyListener.replyReceived(reply); canReceiveLock.unlock(); return; } Logger.println( "Receiving reply from " + reply.getSender() + " with reqId:" + reply.getSequence() + ". Putting on pos=" + pos); if (replies[pos] == null) { receivedReplies++; } replies[pos] = reply; // Compare the reply just received, to the others int sameContent = 1; for (int i = 0; i < replies.length; i++) { if (i != pos && replies[i] != null && (comparator.compare(replies[i].getContent(), reply.getContent()) == 0)) { sameContent++; if (sameContent >= replyQuorum) { response = extractor.extractResponse(replies, sameContent, pos); reqId = -1; this.sm.release(); // resumes the thread that is executing the "invoke" method break; } } } if (response == null) { if (requestType.equals(TOMMessageType.ORDERED_REQUEST)) { if (receivedReplies == getViewManager().getCurrentViewN()) { reqId = -1; this.sm.release(); // resumes the thread that is executing the "invoke" method } } else { // UNORDERED if (receivedReplies != sameContent) { reqId = -1; this.sm.release(); // resumes the thread that is executing the "invoke" method } } } } // Critical section ends here. The semaphore can be released canReceiveLock.unlock(); }
// ******* EDUARDO BEGIN **************// private void reconfigureTo(View v) { Logger.println("Installing a most up-to-date view with id=" + v.getId()); getViewManager().reconfigureTo(v); replies = new TOMMessage[getViewManager().getCurrentViewN()]; getCommunicationSystem().updateConnections(); }
/** * This method sends a request to the replicas, and returns the related reply. If the servers take * more than invokeTimeout seconds the method returns null. This method is thread-safe. * * @param request Request to be sent * @param reqType TOM_NORMAL_REQUESTS for service requests, and other for reconfig requests. * @return The reply from the replicas related to request */ public byte[] invoke(byte[] request, TOMMessageType reqType) { canSendLock.lock(); // Clean all statefull data to prepare for receiving next replies Arrays.fill(replies, null); receivedReplies = 0; response = null; replyListener = null; replyQuorum = (int) Math.ceil( (getViewManager().getCurrentViewN() + getViewManager().getCurrentViewF()) / 2) + 1; // Send the request to the replicas, and get its ID reqId = generateRequestId(reqType); requestType = reqType; TOMulticast(request, reqId, reqType); Logger.println("Sending request (" + reqType + ") with reqId=" + reqId); Logger.println("Expected number of matching replies: " + replyQuorum); // This instruction blocks the thread, until a response is obtained. // The thread will be unblocked when the method replyReceived is invoked // by the client side communication system try { if (!this.sm.tryAcquire(invokeTimeout, TimeUnit.SECONDS)) { Logger.println("###################TIMEOUT#######################"); Logger.println("Reply timeout for reqId=" + reqId); canSendLock.unlock(); System.out.print(getProcessId() + " // " + reqId + " // TIMEOUT // "); System.out.println("Replies received: " + receivedReplies); return null; } } catch (InterruptedException ex) { } Logger.println("Response extracted = " + response); byte[] ret = null; if (response == null) { // the response can be null if n-f replies are received but there isn't // a replyQuorum of matching replies Logger.println("Received n-f replies and no response could be extracted."); canSendLock.unlock(); if (reqType == TOMMessageType.UNORDERED_REQUEST) { // invoke the operation again, whitout the read-only flag Logger.println("###################RETRY#######################"); return invokeOrdered(request); } else { throw new RuntimeException("Received n-f replies without f+1 of them matching."); } } else { // normal operation // ******* EDUARDO BEGIN **************// if (reqType == TOMMessageType.ORDERED_REQUEST) { // Reply to a normal request! if (response.getViewID() == getViewManager().getCurrentViewId()) { ret = response.getContent(); // return the response } else { // if(response.getViewID() > getViewManager().getCurrentViewId()) // updated view received reconfigureTo((View) TOMUtil.getObject(response.getContent())); canSendLock.unlock(); return invoke(request, reqType); } } else { if (response.getViewID() > getViewManager().getCurrentViewId()) { // Reply to a reconfigure request! Logger.println("Reconfiguration request' reply received!"); Object r = TOMUtil.getObject(response.getContent()); if (r instanceof View) { // did not executed the request because it is using an outdated view reconfigureTo((View) r); canSendLock.unlock(); return invoke(request, reqType); } else { // reconfiguration executed! reconfigureTo(((ReconfigureReply) r).getView()); ret = response.getContent(); } } else { // Reply to readonly request ret = response.getContent(); } } } // ******* EDUARDO END **************// canSendLock.unlock(); return ret; }
private byte[][] executeBatch(byte[][] commands, MessageContext[] msgCtx, boolean noop) { int cid = msgCtx[msgCtx.length - 1].getConsensusId(); int[] cids = consensusIds(msgCtx); int checkpointIndex = findCheckpointPosition(cids); byte[][] replies = new byte[commands.length][]; // During the consensus IDs contained in this batch of commands none of the // replicas is supposed to take a checkpoint, so the replica will only execute // the command and return the replies if (checkpointIndex == -1) { if (!noop) { stateLock.lock(); replies = appExecuteBatch(commands, msgCtx); stateLock.unlock(); } Logger.println( "(DurabilityCoordinator.executeBatch) Storing message batch in the state log for consensus " + cid); saveCommands(commands, msgCtx); } else { // there is a replica supposed to take the checkpoint. In this case, the commands // has to be executed in two steps. First the batch of commands containing commands // until the checkpoint period is executed and the log saved or checkpoint taken // if this replica is the one supposed to take the checkpoint. After the checkpoint // or log, the pointer in the log is updated and then the remaining portion of the // commands is executed byte[][] firstHalf = new byte[checkpointIndex + 1][]; MessageContext[] firstHalfMsgCtx = new MessageContext[firstHalf.length]; byte[][] secondHalf = new byte[commands.length - (checkpointIndex + 1)][]; MessageContext[] secondHalfMsgCtx = new MessageContext[secondHalf.length]; System.arraycopy(commands, 0, firstHalf, 0, checkpointIndex + 1); System.arraycopy(msgCtx, 0, firstHalfMsgCtx, 0, checkpointIndex + 1); if (secondHalf.length > 0) { System.arraycopy( commands, checkpointIndex + 1, secondHalf, 0, commands.length - (checkpointIndex + 1)); System.arraycopy( msgCtx, checkpointIndex + 1, secondHalfMsgCtx, 0, commands.length - (checkpointIndex + 1)); } else firstHalfMsgCtx = msgCtx; byte[][] firstHalfReplies = new byte[firstHalf.length][]; byte[][] secondHalfReplies = new byte[secondHalf.length][]; // execute the first half cid = msgCtx[checkpointIndex].getConsensusId(); if (!noop) { stateLock.lock(); firstHalfReplies = appExecuteBatch(firstHalf, firstHalfMsgCtx); stateLock.unlock(); } if (cid % globalCheckpointPeriod == replicaCkpIndex && lastCkpCID < cid) { Logger.println( "(DurabilityCoordinator.executeBatch) Performing checkpoint for consensus " + cid); stateLock.lock(); byte[] snapshot = getSnapshot(); stateLock.unlock(); saveState(snapshot, cid); lastCkpCID = cid; } else { Logger.println( "(DurabilityCoordinator.executeBatch) Storing message batch in the state log for consensus " + cid); saveCommands(firstHalf, firstHalfMsgCtx); } System.arraycopy(firstHalfReplies, 0, replies, 0, firstHalfReplies.length); // execute the second half if it exists if (secondHalf.length > 0) { // System.out.println("----THERE IS A SECOND HALF----"); cid = msgCtx[msgCtx.length - 1].getConsensusId(); if (!noop) { stateLock.lock(); secondHalfReplies = appExecuteBatch(secondHalf, secondHalfMsgCtx); stateLock.unlock(); } Logger.println( "(DurabilityCoordinator.executeBatch) Storing message batch in the state log for consensus " + cid); saveCommands(secondHalf, secondHalfMsgCtx); System.arraycopy( secondHalfReplies, 0, replies, firstHalfReplies.length, secondHalfReplies.length); } } if (cids != null && cids.length > 0) getStateManager().setLastCID(cids[cids.length - 1]); return replies; }