private void sendRequests() {
logger.trace("{} sendRequests determining whether to send more requests", context);
while (operationPolicy.sendMoreRequests(requestsInFlight)) {
ReplicaId replicaIdToSendTo = operationPolicy.getNextReplicaIdForSend();
logger.trace("{} sendRequests sending request to {}", context, replicaIdToSendTo);
requestsInFlight.add(replicaIdToSendTo);
requesterPool.submit(makeOperationRequest(replicaIdToSendTo));
}
}
public Operation(
String datacenterName,
ConnectionPool connectionPool,
ExecutorService requesterPool,
OperationContext context,
BlobId blobId,
long operationTimeoutMs,
OperationPolicy operationPolicy) {
this.datacenterName = datacenterName;
this.connectionPool = connectionPool;
this.requesterPool = requesterPool;
this.context = context;
this.blobId = blobId;
this.operationPolicy = operationPolicy;
this.operationExpirationMs = SystemTime.getInstance().milliseconds() + operationTimeoutMs;
this.operationComplete = new AtomicBoolean(false);
this.responseQueue =
new ArrayBlockingQueue<OperationResponse>(operationPolicy.getReplicaIdCount());
this.requestsInFlight = new HashSet<ReplicaId>();
}
public void execute() throws CoordinatorException {
logger.trace("{} operation beginning execute", context);
sendRequests();
while (true) {
try {
OperationResponse operationResponse =
responseQueue.poll(
operationExpirationMs - SystemTime.getInstance().milliseconds(),
TimeUnit.MILLISECONDS);
logger.trace("{} operation processing a response", context);
logger.trace("Requests in flight {} " + requestsInFlight);
if (operationResponse == null) {
logger.error("{} Operation timed out", context);
throw new CoordinatorException(
"Operation timed out.", CoordinatorError.OperationTimedOut);
}
ReplicaId replicaId = operationResponse.getReplicaId();
logger.trace("Obtained Response from " + replicaId);
if (!requestsInFlight.remove(replicaId)) {
CoordinatorException e =
new CoordinatorException(
"Coordinator received unexpected response",
CoordinatorError.UnexpectedInternalError);
logger.error(
"Response received from replica ("
+ replicaId
+ ") to which no request is in flight: ",
e);
throw e;
}
if (operationResponse.getError() == RequestResponseError.SUCCESS) {
ServerErrorCode errorCode =
processResponseError(replicaId, operationResponse.getResponse());
logger.trace("Error code " + errorCode);
if (errorCode == ServerErrorCode.No_Error) {
operationPolicy.onSuccessfulResponse(replicaId);
logger.trace("Success response from this replica. Operation Success ");
} else {
logger.trace("Failure response from this replica ");
if (errorCode == ServerErrorCode.Data_Corrupt) {
operationPolicy.onCorruptResponse(replicaId);
} else {
operationPolicy.onFailedResponse(replicaId);
}
resolveCoordinatorError(currentError);
logger.trace("Resolved error " + currentError);
}
} else {
if (operationResponse.getError() == RequestResponseError.MESSAGE_FORMAT_ERROR) {
operationPolicy.onCorruptResponse(replicaId);
logger.trace("Corrupt response ");
} else {
operationPolicy.onFailedResponse(replicaId);
logger.trace("Failed response ");
}
setCurrentError(CoordinatorError.UnexpectedInternalError);
resolveCoordinatorError(currentError);
}
if (operationPolicy.isComplete()) {
operationComplete.set(true);
logger.trace("{} operation successfully completing execute", context);
this.onOperationComplete();
return;
}
if (!operationPolicy.mayComplete()) {
if (operationPolicy.isCorrupt()) {
logger.error("{} operation is corrupt.", context);
context.getCoordinatorMetrics().corruptionError.inc();
}
String message = getErrorMessage();
logger.trace("{} {}", context, message);
throw new CoordinatorException(message, getResolvedError());
}
sendRequests();
logger.trace(
"Requests in flight after processing an operation response " + requestsInFlight);
} catch (CoordinatorException e) {
operationComplete.set(true);
if (logger.isTraceEnabled()
|| e.getErrorCode() == CoordinatorError.BlobDoesNotExist
|| e.getErrorCode() == CoordinatorError.BlobExpired) {
logger.trace(context + " operation threw CoordinatorException during execute.", e);
} else {
logger.error(context + " operation threw CoordinatorException during execute: " + e);
}
throw e;
} catch (InterruptedException e) {
operationComplete.set(true);
// Slightly abuse the notion of "unexpected" internal error since InterruptedException does
// not indicate
// something truly unexpected.
logger.error(context + " operation interrupted during execute");
throw new CoordinatorException(
"Operation interrupted.", e, CoordinatorError.UnexpectedInternalError);
}
}
}