public void execute(Pipeline pipeline) {
List<Node> nodes = pipelineData.getNodes();
while (pipelineData.getNodeIndex() < nodes.size()
&& (pipelineData.getZoneResponses().size() + 1) < pipelineData.getZonesRequired()) {
Node node = nodes.get(pipelineData.getNodeIndex());
long start = System.nanoTime();
try {
Store<ByteArray, byte[]> store = stores.get(node.getId());
V result = storeRequest.request(store);
Response<ByteArray, V> response =
new Response<ByteArray, V>(
node, key, result, ((System.nanoTime() - start) / Time.NS_PER_MS));
pipelineData.incrementSuccesses();
pipelineData.getResponses().add(response);
failureDetector.recordSuccess(response.getNode(), response.getRequestTime());
pipelineData.getZoneResponses().add(node.getZoneId());
} catch (Exception e) {
long requestTime = (System.nanoTime() - start) / Time.NS_PER_MS;
if (handleResponseError(e, node, requestTime, pipeline, failureDetector)) return;
}
pipelineData.incrementNodeIndex();
}
int zonesSatisfied = pipelineData.getZoneResponses().size();
if (zonesSatisfied >= (pipelineData.getZonesRequired() + 1)) {
pipeline.addEvent(completeEvent);
} else {
pipelineData.setFatalError(
new InsufficientZoneResponsesException(
(pipelineData.getZonesRequired() + 1)
+ " "
+ pipeline.getOperation().getSimpleName()
+ "s required zone, but only "
+ zonesSatisfied
+ " succeeded"));
pipeline.abort();
}
}
@Override
public void execute(Pipeline pipeline) {
for (Node failedNode : failedNodes) {
int failedNodeId = failedNode.getId();
if (logger.isTraceEnabled())
logger.trace(
"Performing hinted handoff for node "
+ failedNode
+ ", store "
+ pipelineData.getStoreName()
+ "key "
+ key
+ ", version"
+ version);
Slop slop =
new Slop(
pipelineData.getStoreName(),
Slop.Operation.DELETE,
key,
null,
failedNodeId,
new Date());
boolean persisted = hintedHandoff.sendHint(failedNode, version, slop);
Exception e = pipelineData.getFatalError();
if (e != null) {
if (persisted)
pipelineData.setFatalError(
new UnreachableStoreException(
"Delete operation failed on node "
+ failedNodeId
+ ", but has been persisted to slop storage for eventual replication.",
e));
else
pipelineData.setFatalError(
new InsufficientOperationalNodesException(
"All slop servers are unavailable from node " + failedNodeId + ".", e));
}
}
pipeline.addEvent(completeEvent);
}
public void execute(final Pipeline pipeline) {
List<Node> nodes = pipelineData.getNodes();
int attempts = Math.min(preferred, nodes.size());
final Map<Integer, Response<ByteArray, Object>> responses =
new ConcurrentHashMap<Integer, Response<ByteArray, Object>>();
final CountDownLatch latch = new CountDownLatch(attempts);
if (logger.isTraceEnabled())
logger.trace(
"Attempting "
+ attempts
+ " "
+ pipeline.getOperation().getSimpleName()
+ " operations in parallel");
for (int i = 0; i < attempts; i++) {
final Node node = nodes.get(i);
pipelineData.incrementNodeIndex();
NonblockingStoreCallback callback =
new NonblockingStoreCallback() {
public void requestComplete(Object result, long requestTime) {
if (logger.isTraceEnabled())
logger.trace(
pipeline.getOperation().getSimpleName()
+ " response received ("
+ requestTime
+ " ms.) from node "
+ node.getId());
Response<ByteArray, Object> response =
new Response<ByteArray, Object>(node, key, result, requestTime);
responses.put(node.getId(), response);
latch.countDown();
// Note errors that come in after the pipeline has finished.
// These will *not* get a chance to be called in the loop of
// responses below.
if (pipeline.isFinished() && response.getValue() instanceof Exception) {
if (response.getValue() instanceof InvalidMetadataException) {
logger.warn(
"Received invalid metadata problem after a successful "
+ pipeline.getOperation().getSimpleName()
+ " call on node "
+ node.getId()
+ ", store '"
+ pipelineData.getStoreName()
+ "'");
} else {
handleResponseError(response, pipeline, failureDetector);
}
}
}
};
if (logger.isTraceEnabled())
logger.trace(
"Submitting "
+ pipeline.getOperation().getSimpleName()
+ " request on node "
+ node.getId());
NonblockingStore store = nonblockingStores.get(node.getId());
if (pipeline.getOperation() == Operation.GET)
store.submitGetRequest(key, transforms, callback, timeoutMs);
else if (pipeline.getOperation() == Operation.GET_VERSIONS)
store.submitGetVersionsRequest(key, callback, timeoutMs);
else
throw new IllegalStateException(
getClass().getName()
+ " does not support pipeline operation "
+ pipeline.getOperation());
}
try {
latch.await(timeoutMs, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
if (logger.isEnabledFor(Level.WARN)) logger.warn(e, e);
}
for (Response<ByteArray, Object> response : responses.values()) {
if (response.getValue() instanceof Exception) {
if (handleResponseError(response, pipeline, failureDetector)) return;
} else {
pipelineData.incrementSuccesses();
Response<ByteArray, V> rCast = Utils.uncheckedCast(response);
pipelineData.getResponses().add(rCast);
failureDetector.recordSuccess(response.getNode(), response.getRequestTime());
pipelineData.getZoneResponses().add(response.getNode().getZoneId());
}
}
if (pipelineData.getSuccesses() < required) {
if (insufficientSuccessesEvent != null) {
pipeline.addEvent(insufficientSuccessesEvent);
} else {
pipelineData.setFatalError(
new InsufficientOperationalNodesException(
required
+ " "
+ pipeline.getOperation().getSimpleName()
+ "s required, but only "
+ pipelineData.getSuccesses()
+ " succeeded",
pipelineData.getFailures()));
pipeline.abort();
}
} else {
if (pipelineData.getZonesRequired() != null) {
int zonesSatisfied = pipelineData.getZoneResponses().size();
if (zonesSatisfied >= (pipelineData.getZonesRequired() + 1)) {
pipeline.addEvent(completeEvent);
} else {
if (this.insufficientZonesEvent != null) {
pipeline.addEvent(this.insufficientZonesEvent);
} else {
pipelineData.setFatalError(
new InsufficientZoneResponsesException(
(pipelineData.getZonesRequired() + 1)
+ " "
+ pipeline.getOperation().getSimpleName()
+ "s required zone, but only "
+ zonesSatisfied
+ " succeeded"));
}
}
} else {
pipeline.addEvent(completeEvent);
}
}
}