public void execute(Pipeline pipeline) {
    List<Node> nodes = pipelineData.getNodes();

    while (pipelineData.getNodeIndex() < nodes.size()
        && (pipelineData.getZoneResponses().size() + 1) < pipelineData.getZonesRequired()) {
      Node node = nodes.get(pipelineData.getNodeIndex());
      long start = System.nanoTime();

      try {
        Store<ByteArray, byte[]> store = stores.get(node.getId());
        V result = storeRequest.request(store);

        Response<ByteArray, V> response =
            new Response<ByteArray, V>(
                node, key, result, ((System.nanoTime() - start) / Time.NS_PER_MS));

        pipelineData.incrementSuccesses();
        pipelineData.getResponses().add(response);
        failureDetector.recordSuccess(response.getNode(), response.getRequestTime());
        pipelineData.getZoneResponses().add(node.getZoneId());
      } catch (Exception e) {
        long requestTime = (System.nanoTime() - start) / Time.NS_PER_MS;

        if (handleResponseError(e, node, requestTime, pipeline, failureDetector)) return;
      }

      pipelineData.incrementNodeIndex();
    }

    int zonesSatisfied = pipelineData.getZoneResponses().size();
    if (zonesSatisfied >= (pipelineData.getZonesRequired() + 1)) {
      pipeline.addEvent(completeEvent);
    } else {
      pipelineData.setFatalError(
          new InsufficientZoneResponsesException(
              (pipelineData.getZonesRequired() + 1)
                  + " "
                  + pipeline.getOperation().getSimpleName()
                  + "s required zone, but only "
                  + zonesSatisfied
                  + " succeeded"));

      pipeline.abort();
    }
  }
  public void execute(final Pipeline pipeline) {
    List<Node> nodes = pipelineData.getNodes();
    int attempts = Math.min(preferred, nodes.size());
    final Map<Integer, Response<ByteArray, Object>> responses =
        new ConcurrentHashMap<Integer, Response<ByteArray, Object>>();
    final CountDownLatch latch = new CountDownLatch(attempts);

    if (logger.isTraceEnabled())
      logger.trace(
          "Attempting "
              + attempts
              + " "
              + pipeline.getOperation().getSimpleName()
              + " operations in parallel");

    for (int i = 0; i < attempts; i++) {
      final Node node = nodes.get(i);
      pipelineData.incrementNodeIndex();

      NonblockingStoreCallback callback =
          new NonblockingStoreCallback() {

            public void requestComplete(Object result, long requestTime) {
              if (logger.isTraceEnabled())
                logger.trace(
                    pipeline.getOperation().getSimpleName()
                        + " response received ("
                        + requestTime
                        + " ms.) from node "
                        + node.getId());

              Response<ByteArray, Object> response =
                  new Response<ByteArray, Object>(node, key, result, requestTime);
              responses.put(node.getId(), response);
              latch.countDown();

              // Note errors that come in after the pipeline has finished.
              // These will *not* get a chance to be called in the loop of
              // responses below.
              if (pipeline.isFinished() && response.getValue() instanceof Exception) {
                if (response.getValue() instanceof InvalidMetadataException) {
                  logger.warn(
                      "Received invalid metadata problem after a successful "
                          + pipeline.getOperation().getSimpleName()
                          + " call on node "
                          + node.getId()
                          + ", store '"
                          + pipelineData.getStoreName()
                          + "'");
                } else {
                  handleResponseError(response, pipeline, failureDetector);
                }
              }
            }
          };

      if (logger.isTraceEnabled())
        logger.trace(
            "Submitting "
                + pipeline.getOperation().getSimpleName()
                + " request on node "
                + node.getId());

      NonblockingStore store = nonblockingStores.get(node.getId());

      if (pipeline.getOperation() == Operation.GET)
        store.submitGetRequest(key, transforms, callback, timeoutMs);
      else if (pipeline.getOperation() == Operation.GET_VERSIONS)
        store.submitGetVersionsRequest(key, callback, timeoutMs);
      else
        throw new IllegalStateException(
            getClass().getName()
                + " does not support pipeline operation "
                + pipeline.getOperation());
    }

    try {
      latch.await(timeoutMs, TimeUnit.MILLISECONDS);
    } catch (InterruptedException e) {
      if (logger.isEnabledFor(Level.WARN)) logger.warn(e, e);
    }

    for (Response<ByteArray, Object> response : responses.values()) {
      if (response.getValue() instanceof Exception) {
        if (handleResponseError(response, pipeline, failureDetector)) return;
      } else {
        pipelineData.incrementSuccesses();
        Response<ByteArray, V> rCast = Utils.uncheckedCast(response);
        pipelineData.getResponses().add(rCast);
        failureDetector.recordSuccess(response.getNode(), response.getRequestTime());
        pipelineData.getZoneResponses().add(response.getNode().getZoneId());
      }
    }

    if (pipelineData.getSuccesses() < required) {
      if (insufficientSuccessesEvent != null) {
        pipeline.addEvent(insufficientSuccessesEvent);
      } else {
        pipelineData.setFatalError(
            new InsufficientOperationalNodesException(
                required
                    + " "
                    + pipeline.getOperation().getSimpleName()
                    + "s required, but only "
                    + pipelineData.getSuccesses()
                    + " succeeded",
                pipelineData.getFailures()));

        pipeline.abort();
      }

    } else {

      if (pipelineData.getZonesRequired() != null) {

        int zonesSatisfied = pipelineData.getZoneResponses().size();
        if (zonesSatisfied >= (pipelineData.getZonesRequired() + 1)) {
          pipeline.addEvent(completeEvent);
        } else {
          if (this.insufficientZonesEvent != null) {
            pipeline.addEvent(this.insufficientZonesEvent);
          } else {
            pipelineData.setFatalError(
                new InsufficientZoneResponsesException(
                    (pipelineData.getZonesRequired() + 1)
                        + " "
                        + pipeline.getOperation().getSimpleName()
                        + "s required zone, but only "
                        + zonesSatisfied
                        + " succeeded"));
          }
        }

      } else {
        pipeline.addEvent(completeEvent);
      }
    }
  }