public void start() {
   if (expectedSuccessfulOps == 0) {
     // no search shards to search on, bail with empty response
     // (it happens with search across _all with no indices around and consistent with broadcast
     // operations)
     listener.onResponse(
         new SearchResponse(
             InternalSearchResponse.empty(),
             null,
             0,
             0,
             buildTookInMillis(),
             ShardSearchFailure.EMPTY_ARRAY));
     return;
   }
   int shardIndex = -1;
   for (final ShardIterator shardIt : shardsIts) {
     shardIndex++;
     final ShardRouting shard = shardIt.nextOrNull();
     if (shard != null) {
       performFirstPhase(shardIndex, shardIt, shard);
     } else {
       // really, no shards active in this group
       onFirstPhaseResult(
           shardIndex,
           null,
           null,
           shardIt,
           new NoShardAvailableActionException(shardIt.shardId()));
     }
   }
 }
    /** start sending current requests to replicas */
    @Override
    protected void doRun() {
      if (pending.get() == 0) {
        doFinish();
        return;
      }
      ShardRouting shard;
      shardIt.reset(); // reset the iterator
      while ((shard = shardIt.nextOrNull()) != null) {
        // if its unassigned, nothing to do here...
        if (shard.unassigned()) {
          continue;
        }

        // we index on a replica that is initializing as well since we might not have got the event
        // yet that it was started. We will get an exception IllegalShardState exception if its not
        // started
        // and that's fine, we will ignore it
        if (shard.primary()) {
          if (originalPrimaryShard.currentNodeId().equals(shard.currentNodeId()) == false) {
            // there is a new primary, we'll have to replicate to it.
            performOnReplica(shard, shard.currentNodeId());
          }
          if (shard.relocating()) {
            performOnReplica(shard, shard.relocatingNodeId());
          }
        } else if (shouldExecuteReplication(indexMetaData.getSettings())) {
          performOnReplica(shard, shard.currentNodeId());
          if (shard.relocating()) {
            performOnReplica(shard, shard.relocatingNodeId());
          }
        }
      }
    }
 @Override
 protected ShardIterator shards(ClusterState clusterState, UpdateRequest request)
     throws ElasticSearchException {
   if (request.shardId() != -1) {
     return clusterState
         .routingTable()
         .index(request.index())
         .shard(request.shardId())
         .primaryShardIt();
   }
   ShardIterator shardIterator =
       clusterService
           .operationRouting()
           .indexShards(
               clusterService.state(),
               request.index(),
               request.type(),
               request.id(),
               request.routing());
   ShardRouting shard;
   while ((shard = shardIterator.nextOrNull()) != null) {
     if (shard.primary()) {
       return new PlainShardIterator(shardIterator.shardId(), ImmutableList.of(shard));
     }
   }
   return new PlainShardIterator(shardIterator.shardId(), ImmutableList.<ShardRouting>of());
 }
 public void start() {
   if (shardsIts.size() == 0) {
     // no shards
     try {
       listener.onResponse(newResponse(request, new AtomicReferenceArray(0), clusterState));
     } catch (Throwable e) {
       listener.onFailure(e);
     }
     return;
   }
   request.beforeStart();
   // count the local operations, and perform the non local ones
   int shardIndex = -1;
   for (final ShardIterator shardIt : shardsIts) {
     shardIndex++;
     final ShardRouting shard = shardIt.nextOrNull();
     if (shard != null) {
       performOperation(shardIt, shard, shardIndex);
     } else {
       // really, no shards active in this group
       onOperation(
           null, shardIt, shardIndex, new NoShardAvailableActionException(shardIt.shardId()));
     }
   }
 }
 @Override
 protected void doRun() {
   if (checkBlocks() == false) {
     return;
   }
   final ShardIterator shardIt = shards(observer.observedState(), internalRequest);
   final ShardRouting primary = resolvePrimary(shardIt);
   if (primary == null) {
     retryBecauseUnavailable(shardIt.shardId(), "No active shards.");
     return;
   }
   if (primary.active() == false) {
     logger.trace(
         "primary shard [{}] is not yet active, scheduling a retry.", primary.shardId());
     retryBecauseUnavailable(
         shardIt.shardId(), "Primary shard is not active or isn't assigned to a known node.");
     return;
   }
   if (observer.observedState().nodes().nodeExists(primary.currentNodeId()) == false) {
     logger.trace(
         "primary shard [{}] is assigned to anode we do not know the node, scheduling a retry.",
         primary.shardId(),
         primary.currentNodeId());
     retryBecauseUnavailable(
         shardIt.shardId(), "Primary shard is not active or isn't assigned to a known node.");
     return;
   }
   routeRequestOrPerformLocally(primary, shardIt);
 }
 private void mockShard(OperationRouting operationRouting, Integer shardId) {
   ShardIterator shardIterator = mock(ShardIterator.class);
   when(operationRouting.indexShards(
           any(ClusterState.class),
           anyString(),
           anyString(),
           Matchers.eq(shardId.toString()),
           anyString()))
       .thenReturn(shardIterator);
   when(shardIterator.shardId()).thenReturn(new ShardId("foo", shardId));
 }
 @Override
 protected ShardIterator shards(ClusterState clusterState, ShardDeleteRequest request) {
   GroupShardsIterator group =
       clusterService
           .operationRouting()
           .broadcastDeleteShards(clusterService.state(), request.index());
   for (ShardIterator shardIt : group) {
     if (shardIt.shardId().id() == request.shardId()) {
       return shardIt;
     }
   }
   throw new ElasticsearchIllegalStateException(
       "No shards iterator found for shard [" + request.shardId() + "]");
 }
    void retry(@Nullable final Throwable failure) {
      if (observer.isTimedOut()) {
        // we running as a last attempt after a timeout has happened. don't retry
        Throwable listenFailure = failure;
        if (listenFailure == null) {
          if (shardIt == null) {
            listenFailure =
                new UnavailableShardsException(
                    request.concreteIndex(),
                    -1,
                    "Timeout waiting for [{}], request: {}",
                    request.timeout(),
                    actionName);
          } else {
            listenFailure =
                new UnavailableShardsException(
                    shardIt.shardId(),
                    "[{}] shardIt, [{}] active : Timeout waiting for [{}], request: {}",
                    shardIt.size(),
                    shardIt.sizeActive(),
                    request.timeout(),
                    actionName);
          }
        }
        listener.onFailure(listenFailure);
        return;
      }

      observer.waitForNextChange(
          new ClusterStateObserver.Listener() {
            @Override
            public void onNewClusterState(ClusterState state) {
              doStart();
            }

            @Override
            public void onClusterServiceClose() {
              listener.onFailure(new NodeClosedException(nodes.getLocalNode()));
            }

            @Override
            public void onTimeout(TimeValue timeout) {
              // just to be on the safe side, see if we can start it now?
              doStart();
            }
          },
          request.timeout());
    }
    void setFailure(ShardIterator shardIt, int shardIndex, Throwable t) {
      // we don't aggregate shard failures on non active shards (but do keep the header counts
      // right)
      if (TransportActions.isShardNotAvailableException(t)) {
        return;
      }

      if (!(t instanceof BroadcastShardOperationFailedException)) {
        t = new BroadcastShardOperationFailedException(shardIt.shardId(), t);
      }

      Object response = shardsResponses.get(shardIndex);
      if (response == null) {
        // just override it and return
        shardsResponses.set(shardIndex, t);
      }

      if (!(response instanceof Throwable)) {
        // we should never really get here...
        return;
      }

      // the failure is already present, try and not override it with an exception that is less
      // meaningless
      // for example, getting illegal shard state
      if (TransportActions.isReadOverrideException(t)) {
        shardsResponses.set(shardIndex, t);
      }
    }
    private void doFinish() {
      if (finished.compareAndSet(false, true)) {
        Releasables.close(indexShardReference);
        final ShardId shardId = shardIt.shardId();
        final ActionWriteResponse.ShardInfo.Failure[] failuresArray;
        if (!shardReplicaFailures.isEmpty()) {
          int slot = 0;
          failuresArray = new ActionWriteResponse.ShardInfo.Failure[shardReplicaFailures.size()];
          for (Map.Entry<String, Throwable> entry : shardReplicaFailures.entrySet()) {
            RestStatus restStatus = ExceptionsHelper.status(entry.getValue());
            failuresArray[slot++] =
                new ActionWriteResponse.ShardInfo.Failure(
                    shardId.getIndex(),
                    shardId.getId(),
                    entry.getKey(),
                    entry.getValue(),
                    restStatus,
                    false);
          }
        } else {
          failuresArray = ActionWriteResponse.EMPTY;
        }
        finalResponse.setShardInfo(
            new ActionWriteResponse.ShardInfo(totalShards, success.get(), failuresArray));

        listener.onResponse(finalResponse);
      }
    }
 @Override
 public void onFailure(Throwable t) {
   logger.error(
       "unexpected error while replicating for action [{}]. shard [{}]. ",
       t,
       actionName,
       shardIt.shardId());
   forceFinishAsFailed(t);
 }
    @SuppressWarnings({"unchecked"})
    void onOperation(
        @Nullable ShardRouting shard, final ShardIterator shardIt, int shardIndex, Throwable t) {
      // we set the shard failure always, even if its the first in the replication group, and the
      // next one
      // will work (it will just override it...)
      setFailure(shardIt, shardIndex, t);

      ShardRouting nextShard = shardIt.nextOrNull();
      if (nextShard != null) {
        if (t != null) {
          if (logger.isTraceEnabled()) {
            if (!TransportActions.isShardNotAvailableException(t)) {
              if (shard != null) {
                logger.trace(shard.shortSummary() + ": Failed to execute [" + request + "]", t);
              } else {
                logger.trace(shardIt.shardId() + ": Failed to execute [" + request + "]", t);
              }
            }
          }
        }
        // we are not threaded here if we got here from the transport
        // or we possibly threaded if we got from a local threaded one,
        // in which case, the next shard in the partition will not be local one
        // so there is no meaning to this flag
        performOperation(shardIt, nextShard, shardIndex, true);
      } else {
        if (logger.isDebugEnabled()) {
          if (t != null) {
            if (!TransportActions.isShardNotAvailableException(t)) {
              if (shard != null) {
                logger.debug(shard.shortSummary() + ": Failed to execute [" + request + "]", t);
              } else {
                logger.debug(shardIt.shardId() + ": Failed to execute [" + request + "]", t);
              }
            }
          }
        }
        if (expectedOps == counterOps.incrementAndGet()) {
          finishHim();
        }
      }
    }
 private void onFirstPhaseResult(
     int shardIndex, String nodeId, FirstResult result, ShardIterator shardIt) {
   result.shardTarget(new SearchShardTarget(nodeId, shardIt.shardId()));
   processFirstPhaseResult(shardIndex, result);
   // we need to increment successful ops first before we compare the exit condition otherwise if
   // we
   // are fast we could concurrently update totalOps but then preempt one of the threads which can
   // cause the successor to read a wrong value from successfulOps if second phase is very fast ie.
   // count etc.
   successfulOps.incrementAndGet();
   // increment all the "future" shards to update the total ops since we some may work and some may
   // not...
   // and when that happens, we break on total ops, so we must maintain them
   final int xTotalOps = totalOps.addAndGet(shardIt.remaining() + 1);
   if (xTotalOps == expectedTotalOps) {
     try {
       innerMoveToSecondPhase();
     } catch (Exception e) {
       if (logger.isDebugEnabled()) {
         logger.debug(
             (Supplier<?>)
                 () ->
                     new ParameterizedMessage(
                         "{}: Failed to execute [{}] while moving to second phase",
                         shardIt.shardId(),
                         request),
             e);
       }
       raiseEarlyFailure(
           new ReduceSearchPhaseException(firstPhaseName(), "", e, buildShardFailures()));
     }
   } else if (xTotalOps > expectedTotalOps) {
     raiseEarlyFailure(
         new IllegalStateException(
             "unexpected higher total ops ["
                 + xTotalOps
                 + "] compared "
                 + "to expected ["
                 + expectedTotalOps
                 + "]"));
   }
 }
Ejemplo n.º 14
0
  @Override
  public Routing getRouting(WhereClause whereClause, @Nullable String preference) {
    Map<String, Map<String, List<Integer>>> locations = new TreeMap<>();
    GroupShardsIterator shardIterators =
        clusterService
            .operationRouting()
            .searchShards(
                clusterService.state(),
                Strings.EMPTY_ARRAY,
                new String[] {index},
                null,
                preference);
    ShardRouting shardRouting;
    for (ShardIterator shardIterator : shardIterators) {
      shardRouting = shardIterator.nextOrNull();
      processShardRouting(locations, shardRouting, shardIterator.shardId());
    }

    return new Routing(locations);
  }
 protected ShardRouting resolvePrimary(ShardIterator shardIt) {
   // no shardIt, might be in the case between index gateway recovery and shardIt initialization
   ShardRouting shard;
   while ((shard = shardIt.nextOrNull()) != null) {
     // we only deal with primary shardIt here...
     if (shard.primary()) {
       return shard;
     }
   }
   return null;
 }
Ejemplo n.º 16
0
 private ShardRouting corruptRandomPrimaryFile(final boolean includePerCommitFiles)
     throws IOException {
   ClusterState state = client().admin().cluster().prepareState().get().getState();
   Index test = state.metaData().index("test").getIndex();
   GroupShardsIterator shardIterators =
       state.getRoutingTable().activePrimaryShardsGrouped(new String[] {"test"}, false);
   List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
   ShardIterator shardIterator = RandomPicks.randomFrom(random(), iterators);
   ShardRouting shardRouting = shardIterator.nextOrNull();
   assertNotNull(shardRouting);
   assertTrue(shardRouting.primary());
   assertTrue(shardRouting.assignedToNode());
   String nodeId = shardRouting.currentNodeId();
   NodesStatsResponse nodeStatses =
       client().admin().cluster().prepareNodesStats(nodeId).setFs(true).get();
   Set<Path> files = new TreeSet<>(); // treeset makes sure iteration order is deterministic
   for (FsInfo.Path info : nodeStatses.getNodes().get(0).getFs()) {
     String path = info.getPath();
     Path file =
         PathUtils.get(path)
             .resolve("indices")
             .resolve(test.getUUID())
             .resolve(Integer.toString(shardRouting.getId()))
             .resolve("index");
     if (Files.exists(file)) { // multi data path might only have one path in use
       try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) {
         for (Path item : stream) {
           if (Files.isRegularFile(item)
               && "write.lock".equals(item.getFileName().toString()) == false) {
             if (includePerCommitFiles || isPerSegmentFile(item.getFileName().toString())) {
               files.add(item);
             }
           }
         }
       }
     }
   }
   pruneOldDeleteGenerations(files);
   CorruptionUtils.corruptFile(random(), files.toArray(new Path[0]));
   return shardRouting;
 }
  void performFirstPhase(
      final int shardIndex, final ShardIterator shardIt, final ShardRouting shard) {
    if (shard == null) {
      // no more active shards... (we should not really get here, but just for safety)
      onFirstPhaseResult(
          shardIndex, null, null, shardIt, new NoShardAvailableActionException(shardIt.shardId()));
    } else {
      final DiscoveryNode node = nodeIdToDiscoveryNode.apply(shard.currentNodeId());
      if (node == null) {
        onFirstPhaseResult(
            shardIndex,
            shard,
            null,
            shardIt,
            new NoShardAvailableActionException(shardIt.shardId()));
      } else {
        AliasFilter filter = this.aliasFilter.get(shard.index().getUUID());
        assert filter != null;
        ShardSearchTransportRequest transportRequest =
            new ShardSearchTransportRequest(
                request, shardIt.shardId(), shardsIts.size(), filter, startTime());
        sendExecuteFirstPhase(
            node,
            transportRequest,
            new ActionListener<FirstResult>() {
              @Override
              public void onResponse(FirstResult result) {
                onFirstPhaseResult(shardIndex, shard.currentNodeId(), result, shardIt);
              }

              @Override
              public void onFailure(Exception t) {
                onFirstPhaseResult(shardIndex, shard, node.getId(), shardIt, t);
              }
            });
      }
    }
  }
 @SuppressWarnings({"unchecked"})
 void onOperation(
     @Nullable ShardRouting shard, final ShardIterator shardIt, int shardIndex, Throwable t) {
   // we set the shard failure always, even if its the first in the replication group, and the
   // next one
   // will work (it will just override it...)
   setFailure(shardIt, shardIndex, t);
   ShardRouting nextShard = shardIt.nextOrNull();
   if (nextShard != null) {
     if (t != null) {
       if (logger.isTraceEnabled()) {
         if (!TransportActions.isShardNotAvailableException(t)) {
           logger.trace(
               "{}: failed to execute [{}]",
               t,
               shard != null ? shard.shortSummary() : shardIt.shardId(),
               request);
         }
       }
     }
     performOperation(shardIt, nextShard, shardIndex);
   } else {
     if (logger.isDebugEnabled()) {
       if (t != null) {
         if (!TransportActions.isShardNotAvailableException(t)) {
           logger.debug(
               "{}: failed to execute [{}]",
               t,
               shard != null ? shard.shortSummary() : shardIt.shardId(),
               request);
         }
       }
     }
     if (expectedOps == counterOps.incrementAndGet()) {
       finishHim();
     }
   }
 }
 public void start() {
   if (shardsIts.size() == 0) {
     // no shards
     try {
       listener.onResponse(newResponse(request, new AtomicReferenceArray(0), clusterState));
     } catch (Throwable e) {
       listener.onFailure(e);
     }
   }
   request.beforeStart();
   // count the local operations, and perform the non local ones
   int localOperations = 0;
   int shardIndex = -1;
   for (final ShardIterator shardIt : shardsIts) {
     shardIndex++;
     final ShardRouting shard = shardIt.firstOrNull();
     if (shard != null) {
       if (shard.currentNodeId().equals(nodes.localNodeId())) {
         localOperations++;
       } else {
         // do the remote operation here, the localAsync flag is not relevant
         performOperation(shardIt, shardIndex, true);
       }
     } else {
       // really, no shards active in this group
       onOperation(
           null, shardIt, shardIndex, new NoShardAvailableActionException(shardIt.shardId()));
     }
   }
   // we have local operations, perform them now
   if (localOperations > 0) {
     if (request.operationThreading() == BroadcastOperationThreading.SINGLE_THREAD) {
       request.beforeLocalFork();
       threadPool
           .executor(executor)
           .execute(
               new Runnable() {
                 @Override
                 public void run() {
                   int shardIndex = -1;
                   for (final ShardIterator shardIt : shardsIts) {
                     shardIndex++;
                     final ShardRouting shard = shardIt.firstOrNull();
                     if (shard != null) {
                       if (shard.currentNodeId().equals(nodes.localNodeId())) {
                         performOperation(shardIt, shardIndex, false);
                       }
                     }
                   }
                 }
               });
     } else {
       boolean localAsync =
           request.operationThreading() == BroadcastOperationThreading.THREAD_PER_SHARD;
       if (localAsync) {
         request.beforeLocalFork();
       }
       shardIndex = -1;
       for (final ShardIterator shardIt : shardsIts) {
         shardIndex++;
         final ShardRouting shard = shardIt.firstOrNull();
         if (shard != null) {
           if (shard.currentNodeId().equals(nodes.localNodeId())) {
             performOperation(shardIt, shardIndex, localAsync);
           }
         }
       }
     }
   }
 }
    void performOnReplica(
        final PrimaryResponse<Response, ReplicaRequest> response,
        final AtomicInteger counter,
        final ShardRouting shard,
        String nodeId) {
      // if we don't have that node, it means that it might have failed and will be created again,
      // in
      // this case, we don't have to do the operation, and just let it failover
      if (!nodes.nodeExists(nodeId)) {
        if (counter.decrementAndGet() == 0) {
          listener.onResponse(response.response());
        }
        return;
      }

      final ReplicaOperationRequest shardRequest =
          new ReplicaOperationRequest(shardIt.shardId().id(), response.replicaRequest());
      if (!nodeId.equals(nodes.localNodeId())) {
        DiscoveryNode node = nodes.get(nodeId);
        transportService.sendRequest(
            node,
            transportReplicaAction,
            shardRequest,
            transportOptions,
            new VoidTransportResponseHandler(ThreadPool.Names.SAME) {
              @Override
              public void handleResponse(VoidStreamable vResponse) {
                finishIfPossible();
              }

              @Override
              public void handleException(TransportException exp) {
                if (!ignoreReplicaException(exp.unwrapCause())) {
                  logger.warn(
                      "Failed to perform " + transportAction + " on replica " + shardIt.shardId(),
                      exp);
                  shardStateAction.shardFailed(
                      shard,
                      "Failed to perform ["
                          + transportAction
                          + "] on replica, message ["
                          + detailedMessage(exp)
                          + "]");
                }
                finishIfPossible();
              }

              private void finishIfPossible() {
                if (counter.decrementAndGet() == 0) {
                  listener.onResponse(response.response());
                }
              }
            });
      } else {
        if (request.operationThreaded()) {
          request.beforeLocalFork();
          threadPool
              .executor(executor)
              .execute(
                  new Runnable() {
                    @Override
                    public void run() {
                      try {
                        shardOperationOnReplica(shardRequest);
                      } catch (Exception e) {
                        if (!ignoreReplicaException(e)) {
                          logger.warn(
                              "Failed to perform "
                                  + transportAction
                                  + " on replica "
                                  + shardIt.shardId(),
                              e);
                          shardStateAction.shardFailed(
                              shard,
                              "Failed to perform ["
                                  + transportAction
                                  + "] on replica, message ["
                                  + detailedMessage(e)
                                  + "]");
                        }
                      }
                      if (counter.decrementAndGet() == 0) {
                        listener.onResponse(response.response());
                      }
                    }
                  });
        } else {
          try {
            shardOperationOnReplica(shardRequest);
          } catch (Exception e) {
            if (!ignoreReplicaException(e)) {
              logger.warn(
                  "Failed to perform " + transportAction + " on replica" + shardIt.shardId(), e);
              shardStateAction.shardFailed(
                  shard,
                  "Failed to perform ["
                      + transportAction
                      + "] on replica, message ["
                      + detailedMessage(e)
                      + "]");
            }
          }
          if (counter.decrementAndGet() == 0) {
            listener.onResponse(response.response());
          }
        }
      }
    }
    protected boolean doStart() {
      nodes = observer.observedState().nodes();
      try {
        ClusterBlockException blockException = checkGlobalBlock(observer.observedState());
        if (blockException != null) {
          if (blockException.retryable()) {
            retry(blockException);
            return false;
          } else {
            throw blockException;
          }
        }
        internalRequest.concreteIndex(
            observer
                .observedState()
                .metaData()
                .concreteSingleIndex(
                    internalRequest.request().index(), internalRequest.request().indicesOptions()));
        // check if we need to execute, and if not, return
        if (!resolveRequest(observer.observedState(), internalRequest, listener)) {
          return true;
        }
        blockException = checkRequestBlock(observer.observedState(), internalRequest);
        if (blockException != null) {
          if (blockException.retryable()) {
            retry(blockException);
            return false;
          } else {
            throw blockException;
          }
        }
        shardIt = shards(observer.observedState(), internalRequest);
      } catch (Throwable e) {
        listener.onFailure(e);
        return true;
      }

      // no shardIt, might be in the case between index gateway recovery and shardIt initialization
      if (shardIt.size() == 0) {
        retry(null);
        return false;
      }

      // this transport only make sense with an iterator that returns a single shard routing (like
      // primary)
      assert shardIt.size() == 1;

      ShardRouting shard = shardIt.nextOrNull();
      assert shard != null;

      if (!shard.active()) {
        retry(null);
        return false;
      }

      if (!operationStarted.compareAndSet(false, true)) {
        return true;
      }

      internalRequest.request().shardId = shardIt.shardId().id();
      if (shard.currentNodeId().equals(nodes.localNodeId())) {
        internalRequest.request().beforeLocalFork();
        try {
          threadPool
              .executor(executor)
              .execute(
                  new Runnable() {
                    @Override
                    public void run() {
                      try {
                        shardOperation(internalRequest, listener);
                      } catch (Throwable e) {
                        if (retryOnFailure(e)) {
                          operationStarted.set(false);
                          // we already marked it as started when we executed it (removed the
                          // listener) so pass false
                          // to re-add to the cluster listener
                          retry(null);
                        } else {
                          listener.onFailure(e);
                        }
                      }
                    }
                  });
        } catch (Throwable e) {
          if (retryOnFailure(e)) {
            retry(null);
          } else {
            listener.onFailure(e);
          }
        }
      } else {
        DiscoveryNode node = nodes.get(shard.currentNodeId());
        transportService.sendRequest(
            node,
            actionName,
            internalRequest.request(),
            transportOptions(),
            new BaseTransportResponseHandler<Response>() {

              @Override
              public Response newInstance() {
                return newResponse();
              }

              @Override
              public String executor() {
                return ThreadPool.Names.SAME;
              }

              @Override
              public void handleResponse(Response response) {
                listener.onResponse(response);
              }

              @Override
              public void handleException(TransportException exp) {
                // if we got disconnected from the node, or the node / shard is not in the right
                // state (being closed)
                if (exp.unwrapCause() instanceof ConnectTransportException
                    || exp.unwrapCause() instanceof NodeClosedException
                    || retryOnFailure(exp)) {
                  operationStarted.set(false);
                  // we already marked it as started when we executed it (removed the listener) so
                  // pass false
                  // to re-add to the cluster listener
                  retry(null);
                } else {
                  listener.onFailure(exp);
                }
              }
            });
      }
      return true;
    }
Ejemplo n.º 22
0
  /**
   * Tests corruption that happens on a single shard when no replicas are present. We make sure that
   * the primary stays unassigned and all other replicas for the healthy shards happens
   */
  public void testCorruptPrimaryNoReplica()
      throws ExecutionException, InterruptedException, IOException {
    int numDocs = scaledRandomIntBetween(100, 1000);
    internalCluster().ensureAtLeastNumDataNodes(2);

    assertAcked(
        prepareCreate("test")
            .setSettings(
                Settings.builder()
                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
                    .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)
                    .put(
                        MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(),
                        false) // no checkindex - we corrupt shards on purpose
                    .put(
                        IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(),
                        new ByteSizeValue(
                            1,
                            ByteSizeUnit
                                .PB)) // no translog based flush - it might change the .liv /
                // segments.N files
                ));
    ensureGreen();
    IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs];
    for (int i = 0; i < builders.length; i++) {
      builders[i] = client().prepareIndex("test", "type").setSource("field", "value");
    }
    indexRandom(true, builders);
    ensureGreen();
    assertAllSuccessful(
        client()
            .admin()
            .indices()
            .prepareFlush()
            .setForce(true)
            .setWaitIfOngoing(true)
            .execute()
            .actionGet());
    // we have to flush at least once here since we don't corrupt the translog
    SearchResponse countResponse = client().prepareSearch().setSize(0).get();
    assertHitCount(countResponse, numDocs);

    ShardRouting shardRouting = corruptRandomPrimaryFile();
    /*
     * we corrupted the primary shard - now lets make sure we never recover from it successfully
     */
    Settings build = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "1").build();
    client().admin().indices().prepareUpdateSettings("test").setSettings(build).get();
    client().admin().cluster().prepareReroute().get();

    boolean didClusterTurnRed =
        awaitBusy(
            () -> {
              ClusterHealthStatus test =
                  client()
                      .admin()
                      .cluster()
                      .health(Requests.clusterHealthRequest("test"))
                      .actionGet()
                      .getStatus();
              return test == ClusterHealthStatus.RED;
            },
            5,
            TimeUnit
                .MINUTES); // sometimes on slow nodes the replication / recovery is just dead slow
    final ClusterHealthResponse response =
        client().admin().cluster().health(Requests.clusterHealthRequest("test")).get();
    if (response.getStatus() != ClusterHealthStatus.RED) {
      logger.info("Cluster turned red in busy loop: {}", didClusterTurnRed);
      logger.info(
          "cluster state:\n{}\n{}",
          client().admin().cluster().prepareState().get().getState().prettyPrint(),
          client().admin().cluster().preparePendingClusterTasks().get().prettyPrint());
    }
    assertThat(response.getStatus(), is(ClusterHealthStatus.RED));
    ClusterState state = client().admin().cluster().prepareState().get().getState();
    GroupShardsIterator shardIterators =
        state.getRoutingTable().activePrimaryShardsGrouped(new String[] {"test"}, false);
    for (ShardIterator iterator : shardIterators) {
      ShardRouting routing;
      while ((routing = iterator.nextOrNull()) != null) {
        if (routing.getId() == shardRouting.getId()) {
          assertThat(routing.state(), equalTo(ShardRoutingState.UNASSIGNED));
        } else {
          assertThat(
              routing.state(),
              anyOf(equalTo(ShardRoutingState.RELOCATING), equalTo(ShardRoutingState.STARTED)));
        }
      }
    }
    final List<Path> files = listShardFiles(shardRouting);
    Path corruptedFile = null;
    for (Path file : files) {
      if (file.getFileName().toString().startsWith("corrupted_")) {
        corruptedFile = file;
        break;
      }
    }
    assertThat(corruptedFile, notNullValue());
  }
  @Override
  protected void doExecute(final Request request, final ActionListener<Response> listener) {
    ClusterState clusterState = clusterService.state();
    ClusterBlockException blockException = checkGlobalBlock(clusterState, request);
    if (blockException != null) {
      throw blockException;
    }
    // update to concrete index
    request.index(
        clusterState.metaData().concreteSingleIndex(request.index(), request.indicesOptions()));
    blockException = checkRequestBlock(clusterState, request);
    if (blockException != null) {
      throw blockException;
    }

    GroupShardsIterator groups;
    try {
      groups = shards(request);
    } catch (Throwable e) {
      listener.onFailure(e);
      return;
    }
    final AtomicInteger indexCounter = new AtomicInteger();
    final AtomicInteger failureCounter = new AtomicInteger();
    final AtomicInteger completionCounter = new AtomicInteger(groups.size());
    final AtomicReferenceArray<ShardActionResult> shardsResponses =
        new AtomicReferenceArray<>(groups.size());

    for (final ShardIterator shardIt : groups) {
      ShardRequest shardRequest = newShardRequestInstance(request, shardIt.shardId().id());

      // TODO for now, we fork operations on shardIt of the index
      shardRequest.beforeLocalFork(); // optimize for local fork
      shardRequest.operationThreaded(true);

      // no need for threaded listener, we will fork when its done based on the index request
      shardRequest.listenerThreaded(false);
      shardAction.execute(
          shardRequest,
          new ActionListener<ShardResponse>() {
            @Override
            public void onResponse(ShardResponse result) {
              shardsResponses.set(indexCounter.getAndIncrement(), new ShardActionResult(result));
              returnIfNeeded();
            }

            @Override
            public void onFailure(Throwable e) {
              failureCounter.getAndIncrement();
              int index = indexCounter.getAndIncrement();
              if (accumulateExceptions()) {
                shardsResponses.set(
                    index,
                    new ShardActionResult(
                        new DefaultShardOperationFailedException(
                            request.index, shardIt.shardId().id(), e)));
              }
              returnIfNeeded();
            }

            private void returnIfNeeded() {
              if (completionCounter.decrementAndGet() == 0) {
                List<ShardResponse> responses = Lists.newArrayList();
                List<ShardOperationFailedException> failures = Lists.newArrayList();
                for (int i = 0; i < shardsResponses.length(); i++) {
                  ShardActionResult shardActionResult = shardsResponses.get(i);
                  if (shardActionResult == null) {
                    assert !accumulateExceptions();
                    continue;
                  }
                  if (shardActionResult.isFailure()) {
                    assert accumulateExceptions() && shardActionResult.shardFailure != null;
                    failures.add(shardActionResult.shardFailure);
                  } else {
                    responses.add(shardActionResult.shardResponse);
                  }
                }

                assert failures.size() == 0 || failures.size() == failureCounter.get();
                listener.onResponse(
                    newResponseInstance(request, responses, failureCounter.get(), failures));
              }
            }
          });
    }
  }
    void performReplicas(final PrimaryResponse<Response, ReplicaRequest> response) {
      if (ignoreReplicas() || shardIt.size() == 1 /* no replicas */) {
        postPrimaryOperation(request, response);
        listener.onResponse(response.response());
        return;
      }

      // initialize the counter
      int replicaCounter = shardIt.assignedReplicasIncludingRelocating();

      if (replicaCounter == 0) {
        postPrimaryOperation(request, response);
        listener.onResponse(response.response());
        return;
      }

      if (replicationType == ReplicationType.ASYNC) {
        postPrimaryOperation(request, response);
        // async replication, notify the listener
        listener.onResponse(response.response());
        // now, trick the counter so it won't decrease to 0 and notify the listeners
        replicaCounter = Integer.MIN_VALUE;
      }

      // we add one to the replica count to do the postPrimaryOperation
      replicaCounter++;

      AtomicInteger counter = new AtomicInteger(replicaCounter);
      shardIt.reset(); // reset the iterator
      ShardRouting shard;
      while ((shard = shardIt.nextOrNull()) != null) {
        // if its unassigned, nothing to do here...
        if (shard.unassigned()) {
          continue;
        }

        // if the shard is primary and relocating, add one to the counter since we perform it on the
        // replica as well
        // (and we already did it on the primary)
        boolean doOnlyOnRelocating = false;
        if (shard.primary()) {
          if (shard.relocating()) {
            doOnlyOnRelocating = true;
          } else {
            continue;
          }
        }
        // we index on a replica that is initializing as well since we might not have got the event
        // yet that it was started. We will get an exception IllegalShardState exception if its not
        // started
        // and that's fine, we will ignore it
        if (!doOnlyOnRelocating) {
          performOnReplica(response, counter, shard, shard.currentNodeId());
        }
        if (shard.relocating()) {
          performOnReplica(response, counter, shard, shard.relocatingNodeId());
        }
      }

      // now do the postPrimary operation, and check if the listener needs to be invoked
      postPrimaryOperation(request, response);
      // we also invoke here in case replicas finish before postPrimaryAction does
      if (counter.decrementAndGet() == 0) {
        listener.onResponse(response.response());
      }
    }
    ASyncAction(
        MultiPercolateRequest multiPercolateRequest,
        List<Object> percolateRequests,
        ActionListener<MultiPercolateResponse> finalListener,
        ClusterState clusterState) {
      this.finalListener = finalListener;
      this.multiPercolateRequest = multiPercolateRequest;
      this.percolateRequests = percolateRequests;
      responsesByItemAndShard = new AtomicReferenceArray<>(percolateRequests.size());
      expectedOperationsPerItem = new AtomicReferenceArray<>(percolateRequests.size());
      reducedResponses = new AtomicArray<>(percolateRequests.size());

      // Resolving concrete indices and routing and grouping the requests by shard
      requestsByShard = new HashMap<>();
      // Keep track what slots belong to what shard, in case a request to a shard fails on all
      // copies
      shardToSlots = new HashMap<>();
      int expectedResults = 0;
      for (int slot = 0; slot < percolateRequests.size(); slot++) {
        Object element = percolateRequests.get(slot);
        assert element != null;
        if (element instanceof PercolateRequest) {
          PercolateRequest percolateRequest = (PercolateRequest) element;
          String[] concreteIndices;
          try {
            concreteIndices =
                indexNameExpressionResolver.concreteIndices(clusterState, percolateRequest);
          } catch (IndexNotFoundException e) {
            reducedResponses.set(slot, e);
            responsesByItemAndShard.set(slot, new AtomicReferenceArray(0));
            expectedOperationsPerItem.set(slot, new AtomicInteger(0));
            continue;
          }
          Map<String, Set<String>> routing =
              indexNameExpressionResolver.resolveSearchRouting(
                  clusterState, percolateRequest.routing(), percolateRequest.indices());
          // TODO: I only need shardIds, ShardIterator(ShardRouting) is only needed in
          // TransportShardMultiPercolateAction
          GroupShardsIterator shards =
              clusterService
                  .operationRouting()
                  .searchShards(
                      clusterState, concreteIndices, routing, percolateRequest.preference());
          if (shards.size() == 0) {
            reducedResponses.set(slot, new UnavailableShardsException(null, "No shards available"));
            responsesByItemAndShard.set(slot, new AtomicReferenceArray(0));
            expectedOperationsPerItem.set(slot, new AtomicInteger(0));
            continue;
          }

          // The shard id is used as index in the atomic ref array, so we need to find out how many
          // shards there are regardless of routing:
          int numShards =
              clusterService
                  .operationRouting()
                  .searchShardsCount(clusterState, concreteIndices, null);
          responsesByItemAndShard.set(slot, new AtomicReferenceArray(numShards));
          expectedOperationsPerItem.set(slot, new AtomicInteger(shards.size()));
          for (ShardIterator shard : shards) {
            ShardId shardId = shard.shardId();
            TransportShardMultiPercolateAction.Request requests = requestsByShard.get(shardId);
            if (requests == null) {
              requestsByShard.put(
                  shardId,
                  requests =
                      new TransportShardMultiPercolateAction.Request(
                          multiPercolateRequest,
                          shardId.getIndex(),
                          shardId.getId(),
                          percolateRequest.preference()));
            }
            logger.trace("Adding shard[{}] percolate request for item[{}]", shardId, slot);
            requests.add(
                new TransportShardMultiPercolateAction.Request.Item(
                    slot, new PercolateShardRequest(shardId, percolateRequest)));

            IntArrayList items = shardToSlots.get(shardId);
            if (items == null) {
              shardToSlots.put(shardId, items = new IntArrayList());
            }
            items.add(slot);
          }
          expectedResults++;
        } else if (element instanceof Throwable || element instanceof MultiGetResponse.Failure) {
          logger.trace("item[{}] won't be executed, reason: {}", slot, element);
          reducedResponses.set(slot, element);
          responsesByItemAndShard.set(slot, new AtomicReferenceArray(0));
          expectedOperationsPerItem.set(slot, new AtomicInteger(0));
        }
      }
      expectedOperations = new AtomicInteger(expectedResults);
    }
 void performOperation(final ShardIterator shardIt, int shardIndex, boolean localAsync) {
   performOperation(shardIt, shardIt.nextOrNull(), shardIndex, localAsync);
 }
    void performOperation(
        final ShardIterator shardIt,
        final ShardRouting shard,
        final int shardIndex,
        boolean localAsync) {
      if (shard == null) {
        // no more active shards... (we should not really get here, just safety)
        onOperation(
            null, shardIt, shardIndex, new NoShardAvailableActionException(shardIt.shardId()));
      } else {
        try {
          final ShardRequest shardRequest = newShardRequest(shard, request);
          if (shard.currentNodeId().equals(nodes.localNodeId())) {
            if (localAsync) {
              threadPool
                  .executor(executor)
                  .execute(
                      new Runnable() {
                        @Override
                        public void run() {
                          try {
                            onOperation(shard, shardIndex, shardOperation(shardRequest));
                          } catch (Throwable e) {
                            onOperation(shard, shardIt, shardIndex, e);
                          }
                        }
                      });
            } else {
              onOperation(shard, shardIndex, shardOperation(shardRequest));
            }
          } else {
            DiscoveryNode node = nodes.get(shard.currentNodeId());
            if (node == null) {
              // no node connected, act as failure
              onOperation(
                  shard,
                  shardIt,
                  shardIndex,
                  new NoShardAvailableActionException(shardIt.shardId()));
            } else {
              transportService.sendRequest(
                  node,
                  transportShardAction,
                  shardRequest,
                  new BaseTransportResponseHandler<ShardResponse>() {
                    @Override
                    public ShardResponse newInstance() {
                      return newShardResponse();
                    }

                    @Override
                    public String executor() {
                      return ThreadPool.Names.SAME;
                    }

                    @Override
                    public void handleResponse(ShardResponse response) {
                      onOperation(shard, shardIndex, response);
                    }

                    @Override
                    public void handleException(TransportException e) {
                      onOperation(shard, shardIt, shardIndex, e);
                    }
                  });
            }
          }
        } catch (Throwable e) {
          onOperation(shard, shardIt, shardIndex, e);
        }
      }
    }
    protected void doStart() {
      nodes = observer.observedState().nodes();
      try {
        ClusterBlockException blockException = checkGlobalBlock(observer.observedState());
        if (blockException != null) {
          if (blockException.retryable()) {
            retry(blockException);
            return;
          } else {
            throw blockException;
          }
        }
        request.concreteIndex(
            indexNameExpressionResolver
                .concreteSingleIndex(observer.observedState(), request)
                .getName());
        resolveRequest(observer.observedState(), request);
        blockException = checkRequestBlock(observer.observedState(), request);
        if (blockException != null) {
          if (blockException.retryable()) {
            retry(blockException);
            return;
          } else {
            throw blockException;
          }
        }
        shardIt = shards(observer.observedState(), request);
      } catch (Throwable e) {
        listener.onFailure(e);
        return;
      }

      // no shardIt, might be in the case between index gateway recovery and shardIt initialization
      if (shardIt.size() == 0) {
        retry(null);
        return;
      }

      // this transport only make sense with an iterator that returns a single shard routing (like
      // primary)
      assert shardIt.size() == 1;

      ShardRouting shard = shardIt.nextOrNull();
      assert shard != null;

      if (!shard.active()) {
        retry(null);
        return;
      }

      request.shardId = shardIt.shardId();
      DiscoveryNode node = nodes.get(shard.currentNodeId());
      transportService.sendRequest(
          node,
          shardActionName,
          request,
          transportOptions(),
          new BaseTransportResponseHandler<Response>() {

            @Override
            public Response newInstance() {
              return newResponse();
            }

            @Override
            public String executor() {
              return ThreadPool.Names.SAME;
            }

            @Override
            public void handleResponse(Response response) {
              listener.onResponse(response);
            }

            @Override
            public void handleException(TransportException exp) {
              Throwable cause = exp.unwrapCause();
              // if we got disconnected from the node, or the node / shard is not in the right state
              // (being closed)
              if (cause instanceof ConnectTransportException
                  || cause instanceof NodeClosedException
                  || retryOnFailure(exp)) {
                retry(cause);
              } else {
                listener.onFailure(exp);
              }
            }
          });
    }
  private void onFirstPhaseResult(
      final int shardIndex,
      @Nullable ShardRouting shard,
      @Nullable String nodeId,
      final ShardIterator shardIt,
      Exception e) {
    // we always add the shard failure for a specific shard instance
    // we do make sure to clean it on a successful response from a shard
    SearchShardTarget shardTarget = new SearchShardTarget(nodeId, shardIt.shardId());
    addShardFailure(shardIndex, shardTarget, e);

    if (totalOps.incrementAndGet() == expectedTotalOps) {
      if (logger.isDebugEnabled()) {
        if (e != null && !TransportActions.isShardNotAvailableException(e)) {
          logger.debug(
              (Supplier<?>)
                  () ->
                      new ParameterizedMessage(
                          "{}: Failed to execute [{}]",
                          shard != null ? shard.shortSummary() : shardIt.shardId(),
                          request),
              e);
        } else if (logger.isTraceEnabled()) {
          logger.trace(
              (Supplier<?>)
                  () -> new ParameterizedMessage("{}: Failed to execute [{}]", shard, request),
              e);
        }
      }
      final ShardSearchFailure[] shardSearchFailures = buildShardFailures();
      if (successfulOps.get() == 0) {
        if (logger.isDebugEnabled()) {
          logger.debug(
              (Supplier<?>)
                  () ->
                      new ParameterizedMessage(
                          "All shards failed for phase: [{}]", firstPhaseName()),
              e);
        }

        // no successful ops, raise an exception
        raiseEarlyFailure(
            new SearchPhaseExecutionException(
                firstPhaseName(), "all shards failed", e, shardSearchFailures));
      } else {
        try {
          innerMoveToSecondPhase();
        } catch (Exception inner) {
          inner.addSuppressed(e);
          raiseEarlyFailure(
              new ReduceSearchPhaseException(firstPhaseName(), "", inner, shardSearchFailures));
        }
      }
    } else {
      final ShardRouting nextShard = shardIt.nextOrNull();
      final boolean lastShard = nextShard == null;
      // trace log this exception
      logger.trace(
          (Supplier<?>)
              () ->
                  new ParameterizedMessage(
                      "{}: Failed to execute [{}] lastShard [{}]",
                      shard != null ? shard.shortSummary() : shardIt.shardId(),
                      request,
                      lastShard),
          e);
      if (!lastShard) {
        try {
          performFirstPhase(shardIndex, shardIt, nextShard);
        } catch (Exception inner) {
          inner.addSuppressed(e);
          onFirstPhaseResult(shardIndex, shard, shard.currentNodeId(), shardIt, inner);
        }
      } else {
        // no more shards active, add a failure
        if (logger.isDebugEnabled()
            && !logger.isTraceEnabled()) { // do not double log this exception
          if (e != null && !TransportActions.isShardNotAvailableException(e)) {
            logger.debug(
                (Supplier<?>)
                    () ->
                        new ParameterizedMessage(
                            "{}: Failed to execute [{}] lastShard [{}]",
                            shard != null ? shard.shortSummary() : shardIt.shardId(),
                            request,
                            lastShard),
                e);
          }
        }
      }
    }
  }
  private void executeBulk(
      final BulkRequest bulkRequest,
      final long startTime,
      final ActionListener<BulkResponse> listener,
      final AtomicArray<BulkItemResponse> responses) {
    final ClusterState clusterState = clusterService.state();
    // TODO use timeout to wait here if its blocked...
    clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.WRITE);

    final ConcreteIndices concreteIndices =
        new ConcreteIndices(clusterState, indexNameExpressionResolver);
    MetaData metaData = clusterState.metaData();
    for (int i = 0; i < bulkRequest.requests.size(); i++) {
      ActionRequest request = bulkRequest.requests.get(i);
      if (request instanceof DocumentRequest) {
        DocumentRequest req = (DocumentRequest) request;

        if (addFailureIfIndexIsUnavailable(
            req, bulkRequest, responses, i, concreteIndices, metaData)) {
          continue;
        }

        String concreteIndex = concreteIndices.resolveIfAbsent(req);
        if (request instanceof IndexRequest) {
          IndexRequest indexRequest = (IndexRequest) request;
          MappingMetaData mappingMd = null;
          if (metaData.hasIndex(concreteIndex)) {
            mappingMd = metaData.index(concreteIndex).mappingOrDefault(indexRequest.type());
          }
          try {
            indexRequest.process(metaData, mappingMd, allowIdGeneration, concreteIndex);
          } catch (ElasticsearchParseException | RoutingMissingException e) {
            BulkItemResponse.Failure failure =
                new BulkItemResponse.Failure(
                    concreteIndex, indexRequest.type(), indexRequest.id(), e);
            BulkItemResponse bulkItemResponse = new BulkItemResponse(i, "index", failure);
            responses.set(i, bulkItemResponse);
            // make sure the request gets never processed again
            bulkRequest.requests.set(i, null);
          }
        } else {
          concreteIndices.resolveIfAbsent(req);
          req.routing(
              clusterState
                  .metaData()
                  .resolveIndexRouting(req.parent(), req.routing(), req.index()));
        }
      }
    }

    // first, go over all the requests and create a ShardId -> Operations mapping
    Map<ShardId, List<BulkItemRequest>> requestsByShard = new HashMap<>();

    for (int i = 0; i < bulkRequest.requests.size(); i++) {
      ActionRequest request = bulkRequest.requests.get(i);
      if (request instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) request;
        String concreteIndex = concreteIndices.getConcreteIndex(indexRequest.index());
        ShardId shardId =
            clusterService
                .operationRouting()
                .indexShards(
                    clusterState,
                    concreteIndex,
                    indexRequest.type(),
                    indexRequest.id(),
                    indexRequest.routing())
                .shardId();
        List<BulkItemRequest> list = requestsByShard.get(shardId);
        if (list == null) {
          list = new ArrayList<>();
          requestsByShard.put(shardId, list);
        }
        list.add(new BulkItemRequest(i, request));
      } else if (request instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) request;
        String concreteIndex = concreteIndices.getConcreteIndex(deleteRequest.index());
        MappingMetaData mappingMd =
            clusterState.metaData().index(concreteIndex).mappingOrDefault(deleteRequest.type());
        if (mappingMd != null
            && mappingMd.routing().required()
            && deleteRequest.routing() == null) {
          // if routing is required, and no routing on the delete request, we need to broadcast
          // it....
          GroupShardsIterator groupShards =
              clusterService.operationRouting().broadcastDeleteShards(clusterState, concreteIndex);
          for (ShardIterator shardIt : groupShards) {
            List<BulkItemRequest> list = requestsByShard.get(shardIt.shardId());
            if (list == null) {
              list = new ArrayList<>();
              requestsByShard.put(shardIt.shardId(), list);
            }
            list.add(new BulkItemRequest(i, deleteRequest));
          }
        } else {
          ShardId shardId =
              clusterService
                  .operationRouting()
                  .indexShards(
                      clusterState,
                      concreteIndex,
                      deleteRequest.type(),
                      deleteRequest.id(),
                      deleteRequest.routing())
                  .shardId();
          List<BulkItemRequest> list = requestsByShard.get(shardId);
          if (list == null) {
            list = new ArrayList<>();
            requestsByShard.put(shardId, list);
          }
          list.add(new BulkItemRequest(i, request));
        }
      } else if (request instanceof UpdateRequest) {
        UpdateRequest updateRequest = (UpdateRequest) request;
        String concreteIndex = concreteIndices.getConcreteIndex(updateRequest.index());
        MappingMetaData mappingMd =
            clusterState.metaData().index(concreteIndex).mappingOrDefault(updateRequest.type());
        if (mappingMd != null
            && mappingMd.routing().required()
            && updateRequest.routing() == null) {
          BulkItemResponse.Failure failure =
              new BulkItemResponse.Failure(
                  updateRequest.index(),
                  updateRequest.type(),
                  updateRequest.id(),
                  new IllegalArgumentException("routing is required for this item"));
          responses.set(i, new BulkItemResponse(i, updateRequest.type(), failure));
          continue;
        }
        ShardId shardId =
            clusterService
                .operationRouting()
                .indexShards(
                    clusterState,
                    concreteIndex,
                    updateRequest.type(),
                    updateRequest.id(),
                    updateRequest.routing())
                .shardId();
        List<BulkItemRequest> list = requestsByShard.get(shardId);
        if (list == null) {
          list = new ArrayList<>();
          requestsByShard.put(shardId, list);
        }
        list.add(new BulkItemRequest(i, request));
      }
    }

    if (requestsByShard.isEmpty()) {
      listener.onResponse(
          new BulkResponse(
              responses.toArray(new BulkItemResponse[responses.length()]),
              buildTookInMillis(startTime)));
      return;
    }

    final AtomicInteger counter = new AtomicInteger(requestsByShard.size());
    for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) {
      final ShardId shardId = entry.getKey();
      final List<BulkItemRequest> requests = entry.getValue();
      BulkShardRequest bulkShardRequest =
          new BulkShardRequest(
              bulkRequest,
              shardId,
              bulkRequest.refresh(),
              requests.toArray(new BulkItemRequest[requests.size()]));
      bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel());
      bulkShardRequest.timeout(bulkRequest.timeout());
      shardBulkAction.execute(
          bulkShardRequest,
          new ActionListener<BulkShardResponse>() {
            @Override
            public void onResponse(BulkShardResponse bulkShardResponse) {
              for (BulkItemResponse bulkItemResponse : bulkShardResponse.getResponses()) {
                // we may have no response if item failed
                if (bulkItemResponse.getResponse() != null) {
                  bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo());
                }
                responses.set(bulkItemResponse.getItemId(), bulkItemResponse);
              }
              if (counter.decrementAndGet() == 0) {
                finishHim();
              }
            }

            @Override
            public void onFailure(Throwable e) {
              // create failures for all relevant requests
              for (BulkItemRequest request : requests) {
                if (request.request() instanceof IndexRequest) {
                  IndexRequest indexRequest = (IndexRequest) request.request();
                  responses.set(
                      request.id(),
                      new BulkItemResponse(
                          request.id(),
                          indexRequest.opType().toString().toLowerCase(Locale.ENGLISH),
                          new BulkItemResponse.Failure(
                              concreteIndices.getConcreteIndex(indexRequest.index()),
                              indexRequest.type(),
                              indexRequest.id(),
                              e)));
                } else if (request.request() instanceof DeleteRequest) {
                  DeleteRequest deleteRequest = (DeleteRequest) request.request();
                  responses.set(
                      request.id(),
                      new BulkItemResponse(
                          request.id(),
                          "delete",
                          new BulkItemResponse.Failure(
                              concreteIndices.getConcreteIndex(deleteRequest.index()),
                              deleteRequest.type(),
                              deleteRequest.id(),
                              e)));
                } else if (request.request() instanceof UpdateRequest) {
                  UpdateRequest updateRequest = (UpdateRequest) request.request();
                  responses.set(
                      request.id(),
                      new BulkItemResponse(
                          request.id(),
                          "update",
                          new BulkItemResponse.Failure(
                              concreteIndices.getConcreteIndex(updateRequest.index()),
                              updateRequest.type(),
                              updateRequest.id(),
                              e)));
                }
              }
              if (counter.decrementAndGet() == 0) {
                finishHim();
              }
            }

            private void finishHim() {
              listener.onResponse(
                  new BulkResponse(
                      responses.toArray(new BulkItemResponse[responses.length()]),
                      buildTookInMillis(startTime)));
            }
          });
    }
  }