コード例 #1
0
 protected AbstractSearchAsyncAction(
     Logger logger,
     SearchTransportService searchTransportService,
     Function<String, DiscoveryNode> nodeIdToDiscoveryNode,
     Map<String, AliasFilter> aliasFilter,
     Executor executor,
     SearchRequest request,
     ActionListener<SearchResponse> listener,
     GroupShardsIterator shardsIts,
     long startTime,
     long clusterStateVersion,
     SearchTask task) {
   super(startTime);
   this.logger = logger;
   this.searchTransportService = searchTransportService;
   this.executor = executor;
   this.request = request;
   this.task = task;
   this.listener = listener;
   this.nodeIdToDiscoveryNode = nodeIdToDiscoveryNode;
   this.clusterStateVersion = clusterStateVersion;
   this.shardsIts = shardsIts;
   expectedSuccessfulOps = shardsIts.size();
   // we need to add 1 for non active partition, since we count it in the total!
   expectedTotalOps = shardsIts.totalSizeWith1ForEmpty();
   firstResults = new AtomicArray<>(shardsIts.size());
   this.aliasFilter = aliasFilter;
 }
コード例 #2
0
 public void start() {
   if (shardsIts.size() == 0) {
     // no shards
     try {
       listener.onResponse(newResponse(request, new AtomicReferenceArray(0), clusterState));
     } catch (Throwable e) {
       listener.onFailure(e);
     }
     return;
   }
   request.beforeStart();
   // count the local operations, and perform the non local ones
   int shardIndex = -1;
   for (final ShardIterator shardIt : shardsIts) {
     shardIndex++;
     final ShardRouting shard = shardIt.nextOrNull();
     if (shard != null) {
       performOperation(shardIt, shard, shardIndex);
     } else {
       // really, no shards active in this group
       onOperation(
           null, shardIt, shardIndex, new NoShardAvailableActionException(shardIt.shardId()));
     }
   }
 }
コード例 #3
0
    protected AsyncBroadcastAction(Request request, ActionListener<Response> listener) {
      this.request = request;
      this.listener = listener;

      clusterState = clusterService.state();

      ClusterBlockException blockException = checkGlobalBlock(clusterState, request);
      if (blockException != null) {
        throw blockException;
      }
      // update to concrete indices
      String[] concreteIndices =
          clusterState.metaData().concreteIndices(request.indicesOptions(), request.indices());
      blockException = checkRequestBlock(clusterState, request, concreteIndices);
      if (blockException != null) {
        throw blockException;
      }

      nodes = clusterState.nodes();
      logger.trace("resolving shards based on cluster state version [{}]", clusterState.version());
      shardsIts = shards(clusterState, request, concreteIndices);
      expectedOps = shardsIts.size();

      shardsResponses = new AtomicReferenceArray<Object>(expectedOps);
    }
コード例 #4
0
  protected final void addShardFailure(
      final int shardIndex, @Nullable SearchShardTarget shardTarget, Exception e) {
    // we don't aggregate shard failures on non active shards (but do keep the header counts right)
    if (TransportActions.isShardNotAvailableException(e)) {
      return;
    }

    // lazily create shard failures, so we can early build the empty shard failure list in most
    // cases (no failures)
    if (shardFailures == null) {
      synchronized (shardFailuresMutex) {
        if (shardFailures == null) {
          shardFailures = new AtomicArray<>(shardsIts.size());
        }
      }
    }
    ShardSearchFailure failure = shardFailures.get(shardIndex);
    if (failure == null) {
      shardFailures.set(shardIndex, new ShardSearchFailure(e, shardTarget));
    } else {
      // the failure is already present, try and not override it with an exception that is less
      // meaningless
      // for example, getting illegal shard state
      if (TransportActions.isReadOverrideException(e)) {
        shardFailures.set(shardIndex, new ShardSearchFailure(e, shardTarget));
      }
    }
  }
コード例 #5
0
  void performFirstPhase(
      final int shardIndex, final ShardIterator shardIt, final ShardRouting shard) {
    if (shard == null) {
      // no more active shards... (we should not really get here, but just for safety)
      onFirstPhaseResult(
          shardIndex, null, null, shardIt, new NoShardAvailableActionException(shardIt.shardId()));
    } else {
      final DiscoveryNode node = nodeIdToDiscoveryNode.apply(shard.currentNodeId());
      if (node == null) {
        onFirstPhaseResult(
            shardIndex,
            shard,
            null,
            shardIt,
            new NoShardAvailableActionException(shardIt.shardId()));
      } else {
        AliasFilter filter = this.aliasFilter.get(shard.index().getUUID());
        assert filter != null;
        ShardSearchTransportRequest transportRequest =
            new ShardSearchTransportRequest(
                request, shardIt.shardId(), shardsIts.size(), filter, startTime());
        sendExecuteFirstPhase(
            node,
            transportRequest,
            new ActionListener<FirstResult>() {
              @Override
              public void onResponse(FirstResult result) {
                onFirstPhaseResult(shardIndex, shard.currentNodeId(), result, shardIt);
              }

              @Override
              public void onFailure(Exception t) {
                onFirstPhaseResult(shardIndex, shard, node.getId(), shardIt, t);
              }
            });
      }
    }
  }
コード例 #6
0
 public void start() {
   if (shardsIts.size() == 0) {
     // no shards
     try {
       listener.onResponse(newResponse(request, new AtomicReferenceArray(0), clusterState));
     } catch (Throwable e) {
       listener.onFailure(e);
     }
   }
   request.beforeStart();
   // count the local operations, and perform the non local ones
   int localOperations = 0;
   int shardIndex = -1;
   for (final ShardIterator shardIt : shardsIts) {
     shardIndex++;
     final ShardRouting shard = shardIt.firstOrNull();
     if (shard != null) {
       if (shard.currentNodeId().equals(nodes.localNodeId())) {
         localOperations++;
       } else {
         // do the remote operation here, the localAsync flag is not relevant
         performOperation(shardIt, shardIndex, true);
       }
     } else {
       // really, no shards active in this group
       onOperation(
           null, shardIt, shardIndex, new NoShardAvailableActionException(shardIt.shardId()));
     }
   }
   // we have local operations, perform them now
   if (localOperations > 0) {
     if (request.operationThreading() == BroadcastOperationThreading.SINGLE_THREAD) {
       request.beforeLocalFork();
       threadPool
           .executor(executor)
           .execute(
               new Runnable() {
                 @Override
                 public void run() {
                   int shardIndex = -1;
                   for (final ShardIterator shardIt : shardsIts) {
                     shardIndex++;
                     final ShardRouting shard = shardIt.firstOrNull();
                     if (shard != null) {
                       if (shard.currentNodeId().equals(nodes.localNodeId())) {
                         performOperation(shardIt, shardIndex, false);
                       }
                     }
                   }
                 }
               });
     } else {
       boolean localAsync =
           request.operationThreading() == BroadcastOperationThreading.THREAD_PER_SHARD;
       if (localAsync) {
         request.beforeLocalFork();
       }
       shardIndex = -1;
       for (final ShardIterator shardIt : shardsIts) {
         shardIndex++;
         final ShardRouting shard = shardIt.firstOrNull();
         if (shard != null) {
           if (shard.currentNodeId().equals(nodes.localNodeId())) {
             performOperation(shardIt, shardIndex, localAsync);
           }
         }
       }
     }
   }
 }
コード例 #7
0
 private int numShards(String... index) {
   ClusterState state = client().admin().cluster().prepareState().get().getState();
   GroupShardsIterator shardIterators =
       state.getRoutingTable().activePrimaryShardsGrouped(index, false);
   return shardIterators.size();
 }
コード例 #8
0
  @Override
  protected void doExecute(final Request request, final ActionListener<Response> listener) {
    ClusterState clusterState = clusterService.state();
    ClusterBlockException blockException = checkGlobalBlock(clusterState, request);
    if (blockException != null) {
      throw blockException;
    }
    // update to concrete index
    request.index(
        clusterState.metaData().concreteSingleIndex(request.index(), request.indicesOptions()));
    blockException = checkRequestBlock(clusterState, request);
    if (blockException != null) {
      throw blockException;
    }

    GroupShardsIterator groups;
    try {
      groups = shards(request);
    } catch (Throwable e) {
      listener.onFailure(e);
      return;
    }
    final AtomicInteger indexCounter = new AtomicInteger();
    final AtomicInteger failureCounter = new AtomicInteger();
    final AtomicInteger completionCounter = new AtomicInteger(groups.size());
    final AtomicReferenceArray<ShardActionResult> shardsResponses =
        new AtomicReferenceArray<>(groups.size());

    for (final ShardIterator shardIt : groups) {
      ShardRequest shardRequest = newShardRequestInstance(request, shardIt.shardId().id());

      // TODO for now, we fork operations on shardIt of the index
      shardRequest.beforeLocalFork(); // optimize for local fork
      shardRequest.operationThreaded(true);

      // no need for threaded listener, we will fork when its done based on the index request
      shardRequest.listenerThreaded(false);
      shardAction.execute(
          shardRequest,
          new ActionListener<ShardResponse>() {
            @Override
            public void onResponse(ShardResponse result) {
              shardsResponses.set(indexCounter.getAndIncrement(), new ShardActionResult(result));
              returnIfNeeded();
            }

            @Override
            public void onFailure(Throwable e) {
              failureCounter.getAndIncrement();
              int index = indexCounter.getAndIncrement();
              if (accumulateExceptions()) {
                shardsResponses.set(
                    index,
                    new ShardActionResult(
                        new DefaultShardOperationFailedException(
                            request.index, shardIt.shardId().id(), e)));
              }
              returnIfNeeded();
            }

            private void returnIfNeeded() {
              if (completionCounter.decrementAndGet() == 0) {
                List<ShardResponse> responses = Lists.newArrayList();
                List<ShardOperationFailedException> failures = Lists.newArrayList();
                for (int i = 0; i < shardsResponses.length(); i++) {
                  ShardActionResult shardActionResult = shardsResponses.get(i);
                  if (shardActionResult == null) {
                    assert !accumulateExceptions();
                    continue;
                  }
                  if (shardActionResult.isFailure()) {
                    assert accumulateExceptions() && shardActionResult.shardFailure != null;
                    failures.add(shardActionResult.shardFailure);
                  } else {
                    responses.add(shardActionResult.shardResponse);
                  }
                }

                assert failures.size() == 0 || failures.size() == failureCounter.get();
                listener.onResponse(
                    newResponseInstance(request, responses, failureCounter.get(), failures));
              }
            }
          });
    }
  }
コード例 #9
0
    ASyncAction(
        MultiPercolateRequest multiPercolateRequest,
        List<Object> percolateRequests,
        ActionListener<MultiPercolateResponse> finalListener,
        ClusterState clusterState) {
      this.finalListener = finalListener;
      this.multiPercolateRequest = multiPercolateRequest;
      this.percolateRequests = percolateRequests;
      responsesByItemAndShard = new AtomicReferenceArray<>(percolateRequests.size());
      expectedOperationsPerItem = new AtomicReferenceArray<>(percolateRequests.size());
      reducedResponses = new AtomicArray<>(percolateRequests.size());

      // Resolving concrete indices and routing and grouping the requests by shard
      requestsByShard = new HashMap<>();
      // Keep track what slots belong to what shard, in case a request to a shard fails on all
      // copies
      shardToSlots = new HashMap<>();
      int expectedResults = 0;
      for (int slot = 0; slot < percolateRequests.size(); slot++) {
        Object element = percolateRequests.get(slot);
        assert element != null;
        if (element instanceof PercolateRequest) {
          PercolateRequest percolateRequest = (PercolateRequest) element;
          String[] concreteIndices;
          try {
            concreteIndices =
                indexNameExpressionResolver.concreteIndices(clusterState, percolateRequest);
          } catch (IndexNotFoundException e) {
            reducedResponses.set(slot, e);
            responsesByItemAndShard.set(slot, new AtomicReferenceArray(0));
            expectedOperationsPerItem.set(slot, new AtomicInteger(0));
            continue;
          }
          Map<String, Set<String>> routing =
              indexNameExpressionResolver.resolveSearchRouting(
                  clusterState, percolateRequest.routing(), percolateRequest.indices());
          // TODO: I only need shardIds, ShardIterator(ShardRouting) is only needed in
          // TransportShardMultiPercolateAction
          GroupShardsIterator shards =
              clusterService
                  .operationRouting()
                  .searchShards(
                      clusterState, concreteIndices, routing, percolateRequest.preference());
          if (shards.size() == 0) {
            reducedResponses.set(slot, new UnavailableShardsException(null, "No shards available"));
            responsesByItemAndShard.set(slot, new AtomicReferenceArray(0));
            expectedOperationsPerItem.set(slot, new AtomicInteger(0));
            continue;
          }

          // The shard id is used as index in the atomic ref array, so we need to find out how many
          // shards there are regardless of routing:
          int numShards =
              clusterService
                  .operationRouting()
                  .searchShardsCount(clusterState, concreteIndices, null);
          responsesByItemAndShard.set(slot, new AtomicReferenceArray(numShards));
          expectedOperationsPerItem.set(slot, new AtomicInteger(shards.size()));
          for (ShardIterator shard : shards) {
            ShardId shardId = shard.shardId();
            TransportShardMultiPercolateAction.Request requests = requestsByShard.get(shardId);
            if (requests == null) {
              requestsByShard.put(
                  shardId,
                  requests =
                      new TransportShardMultiPercolateAction.Request(
                          multiPercolateRequest,
                          shardId.getIndex(),
                          shardId.getId(),
                          percolateRequest.preference()));
            }
            logger.trace("Adding shard[{}] percolate request for item[{}]", shardId, slot);
            requests.add(
                new TransportShardMultiPercolateAction.Request.Item(
                    slot, new PercolateShardRequest(shardId, percolateRequest)));

            IntArrayList items = shardToSlots.get(shardId);
            if (items == null) {
              shardToSlots.put(shardId, items = new IntArrayList());
            }
            items.add(slot);
          }
          expectedResults++;
        } else if (element instanceof Throwable || element instanceof MultiGetResponse.Failure) {
          logger.trace("item[{}] won't be executed, reason: {}", slot, element);
          reducedResponses.set(slot, element);
          responsesByItemAndShard.set(slot, new AtomicReferenceArray(0));
          expectedOperationsPerItem.set(slot, new AtomicInteger(0));
        }
      }
      expectedOperations = new AtomicInteger(expectedResults);
    }