Exemplo n.º 1
0
  public void projectivize() {
    IntArrayList ids = new IntArrayList();
    int i, size = size();
    DEPNode nonProj;

    for (i = 1; i < size; i++) ids.add(i);

    while ((nonProj = getSmallestNonProjectiveArc(ids)) != null)
      nonProj.setHead(nonProj.getHead().getHead(), DEPLib.DEP_NON_PROJ);
  }
 /** Builds an array, with potential null elements, with docs to load. */
 public void fillDocIdsToLoad(AtomicArray<IntArrayList> docsIdsToLoad, ScoreDoc[] shardDocs) {
   for (ScoreDoc shardDoc : shardDocs) {
     IntArrayList list = docsIdsToLoad.get(shardDoc.shardIndex);
     if (list == null) {
       list = new IntArrayList(); // can't be shared!, uses unsafe on it later on
       docsIdsToLoad.set(shardDoc.shardIndex, list);
     }
     list.add(shardDoc.doc);
   }
 }
    @SuppressWarnings("unchecked")
    void onShardFailure(ShardId shardId, Throwable e) {
      logger.debug("{} Shard multi percolate failure", e, shardId);
      try {
        IntArrayList slots = shardToSlots.get(shardId);
        for (int i = 0; i < slots.size(); i++) {
          int slot = slots.get(i);
          AtomicReferenceArray shardResults = responsesByItemAndShard.get(slot);
          if (shardResults == null) {
            continue;
          }

          shardResults.set(shardId.id(), new BroadcastShardOperationFailedException(shardId, e));
          assert expectedOperationsPerItem.get(slot).get() >= 1
              : "slot[" + slot + "] can't be lower than one. Caused by: " + e.getMessage();
          if (expectedOperationsPerItem.get(slot).decrementAndGet() == 0) {
            reduce(slot);
          }
        }
      } catch (Throwable t) {
        logger.error("{} Percolate original reduce error, original error {}", t, shardId, e);
        finalListener.onFailure(t);
      }
    }
Exemplo n.º 4
0
  /** Called by {@link DEPTree#projectivize()}. */
  private DEPNode getSmallestNonProjectiveArc(IntArrayList ids) {
    IntOpenHashSet remove = new IntOpenHashSet();
    DEPNode wk, nonProj = null;
    int np, max = 0;

    for (IntCursor cur : ids) {
      wk = get(cur.value);
      np = isNonProjective(wk);

      if (np == 0) {
        remove.add(cur.value);
      } else if (np > max) {
        nonProj = wk;
        max = np;
      }
    }

    ids.removeAll(remove);
    return nonProj;
  }
  @Override
  protected void doExecute(
      final MultiPercolateRequest request, final ActionListener<MultiPercolateResponse> listener) {
    final ClusterState clusterState = clusterService.state();
    clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ);

    final List<Object> percolateRequests = new ArrayList<>(request.requests().size());
    // Can have a mixture of percolate requests. (normal percolate requests & percolate existing
    // doc),
    // so we need to keep track for what percolate request we had a get request
    final IntArrayList getRequestSlots = new IntArrayList();
    List<GetRequest> existingDocsRequests = new ArrayList<>();
    for (int slot = 0; slot < request.requests().size(); slot++) {
      PercolateRequest percolateRequest = request.requests().get(slot);
      percolateRequest.startTime = System.currentTimeMillis();
      percolateRequests.add(percolateRequest);
      if (percolateRequest.getRequest() != null) {
        existingDocsRequests.add(percolateRequest.getRequest());
        getRequestSlots.add(slot);
      }
    }

    if (!existingDocsRequests.isEmpty()) {
      final MultiGetRequest multiGetRequest = new MultiGetRequest(request);
      for (GetRequest getRequest : existingDocsRequests) {
        multiGetRequest.add(
            new MultiGetRequest.Item(getRequest.index(), getRequest.type(), getRequest.id())
                .routing(getRequest.routing()));
      }

      multiGetAction.execute(
          multiGetRequest,
          new ActionListener<MultiGetResponse>() {

            @Override
            public void onResponse(MultiGetResponse multiGetItemResponses) {
              for (int i = 0; i < multiGetItemResponses.getResponses().length; i++) {
                MultiGetItemResponse itemResponse = multiGetItemResponses.getResponses()[i];
                int slot = getRequestSlots.get(i);
                if (!itemResponse.isFailed()) {
                  GetResponse getResponse = itemResponse.getResponse();
                  if (getResponse.isExists()) {
                    PercolateRequest originalRequest =
                        (PercolateRequest) percolateRequests.get(slot);
                    percolateRequests.set(
                        slot,
                        new PercolateRequest(originalRequest, getResponse.getSourceAsBytesRef()));
                  } else {
                    logger.trace("mpercolate existing doc, item[{}] doesn't exist", slot);
                    percolateRequests.set(
                        slot,
                        new DocumentMissingException(
                            null, getResponse.getType(), getResponse.getId()));
                  }
                } else {
                  logger.trace(
                      "mpercolate existing doc, item[{}] failure {}",
                      slot,
                      itemResponse.getFailure());
                  percolateRequests.set(slot, itemResponse.getFailure());
                }
              }
              new ASyncAction(request, percolateRequests, listener, clusterState).run();
            }

            @Override
            public void onFailure(Throwable e) {
              listener.onFailure(e);
            }
          });
    } else {
      new ASyncAction(request, percolateRequests, listener, clusterState).run();
    }
  }
    ASyncAction(
        MultiPercolateRequest multiPercolateRequest,
        List<Object> percolateRequests,
        ActionListener<MultiPercolateResponse> finalListener,
        ClusterState clusterState) {
      this.finalListener = finalListener;
      this.multiPercolateRequest = multiPercolateRequest;
      this.percolateRequests = percolateRequests;
      responsesByItemAndShard = new AtomicReferenceArray<>(percolateRequests.size());
      expectedOperationsPerItem = new AtomicReferenceArray<>(percolateRequests.size());
      reducedResponses = new AtomicArray<>(percolateRequests.size());

      // Resolving concrete indices and routing and grouping the requests by shard
      requestsByShard = new HashMap<>();
      // Keep track what slots belong to what shard, in case a request to a shard fails on all
      // copies
      shardToSlots = new HashMap<>();
      int expectedResults = 0;
      for (int slot = 0; slot < percolateRequests.size(); slot++) {
        Object element = percolateRequests.get(slot);
        assert element != null;
        if (element instanceof PercolateRequest) {
          PercolateRequest percolateRequest = (PercolateRequest) element;
          String[] concreteIndices;
          try {
            concreteIndices =
                indexNameExpressionResolver.concreteIndices(clusterState, percolateRequest);
          } catch (IndexNotFoundException e) {
            reducedResponses.set(slot, e);
            responsesByItemAndShard.set(slot, new AtomicReferenceArray(0));
            expectedOperationsPerItem.set(slot, new AtomicInteger(0));
            continue;
          }
          Map<String, Set<String>> routing =
              indexNameExpressionResolver.resolveSearchRouting(
                  clusterState, percolateRequest.routing(), percolateRequest.indices());
          // TODO: I only need shardIds, ShardIterator(ShardRouting) is only needed in
          // TransportShardMultiPercolateAction
          GroupShardsIterator shards =
              clusterService
                  .operationRouting()
                  .searchShards(
                      clusterState, concreteIndices, routing, percolateRequest.preference());
          if (shards.size() == 0) {
            reducedResponses.set(slot, new UnavailableShardsException(null, "No shards available"));
            responsesByItemAndShard.set(slot, new AtomicReferenceArray(0));
            expectedOperationsPerItem.set(slot, new AtomicInteger(0));
            continue;
          }

          // The shard id is used as index in the atomic ref array, so we need to find out how many
          // shards there are regardless of routing:
          int numShards =
              clusterService
                  .operationRouting()
                  .searchShardsCount(clusterState, concreteIndices, null);
          responsesByItemAndShard.set(slot, new AtomicReferenceArray(numShards));
          expectedOperationsPerItem.set(slot, new AtomicInteger(shards.size()));
          for (ShardIterator shard : shards) {
            ShardId shardId = shard.shardId();
            TransportShardMultiPercolateAction.Request requests = requestsByShard.get(shardId);
            if (requests == null) {
              requestsByShard.put(
                  shardId,
                  requests =
                      new TransportShardMultiPercolateAction.Request(
                          multiPercolateRequest,
                          shardId.getIndex(),
                          shardId.getId(),
                          percolateRequest.preference()));
            }
            logger.trace("Adding shard[{}] percolate request for item[{}]", shardId, slot);
            requests.add(
                new TransportShardMultiPercolateAction.Request.Item(
                    slot, new PercolateShardRequest(shardId, percolateRequest)));

            IntArrayList items = shardToSlots.get(shardId);
            if (items == null) {
              shardToSlots.put(shardId, items = new IntArrayList());
            }
            items.add(slot);
          }
          expectedResults++;
        } else if (element instanceof Throwable || element instanceof MultiGetResponse.Failure) {
          logger.trace("item[{}] won't be executed, reason: {}", slot, element);
          reducedResponses.set(slot, element);
          responsesByItemAndShard.set(slot, new AtomicReferenceArray(0));
          expectedOperationsPerItem.set(slot, new AtomicInteger(0));
        }
      }
      expectedOperations = new AtomicInteger(expectedResults);
    }
Exemplo n.º 7
0
  /** Performs tokenization and saves the results to the <code>context</code>. */
  public void tokenize(PreprocessingContext context) {
    // Documents to tokenize
    final List<Document> documents = context.documents;

    // Fields to tokenize
    final String[] fieldNames = documentFields.toArray(new String[documentFields.size()]);

    if (fieldNames.length > 8) {
      throw new ProcessingException("Maximum number of tokenized fields is 8.");
    }

    // Prepare arrays
    images = Lists.newArrayList();
    tokenTypes = new ShortArrayList();
    documentIndices = new IntArrayList();
    fieldIndices = new ByteArrayList();

    final Iterator<Document> docIterator = documents.iterator();
    int documentIndex = 0;
    final ITokenizer ts = context.language.getTokenizer();
    final MutableCharArray wrapper = new MutableCharArray(CharArrayUtils.EMPTY_ARRAY);

    while (docIterator.hasNext()) {
      final Document doc = docIterator.next();

      boolean hadTokens = false;
      for (int i = 0; i < fieldNames.length; i++) {
        final byte fieldIndex = (byte) i;
        final String fieldName = fieldNames[i];
        final String fieldValue = doc.getField(fieldName);

        if (!StringUtils.isEmpty(fieldValue)) {
          try {
            short tokenType;

            ts.reset(new StringReader(fieldValue));
            if ((tokenType = ts.nextToken()) != ITokenizer.TT_EOF) {
              if (hadTokens) addFieldSeparator(documentIndex);
              do {
                ts.setTermBuffer(wrapper);
                add(documentIndex, fieldIndex, context.intern(wrapper), tokenType);
              } while ((tokenType = ts.nextToken()) != ITokenizer.TT_EOF);
              hadTokens = true;
            }
          } catch (IOException e) {
            // Not possible (StringReader above)?
            throw ExceptionUtils.wrapAsRuntimeException(e);
          }
        }
      }

      if (docIterator.hasNext()) {
        addDocumentSeparator();
      }

      documentIndex++;
    }

    addTerminator();

    // Save results in the PreprocessingContext
    context.allTokens.documentIndex = documentIndices.toArray();
    context.allTokens.fieldIndex = fieldIndices.toArray();
    context.allTokens.image = images.toArray(new char[images.size()][]);
    context.allTokens.type = tokenTypes.toArray();
    context.allFields.name = fieldNames;

    // Clean up
    images = null;
    fieldIndices = null;
    tokenTypes = null;
    documentIndices = null;
  }
Exemplo n.º 8
0
 /** Adds custom token code to the sequence. May be used to add separator constants. */
 void add(int documentIndex, byte fieldIndex, char[] image, short tokenTypeCode) {
   documentIndices.add(documentIndex);
   fieldIndices.add(fieldIndex);
   images.add(image);
   tokenTypes.add(tokenTypeCode);
 }