Пример #1
0
 @Override
 protected ShardSuggestResponse shardOperation(ShardSuggestRequest request)
     throws ElasticSearchException {
   IndexService indexService = indicesService.indexServiceSafe(request.index());
   IndexShard indexShard = indexService.shardSafe(request.shardId());
   final Engine.Searcher searcher = indexShard.searcher();
   XContentParser parser = null;
   try {
     BytesReference suggest = request.suggest();
     if (suggest != null && suggest.length() > 0) {
       parser = XContentFactory.xContent(suggest).createParser(suggest);
       if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
         throw new ElasticSearchIllegalArgumentException("suggest content missing");
       }
       final SuggestionSearchContext context =
           suggestPhase
               .parseElement()
               .parseInternal(
                   parser, indexService.mapperService(), request.index(), request.shardId());
       final Suggest result = suggestPhase.execute(context, searcher.reader());
       return new ShardSuggestResponse(request.index(), request.shardId(), result);
     }
     return new ShardSuggestResponse(request.index(), request.shardId(), new Suggest());
   } catch (Throwable ex) {
     throw new ElasticSearchException("failed to execute suggest", ex);
   } finally {
     searcher.release();
     if (parser != null) {
       parser.close();
     }
   }
 }
Пример #2
0
 public ContextIndexSearcher(SearchContext searchContext, Engine.Searcher searcher) {
   super(searcher.reader());
   in = searcher.searcher();
   engineSearcher = searcher;
   setSimilarity(searcher.searcher().getSimilarity(true));
   setQueryCache(searchContext.indexShard().indexService().cache().query());
   setQueryCachingPolicy(searchContext.indexShard().getQueryCachingPolicy());
 }
 private Set<Uid> getShardDocUIDs(final IndexShard shard) throws IOException {
   shard.refresh("get_uids");
   try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
     Set<Uid> ids = new HashSet<>();
     for (LeafReaderContext leafContext : searcher.reader().leaves()) {
       LeafReader reader = leafContext.reader();
       Bits liveDocs = reader.getLiveDocs();
       for (int i = 0; i < reader.maxDoc(); i++) {
         if (liveDocs == null || liveDocs.get(i)) {
           Document uuid = reader.document(i, Collections.singleton(UidFieldMapper.NAME));
           ids.add(Uid.createUid(uuid.get(UidFieldMapper.NAME)));
         }
       }
     }
     return ids;
   }
 }
Пример #4
0
  public void initialize(Engine.Searcher docSearcher, ParsedDocument parsedDocument) {
    this.docSearcher = docSearcher;

    IndexReader indexReader = docSearcher.reader();
    LeafReaderContext atomicReaderContext = indexReader.leaves().get(0);
    LeafSearchLookup leafLookup = lookup().getLeafSearchLookup(atomicReaderContext);
    leafLookup.setDocument(0);
    leafLookup.source().setSource(parsedDocument.source());

    Map<String, SearchHitField> fields = new HashMap<>();
    for (IndexableField field : parsedDocument.rootDoc().getFields()) {
      fields.put(field.name(), new InternalSearchHitField(field.name(), Collections.emptyList()));
    }
    hitContext()
        .reset(
            new InternalSearchHit(0, "unknown", new StringText(parsedDocument.type()), fields),
            atomicReaderContext,
            0,
            docSearcher.searcher());
  }
  @Override
  protected ShardTermlistResponse shardOperation(ShardTermlistRequest request)
      throws ElasticSearchException {
    synchronized (termlistMutex) {
      InternalIndexShard indexShard =
          (InternalIndexShard)
              indicesService.indexServiceSafe(request.index()).shardSafe(request.shardId());
      indexShard.store().directory();
      Engine.Searcher searcher = indexShard.searcher();
      try {
        Set<String> set = new CompactHashSet();

        Fields fields = MultiFields.getFields(searcher.reader());
        if (fields != null) {
          for (Iterator<String> it = fields.iterator(); it.hasNext(); ) {
            String field = it.next();
            if (field.charAt(0) == '_') {
              continue;
            }
            if (request.getField() == null || field.equals(request.getField())) {
              Terms terms = fields.terms(field);
              if (terms != null) {
                TermsEnum termsEnum = terms.iterator(null);
                BytesRef text;
                while ((text = termsEnum.next()) != null) {
                  set.add(text.utf8ToString());
                  System.out.println("field=" + field + "; text=" + text.utf8ToString());
                }
              }
            }
          }
        }
        return new ShardTermlistResponse(request.index(), request.shardId(), set);
      } catch (IOException ex) {
        throw new ElasticSearchException(ex.getMessage(), ex);
      }
    }
  }
Пример #6
0
 void warm(Engine.Searcher searcher, IndexShard shard, IndexSettings settings) {
   if (shard.state() == IndexShardState.CLOSED) {
     return;
   }
   if (settings.isWarmerEnabled() == false) {
     return;
   }
   if (logger.isTraceEnabled()) {
     logger.trace("{} top warming [{}]", shard.shardId(), searcher.reader());
   }
   shard.warmerService().onPreWarm();
   long time = System.nanoTime();
   final List<TerminationHandle> terminationHandles = new ArrayList<>();
   // get a handle on pending tasks
   for (final Listener listener : listeners) {
     terminationHandles.add(listener.warmReader(shard, searcher));
   }
   // wait for termination
   for (TerminationHandle terminationHandle : terminationHandles) {
     try {
       terminationHandle.awaitTermination();
     } catch (InterruptedException e) {
       Thread.currentThread().interrupt();
       logger.warn("top warming has been interrupted", e);
       break;
     }
   }
   long took = System.nanoTime() - time;
   shard.warmerService().onPostWarm(took);
   if (shard.warmerService().logger().isTraceEnabled()) {
     shard
         .warmerService()
         .logger()
         .trace("top warming took [{}]", new TimeValue(took, TimeUnit.NANOSECONDS));
   }
 }
 public ContextIndexSearcher(SearchContext searchContext, Engine.Searcher searcher) {
   super(searcher.reader());
   in = searcher.searcher();
   this.searchContext = searchContext;
   setSimilarity(searcher.searcher().getSimilarity());
 }
Пример #8
0
        @Override
        public PercolateShardResponse doPercolate(
            PercolateShardRequest request, PercolateContext context) {
          Engine.Searcher percolatorSearcher = context.indexShard().acquireSearcher("percolate");
          try {
            MatchAndSort matchAndSort = QueryCollector.matchAndSort(logger, context);
            queryBasedPercolating(percolatorSearcher, context, matchAndSort);
            TopDocs topDocs = matchAndSort.topDocs();
            long count = topDocs.totalHits;
            List<BytesRef> matches = new ArrayList<BytesRef>(topDocs.scoreDocs.length);
            float[] scores = new float[topDocs.scoreDocs.length];
            List<Map<String, HighlightField>> hls = null;
            if (context.highlight() != null) {
              hls = new ArrayList<Map<String, HighlightField>>(topDocs.scoreDocs.length);
            }

            final FieldMapper<?> idMapper =
                context.mapperService().smartNameFieldMapper(IdFieldMapper.NAME);
            final IndexFieldData<?> idFieldData = context.fieldData().getForField(idMapper);
            int i = 0;
            final HashedBytesRef spare = new HashedBytesRef(new BytesRef());
            for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
              int segmentIdx =
                  ReaderUtil.subIndex(scoreDoc.doc, percolatorSearcher.reader().leaves());
              AtomicReaderContext atomicReaderContext =
                  percolatorSearcher.reader().leaves().get(segmentIdx);
              BytesValues values = idFieldData.load(atomicReaderContext).getBytesValues(true);
              final int localDocId = scoreDoc.doc - atomicReaderContext.docBase;
              final int numValues = values.setDocument(localDocId);
              assert numValues == 1;
              spare.bytes = values.nextValue();
              spare.hash = values.currentValueHash();
              matches.add(values.copyShared());
              if (hls != null) {
                Query query = context.percolateQueries().get(spare);
                context.parsedQuery(new ParsedQuery(query, ImmutableMap.<String, Filter>of()));
                context.hitContext().cache().clear();
                highlightPhase.hitExecute(context, context.hitContext());
                hls.add(i, context.hitContext().hit().getHighlightFields());
              }
              scores[i++] = scoreDoc.score;
            }
            if (hls != null) {
              return new PercolateShardResponse(
                  matches.toArray(new BytesRef[matches.size()]),
                  hls,
                  count,
                  scores,
                  context,
                  request.index(),
                  request.shardId());
            } else {
              return new PercolateShardResponse(
                  matches.toArray(new BytesRef[matches.size()]),
                  count,
                  scores,
                  context,
                  request.index(),
                  request.shardId());
            }
          } catch (Throwable e) {
            logger.debug("failed to execute", e);
            throw new PercolateException(context.indexShard().shardId(), "failed to execute", e);
          } finally {
            percolatorSearcher.release();
          }
        }
  public TermVectorResponse getTermVector(TermVectorRequest request, String concreteIndex) {
    final Engine.Searcher searcher = indexShard.acquireSearcher("term_vector");
    IndexReader topLevelReader = searcher.reader();
    final TermVectorResponse termVectorResponse =
        new TermVectorResponse(concreteIndex, request.type(), request.id());

    final Term uidTerm =
        new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(request.type(), request.id()));
    Engine.GetResult get = indexShard.get(new Engine.Get(request.realtime(), uidTerm));
    boolean docFromTranslog = get.source() != null;
    AggregatedDfs dfs = null;

    /* fetched from translog is treated as an artificial document */
    if (docFromTranslog) {
      request.doc(get.source().source, false);
      termVectorResponse.setDocVersion(get.version());
    }

    /* handle potential wildcards in fields */
    if (request.selectedFields() != null) {
      handleFieldWildcards(request);
    }

    try {
      Fields topLevelFields = MultiFields.getFields(topLevelReader);
      Versions.DocIdAndVersion docIdAndVersion = get.docIdAndVersion();
      /* from an artificial document */
      if (request.doc() != null) {
        Fields termVectorsByField = generateTermVectorsFromDoc(request, !docFromTranslog);
        // if no document indexed in shard, take the queried document itself for stats
        if (topLevelFields == null) {
          topLevelFields = termVectorsByField;
        }
        if (termVectorsByField != null && useDfs(request)) {
          dfs = getAggregatedDfs(termVectorsByField, request);
        }
        termVectorResponse.setFields(
            termVectorsByField, request.selectedFields(), request.getFlags(), topLevelFields, dfs);
        termVectorResponse.setExists(true);
        termVectorResponse.setArtificial(!docFromTranslog);
      }
      /* or from an existing document */
      else if (docIdAndVersion != null) {
        // fields with stored term vectors
        Fields termVectorsByField =
            docIdAndVersion.context.reader().getTermVectors(docIdAndVersion.docId);
        Set<String> selectedFields = request.selectedFields();
        // generate tvs for fields where analyzer is overridden
        if (selectedFields == null && request.perFieldAnalyzer() != null) {
          selectedFields = getFieldsToGenerate(request.perFieldAnalyzer(), termVectorsByField);
        }
        // fields without term vectors
        if (selectedFields != null) {
          termVectorsByField =
              addGeneratedTermVectors(get, termVectorsByField, request, selectedFields);
        }
        if (termVectorsByField != null && useDfs(request)) {
          dfs = getAggregatedDfs(termVectorsByField, request);
        }
        termVectorResponse.setFields(
            termVectorsByField, request.selectedFields(), request.getFlags(), topLevelFields, dfs);
        termVectorResponse.setDocVersion(docIdAndVersion.version);
        termVectorResponse.setExists(true);
      } else {
        termVectorResponse.setExists(false);
      }
    } catch (Throwable ex) {
      throw new ElasticsearchException("failed to execute term vector request", ex);
    } finally {
      searcher.close();
      get.release();
    }
    return termVectorResponse;
  }
  @Override
  protected ShardStatus shardOperation(IndexShardStatusRequest request)
      throws ElasticSearchException {
    InternalIndexService indexService =
        (InternalIndexService) indicesService.indexServiceSafe(request.index());
    InternalIndexShard indexShard = (InternalIndexShard) indexService.shardSafe(request.shardId());
    ShardStatus shardStatus = new ShardStatus(indexShard.routingEntry());
    shardStatus.state = indexShard.state();
    try {
      shardStatus.storeSize = indexShard.store().estimateSize();
    } catch (IOException e) {
      // failure to get the store size...
    }
    if (indexShard.state() == IndexShardState.STARTED) {
      //            shardStatus.estimatedFlushableMemorySize =
      // indexShard.estimateFlushableMemorySize();
      shardStatus.translogId = indexShard.translog().currentId();
      shardStatus.translogOperations = indexShard.translog().estimatedNumberOfOperations();
      Engine.Searcher searcher = indexShard.searcher();
      try {
        shardStatus.docs = new DocsStatus();
        shardStatus.docs.numDocs = searcher.reader().numDocs();
        shardStatus.docs.maxDoc = searcher.reader().maxDoc();
        shardStatus.docs.deletedDocs = searcher.reader().numDeletedDocs();
      } finally {
        searcher.release();
      }

      shardStatus.mergeStats = indexShard.mergeScheduler().stats();
      shardStatus.refreshStats = indexShard.refreshStats();
      shardStatus.flushStats = indexShard.flushStats();
    }

    if (request.recovery) {
      // check on going recovery (from peer or gateway)
      RecoveryStatus peerRecoveryStatus = indexShard.peerRecoveryStatus();
      if (peerRecoveryStatus == null) {
        peerRecoveryStatus = peerRecoveryTarget.peerRecoveryStatus(indexShard.shardId());
      }
      if (peerRecoveryStatus != null) {
        PeerRecoveryStatus.Stage stage;
        switch (peerRecoveryStatus.stage()) {
          case INIT:
            stage = PeerRecoveryStatus.Stage.INIT;
            break;
          case INDEX:
            stage = PeerRecoveryStatus.Stage.INDEX;
            break;
          case TRANSLOG:
            stage = PeerRecoveryStatus.Stage.TRANSLOG;
            break;
          case FINALIZE:
            stage = PeerRecoveryStatus.Stage.FINALIZE;
            break;
          case DONE:
            stage = PeerRecoveryStatus.Stage.DONE;
            break;
          default:
            stage = PeerRecoveryStatus.Stage.INIT;
        }
        shardStatus.peerRecoveryStatus =
            new PeerRecoveryStatus(
                stage,
                peerRecoveryStatus.startTime(),
                peerRecoveryStatus.time(),
                peerRecoveryStatus.phase1TotalSize(),
                peerRecoveryStatus.phase1ExistingTotalSize(),
                peerRecoveryStatus.currentFilesSize(),
                peerRecoveryStatus.currentTranslogOperations());
      }

      IndexShardGatewayService gatewayService =
          indexService.shardInjector(request.shardId()).getInstance(IndexShardGatewayService.class);
      org.elasticsearch.index.gateway.RecoveryStatus gatewayRecoveryStatus =
          gatewayService.recoveryStatus();
      if (gatewayRecoveryStatus != null) {
        GatewayRecoveryStatus.Stage stage;
        switch (gatewayRecoveryStatus.stage()) {
          case INIT:
            stage = GatewayRecoveryStatus.Stage.INIT;
            break;
          case INDEX:
            stage = GatewayRecoveryStatus.Stage.INDEX;
            break;
          case TRANSLOG:
            stage = GatewayRecoveryStatus.Stage.TRANSLOG;
            break;
          case DONE:
            stage = GatewayRecoveryStatus.Stage.DONE;
            break;
          default:
            stage = GatewayRecoveryStatus.Stage.INIT;
        }
        shardStatus.gatewayRecoveryStatus =
            new GatewayRecoveryStatus(
                stage,
                gatewayRecoveryStatus.startTime(),
                gatewayRecoveryStatus.time(),
                gatewayRecoveryStatus.index().totalSize(),
                gatewayRecoveryStatus.index().reusedTotalSize(),
                gatewayRecoveryStatus.index().currentFilesSize(),
                gatewayRecoveryStatus.translog().currentTranslogOperations());
      }
    }

    if (request.snapshot) {
      IndexShardGatewayService gatewayService =
          indexService.shardInjector(request.shardId()).getInstance(IndexShardGatewayService.class);
      SnapshotStatus snapshotStatus = gatewayService.snapshotStatus();
      if (snapshotStatus != null) {
        GatewaySnapshotStatus.Stage stage;
        switch (snapshotStatus.stage()) {
          case DONE:
            stage = GatewaySnapshotStatus.Stage.DONE;
            break;
          case FAILURE:
            stage = GatewaySnapshotStatus.Stage.FAILURE;
            break;
          case TRANSLOG:
            stage = GatewaySnapshotStatus.Stage.TRANSLOG;
            break;
          case FINALIZE:
            stage = GatewaySnapshotStatus.Stage.FINALIZE;
            break;
          case INDEX:
            stage = GatewaySnapshotStatus.Stage.INDEX;
            break;
          default:
            stage = GatewaySnapshotStatus.Stage.NONE;
            break;
        }
        shardStatus.gatewaySnapshotStatus =
            new GatewaySnapshotStatus(
                stage,
                snapshotStatus.startTime(),
                snapshotStatus.time(),
                snapshotStatus.index().totalSize(),
                snapshotStatus.translog().expectedNumberOfOperations());
      }
    }

    return shardStatus;
  }