@RequestMapping(
     value = "/reports/eventSearch",
     method = RequestMethod.POST,
     produces = MediaType.APPLICATION_JSON_VALUE)
 public void eventSearch(final HttpEntity<String> httpEntity, final HttpServletResponse response)
     throws IOException {
   long start = System.currentTimeMillis();
   String jsonRequest = httpEntity.getBody();
   final SearchRequest searchRequest = new SearchRequest(indexName);
   try {
     searchRequest.source(jsonRequest);
     searchRequest.types("event");
     SearchResponse r =
         clientTpl.executeWithClient(
             new ClientCallback<SearchResponse>() {
               @Override
               public SearchResponse execute(final Client client) {
                 return client.search(searchRequest).actionGet();
               }
             });
     response.setContentType(MediaType.APPLICATION_JSON_VALUE);
     OutputStream responseStream = response.getOutputStream();
     XContentBuilder builder = XContentFactory.jsonBuilder(responseStream);
     builder.startObject();
     r.toXContent(builder, ToXContent.EMPTY_PARAMS);
     builder.endObject();
     builder.close();
     responseStream.close();
   } finally {
     logger.debug("Executed search in {}ms: {}", System.currentTimeMillis() - start, jsonRequest);
   }
 }
Пример #2
0
  public HistogramResult histogram(
      String query, Indexer.DateHistogramInterval interval, String filter, TimeRange range)
      throws IndexHelper.InvalidRangeFormatException {
    DateHistogramFacetBuilder fb =
        FacetBuilders.dateHistogramFacet("histogram")
            .field("timestamp")
            .interval(interval.toString().toLowerCase());

    fb.facetFilter(standardFilters(range, filter));

    QueryStringQueryBuilder qs = queryString(query);
    qs.allowLeadingWildcard(server.getConfiguration().isAllowLeadingWildcardSearches());

    SearchRequestBuilder srb = c.prepareSearch();
    srb.setIndices(IndexHelper.determineAffectedIndices(server, range).toArray(new String[] {}));
    srb.setQuery(qs);
    srb.addFacet(fb);

    final SearchRequest request = srb.request();
    SearchResponse r = c.search(request).actionGet();
    return new DateHistogramResult(
        (DateHistogramFacet) r.getFacets().facet("histogram"),
        query,
        request.source(),
        interval,
        r.getTook());
  }
Пример #3
0
  public TermsResult terms(String field, int size, String query, String filter, TimeRange range)
      throws IndexHelper.InvalidRangeFormatException {
    if (size == 0) {
      size = 50;
    }

    SearchRequestBuilder srb;
    if (filter == null) {
      srb = standardSearchRequest(query, IndexHelper.determineAffectedIndices(server, range));
    } else {
      srb =
          filteredSearchRequest(query, filter, IndexHelper.determineAffectedIndices(server, range));
    }

    TermsFacetBuilder terms = new TermsFacetBuilder(TERMS_FACET_NAME);
    terms.global(false);
    terms.field(field);
    terms.size(size);

    terms.facetFilter(standardFilters(range, filter));

    srb.addFacet(terms);

    final SearchRequest request = srb.request();
    SearchResponse r = c.search(request).actionGet();

    return new TermsResult(
        (TermsFacet) r.getFacets().facet(TERMS_FACET_NAME), query, request.source(), r.getTook());
  }
Пример #4
0
  public FieldStatsResult fieldStats(String field, String query, String filter, TimeRange range)
      throws FieldTypeException, IndexHelper.InvalidRangeFormatException {
    SearchRequestBuilder srb;

    if (filter == null) {
      srb = standardSearchRequest(query, IndexHelper.determineAffectedIndices(server, range));
    } else {
      srb =
          filteredSearchRequest(query, filter, IndexHelper.determineAffectedIndices(server, range));
    }

    StatisticalFacetBuilder stats = new StatisticalFacetBuilder(STATS_FACET_NAME);
    stats.global(false);

    stats.facetFilter(standardFilters(range, filter));

    stats.field(field);

    srb.addFacet(stats);

    SearchResponse r;
    final SearchRequest request;
    try {
      request = srb.request();
      r = c.search(request).actionGet();
    } catch (org.elasticsearch.action.search.SearchPhaseExecutionException e) {
      throw new FieldTypeException(e);
    }

    return new FieldStatsResult(
        (StatisticalFacet) r.getFacets().facet(STATS_FACET_NAME),
        query,
        request.source(),
        r.getTook());
  }
 /**
  * Build a random search request.
  *
  * @param randomSearchSourceBuilder builds a random {@link SearchSourceBuilder}. You can use
  *     {@link #randomSearchSourceBuilder(Supplier, Supplier, Supplier, Supplier)}.
  */
 public static SearchRequest randomSearchRequest(
     Supplier<SearchSourceBuilder> randomSearchSourceBuilder) throws IOException {
   SearchRequest searchRequest = new SearchRequest();
   if (randomBoolean()) {
     searchRequest.indices(generateRandomStringArray(10, 10, false, false));
   }
   if (randomBoolean()) {
     searchRequest.indicesOptions(
         IndicesOptions.fromOptions(
             randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
   }
   if (randomBoolean()) {
     searchRequest.types(generateRandomStringArray(10, 10, false, false));
   }
   if (randomBoolean()) {
     searchRequest.preference(randomAsciiOfLengthBetween(3, 10));
   }
   if (randomBoolean()) {
     searchRequest.requestCache(randomBoolean());
   }
   if (randomBoolean()) {
     searchRequest.routing(randomAsciiOfLengthBetween(3, 10));
   }
   if (randomBoolean()) {
     searchRequest.scroll(randomPositiveTimeValue());
   }
   if (randomBoolean()) {
     searchRequest.searchType(randomFrom(SearchType.values()));
   }
   if (randomBoolean()) {
     searchRequest.source(randomSearchSourceBuilder.get());
   }
   return searchRequest;
 }
  public void testInitialSearchParamsMisc() {
    SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder());
    Version remoteVersion = Version.fromId(between(0, Version.CURRENT.id));

    TimeValue scroll = null;
    if (randomBoolean()) {
      scroll = TimeValue.parseTimeValue(randomPositiveTimeValue(), "test");
      searchRequest.scroll(scroll);
    }
    int size = between(0, Integer.MAX_VALUE);
    searchRequest.source().size(size);
    Boolean fetchVersion = null;
    if (randomBoolean()) {
      fetchVersion = randomBoolean();
      searchRequest.source().version(fetchVersion);
    }

    Map<String, String> params = initialSearchParams(searchRequest, remoteVersion);

    assertThat(
        params, scroll == null ? not(hasKey("scroll")) : hasEntry("scroll", scroll.toString()));
    assertThat(params, hasEntry("size", Integer.toString(size)));
    assertThat(
        params,
        fetchVersion == null || fetchVersion == true
            ? hasEntry("version", null)
            : not(hasEntry("version", null)));
    assertThat(params, hasEntry("_source", "true"));
  }
 @Override
 public void doStart(Consumer<? super Response> onResponse) {
   if (logger.isDebugEnabled()) {
     logger.debug(
         "executing initial scroll against {}{}",
         isEmpty(firstSearchRequest.indices()) ? "all indices" : firstSearchRequest.indices(),
         isEmpty(firstSearchRequest.types()) ? "" : firstSearchRequest.types());
   }
   searchWithRetry(
       listener -> client.search(firstSearchRequest, listener), r -> consume(r, onResponse));
 }
Пример #8
0
  public CountResult count(String query, TimeRange range, String filter)
      throws IndexHelper.InvalidRangeFormatException {
    Set<String> indices = IndexHelper.determineAffectedIndices(server, range);

    SearchRequest request;
    if (filter == null) {
      request = standardSearchRequest(query, indices, range).request();
    } else {
      request = filteredSearchRequest(query, filter, indices, range).request();
    }
    request.searchType(SearchType.COUNT);

    SearchResponse r = c.search(request).actionGet();
    return new CountResult(r.getHits().getTotalHits(), r.getTookInMillis());
  }
Пример #9
0
 @Override
 public String[] indices() {
   if (searchRequest == null) {
     throw new IllegalStateException("unable to retrieve indices, search request is null");
   }
   return searchRequest.indices();
 }
Пример #10
0
 @Override
 public IndicesRequest indices(String[] indices) {
   if (searchRequest == null) {
     throw new IllegalStateException("unable to set indices, search request is null");
   }
   searchRequest.indices(indices);
   return this;
 }
 public ScoreDoc[] getLastEmittedDocPerShard(
     SearchRequest request, ScoreDoc[] sortedShardList, int numShards) {
   if (request.scroll() != null) {
     return getLastEmittedDocPerShard(sortedShardList, numShards);
   } else {
     return null;
   }
 }
  public void testInitialSearchParamsSort() {
    SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder());

    // Test sort:_doc for versions that support it.
    Version remoteVersion = Version.fromId(between(Version.V_2_1_0_ID, Version.CURRENT.id));
    searchRequest.source().sort("_doc");
    assertThat(initialSearchParams(searchRequest, remoteVersion), hasEntry("sort", "_doc:asc"));

    // Test search_type scan for versions that don't support sort:_doc.
    remoteVersion = Version.fromId(between(0, Version.V_2_1_0_ID - 1));
    assertThat(initialSearchParams(searchRequest, remoteVersion), hasEntry("search_type", "scan"));

    // Test sorting by some field. Version doesn't matter.
    remoteVersion = Version.fromId(between(0, Version.CURRENT.id));
    searchRequest.source().sorts().clear();
    searchRequest.source().sort("foo");
    assertThat(initialSearchParams(searchRequest, remoteVersion), hasEntry("sort", "foo:asc"));
  }
Пример #13
0
 @Override
 public void readFrom(StreamInput in) throws IOException {
   super.readFrom(in);
   name = in.readString();
   if (in.readBoolean()) {
     searchRequest = new SearchRequest();
     searchRequest.readFrom(in);
   }
   readTimeout(in);
 }
 @Override
 public void handleRequest(final RestRequest request, final RestChannel channel) {
   PutWarmerRequest putWarmerRequest = new PutWarmerRequest(request.param("name"));
   putWarmerRequest.listenerThreaded(false);
   SearchRequest searchRequest =
       new SearchRequest(Strings.splitStringByCommaToArray(request.param("index")))
           .types(Strings.splitStringByCommaToArray(request.param("type")))
           .source(request.content(), request.contentUnsafe());
   searchRequest.indicesOptions(
       IndicesOptions.fromRequest(request, searchRequest.indicesOptions()));
   putWarmerRequest.searchRequest(searchRequest);
   putWarmerRequest.timeout(request.paramAsTime("timeout", putWarmerRequest.timeout()));
   putWarmerRequest.masterNodeTimeout(
       request.paramAsTime("master_timeout", putWarmerRequest.masterNodeTimeout()));
   client
       .admin()
       .indices()
       .putWarmer(
           putWarmerRequest, new AcknowledgedRestResponseActionListener(request, channel, logger));
 }
 public static InternalSearchRequest internalSearchRequest(
     ShardRouting shardRouting,
     int numberOfShards,
     SearchRequest request,
     String[] filteringAliases,
     long nowInMillis) {
   InternalSearchRequest internalRequest =
       new InternalSearchRequest(shardRouting, numberOfShards, request.searchType());
   internalRequest.source(request.source(), request.sourceOffset(), request.sourceLength());
   internalRequest.extraSource(
       request.extraSource(), request.extraSourceOffset(), request.extraSourceLength());
   internalRequest.scroll(request.scroll());
   internalRequest.filteringAliases(filteringAliases);
   internalRequest.types(request.types());
   internalRequest.nowInMillis(nowInMillis);
   return internalRequest;
 }
Пример #16
0
  public SearchResult search(
      String query, String filter, TimeRange range, int limit, int offset, Sorting sorting)
      throws IndexHelper.InvalidRangeFormatException {
    if (limit <= 0) {
      limit = LIMIT;
    }

    Set<String> indices = IndexHelper.determineAffectedIndices(server, range);

    SearchRequest request;

    if (filter == null) {
      request = standardSearchRequest(query, indices, limit, offset, range, sorting).request();
    } else {
      request =
          filteredSearchRequest(query, filter, indices, limit, offset, range, sorting).request();
    }

    SearchResponse r = c.search(request).actionGet();
    return new SearchResult(r.getHits(), indices, query, request.source(), r.getTook());
  }
Пример #17
0
 @Override
 public void writeTo(StreamOutput out) throws IOException {
   super.writeTo(out);
   out.writeString(name);
   if (searchRequest == null) {
     out.writeBoolean(false);
   } else {
     out.writeBoolean(true);
     searchRequest.writeTo(out);
   }
   writeTimeout(out);
 }
 @Override
 protected void doStartNextScroll(
     String scrollId, TimeValue extraKeepAlive, Consumer<? super Response> onResponse) {
   execute(
       "POST",
       scrollPath(),
       scrollParams(
           timeValueNanos(searchRequest.scroll().keepAlive().nanos() + extraKeepAlive.nanos())),
       scrollEntity(scrollId),
       RESPONSE_PARSER,
       onResponse);
 }
Пример #19
0
 @Override
 public ActionRequestValidationException validate() {
   ActionRequestValidationException validationException = null;
   if (searchRequest == null) {
     validationException = addValidationError("search request is missing", validationException);
   } else {
     validationException = searchRequest.validate();
   }
   if (name == null) {
     validationException = addValidationError("name is missing", validationException);
   }
   return validationException;
 }
 @Override
 protected void doStartNextScroll(
     String scrollId, TimeValue extraKeepAlive, Consumer<? super Response> onResponse) {
   SearchScrollRequest request = new SearchScrollRequest();
   // Add the wait time into the scroll timeout so it won't timeout while we wait for throttling
   request
       .scrollId(scrollId)
       .scroll(
           timeValueNanos(
               firstSearchRequest.scroll().keepAlive().nanos() + extraKeepAlive.nanos()));
   searchWithRetry(
       listener -> client.searchScroll(request, listener), r -> consume(r, onResponse));
 }
  @Before
  @Override
  public void setUp() throws Exception {
    super.setUp();
    threadPool =
        new TestThreadPool(getTestName()) {
          @Override
          public Executor executor(String name) {
            return Runnable::run;
          }

          @Override
          public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable command) {
            command.run();
            return null;
          }
        };
    retries = 0;
    searchRequest = new SearchRequest();
    searchRequest.scroll(timeValueMinutes(5));
    searchRequest.source(new SearchSourceBuilder().size(10).version(true).sort("_doc").size(123));
    retriesAllowed = 0;
  }
  public void testInitialSearchParamsFields() {
    SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder());

    // Test request without any fields
    Version remoteVersion = VersionUtils.randomVersion(random());
    assertThat(
        initialSearchParams(searchRequest, remoteVersion),
        not(either(hasKey("stored_fields")).or(hasKey("fields"))));

    // Setup some fields for the next two tests
    searchRequest.source().storedField("_source").storedField("_id");

    // Test stored_fields for versions that support it
    remoteVersion = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0_alpha4, null);
    assertThat(
        initialSearchParams(searchRequest, remoteVersion),
        hasEntry("stored_fields", "_source,_id"));

    // Test fields for versions that support it
    remoteVersion = VersionUtils.randomVersionBetween(random(), null, Version.V_5_0_0_alpha3);
    assertThat(
        initialSearchParams(searchRequest, remoteVersion), hasEntry("fields", "_source,_id"));
  }
Пример #23
0
 @Override
 public void handleRequest(
     final RestRequest request, final RestChannel channel, final Client client)
     throws IOException {
   final SearchRequest searchRequest =
       new SearchRequest(
           Strings.splitStringByCommaToArray(request.param("index")), new SearchSourceBuilder());
   searchRequest.indicesOptions(
       IndicesOptions.fromRequest(request, searchRequest.indicesOptions()));
   if (RestActions.hasBodyContent(request)) {
     final BytesReference sourceBytes = RestActions.getRestContent(request);
     try (XContentParser parser =
         XContentFactory.xContent(sourceBytes).createParser(sourceBytes)) {
       final QueryParseContext context =
           new QueryParseContext(queryRegistry, parser, parseFieldMatcher);
       searchRequest.source().suggest(SuggestBuilder.fromXContent(context, suggesters));
     }
   } else {
     throw new IllegalArgumentException("no content or source provided to execute suggestion");
   }
   searchRequest.routing(request.param("routing"));
   searchRequest.preference(request.param("preference"));
   client.search(
       searchRequest,
       new RestBuilderListener<SearchResponse>(channel) {
         @Override
         public RestResponse buildResponse(SearchResponse response, XContentBuilder builder)
             throws Exception {
           RestStatus restStatus =
               RestStatus.status(
                   response.getSuccessfulShards(),
                   response.getTotalShards(),
                   response.getShardFailures());
           builder.startObject();
           buildBroadcastShardsHeader(
               builder,
               request,
               response.getTotalShards(),
               response.getSuccessfulShards(),
               response.getFailedShards(),
               response.getShardFailures());
           Suggest suggest = response.getSuggest();
           if (suggest != null) {
             suggest.toInnerXContent(builder, request);
           }
           builder.endObject();
           return new BytesRestResponse(restStatus, builder);
         }
       });
 }
Пример #24
0
  public static SearchRequest parseSearchRequest(RestRequest request) {
    String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
    SearchRequest searchRequest = new SearchRequest(indices);
    // get the content, and put it in the body
    // add content/source as template if template flag is set
    boolean isTemplateRequest = request.path().endsWith("/template");
    if (RestActions.hasBodyContent(request)) {
      if (isTemplateRequest) {
        searchRequest.templateSource(RestActions.getRestContent(request));
      } else {
        searchRequest.source(RestActions.getRestContent(request));
      }
    }

    // do not allow 'query_and_fetch' or 'dfs_query_and_fetch' search types
    // from the REST layer. these modes are an internal optimization and should
    // not be specified explicitly by the user.
    String searchType = request.param("search_type");
    if (SearchType.fromString(searchType).equals(SearchType.QUERY_AND_FETCH)
        || SearchType.fromString(searchType).equals(SearchType.DFS_QUERY_AND_FETCH)) {
      throw new IllegalArgumentException("Unsupported search type [" + searchType + "]");
    } else {
      searchRequest.searchType(searchType);
    }

    searchRequest.extraSource(parseSearchSource(request));
    searchRequest.queryCache(request.paramAsBoolean("query_cache", null));

    String scroll = request.param("scroll");
    if (scroll != null) {
      searchRequest.scroll(new Scroll(parseTimeValue(scroll, null, "scroll")));
    }

    searchRequest.types(Strings.splitStringByCommaToArray(request.param("type")));
    searchRequest.routing(request.param("routing"));
    searchRequest.preference(request.param("preference"));
    searchRequest.indicesOptions(
        IndicesOptions.fromRequest(request, searchRequest.indicesOptions()));

    return searchRequest;
  }
  public void testIntialSearchPath() {
    SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder());

    assertEquals("/_search", initialSearchPath(searchRequest));
    searchRequest.indices("a");
    searchRequest.types("b");
    assertEquals("/a/b/_search", initialSearchPath(searchRequest));
    searchRequest.indices("a", "b");
    searchRequest.types("c", "d");
    assertEquals("/a,b/c,d/_search", initialSearchPath(searchRequest));

    searchRequest.indices("cat,");
    expectBadStartRequest(searchRequest, "Index", ",", "cat,");
    searchRequest.indices("cat,", "dog");
    expectBadStartRequest(searchRequest, "Index", ",", "cat,");
    searchRequest.indices("dog", "cat,");
    expectBadStartRequest(searchRequest, "Index", ",", "cat,");
    searchRequest.indices("cat/");
    expectBadStartRequest(searchRequest, "Index", "/", "cat/");
    searchRequest.indices("cat/", "dog");
    expectBadStartRequest(searchRequest, "Index", "/", "cat/");
    searchRequest.indices("dog", "cat/");
    expectBadStartRequest(searchRequest, "Index", "/", "cat/");

    searchRequest.indices("ok");
    searchRequest.types("cat,");
    expectBadStartRequest(searchRequest, "Type", ",", "cat,");
    searchRequest.types("cat,", "dog");
    expectBadStartRequest(searchRequest, "Type", ",", "cat,");
    searchRequest.types("dog", "cat,");
    expectBadStartRequest(searchRequest, "Type", ",", "cat,");
    searchRequest.types("cat/");
    expectBadStartRequest(searchRequest, "Type", "/", "cat/");
    searchRequest.types("cat/", "dog");
    expectBadStartRequest(searchRequest, "Type", "/", "cat/");
    searchRequest.types("dog", "cat/");
    expectBadStartRequest(searchRequest, "Type", "/", "cat/");
  }
 /**
  * Pull action thread
  *
  * @param request request
  * @param state state
  * @param transportClient bulk client for remote cluster access
  * @param nodeClient bulk client for local cluster access
  */
 final void performPull(
     final KnapsackPullRequest request,
     final KnapsackState state,
     final BulkTransportClient transportClient,
     final BulkNodeClient nodeClient) {
   try {
     logger.info("start of pull: {}", state);
     long count = 0L;
     Map<String, Set<String>> indices = new HashMap<>();
     for (String s : Strings.commaDelimitedListToSet(request.getIndex())) {
       indices.put(s, Strings.commaDelimitedListToSet(request.getType()));
     }
     if (request.withMetadata()) {
       // renaming indices/types
       if (request.getIndexTypeNames() != null) {
         for (Object spec : request.getIndexTypeNames().keySet()) {
           if (spec == null) {
             continue;
           }
           String[] s = spec.toString().split("/");
           String index = s[0];
           String type = s.length > 1 ? s[1] : null;
           if (!"_all".equals(index)) {
             Set<String> types = indices.get(index);
             if (types == null) {
               types = new HashSet<>();
             }
             if (type != null) {
               types.add(type);
             }
             indices.put(index, types);
           }
         }
       }
       // get settings for all indices
       logger.info("getting settings for indices {}", indices.keySet());
       Set<String> settingsIndices = new HashSet<>(indices.keySet());
       settingsIndices.remove("_all");
       Map<String, String> settings =
           getSettings(
               transportClient.client(),
               settingsIndices.toArray(new String[settingsIndices.size()]));
       logger.info("found indices: {}", settings.keySet());
       // we resolved the specs in indices to the real indices in the settings
       // get mapping and alias per index and create index if copy mode is enabled
       for (String index : settings.keySet()) {
         CreateIndexRequest createIndexRequest = createIndexRequest(mapIndex(request, index));
         Set<String> types = indices.get(index);
         createIndexRequest.settings(settings.get(index));
         logger.info("getting mappings for index {} and types {}", index, types);
         Map<String, String> mappings =
             getMapping(
                 transportClient.client(), index, types != null ? new HashSet<>(types) : null);
         logger.info("found mappings: {}", mappings.keySet());
         for (String type : mappings.keySet()) {
           logger.info("adding mapping: {}", mapType(request, index, type));
           createIndexRequest.mapping(mapType(request, index, type), mappings.get(type));
         }
         // create index
         logger.info("creating index: {}", mapIndex(request, index));
         nodeClient.client().execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
         logger.info("index created: {}", mapIndex(request, index));
         logger.info("getting aliases for index {}", index);
         Map<String, String> aliases = getAliases(client, index);
         logger.info("found {} aliases", aliases.size());
         if (!aliases.isEmpty()) {
           IndicesAliasesRequestBuilder requestBuilder =
               new IndicesAliasesRequestBuilder(
                   nodeClient.client(), IndicesAliasesAction.INSTANCE);
           for (String alias : aliases.keySet()) {
             if (aliases.get(alias).isEmpty()) {
               requestBuilder.addAlias(index, alias);
             } else {
               requestBuilder.addAlias(index, alias, aliases.get(alias)); // with filter
             }
           }
           requestBuilder.execute().actionGet();
           logger.info("aliases created", aliases.size());
         }
       }
     }
     SearchRequest searchRequest = request.getSearchRequest();
     if (searchRequest == null) {
       searchRequest =
           new SearchRequestBuilder(transportClient.client(), SearchAction.INSTANCE)
               .setQuery(QueryBuilders.matchAllQuery())
               .addSort(SortBuilders.fieldSort("_doc"))
               .request();
     }
     long total = 0L;
     for (String index : indices.keySet()) {
       if (!"_all".equals(index)) {
         searchRequest.indices(index);
       }
       Set<String> types = indices.get(index);
       if (types != null) {
         searchRequest.types(types.toArray(new String[types.size()]));
       }
       searchRequest.scroll(request.getTimeout());
       SearchResponse searchResponse =
           transportClient.client().execute(SearchAction.INSTANCE, searchRequest).actionGet();
       do {
         total += searchResponse.getHits().getHits().length;
         logger.debug(
             "total={} hits={} took={}",
             total,
             searchResponse.getHits().getHits().length,
             searchResponse.getTookInMillis());
         for (SearchHit hit : searchResponse.getHits()) {
           indexSearchHit(nodeClient, request, hit);
           count++;
         }
         searchResponse =
             transportClient
                 .client()
                 .execute(
                     SearchScrollAction.INSTANCE,
                     new SearchScrollRequest(searchResponse.getScrollId())
                         .scroll(request.getTimeout()))
                 .actionGet();
       } while (searchResponse.getHits().getHits().length > 0 && !Thread.interrupted());
     }
     nodeClient.flushIngest();
     nodeClient.waitForResponses(TimeValue.timeValueSeconds(60));
     for (String index : indices.keySet()) {
       nodeClient.refreshIndex(index);
     }
     nodeClient.shutdown();
     transportClient.shutdown();
     logger.info("end of pull: {}, docs = {}, count = {}", state, total, count);
   } catch (Throwable e) {
     logger.error(e.getMessage(), e);
   } finally {
     try {
       knapsack.removeImport(state);
     } catch (IOException e) {
       logger.error(e.getMessage(), e);
     }
   }
 }