@RequestMapping( value = "/reports/eventSearch", method = RequestMethod.POST, produces = MediaType.APPLICATION_JSON_VALUE) public void eventSearch(final HttpEntity<String> httpEntity, final HttpServletResponse response) throws IOException { long start = System.currentTimeMillis(); String jsonRequest = httpEntity.getBody(); final SearchRequest searchRequest = new SearchRequest(indexName); try { searchRequest.source(jsonRequest); searchRequest.types("event"); SearchResponse r = clientTpl.executeWithClient( new ClientCallback<SearchResponse>() { @Override public SearchResponse execute(final Client client) { return client.search(searchRequest).actionGet(); } }); response.setContentType(MediaType.APPLICATION_JSON_VALUE); OutputStream responseStream = response.getOutputStream(); XContentBuilder builder = XContentFactory.jsonBuilder(responseStream); builder.startObject(); r.toXContent(builder, ToXContent.EMPTY_PARAMS); builder.endObject(); builder.close(); responseStream.close(); } finally { logger.debug("Executed search in {}ms: {}", System.currentTimeMillis() - start, jsonRequest); } }
public HistogramResult histogram( String query, Indexer.DateHistogramInterval interval, String filter, TimeRange range) throws IndexHelper.InvalidRangeFormatException { DateHistogramFacetBuilder fb = FacetBuilders.dateHistogramFacet("histogram") .field("timestamp") .interval(interval.toString().toLowerCase()); fb.facetFilter(standardFilters(range, filter)); QueryStringQueryBuilder qs = queryString(query); qs.allowLeadingWildcard(server.getConfiguration().isAllowLeadingWildcardSearches()); SearchRequestBuilder srb = c.prepareSearch(); srb.setIndices(IndexHelper.determineAffectedIndices(server, range).toArray(new String[] {})); srb.setQuery(qs); srb.addFacet(fb); final SearchRequest request = srb.request(); SearchResponse r = c.search(request).actionGet(); return new DateHistogramResult( (DateHistogramFacet) r.getFacets().facet("histogram"), query, request.source(), interval, r.getTook()); }
public TermsResult terms(String field, int size, String query, String filter, TimeRange range) throws IndexHelper.InvalidRangeFormatException { if (size == 0) { size = 50; } SearchRequestBuilder srb; if (filter == null) { srb = standardSearchRequest(query, IndexHelper.determineAffectedIndices(server, range)); } else { srb = filteredSearchRequest(query, filter, IndexHelper.determineAffectedIndices(server, range)); } TermsFacetBuilder terms = new TermsFacetBuilder(TERMS_FACET_NAME); terms.global(false); terms.field(field); terms.size(size); terms.facetFilter(standardFilters(range, filter)); srb.addFacet(terms); final SearchRequest request = srb.request(); SearchResponse r = c.search(request).actionGet(); return new TermsResult( (TermsFacet) r.getFacets().facet(TERMS_FACET_NAME), query, request.source(), r.getTook()); }
public FieldStatsResult fieldStats(String field, String query, String filter, TimeRange range) throws FieldTypeException, IndexHelper.InvalidRangeFormatException { SearchRequestBuilder srb; if (filter == null) { srb = standardSearchRequest(query, IndexHelper.determineAffectedIndices(server, range)); } else { srb = filteredSearchRequest(query, filter, IndexHelper.determineAffectedIndices(server, range)); } StatisticalFacetBuilder stats = new StatisticalFacetBuilder(STATS_FACET_NAME); stats.global(false); stats.facetFilter(standardFilters(range, filter)); stats.field(field); srb.addFacet(stats); SearchResponse r; final SearchRequest request; try { request = srb.request(); r = c.search(request).actionGet(); } catch (org.elasticsearch.action.search.SearchPhaseExecutionException e) { throw new FieldTypeException(e); } return new FieldStatsResult( (StatisticalFacet) r.getFacets().facet(STATS_FACET_NAME), query, request.source(), r.getTook()); }
/** * Build a random search request. * * @param randomSearchSourceBuilder builds a random {@link SearchSourceBuilder}. You can use * {@link #randomSearchSourceBuilder(Supplier, Supplier, Supplier, Supplier)}. */ public static SearchRequest randomSearchRequest( Supplier<SearchSourceBuilder> randomSearchSourceBuilder) throws IOException { SearchRequest searchRequest = new SearchRequest(); if (randomBoolean()) { searchRequest.indices(generateRandomStringArray(10, 10, false, false)); } if (randomBoolean()) { searchRequest.indicesOptions( IndicesOptions.fromOptions( randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())); } if (randomBoolean()) { searchRequest.types(generateRandomStringArray(10, 10, false, false)); } if (randomBoolean()) { searchRequest.preference(randomAsciiOfLengthBetween(3, 10)); } if (randomBoolean()) { searchRequest.requestCache(randomBoolean()); } if (randomBoolean()) { searchRequest.routing(randomAsciiOfLengthBetween(3, 10)); } if (randomBoolean()) { searchRequest.scroll(randomPositiveTimeValue()); } if (randomBoolean()) { searchRequest.searchType(randomFrom(SearchType.values())); } if (randomBoolean()) { searchRequest.source(randomSearchSourceBuilder.get()); } return searchRequest; }
public void testInitialSearchParamsMisc() { SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder()); Version remoteVersion = Version.fromId(between(0, Version.CURRENT.id)); TimeValue scroll = null; if (randomBoolean()) { scroll = TimeValue.parseTimeValue(randomPositiveTimeValue(), "test"); searchRequest.scroll(scroll); } int size = between(0, Integer.MAX_VALUE); searchRequest.source().size(size); Boolean fetchVersion = null; if (randomBoolean()) { fetchVersion = randomBoolean(); searchRequest.source().version(fetchVersion); } Map<String, String> params = initialSearchParams(searchRequest, remoteVersion); assertThat( params, scroll == null ? not(hasKey("scroll")) : hasEntry("scroll", scroll.toString())); assertThat(params, hasEntry("size", Integer.toString(size))); assertThat( params, fetchVersion == null || fetchVersion == true ? hasEntry("version", null) : not(hasEntry("version", null))); assertThat(params, hasEntry("_source", "true")); }
@Override public void doStart(Consumer<? super Response> onResponse) { if (logger.isDebugEnabled()) { logger.debug( "executing initial scroll against {}{}", isEmpty(firstSearchRequest.indices()) ? "all indices" : firstSearchRequest.indices(), isEmpty(firstSearchRequest.types()) ? "" : firstSearchRequest.types()); } searchWithRetry( listener -> client.search(firstSearchRequest, listener), r -> consume(r, onResponse)); }
public CountResult count(String query, TimeRange range, String filter) throws IndexHelper.InvalidRangeFormatException { Set<String> indices = IndexHelper.determineAffectedIndices(server, range); SearchRequest request; if (filter == null) { request = standardSearchRequest(query, indices, range).request(); } else { request = filteredSearchRequest(query, filter, indices, range).request(); } request.searchType(SearchType.COUNT); SearchResponse r = c.search(request).actionGet(); return new CountResult(r.getHits().getTotalHits(), r.getTookInMillis()); }
@Override public String[] indices() { if (searchRequest == null) { throw new IllegalStateException("unable to retrieve indices, search request is null"); } return searchRequest.indices(); }
@Override public IndicesRequest indices(String[] indices) { if (searchRequest == null) { throw new IllegalStateException("unable to set indices, search request is null"); } searchRequest.indices(indices); return this; }
public ScoreDoc[] getLastEmittedDocPerShard( SearchRequest request, ScoreDoc[] sortedShardList, int numShards) { if (request.scroll() != null) { return getLastEmittedDocPerShard(sortedShardList, numShards); } else { return null; } }
public void testInitialSearchParamsSort() { SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder()); // Test sort:_doc for versions that support it. Version remoteVersion = Version.fromId(between(Version.V_2_1_0_ID, Version.CURRENT.id)); searchRequest.source().sort("_doc"); assertThat(initialSearchParams(searchRequest, remoteVersion), hasEntry("sort", "_doc:asc")); // Test search_type scan for versions that don't support sort:_doc. remoteVersion = Version.fromId(between(0, Version.V_2_1_0_ID - 1)); assertThat(initialSearchParams(searchRequest, remoteVersion), hasEntry("search_type", "scan")); // Test sorting by some field. Version doesn't matter. remoteVersion = Version.fromId(between(0, Version.CURRENT.id)); searchRequest.source().sorts().clear(); searchRequest.source().sort("foo"); assertThat(initialSearchParams(searchRequest, remoteVersion), hasEntry("sort", "foo:asc")); }
@Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); name = in.readString(); if (in.readBoolean()) { searchRequest = new SearchRequest(); searchRequest.readFrom(in); } readTimeout(in); }
@Override public void handleRequest(final RestRequest request, final RestChannel channel) { PutWarmerRequest putWarmerRequest = new PutWarmerRequest(request.param("name")); putWarmerRequest.listenerThreaded(false); SearchRequest searchRequest = new SearchRequest(Strings.splitStringByCommaToArray(request.param("index"))) .types(Strings.splitStringByCommaToArray(request.param("type"))) .source(request.content(), request.contentUnsafe()); searchRequest.indicesOptions( IndicesOptions.fromRequest(request, searchRequest.indicesOptions())); putWarmerRequest.searchRequest(searchRequest); putWarmerRequest.timeout(request.paramAsTime("timeout", putWarmerRequest.timeout())); putWarmerRequest.masterNodeTimeout( request.paramAsTime("master_timeout", putWarmerRequest.masterNodeTimeout())); client .admin() .indices() .putWarmer( putWarmerRequest, new AcknowledgedRestResponseActionListener(request, channel, logger)); }
public static InternalSearchRequest internalSearchRequest( ShardRouting shardRouting, int numberOfShards, SearchRequest request, String[] filteringAliases, long nowInMillis) { InternalSearchRequest internalRequest = new InternalSearchRequest(shardRouting, numberOfShards, request.searchType()); internalRequest.source(request.source(), request.sourceOffset(), request.sourceLength()); internalRequest.extraSource( request.extraSource(), request.extraSourceOffset(), request.extraSourceLength()); internalRequest.scroll(request.scroll()); internalRequest.filteringAliases(filteringAliases); internalRequest.types(request.types()); internalRequest.nowInMillis(nowInMillis); return internalRequest; }
public SearchResult search( String query, String filter, TimeRange range, int limit, int offset, Sorting sorting) throws IndexHelper.InvalidRangeFormatException { if (limit <= 0) { limit = LIMIT; } Set<String> indices = IndexHelper.determineAffectedIndices(server, range); SearchRequest request; if (filter == null) { request = standardSearchRequest(query, indices, limit, offset, range, sorting).request(); } else { request = filteredSearchRequest(query, filter, indices, limit, offset, range, sorting).request(); } SearchResponse r = c.search(request).actionGet(); return new SearchResult(r.getHits(), indices, query, request.source(), r.getTook()); }
@Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(name); if (searchRequest == null) { out.writeBoolean(false); } else { out.writeBoolean(true); searchRequest.writeTo(out); } writeTimeout(out); }
@Override protected void doStartNextScroll( String scrollId, TimeValue extraKeepAlive, Consumer<? super Response> onResponse) { execute( "POST", scrollPath(), scrollParams( timeValueNanos(searchRequest.scroll().keepAlive().nanos() + extraKeepAlive.nanos())), scrollEntity(scrollId), RESPONSE_PARSER, onResponse); }
@Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; if (searchRequest == null) { validationException = addValidationError("search request is missing", validationException); } else { validationException = searchRequest.validate(); } if (name == null) { validationException = addValidationError("name is missing", validationException); } return validationException; }
@Override protected void doStartNextScroll( String scrollId, TimeValue extraKeepAlive, Consumer<? super Response> onResponse) { SearchScrollRequest request = new SearchScrollRequest(); // Add the wait time into the scroll timeout so it won't timeout while we wait for throttling request .scrollId(scrollId) .scroll( timeValueNanos( firstSearchRequest.scroll().keepAlive().nanos() + extraKeepAlive.nanos())); searchWithRetry( listener -> client.searchScroll(request, listener), r -> consume(r, onResponse)); }
@Before @Override public void setUp() throws Exception { super.setUp(); threadPool = new TestThreadPool(getTestName()) { @Override public Executor executor(String name) { return Runnable::run; } @Override public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable command) { command.run(); return null; } }; retries = 0; searchRequest = new SearchRequest(); searchRequest.scroll(timeValueMinutes(5)); searchRequest.source(new SearchSourceBuilder().size(10).version(true).sort("_doc").size(123)); retriesAllowed = 0; }
public void testInitialSearchParamsFields() { SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder()); // Test request without any fields Version remoteVersion = VersionUtils.randomVersion(random()); assertThat( initialSearchParams(searchRequest, remoteVersion), not(either(hasKey("stored_fields")).or(hasKey("fields")))); // Setup some fields for the next two tests searchRequest.source().storedField("_source").storedField("_id"); // Test stored_fields for versions that support it remoteVersion = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0_alpha4, null); assertThat( initialSearchParams(searchRequest, remoteVersion), hasEntry("stored_fields", "_source,_id")); // Test fields for versions that support it remoteVersion = VersionUtils.randomVersionBetween(random(), null, Version.V_5_0_0_alpha3); assertThat( initialSearchParams(searchRequest, remoteVersion), hasEntry("fields", "_source,_id")); }
@Override public void handleRequest( final RestRequest request, final RestChannel channel, final Client client) throws IOException { final SearchRequest searchRequest = new SearchRequest( Strings.splitStringByCommaToArray(request.param("index")), new SearchSourceBuilder()); searchRequest.indicesOptions( IndicesOptions.fromRequest(request, searchRequest.indicesOptions())); if (RestActions.hasBodyContent(request)) { final BytesReference sourceBytes = RestActions.getRestContent(request); try (XContentParser parser = XContentFactory.xContent(sourceBytes).createParser(sourceBytes)) { final QueryParseContext context = new QueryParseContext(queryRegistry, parser, parseFieldMatcher); searchRequest.source().suggest(SuggestBuilder.fromXContent(context, suggesters)); } } else { throw new IllegalArgumentException("no content or source provided to execute suggestion"); } searchRequest.routing(request.param("routing")); searchRequest.preference(request.param("preference")); client.search( searchRequest, new RestBuilderListener<SearchResponse>(channel) { @Override public RestResponse buildResponse(SearchResponse response, XContentBuilder builder) throws Exception { RestStatus restStatus = RestStatus.status( response.getSuccessfulShards(), response.getTotalShards(), response.getShardFailures()); builder.startObject(); buildBroadcastShardsHeader( builder, request, response.getTotalShards(), response.getSuccessfulShards(), response.getFailedShards(), response.getShardFailures()); Suggest suggest = response.getSuggest(); if (suggest != null) { suggest.toInnerXContent(builder, request); } builder.endObject(); return new BytesRestResponse(restStatus, builder); } }); }
public static SearchRequest parseSearchRequest(RestRequest request) { String[] indices = Strings.splitStringByCommaToArray(request.param("index")); SearchRequest searchRequest = new SearchRequest(indices); // get the content, and put it in the body // add content/source as template if template flag is set boolean isTemplateRequest = request.path().endsWith("/template"); if (RestActions.hasBodyContent(request)) { if (isTemplateRequest) { searchRequest.templateSource(RestActions.getRestContent(request)); } else { searchRequest.source(RestActions.getRestContent(request)); } } // do not allow 'query_and_fetch' or 'dfs_query_and_fetch' search types // from the REST layer. these modes are an internal optimization and should // not be specified explicitly by the user. String searchType = request.param("search_type"); if (SearchType.fromString(searchType).equals(SearchType.QUERY_AND_FETCH) || SearchType.fromString(searchType).equals(SearchType.DFS_QUERY_AND_FETCH)) { throw new IllegalArgumentException("Unsupported search type [" + searchType + "]"); } else { searchRequest.searchType(searchType); } searchRequest.extraSource(parseSearchSource(request)); searchRequest.queryCache(request.paramAsBoolean("query_cache", null)); String scroll = request.param("scroll"); if (scroll != null) { searchRequest.scroll(new Scroll(parseTimeValue(scroll, null, "scroll"))); } searchRequest.types(Strings.splitStringByCommaToArray(request.param("type"))); searchRequest.routing(request.param("routing")); searchRequest.preference(request.param("preference")); searchRequest.indicesOptions( IndicesOptions.fromRequest(request, searchRequest.indicesOptions())); return searchRequest; }
public void testIntialSearchPath() { SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder()); assertEquals("/_search", initialSearchPath(searchRequest)); searchRequest.indices("a"); searchRequest.types("b"); assertEquals("/a/b/_search", initialSearchPath(searchRequest)); searchRequest.indices("a", "b"); searchRequest.types("c", "d"); assertEquals("/a,b/c,d/_search", initialSearchPath(searchRequest)); searchRequest.indices("cat,"); expectBadStartRequest(searchRequest, "Index", ",", "cat,"); searchRequest.indices("cat,", "dog"); expectBadStartRequest(searchRequest, "Index", ",", "cat,"); searchRequest.indices("dog", "cat,"); expectBadStartRequest(searchRequest, "Index", ",", "cat,"); searchRequest.indices("cat/"); expectBadStartRequest(searchRequest, "Index", "/", "cat/"); searchRequest.indices("cat/", "dog"); expectBadStartRequest(searchRequest, "Index", "/", "cat/"); searchRequest.indices("dog", "cat/"); expectBadStartRequest(searchRequest, "Index", "/", "cat/"); searchRequest.indices("ok"); searchRequest.types("cat,"); expectBadStartRequest(searchRequest, "Type", ",", "cat,"); searchRequest.types("cat,", "dog"); expectBadStartRequest(searchRequest, "Type", ",", "cat,"); searchRequest.types("dog", "cat,"); expectBadStartRequest(searchRequest, "Type", ",", "cat,"); searchRequest.types("cat/"); expectBadStartRequest(searchRequest, "Type", "/", "cat/"); searchRequest.types("cat/", "dog"); expectBadStartRequest(searchRequest, "Type", "/", "cat/"); searchRequest.types("dog", "cat/"); expectBadStartRequest(searchRequest, "Type", "/", "cat/"); }
/** * Pull action thread * * @param request request * @param state state * @param transportClient bulk client for remote cluster access * @param nodeClient bulk client for local cluster access */ final void performPull( final KnapsackPullRequest request, final KnapsackState state, final BulkTransportClient transportClient, final BulkNodeClient nodeClient) { try { logger.info("start of pull: {}", state); long count = 0L; Map<String, Set<String>> indices = new HashMap<>(); for (String s : Strings.commaDelimitedListToSet(request.getIndex())) { indices.put(s, Strings.commaDelimitedListToSet(request.getType())); } if (request.withMetadata()) { // renaming indices/types if (request.getIndexTypeNames() != null) { for (Object spec : request.getIndexTypeNames().keySet()) { if (spec == null) { continue; } String[] s = spec.toString().split("/"); String index = s[0]; String type = s.length > 1 ? s[1] : null; if (!"_all".equals(index)) { Set<String> types = indices.get(index); if (types == null) { types = new HashSet<>(); } if (type != null) { types.add(type); } indices.put(index, types); } } } // get settings for all indices logger.info("getting settings for indices {}", indices.keySet()); Set<String> settingsIndices = new HashSet<>(indices.keySet()); settingsIndices.remove("_all"); Map<String, String> settings = getSettings( transportClient.client(), settingsIndices.toArray(new String[settingsIndices.size()])); logger.info("found indices: {}", settings.keySet()); // we resolved the specs in indices to the real indices in the settings // get mapping and alias per index and create index if copy mode is enabled for (String index : settings.keySet()) { CreateIndexRequest createIndexRequest = createIndexRequest(mapIndex(request, index)); Set<String> types = indices.get(index); createIndexRequest.settings(settings.get(index)); logger.info("getting mappings for index {} and types {}", index, types); Map<String, String> mappings = getMapping( transportClient.client(), index, types != null ? new HashSet<>(types) : null); logger.info("found mappings: {}", mappings.keySet()); for (String type : mappings.keySet()) { logger.info("adding mapping: {}", mapType(request, index, type)); createIndexRequest.mapping(mapType(request, index, type), mappings.get(type)); } // create index logger.info("creating index: {}", mapIndex(request, index)); nodeClient.client().execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet(); logger.info("index created: {}", mapIndex(request, index)); logger.info("getting aliases for index {}", index); Map<String, String> aliases = getAliases(client, index); logger.info("found {} aliases", aliases.size()); if (!aliases.isEmpty()) { IndicesAliasesRequestBuilder requestBuilder = new IndicesAliasesRequestBuilder( nodeClient.client(), IndicesAliasesAction.INSTANCE); for (String alias : aliases.keySet()) { if (aliases.get(alias).isEmpty()) { requestBuilder.addAlias(index, alias); } else { requestBuilder.addAlias(index, alias, aliases.get(alias)); // with filter } } requestBuilder.execute().actionGet(); logger.info("aliases created", aliases.size()); } } } SearchRequest searchRequest = request.getSearchRequest(); if (searchRequest == null) { searchRequest = new SearchRequestBuilder(transportClient.client(), SearchAction.INSTANCE) .setQuery(QueryBuilders.matchAllQuery()) .addSort(SortBuilders.fieldSort("_doc")) .request(); } long total = 0L; for (String index : indices.keySet()) { if (!"_all".equals(index)) { searchRequest.indices(index); } Set<String> types = indices.get(index); if (types != null) { searchRequest.types(types.toArray(new String[types.size()])); } searchRequest.scroll(request.getTimeout()); SearchResponse searchResponse = transportClient.client().execute(SearchAction.INSTANCE, searchRequest).actionGet(); do { total += searchResponse.getHits().getHits().length; logger.debug( "total={} hits={} took={}", total, searchResponse.getHits().getHits().length, searchResponse.getTookInMillis()); for (SearchHit hit : searchResponse.getHits()) { indexSearchHit(nodeClient, request, hit); count++; } searchResponse = transportClient .client() .execute( SearchScrollAction.INSTANCE, new SearchScrollRequest(searchResponse.getScrollId()) .scroll(request.getTimeout())) .actionGet(); } while (searchResponse.getHits().getHits().length > 0 && !Thread.interrupted()); } nodeClient.flushIngest(); nodeClient.waitForResponses(TimeValue.timeValueSeconds(60)); for (String index : indices.keySet()) { nodeClient.refreshIndex(index); } nodeClient.shutdown(); transportClient.shutdown(); logger.info("end of pull: {}, docs = {}, count = {}", state, total, count); } catch (Throwable e) { logger.error(e.getMessage(), e); } finally { try { knapsack.removeImport(state); } catch (IOException e) { logger.error(e.getMessage(), e); } } }