@RequestMapping(
value = "/reports/eventSearch",
method = RequestMethod.POST,
produces = MediaType.APPLICATION_JSON_VALUE)
public void eventSearch(final HttpEntity<String> httpEntity, final HttpServletResponse response)
throws IOException {
long start = System.currentTimeMillis();
String jsonRequest = httpEntity.getBody();
final SearchRequest searchRequest = new SearchRequest(indexName);
try {
searchRequest.source(jsonRequest);
searchRequest.types("event");
SearchResponse r =
clientTpl.executeWithClient(
new ClientCallback<SearchResponse>() {
@Override
public SearchResponse execute(final Client client) {
return client.search(searchRequest).actionGet();
}
});
response.setContentType(MediaType.APPLICATION_JSON_VALUE);
OutputStream responseStream = response.getOutputStream();
XContentBuilder builder = XContentFactory.jsonBuilder(responseStream);
builder.startObject();
r.toXContent(builder, ToXContent.EMPTY_PARAMS);
builder.endObject();
builder.close();
responseStream.close();
} finally {
logger.debug("Executed search in {}ms: {}", System.currentTimeMillis() - start, jsonRequest);
}
}
public HistogramResult histogram(
String query, Indexer.DateHistogramInterval interval, String filter, TimeRange range)
throws IndexHelper.InvalidRangeFormatException {
DateHistogramFacetBuilder fb =
FacetBuilders.dateHistogramFacet("histogram")
.field("timestamp")
.interval(interval.toString().toLowerCase());
fb.facetFilter(standardFilters(range, filter));
QueryStringQueryBuilder qs = queryString(query);
qs.allowLeadingWildcard(server.getConfiguration().isAllowLeadingWildcardSearches());
SearchRequestBuilder srb = c.prepareSearch();
srb.setIndices(IndexHelper.determineAffectedIndices(server, range).toArray(new String[] {}));
srb.setQuery(qs);
srb.addFacet(fb);
final SearchRequest request = srb.request();
SearchResponse r = c.search(request).actionGet();
return new DateHistogramResult(
(DateHistogramFacet) r.getFacets().facet("histogram"),
query,
request.source(),
interval,
r.getTook());
}
public TermsResult terms(String field, int size, String query, String filter, TimeRange range)
throws IndexHelper.InvalidRangeFormatException {
if (size == 0) {
size = 50;
}
SearchRequestBuilder srb;
if (filter == null) {
srb = standardSearchRequest(query, IndexHelper.determineAffectedIndices(server, range));
} else {
srb =
filteredSearchRequest(query, filter, IndexHelper.determineAffectedIndices(server, range));
}
TermsFacetBuilder terms = new TermsFacetBuilder(TERMS_FACET_NAME);
terms.global(false);
terms.field(field);
terms.size(size);
terms.facetFilter(standardFilters(range, filter));
srb.addFacet(terms);
final SearchRequest request = srb.request();
SearchResponse r = c.search(request).actionGet();
return new TermsResult(
(TermsFacet) r.getFacets().facet(TERMS_FACET_NAME), query, request.source(), r.getTook());
}
public FieldStatsResult fieldStats(String field, String query, String filter, TimeRange range)
throws FieldTypeException, IndexHelper.InvalidRangeFormatException {
SearchRequestBuilder srb;
if (filter == null) {
srb = standardSearchRequest(query, IndexHelper.determineAffectedIndices(server, range));
} else {
srb =
filteredSearchRequest(query, filter, IndexHelper.determineAffectedIndices(server, range));
}
StatisticalFacetBuilder stats = new StatisticalFacetBuilder(STATS_FACET_NAME);
stats.global(false);
stats.facetFilter(standardFilters(range, filter));
stats.field(field);
srb.addFacet(stats);
SearchResponse r;
final SearchRequest request;
try {
request = srb.request();
r = c.search(request).actionGet();
} catch (org.elasticsearch.action.search.SearchPhaseExecutionException e) {
throw new FieldTypeException(e);
}
return new FieldStatsResult(
(StatisticalFacet) r.getFacets().facet(STATS_FACET_NAME),
query,
request.source(),
r.getTook());
}
/**
* Build a random search request.
*
* @param randomSearchSourceBuilder builds a random {@link SearchSourceBuilder}. You can use
* {@link #randomSearchSourceBuilder(Supplier, Supplier, Supplier, Supplier)}.
*/
public static SearchRequest randomSearchRequest(
Supplier<SearchSourceBuilder> randomSearchSourceBuilder) throws IOException {
SearchRequest searchRequest = new SearchRequest();
if (randomBoolean()) {
searchRequest.indices(generateRandomStringArray(10, 10, false, false));
}
if (randomBoolean()) {
searchRequest.indicesOptions(
IndicesOptions.fromOptions(
randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()));
}
if (randomBoolean()) {
searchRequest.types(generateRandomStringArray(10, 10, false, false));
}
if (randomBoolean()) {
searchRequest.preference(randomAsciiOfLengthBetween(3, 10));
}
if (randomBoolean()) {
searchRequest.requestCache(randomBoolean());
}
if (randomBoolean()) {
searchRequest.routing(randomAsciiOfLengthBetween(3, 10));
}
if (randomBoolean()) {
searchRequest.scroll(randomPositiveTimeValue());
}
if (randomBoolean()) {
searchRequest.searchType(randomFrom(SearchType.values()));
}
if (randomBoolean()) {
searchRequest.source(randomSearchSourceBuilder.get());
}
return searchRequest;
}
public void testInitialSearchParamsMisc() {
SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder());
Version remoteVersion = Version.fromId(between(0, Version.CURRENT.id));
TimeValue scroll = null;
if (randomBoolean()) {
scroll = TimeValue.parseTimeValue(randomPositiveTimeValue(), "test");
searchRequest.scroll(scroll);
}
int size = between(0, Integer.MAX_VALUE);
searchRequest.source().size(size);
Boolean fetchVersion = null;
if (randomBoolean()) {
fetchVersion = randomBoolean();
searchRequest.source().version(fetchVersion);
}
Map<String, String> params = initialSearchParams(searchRequest, remoteVersion);
assertThat(
params, scroll == null ? not(hasKey("scroll")) : hasEntry("scroll", scroll.toString()));
assertThat(params, hasEntry("size", Integer.toString(size)));
assertThat(
params,
fetchVersion == null || fetchVersion == true
? hasEntry("version", null)
: not(hasEntry("version", null)));
assertThat(params, hasEntry("_source", "true"));
}
@Override
public void doStart(Consumer<? super Response> onResponse) {
if (logger.isDebugEnabled()) {
logger.debug(
"executing initial scroll against {}{}",
isEmpty(firstSearchRequest.indices()) ? "all indices" : firstSearchRequest.indices(),
isEmpty(firstSearchRequest.types()) ? "" : firstSearchRequest.types());
}
searchWithRetry(
listener -> client.search(firstSearchRequest, listener), r -> consume(r, onResponse));
}
public CountResult count(String query, TimeRange range, String filter)
throws IndexHelper.InvalidRangeFormatException {
Set<String> indices = IndexHelper.determineAffectedIndices(server, range);
SearchRequest request;
if (filter == null) {
request = standardSearchRequest(query, indices, range).request();
} else {
request = filteredSearchRequest(query, filter, indices, range).request();
}
request.searchType(SearchType.COUNT);
SearchResponse r = c.search(request).actionGet();
return new CountResult(r.getHits().getTotalHits(), r.getTookInMillis());
}
@Override
public String[] indices() {
if (searchRequest == null) {
throw new IllegalStateException("unable to retrieve indices, search request is null");
}
return searchRequest.indices();
}
@Override
public IndicesRequest indices(String[] indices) {
if (searchRequest == null) {
throw new IllegalStateException("unable to set indices, search request is null");
}
searchRequest.indices(indices);
return this;
}
public ScoreDoc[] getLastEmittedDocPerShard(
SearchRequest request, ScoreDoc[] sortedShardList, int numShards) {
if (request.scroll() != null) {
return getLastEmittedDocPerShard(sortedShardList, numShards);
} else {
return null;
}
}
public void testInitialSearchParamsSort() {
SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder());
// Test sort:_doc for versions that support it.
Version remoteVersion = Version.fromId(between(Version.V_2_1_0_ID, Version.CURRENT.id));
searchRequest.source().sort("_doc");
assertThat(initialSearchParams(searchRequest, remoteVersion), hasEntry("sort", "_doc:asc"));
// Test search_type scan for versions that don't support sort:_doc.
remoteVersion = Version.fromId(between(0, Version.V_2_1_0_ID - 1));
assertThat(initialSearchParams(searchRequest, remoteVersion), hasEntry("search_type", "scan"));
// Test sorting by some field. Version doesn't matter.
remoteVersion = Version.fromId(between(0, Version.CURRENT.id));
searchRequest.source().sorts().clear();
searchRequest.source().sort("foo");
assertThat(initialSearchParams(searchRequest, remoteVersion), hasEntry("sort", "foo:asc"));
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
name = in.readString();
if (in.readBoolean()) {
searchRequest = new SearchRequest();
searchRequest.readFrom(in);
}
readTimeout(in);
}
@Override
public void handleRequest(final RestRequest request, final RestChannel channel) {
PutWarmerRequest putWarmerRequest = new PutWarmerRequest(request.param("name"));
putWarmerRequest.listenerThreaded(false);
SearchRequest searchRequest =
new SearchRequest(Strings.splitStringByCommaToArray(request.param("index")))
.types(Strings.splitStringByCommaToArray(request.param("type")))
.source(request.content(), request.contentUnsafe());
searchRequest.indicesOptions(
IndicesOptions.fromRequest(request, searchRequest.indicesOptions()));
putWarmerRequest.searchRequest(searchRequest);
putWarmerRequest.timeout(request.paramAsTime("timeout", putWarmerRequest.timeout()));
putWarmerRequest.masterNodeTimeout(
request.paramAsTime("master_timeout", putWarmerRequest.masterNodeTimeout()));
client
.admin()
.indices()
.putWarmer(
putWarmerRequest, new AcknowledgedRestResponseActionListener(request, channel, logger));
}
public static InternalSearchRequest internalSearchRequest(
ShardRouting shardRouting,
int numberOfShards,
SearchRequest request,
String[] filteringAliases,
long nowInMillis) {
InternalSearchRequest internalRequest =
new InternalSearchRequest(shardRouting, numberOfShards, request.searchType());
internalRequest.source(request.source(), request.sourceOffset(), request.sourceLength());
internalRequest.extraSource(
request.extraSource(), request.extraSourceOffset(), request.extraSourceLength());
internalRequest.scroll(request.scroll());
internalRequest.filteringAliases(filteringAliases);
internalRequest.types(request.types());
internalRequest.nowInMillis(nowInMillis);
return internalRequest;
}
public SearchResult search(
String query, String filter, TimeRange range, int limit, int offset, Sorting sorting)
throws IndexHelper.InvalidRangeFormatException {
if (limit <= 0) {
limit = LIMIT;
}
Set<String> indices = IndexHelper.determineAffectedIndices(server, range);
SearchRequest request;
if (filter == null) {
request = standardSearchRequest(query, indices, limit, offset, range, sorting).request();
} else {
request =
filteredSearchRequest(query, filter, indices, limit, offset, range, sorting).request();
}
SearchResponse r = c.search(request).actionGet();
return new SearchResult(r.getHits(), indices, query, request.source(), r.getTook());
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeString(name);
if (searchRequest == null) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
searchRequest.writeTo(out);
}
writeTimeout(out);
}
@Override
protected void doStartNextScroll(
String scrollId, TimeValue extraKeepAlive, Consumer<? super Response> onResponse) {
execute(
"POST",
scrollPath(),
scrollParams(
timeValueNanos(searchRequest.scroll().keepAlive().nanos() + extraKeepAlive.nanos())),
scrollEntity(scrollId),
RESPONSE_PARSER,
onResponse);
}
@Override
public ActionRequestValidationException validate() {
ActionRequestValidationException validationException = null;
if (searchRequest == null) {
validationException = addValidationError("search request is missing", validationException);
} else {
validationException = searchRequest.validate();
}
if (name == null) {
validationException = addValidationError("name is missing", validationException);
}
return validationException;
}
@Override
protected void doStartNextScroll(
String scrollId, TimeValue extraKeepAlive, Consumer<? super Response> onResponse) {
SearchScrollRequest request = new SearchScrollRequest();
// Add the wait time into the scroll timeout so it won't timeout while we wait for throttling
request
.scrollId(scrollId)
.scroll(
timeValueNanos(
firstSearchRequest.scroll().keepAlive().nanos() + extraKeepAlive.nanos()));
searchWithRetry(
listener -> client.searchScroll(request, listener), r -> consume(r, onResponse));
}
@Before
@Override
public void setUp() throws Exception {
super.setUp();
threadPool =
new TestThreadPool(getTestName()) {
@Override
public Executor executor(String name) {
return Runnable::run;
}
@Override
public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable command) {
command.run();
return null;
}
};
retries = 0;
searchRequest = new SearchRequest();
searchRequest.scroll(timeValueMinutes(5));
searchRequest.source(new SearchSourceBuilder().size(10).version(true).sort("_doc").size(123));
retriesAllowed = 0;
}
public void testInitialSearchParamsFields() {
SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder());
// Test request without any fields
Version remoteVersion = VersionUtils.randomVersion(random());
assertThat(
initialSearchParams(searchRequest, remoteVersion),
not(either(hasKey("stored_fields")).or(hasKey("fields"))));
// Setup some fields for the next two tests
searchRequest.source().storedField("_source").storedField("_id");
// Test stored_fields for versions that support it
remoteVersion = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0_alpha4, null);
assertThat(
initialSearchParams(searchRequest, remoteVersion),
hasEntry("stored_fields", "_source,_id"));
// Test fields for versions that support it
remoteVersion = VersionUtils.randomVersionBetween(random(), null, Version.V_5_0_0_alpha3);
assertThat(
initialSearchParams(searchRequest, remoteVersion), hasEntry("fields", "_source,_id"));
}
@Override
public void handleRequest(
final RestRequest request, final RestChannel channel, final Client client)
throws IOException {
final SearchRequest searchRequest =
new SearchRequest(
Strings.splitStringByCommaToArray(request.param("index")), new SearchSourceBuilder());
searchRequest.indicesOptions(
IndicesOptions.fromRequest(request, searchRequest.indicesOptions()));
if (RestActions.hasBodyContent(request)) {
final BytesReference sourceBytes = RestActions.getRestContent(request);
try (XContentParser parser =
XContentFactory.xContent(sourceBytes).createParser(sourceBytes)) {
final QueryParseContext context =
new QueryParseContext(queryRegistry, parser, parseFieldMatcher);
searchRequest.source().suggest(SuggestBuilder.fromXContent(context, suggesters));
}
} else {
throw new IllegalArgumentException("no content or source provided to execute suggestion");
}
searchRequest.routing(request.param("routing"));
searchRequest.preference(request.param("preference"));
client.search(
searchRequest,
new RestBuilderListener<SearchResponse>(channel) {
@Override
public RestResponse buildResponse(SearchResponse response, XContentBuilder builder)
throws Exception {
RestStatus restStatus =
RestStatus.status(
response.getSuccessfulShards(),
response.getTotalShards(),
response.getShardFailures());
builder.startObject();
buildBroadcastShardsHeader(
builder,
request,
response.getTotalShards(),
response.getSuccessfulShards(),
response.getFailedShards(),
response.getShardFailures());
Suggest suggest = response.getSuggest();
if (suggest != null) {
suggest.toInnerXContent(builder, request);
}
builder.endObject();
return new BytesRestResponse(restStatus, builder);
}
});
}
public static SearchRequest parseSearchRequest(RestRequest request) {
String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
SearchRequest searchRequest = new SearchRequest(indices);
// get the content, and put it in the body
// add content/source as template if template flag is set
boolean isTemplateRequest = request.path().endsWith("/template");
if (RestActions.hasBodyContent(request)) {
if (isTemplateRequest) {
searchRequest.templateSource(RestActions.getRestContent(request));
} else {
searchRequest.source(RestActions.getRestContent(request));
}
}
// do not allow 'query_and_fetch' or 'dfs_query_and_fetch' search types
// from the REST layer. these modes are an internal optimization and should
// not be specified explicitly by the user.
String searchType = request.param("search_type");
if (SearchType.fromString(searchType).equals(SearchType.QUERY_AND_FETCH)
|| SearchType.fromString(searchType).equals(SearchType.DFS_QUERY_AND_FETCH)) {
throw new IllegalArgumentException("Unsupported search type [" + searchType + "]");
} else {
searchRequest.searchType(searchType);
}
searchRequest.extraSource(parseSearchSource(request));
searchRequest.queryCache(request.paramAsBoolean("query_cache", null));
String scroll = request.param("scroll");
if (scroll != null) {
searchRequest.scroll(new Scroll(parseTimeValue(scroll, null, "scroll")));
}
searchRequest.types(Strings.splitStringByCommaToArray(request.param("type")));
searchRequest.routing(request.param("routing"));
searchRequest.preference(request.param("preference"));
searchRequest.indicesOptions(
IndicesOptions.fromRequest(request, searchRequest.indicesOptions()));
return searchRequest;
}
public void testIntialSearchPath() {
SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder());
assertEquals("/_search", initialSearchPath(searchRequest));
searchRequest.indices("a");
searchRequest.types("b");
assertEquals("/a/b/_search", initialSearchPath(searchRequest));
searchRequest.indices("a", "b");
searchRequest.types("c", "d");
assertEquals("/a,b/c,d/_search", initialSearchPath(searchRequest));
searchRequest.indices("cat,");
expectBadStartRequest(searchRequest, "Index", ",", "cat,");
searchRequest.indices("cat,", "dog");
expectBadStartRequest(searchRequest, "Index", ",", "cat,");
searchRequest.indices("dog", "cat,");
expectBadStartRequest(searchRequest, "Index", ",", "cat,");
searchRequest.indices("cat/");
expectBadStartRequest(searchRequest, "Index", "/", "cat/");
searchRequest.indices("cat/", "dog");
expectBadStartRequest(searchRequest, "Index", "/", "cat/");
searchRequest.indices("dog", "cat/");
expectBadStartRequest(searchRequest, "Index", "/", "cat/");
searchRequest.indices("ok");
searchRequest.types("cat,");
expectBadStartRequest(searchRequest, "Type", ",", "cat,");
searchRequest.types("cat,", "dog");
expectBadStartRequest(searchRequest, "Type", ",", "cat,");
searchRequest.types("dog", "cat,");
expectBadStartRequest(searchRequest, "Type", ",", "cat,");
searchRequest.types("cat/");
expectBadStartRequest(searchRequest, "Type", "/", "cat/");
searchRequest.types("cat/", "dog");
expectBadStartRequest(searchRequest, "Type", "/", "cat/");
searchRequest.types("dog", "cat/");
expectBadStartRequest(searchRequest, "Type", "/", "cat/");
}
/**
* Pull action thread
*
* @param request request
* @param state state
* @param transportClient bulk client for remote cluster access
* @param nodeClient bulk client for local cluster access
*/
final void performPull(
final KnapsackPullRequest request,
final KnapsackState state,
final BulkTransportClient transportClient,
final BulkNodeClient nodeClient) {
try {
logger.info("start of pull: {}", state);
long count = 0L;
Map<String, Set<String>> indices = new HashMap<>();
for (String s : Strings.commaDelimitedListToSet(request.getIndex())) {
indices.put(s, Strings.commaDelimitedListToSet(request.getType()));
}
if (request.withMetadata()) {
// renaming indices/types
if (request.getIndexTypeNames() != null) {
for (Object spec : request.getIndexTypeNames().keySet()) {
if (spec == null) {
continue;
}
String[] s = spec.toString().split("/");
String index = s[0];
String type = s.length > 1 ? s[1] : null;
if (!"_all".equals(index)) {
Set<String> types = indices.get(index);
if (types == null) {
types = new HashSet<>();
}
if (type != null) {
types.add(type);
}
indices.put(index, types);
}
}
}
// get settings for all indices
logger.info("getting settings for indices {}", indices.keySet());
Set<String> settingsIndices = new HashSet<>(indices.keySet());
settingsIndices.remove("_all");
Map<String, String> settings =
getSettings(
transportClient.client(),
settingsIndices.toArray(new String[settingsIndices.size()]));
logger.info("found indices: {}", settings.keySet());
// we resolved the specs in indices to the real indices in the settings
// get mapping and alias per index and create index if copy mode is enabled
for (String index : settings.keySet()) {
CreateIndexRequest createIndexRequest = createIndexRequest(mapIndex(request, index));
Set<String> types = indices.get(index);
createIndexRequest.settings(settings.get(index));
logger.info("getting mappings for index {} and types {}", index, types);
Map<String, String> mappings =
getMapping(
transportClient.client(), index, types != null ? new HashSet<>(types) : null);
logger.info("found mappings: {}", mappings.keySet());
for (String type : mappings.keySet()) {
logger.info("adding mapping: {}", mapType(request, index, type));
createIndexRequest.mapping(mapType(request, index, type), mappings.get(type));
}
// create index
logger.info("creating index: {}", mapIndex(request, index));
nodeClient.client().execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
logger.info("index created: {}", mapIndex(request, index));
logger.info("getting aliases for index {}", index);
Map<String, String> aliases = getAliases(client, index);
logger.info("found {} aliases", aliases.size());
if (!aliases.isEmpty()) {
IndicesAliasesRequestBuilder requestBuilder =
new IndicesAliasesRequestBuilder(
nodeClient.client(), IndicesAliasesAction.INSTANCE);
for (String alias : aliases.keySet()) {
if (aliases.get(alias).isEmpty()) {
requestBuilder.addAlias(index, alias);
} else {
requestBuilder.addAlias(index, alias, aliases.get(alias)); // with filter
}
}
requestBuilder.execute().actionGet();
logger.info("aliases created", aliases.size());
}
}
}
SearchRequest searchRequest = request.getSearchRequest();
if (searchRequest == null) {
searchRequest =
new SearchRequestBuilder(transportClient.client(), SearchAction.INSTANCE)
.setQuery(QueryBuilders.matchAllQuery())
.addSort(SortBuilders.fieldSort("_doc"))
.request();
}
long total = 0L;
for (String index : indices.keySet()) {
if (!"_all".equals(index)) {
searchRequest.indices(index);
}
Set<String> types = indices.get(index);
if (types != null) {
searchRequest.types(types.toArray(new String[types.size()]));
}
searchRequest.scroll(request.getTimeout());
SearchResponse searchResponse =
transportClient.client().execute(SearchAction.INSTANCE, searchRequest).actionGet();
do {
total += searchResponse.getHits().getHits().length;
logger.debug(
"total={} hits={} took={}",
total,
searchResponse.getHits().getHits().length,
searchResponse.getTookInMillis());
for (SearchHit hit : searchResponse.getHits()) {
indexSearchHit(nodeClient, request, hit);
count++;
}
searchResponse =
transportClient
.client()
.execute(
SearchScrollAction.INSTANCE,
new SearchScrollRequest(searchResponse.getScrollId())
.scroll(request.getTimeout()))
.actionGet();
} while (searchResponse.getHits().getHits().length > 0 && !Thread.interrupted());
}
nodeClient.flushIngest();
nodeClient.waitForResponses(TimeValue.timeValueSeconds(60));
for (String index : indices.keySet()) {
nodeClient.refreshIndex(index);
}
nodeClient.shutdown();
transportClient.shutdown();
logger.info("end of pull: {}, docs = {}, count = {}", state, total, count);
} catch (Throwable e) {
logger.error(e.getMessage(), e);
} finally {
try {
knapsack.removeImport(state);
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
}
}
