private SearchRequestBuilder standardSearchRequest(
String query, Set<String> indices, int limit, int offset, TimeRange range, Sorting sort)
throws IndexHelper.InvalidRangeFormatException {
if (query == null || query.trim().isEmpty()) {
query = "*";
}
SearchRequestBuilder srb = c.prepareSearch();
srb.setIndices(indices.toArray(new String[] {}));
if (query.trim().equals("*")) {
srb.setQuery(matchAllQuery());
} else {
QueryStringQueryBuilder qs = queryString(query);
qs.allowLeadingWildcard(server.getConfiguration().isAllowLeadingWildcardSearches());
srb.setQuery(qs);
}
srb.setFrom(offset);
if (limit > 0) {
srb.setSize(limit);
}
if (range != null) {
srb.setFilter(IndexHelper.getTimestampRangeFilter(range));
}
if (sort != null) {
srb.addSort(sort.getField(), sort.asElastic());
}
return srb;
}
private SearchResponse doSearch(SearchRequestBuilder searchRequest, SearchQuery searchQuery) {
if (searchQuery.getFilter() != null) {
searchRequest.setPostFilter(searchQuery.getFilter());
}
if (CollectionUtils.isNotEmpty(searchQuery.getElasticsearchSorts())) {
for (SortBuilder sort : searchQuery.getElasticsearchSorts()) {
searchRequest.addSort(sort);
}
}
if (CollectionUtils.isNotEmpty(searchQuery.getFacets())) {
for (FacetRequest facetRequest : searchQuery.getFacets()) {
FacetBuilder facet = facetRequest.getFacet();
if (facetRequest.applyQueryFilter() && searchQuery.getFilter() != null) {
facet.facetFilter(searchQuery.getFilter());
}
searchRequest.addFacet(facet);
}
}
if (searchQuery.getHighlightFields() != null) {
for (HighlightBuilder.Field highlightField : searchQuery.getHighlightFields()) {
searchRequest.addHighlightedField(highlightField);
}
}
if (CollectionUtils.isNotEmpty(searchQuery.getAggregations())) {
for (AbstractAggregationBuilder aggregationBuilder : searchQuery.getAggregations()) {
searchRequest.addAggregation(aggregationBuilder);
}
}
return searchRequest.setQuery(searchQuery.getQuery()).execute().actionGet();
}
public SearchResponse executeQuery(
MetricConfig conf, String type, List<String> fields, BuilderCustomizer customizer) {
SearchRequestBuilder builder =
client
.prepareSearch(MetricUtils.getIndexes(conf.getStart(), conf.getStop(), client))
.setTypes(type)
.setSearchType(SearchType.QUERY_AND_FETCH)
.setSize((int) (conf.getStop() - conf.getStart()) / 10);
for (String next : fields) {
builder.addField(next);
}
BoolQueryBuilder query =
QueryBuilders.boolQuery()
.must(
QueryBuilders.rangeQuery("timestamp")
.from(conf.getStart() - conf.getStep())
.to(conf.getStop())
.includeLower(false)
.includeUpper(true));
for (String tag : conf.getTags()) {
query = query.must(QueryBuilders.termQuery("tags", tag));
}
builder.setQuery(query);
if (customizer != null) {
customizer.customize(builder);
}
return builder.get();
}
private SearchRequestBuilder criteria2builder(ElasticSearchCriteria criteria) {
String[] indices = criteria.getIndices();
if (indices == null || indices.length == 0)
indices = new String[] {indexManager.getIndexName()};
SearchRequestBuilder srb = client.prepareSearch(indices);
srb.setTimeout(new TimeValue(10, TimeUnit.SECONDS));
String[] types = criteria.getTypes();
if (types != null && types.length > 0) srb.setTypes(types);
QueryBuilder qb = criteria.getQueryBuilder();
String query = criteria.getQuery();
if (qb == null && StringUtils.isBlank(query))
throw new NullPointerException("queryBuilder is null and queryString is blank");
if (qb == null && StringUtils.isNotBlank(query)) {
if (wildcardQueryPattern.matcher(query).matches()) {
String[] arr = query.split(":", 2);
qb = QueryBuilders.wildcardQuery(arr[0], arr[1]);
} else {
QueryStringQueryBuilder qsqb = new QueryStringQueryBuilder(query);
qsqb.defaultOperator(Operator.AND);
qb = qsqb;
}
}
srb.setQuery(qb);
Map<String, Boolean> sorts = criteria.getSorts();
for (Map.Entry<String, Boolean> entry : sorts.entrySet())
srb.addSort(entry.getKey(), entry.getValue() ? SortOrder.DESC : SortOrder.ASC);
return srb;
}
public HistogramResult histogram(
String query, Indexer.DateHistogramInterval interval, String filter, TimeRange range)
throws IndexHelper.InvalidRangeFormatException {
DateHistogramFacetBuilder fb =
FacetBuilders.dateHistogramFacet("histogram")
.field("timestamp")
.interval(interval.toString().toLowerCase());
fb.facetFilter(standardFilters(range, filter));
QueryStringQueryBuilder qs = queryString(query);
qs.allowLeadingWildcard(server.getConfiguration().isAllowLeadingWildcardSearches());
SearchRequestBuilder srb = c.prepareSearch();
srb.setIndices(IndexHelper.determineAffectedIndices(server, range).toArray(new String[] {}));
srb.setQuery(qs);
srb.addFacet(fb);
final SearchRequest request = srb.request();
SearchResponse r = c.search(request).actionGet();
return new DateHistogramResult(
(DateHistogramFacet) r.getFacets().facet("histogram"),
query,
request.source(),
interval,
r.getTook());
}
public static SearchRequest randomSearch(Client client, String[] indices) {
final SearchRequestBuilder builder = new SearchRequestBuilder(client);
builder.setIndices(indices);
builder.setTypes(INDEX_TYPE);
builder.setQuery(randomFrom(TestQueryType.values()).getQuery());
return builder.request();
}
private SearchHit oneOfIndex(String index, QueryBuilder q, SortOrder sort) {
SearchRequestBuilder srb = c.prepareSearch();
srb.setIndices(index);
srb.setQuery(q);
srb.setSize(1);
srb.addSort("timestamp", sort);
SearchResponse r = c.search(srb.request()).actionGet();
if (r.getHits() != null && r.getHits().totalHits() > 0) {
return r.getHits().getAt(0);
} else {
return null;
}
}
@Override
public <T> Page<T> queryForPage(CriteriaQuery criteriaQuery, Class<T> clazz) {
QueryBuilder elasticsearchQuery =
new CriteriaQueryProcessor().createQueryFromCriteria(criteriaQuery.getCriteria());
FilterBuilder elasticsearchFilter =
new CriteriaFilterProcessor().createFilterFromCriteria(criteriaQuery.getCriteria());
SearchRequestBuilder searchRequestBuilder = prepareSearch(criteriaQuery, clazz);
if (elasticsearchQuery != null) {
searchRequestBuilder.setQuery(elasticsearchQuery);
} else {
searchRequestBuilder.setQuery(QueryBuilders.matchAllQuery());
}
if (criteriaQuery.getMinScore() > 0) {
searchRequestBuilder.setMinScore(criteriaQuery.getMinScore());
}
if (elasticsearchFilter != null) searchRequestBuilder.setPostFilter(elasticsearchFilter);
SearchResponse response = searchRequestBuilder.execute().actionGet();
return resultsMapper.mapResults(response, clazz, criteriaQuery.getPageable());
}
/**
* Perform filter query into one ES index and type to get records with any value stored in one
* field.
*
* @param indexName to search in
* @param indexType to search
* @param fieldName name of field to search for any value in it
* @return
*/
public SearchResponse performQueryByOneFieldAnyValue(
String indexName, String indexType, String fieldName) throws SearchIndexMissingException {
try {
SearchRequestBuilder searchBuilder = getClient().prepareSearch(indexName).setTypes(indexType);
searchBuilder.setFilter(FilterBuilders.notFilter(FilterBuilders.missingFilter(fieldName)));
searchBuilder.setQuery(QueryBuilders.matchAllQuery());
final SearchResponse response = searchBuilder.execute().actionGet();
return response;
} catch (IndexMissingException e) {
log.log(Level.WARNING, e.getMessage());
throw new SearchIndexMissingException(e);
}
}
private SearchRequestBuilder buildRequest(String script, Object... params) {
ensureGreen("test");
Map<String, Object> paramsMap = new HashMap<>();
assert (params.length % 2 == 0);
for (int i = 0; i < params.length; i += 2) {
paramsMap.put(params[i].toString(), params[i + 1]);
}
SearchRequestBuilder req = client().prepareSearch().setIndices("test");
req.setQuery(QueryBuilders.matchAllQuery())
.addSort(SortBuilders.fieldSort("_uid").order(SortOrder.ASC))
.addScriptField("foo", new Script(script, ScriptType.INLINE, "expression", paramsMap));
return req;
}
public SearchResponse doSearch(ActivityQuery query, SearchOptions options) {
SearchRequestBuilder requestBuilder =
getClient()
.prepareSearch(ActivityIndexDefinition.INDEX)
.setTypes(ActivityIndexDefinition.TYPE);
requestBuilder.setFrom(options.getOffset());
requestBuilder.setSize(options.getLimit());
requestBuilder.addSort(ActivityIndexDefinition.FIELD_CREATED_AT, SortOrder.DESC);
AndFilterBuilder filter = FilterBuilders.andFilter();
if (!query.getTypes().isEmpty()) {
OrFilterBuilder typeFilter = FilterBuilders.orFilter();
for (String type : query.getTypes()) {
typeFilter.add(FilterBuilders.termFilter(ActivityIndexDefinition.FIELD_TYPE, type));
}
filter.add(typeFilter);
}
if (!query.getDataOrFilters().isEmpty()) {
for (Map.Entry<String, Object> entry : query.getDataOrFilters().entrySet()) {
OrFilterBuilder orFilter = FilterBuilders.orFilter();
orFilter.add(
FilterBuilders.nestedFilter(
ActivityIndexDefinition.FIELD_DETAILS,
FilterBuilders.termFilter(
ActivityIndexDefinition.FIELD_DETAILS + "." + entry.getKey(),
entry.getValue())));
filter.add(orFilter);
}
}
Date since = query.getSince();
if (since != null) {
filter.add(
FilterBuilders.rangeFilter(ActivityIndexDefinition.FIELD_CREATED_AT)
.gt(since)
.cache(false));
}
Date to = query.getTo();
if (to != null) {
filter.add(
FilterBuilders.rangeFilter(ActivityIndexDefinition.FIELD_CREATED_AT).lt(to).cache(false));
}
requestBuilder.setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), filter));
return requestBuilder.get();
}
private static SearchResponse searchByAddress(
final String aIndex,
final Client aClient,
final String aType,
SearchType aSearchType,
QueryBuilder aQuery) {
SearchRequestBuilder searchRequest = aClient.prepareSearch(aIndex);
if (aType != null) {
searchRequest.setTypes(aType);
}
if (aSearchType != null) {
searchRequest.setSearchType(aSearchType);
}
searchRequest.setQuery(aQuery);
return searchRequest.execute().actionGet();
}
public List<String> autocomplete(String prefix, int limit) {
SearchRequestBuilder builder = new SearchRequestBuilder(factory.client());
builder.setIndices(Constants.INDEX).setTypes(Constants.TYPE);
builder.setSearchType(SearchType.DEFAULT);
builder.setFrom(0).setSize(limit).setExplain(IS_EXPLAIN);
QueryBuilder queryBuilder = QueryBuilders.matchPhraseQuery("title", prefix);
builder.setQuery(queryBuilder.toString());
logger.info("Search qeury");
logger.info(builder.toString());
SearchResponse sr = builder.execute().actionGet();
logger.info("Search response");
logger.info(sr.toString());
Iterator<SearchHit> it = sr.getHits().iterator();
while (it.hasNext()) {
SearchHit hit = it.next();
hit.getSource();
}
return Arrays.asList("a", "b", "c");
}
public void testStringSpecialValueVariable() throws Exception {
// i.e. expression script for term aggregations, which is not allowed
createIndex("test");
ensureGreen("test");
indexRandom(
true,
client().prepareIndex("test", "doc", "1").setSource("text", "hello"),
client().prepareIndex("test", "doc", "2").setSource("text", "goodbye"),
client().prepareIndex("test", "doc", "3").setSource("text", "hello"));
SearchRequestBuilder req = client().prepareSearch().setIndices("test");
req.setQuery(QueryBuilders.matchAllQuery())
.addAggregation(
AggregationBuilders.terms("term_agg")
.field("text")
.script(
new Script(
"_value", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null)));
String message;
try {
// shards that don't have docs with the "text" field will not fail,
// so we may or may not get a total failure
SearchResponse rsp = req.get();
assertThat(
rsp.getShardFailures().length,
greaterThan(0)); // at least the shards containing the docs should have failed
message = rsp.getShardFailures()[0].reason();
} catch (SearchPhaseExecutionException e) {
message = e.toString();
}
assertThat(
message + "should have contained ExpressionScriptExecutionException",
message.contains("ExpressionScriptExecutionException"),
equalTo(true));
assertThat(
message + "should have contained text variable error",
message.contains("text variable"),
equalTo(true));
}
/**
* @param req
* @return
* @throws IOException
*/
public AgroAutocompleteResponse autocomplete(AgroAutocompleteRequest req) throws IOException {
AgroAutocompleteResponse res = new AgroAutocompleteResponse();
// create the SearchRequestBuilder
SearchRequestBuilder searchReqBuilder = esClient.prepareSearch("history").setTypes("query");
// setup query
QueryBuilder queryBuilder;
queryBuilder = QueryBuilders.prefixQuery("query.raw", req.getQuery());
searchReqBuilder.setQuery(queryBuilder);
// setup paging
searchReqBuilder.setFrom(0);
searchReqBuilder.setSize(200);
// execute the request and get the response
SearchResponse response = searchReqBuilder.execute().actionGet();
// parse the response into AgroAutocompleteResponse
Iterator<SearchHit> hitsIter = response.getHits().iterator();
SearchHit hit;
int i = 0;
while (hitsIter.hasNext() && i < 10) {
hit = hitsIter.next();
String source = hit.getSourceAsString();
UserQuery uq = objectMapper.reader(UserQuery.class).readValue(source);
if (res.addResult(uq.getQuery())) {
i++;
}
}
// add query (meta)data to res
res.setTotal(response.getHits().getTotalHits());
res.setTime(response.getTookInMillis());
// ..and
return res;
}
@Override
public SearchRequestBuilder build(SearchRequestBuilder builder) {
if (_conditionQuery != null) {
QueryBuilder queryBuilder = _conditionQuery.getQuery();
if (queryBuilder != null) {
builder.setQuery(queryBuilder);
}
_conditionQuery
.getFieldSortBuilderList()
.forEach(
sort -> {
builder.addSort(sort);
});
}
if (_specification != null) {
builder.setFetchSource(
_specification.columnList.toArray(new String[_specification.columnList.size()]), null);
}
return builder;
}
public Iterator<Blog> search(String keyword, int start, int size) {
SearchRequestBuilder builder = new SearchRequestBuilder(factory.client());
builder.setIndices(Constants.INDEX).setTypes(Constants.TYPE);
builder.setSearchType(SearchType.DEFAULT);
builder.setFrom(start).setSize(size).setExplain(IS_EXPLAIN);
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().minimumNumberShouldMatch(1);
queryBuilder.should(QueryBuilders.matchPhraseQuery("title", keyword));
queryBuilder.should(QueryBuilders.matchPhraseQuery("content", keyword));
queryBuilder.should(QueryBuilders.matchPhraseQuery("createPlace", keyword));
builder.setQuery(queryBuilder.toString());
logger.info("Search qeury");
logger.info(builder.toString());
SearchResponse sr = builder.execute().actionGet();
logger.info("Search response");
logger.info(sr.toString());
try {
return new IteratorImpl<Blog>(
sr.getHits().iterator(), Blog.class.getConstructor(SearchHit.class));
} catch (NoSuchMethodException | SecurityException e) {
throw new RuntimeException(e);
}
}
public void testSpecialValueVariable() throws Exception {
// i.e. _value for aggregations
createIndex("test");
ensureGreen("test");
indexRandom(
true,
client().prepareIndex("test", "doc", "1").setSource("x", 5, "y", 1.2),
client().prepareIndex("test", "doc", "2").setSource("x", 10, "y", 1.4),
client().prepareIndex("test", "doc", "3").setSource("x", 13, "y", 1.8));
SearchRequestBuilder req = client().prepareSearch().setIndices("test");
req.setQuery(QueryBuilders.matchAllQuery())
.addAggregation(
AggregationBuilders.stats("int_agg")
.field("x")
.script(
new Script(
"_value * 3", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null)))
.addAggregation(
AggregationBuilders.stats("double_agg")
.field("y")
.script(
new Script(
"_value - 1.1",
ScriptType.INLINE,
ExpressionScriptEngineService.NAME,
null)));
SearchResponse rsp = req.get();
assertEquals(3, rsp.getHits().getTotalHits());
Stats stats = rsp.getAggregations().get("int_agg");
assertEquals(39.0, stats.getMax(), 0.0001);
assertEquals(15.0, stats.getMin(), 0.0001);
stats = rsp.getAggregations().get("double_agg");
assertEquals(0.7, stats.getMax(), 0.0001);
assertEquals(0.1, stats.getMin(), 0.0001);
}
public void testScore() throws Exception {
createIndex("test");
ensureGreen("test");
indexRandom(
true,
client().prepareIndex("test", "doc", "1").setSource("text", "hello goodbye"),
client().prepareIndex("test", "doc", "2").setSource("text", "hello hello hello goodbye"),
client().prepareIndex("test", "doc", "3").setSource("text", "hello hello goodebye"));
ScoreFunctionBuilder score =
ScoreFunctionBuilders.scriptFunction(
new Script("1 / _score", ScriptType.INLINE, "expression", null));
SearchRequestBuilder req = client().prepareSearch().setIndices("test");
req.setQuery(
QueryBuilders.functionScoreQuery(QueryBuilders.termQuery("text", "hello"), score)
.boostMode("replace"));
req.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); // make sure DF is consistent
SearchResponse rsp = req.get();
assertSearchResponse(rsp);
SearchHits hits = rsp.getHits();
assertEquals(3, hits.getTotalHits());
assertEquals("1", hits.getAt(0).getId());
assertEquals("3", hits.getAt(1).getId());
assertEquals("2", hits.getAt(2).getId());
}
/**
* Create filters based on the Where clause.
*
* @param where the 'WHERE' part of the SQL query.
* @throws SqlParseException
*/
private void setWhere(Where where) throws SqlParseException {
if (where != null) {
BoolFilterBuilder boolFilter = FilterMaker.explan(where);
request.setQuery(QueryBuilders.filteredQuery(null, boolFilter));
}
}