@Override
public String scan(SearchQuery searchQuery, long scrollTimeInMillis, boolean noFields) {
Assert.notNull(searchQuery.getIndices(), "No index defined for Query");
Assert.notNull(searchQuery.getTypes(), "No type define for Query");
Assert.notNull(searchQuery.getPageable(), "Query.pageable is required for scan & scroll");
SearchRequestBuilder requestBuilder =
client
.prepareSearch(toArray(searchQuery.getIndices()))
.setSearchType(SCAN)
.setQuery(searchQuery.getQuery())
.setTypes(toArray(searchQuery.getTypes()))
.setScroll(TimeValue.timeValueMillis(scrollTimeInMillis))
.setFrom(0)
.setSize(searchQuery.getPageable().getPageSize());
if (searchQuery.getFilter() != null) {
requestBuilder.setPostFilter(searchQuery.getFilter());
}
if (isNotEmpty(searchQuery.getFields())) {
requestBuilder.addFields(toArray(searchQuery.getFields()));
}
if (noFields) {
requestBuilder.setNoFields();
}
return requestBuilder.execute().actionGet().getScrollId();
}
public SearchResponse executeQuery(
MetricConfig conf, String type, List<String> fields, BuilderCustomizer customizer) {
SearchRequestBuilder builder =
client
.prepareSearch(MetricUtils.getIndexes(conf.getStart(), conf.getStop(), client))
.setTypes(type)
.setSearchType(SearchType.QUERY_AND_FETCH)
.setSize((int) (conf.getStop() - conf.getStart()) / 10);
for (String next : fields) {
builder.addField(next);
}
BoolQueryBuilder query =
QueryBuilders.boolQuery()
.must(
QueryBuilders.rangeQuery("timestamp")
.from(conf.getStart() - conf.getStep())
.to(conf.getStop())
.includeLower(false)
.includeUpper(true));
for (String tag : conf.getTags()) {
query = query.must(QueryBuilders.termQuery("tags", tag));
}
builder.setQuery(query);
if (customizer != null) {
customizer.customize(builder);
}
return builder.get();
}
void assertRealtimeGetWorks(String indexName) {
assertAcked(
client()
.admin()
.indices()
.prepareUpdateSettings(indexName)
.setSettings(Settings.builder().put("refresh_interval", -1).build()));
SearchRequestBuilder searchReq =
client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery());
SearchHit hit = searchReq.get().getHits().getAt(0);
String docId = hit.getId();
// foo is new, it is not a field in the generated index
client().prepareUpdate(indexName, "doc", docId).setDoc("foo", "bar").get();
GetResponse getRsp = client().prepareGet(indexName, "doc", docId).get();
Map<String, Object> source = getRsp.getSourceAsMap();
assertThat(source, Matchers.hasKey("foo"));
assertAcked(
client()
.admin()
.indices()
.prepareUpdateSettings(indexName)
.setSettings(
Settings.builder()
.put("refresh_interval", EngineConfig.DEFAULT_REFRESH_INTERVAL)
.build()));
}
public TermsResult terms(String field, int size, String query, String filter, TimeRange range)
throws IndexHelper.InvalidRangeFormatException {
if (size == 0) {
size = 50;
}
SearchRequestBuilder srb;
if (filter == null) {
srb = standardSearchRequest(query, IndexHelper.determineAffectedIndices(server, range));
} else {
srb =
filteredSearchRequest(query, filter, IndexHelper.determineAffectedIndices(server, range));
}
TermsFacetBuilder terms = new TermsFacetBuilder(TERMS_FACET_NAME);
terms.global(false);
terms.field(field);
terms.size(size);
terms.facetFilter(standardFilters(range, filter));
srb.addFacet(terms);
final SearchRequest request = srb.request();
SearchResponse r = c.search(request).actionGet();
return new TermsResult(
(TermsFacet) r.getFacets().facet(TERMS_FACET_NAME), query, request.source(), r.getTook());
}
public Map<String, Long> countByField(IndexField indexField, FilterBuilder filter) {
Map<String, Long> counts = new HashMap<>();
SearchRequestBuilder request =
client
.prepareSearch(this.getIndexName())
.setTypes(this.getIndexType())
.setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), filter))
.setSize(0)
.addAggregation(
AggregationBuilders.terms(indexField.field())
.field(indexField.field())
.order(Terms.Order.count(false))
.size(Integer.MAX_VALUE)
.minDocCount(0));
SearchResponse response = request.get();
Terms values = response.getAggregations().get(indexField.field());
for (Terms.Bucket value : values.getBuckets()) {
counts.put(value.getKey(), value.getDocCount());
}
return counts;
}
public FieldStatsResult fieldStats(String field, String query, String filter, TimeRange range)
throws FieldTypeException, IndexHelper.InvalidRangeFormatException {
SearchRequestBuilder srb;
if (filter == null) {
srb = standardSearchRequest(query, IndexHelper.determineAffectedIndices(server, range));
} else {
srb =
filteredSearchRequest(query, filter, IndexHelper.determineAffectedIndices(server, range));
}
StatisticalFacetBuilder stats = new StatisticalFacetBuilder(STATS_FACET_NAME);
stats.global(false);
stats.facetFilter(standardFilters(range, filter));
stats.field(field);
srb.addFacet(stats);
SearchResponse r;
final SearchRequest request;
try {
request = srb.request();
r = c.search(request).actionGet();
} catch (org.elasticsearch.action.search.SearchPhaseExecutionException e) {
throw new FieldTypeException(e);
}
return new FieldStatsResult(
(StatisticalFacet) r.getFacets().facet(STATS_FACET_NAME),
query,
request.source(),
r.getTook());
}
public HistogramResult histogram(
String query, Indexer.DateHistogramInterval interval, String filter, TimeRange range)
throws IndexHelper.InvalidRangeFormatException {
DateHistogramFacetBuilder fb =
FacetBuilders.dateHistogramFacet("histogram")
.field("timestamp")
.interval(interval.toString().toLowerCase());
fb.facetFilter(standardFilters(range, filter));
QueryStringQueryBuilder qs = queryString(query);
qs.allowLeadingWildcard(server.getConfiguration().isAllowLeadingWildcardSearches());
SearchRequestBuilder srb = c.prepareSearch();
srb.setIndices(IndexHelper.determineAffectedIndices(server, range).toArray(new String[] {}));
srb.setQuery(qs);
srb.addFacet(fb);
final SearchRequest request = srb.request();
SearchResponse r = c.search(request).actionGet();
return new DateHistogramResult(
(DateHistogramFacet) r.getFacets().facet("histogram"),
query,
request.source(),
interval,
r.getTook());
}
@Override
@CheckForNull
public Date getLastSynchronization(Map<String, String> params) {
SearchRequestBuilder request =
client
.prepareSearch(this.getIndexName())
.setTypes(this.getIndexType())
.setQuery(
QueryBuilders.filteredQuery(
QueryBuilders.matchAllQuery(), getLastSynchronizationBuilder(params)))
.setSize(0)
.addAggregation(
AggregationBuilders.max("latest").field(BaseNormalizer.UPDATED_AT_FIELD));
SearchResponse response = request.get();
Max max = response.getAggregations().get("latest");
if (max.getValue() > 0) {
Date date = new DateTime((long) max.getValue()).toDate();
LOG.debug(
"Index {}:{} has last update of {}", this.getIndexName(), this.getIndexType(), date);
return date;
} else {
LOG.debug("Index {}:{} has no last update date", this.getIndexName(), this.getIndexType());
return null;
}
}
private SearchRequestBuilder criteria2builder(ElasticSearchCriteria criteria) {
String[] indices = criteria.getIndices();
if (indices == null || indices.length == 0)
indices = new String[] {indexManager.getIndexName()};
SearchRequestBuilder srb = client.prepareSearch(indices);
srb.setTimeout(new TimeValue(10, TimeUnit.SECONDS));
String[] types = criteria.getTypes();
if (types != null && types.length > 0) srb.setTypes(types);
QueryBuilder qb = criteria.getQueryBuilder();
String query = criteria.getQuery();
if (qb == null && StringUtils.isBlank(query))
throw new NullPointerException("queryBuilder is null and queryString is blank");
if (qb == null && StringUtils.isNotBlank(query)) {
if (wildcardQueryPattern.matcher(query).matches()) {
String[] arr = query.split(":", 2);
qb = QueryBuilders.wildcardQuery(arr[0], arr[1]);
} else {
QueryStringQueryBuilder qsqb = new QueryStringQueryBuilder(query);
qsqb.defaultOperator(Operator.AND);
qb = qsqb;
}
}
srb.setQuery(qb);
Map<String, Boolean> sorts = criteria.getSorts();
for (Map.Entry<String, Boolean> entry : sorts.entrySet())
srb.addSort(entry.getKey(), entry.getValue() ? SortOrder.DESC : SortOrder.ASC);
return srb;
}
public void addDeletion(SearchRequestBuilder searchRequest) {
searchRequest
.addSort("_doc", SortOrder.ASC)
.setScroll(TimeValue.timeValueMinutes(5))
.setSize(100)
// load only doc ids, not _source fields
.setFetchSource(false);
// this search is synchronous. An optimization would be to be non-blocking,
// but it requires to tracking pending requests in close().
// Same semaphore can't be reused because of potential deadlock (requires to acquire
// two locks)
SearchResponse searchResponse = searchRequest.get();
while (true) {
SearchHit[] hits = searchResponse.getHits().getHits();
for (SearchHit hit : hits) {
DeleteRequestBuilder deleteRequestBuilder =
client.prepareDelete(hit.index(), hit.type(), hit.getId());
SearchHitField routing = hit.field("_routing");
if (routing != null) {
deleteRequestBuilder.setRouting(routing.getValue());
}
add(deleteRequestBuilder.request());
}
String scrollId = searchResponse.getScrollId();
searchResponse =
client.prepareSearchScroll(scrollId).setScroll(TimeValue.timeValueMinutes(5)).get();
if (hits.length == 0) {
client.nativeClient().prepareClearScroll().addScrollId(scrollId).get();
break;
}
}
}
public void testDeepScrollingDoesNotBlowUp() throws Exception {
client()
.prepareIndex("index", "type", "1")
.setSource("field", "value")
.setRefresh(true)
.execute()
.get();
for (SearchType searchType : SearchType.values()) {
SearchRequestBuilder builder =
client()
.prepareSearch("index")
.setSearchType(searchType)
.setQuery(QueryBuilders.matchAllQuery())
.setSize(Integer.MAX_VALUE)
.setScroll("1m");
SearchResponse response = builder.execute().actionGet();
try {
ElasticsearchAssertions.assertHitCount(response, 1l);
} finally {
String scrollId = response.getScrollId();
if (scrollId != null) {
clearScroll(scrollId);
}
}
}
}
/** Set indices and types to the search request. */
private void setIndicesAndTypes() {
request.setIndices(query.getIndexArr());
String[] typeArr = query.getTypeArr();
if (typeArr != null) {
request.setTypes(typeArr);
}
}
private void addOrder(ProfileRuleQuery query, SearchRequestBuilder builder) {
SortOrder sortOrder = query.asc() ? SortOrder.ASC : SortOrder.DESC;
if (query.sort().equals(ProfileRuleQuery.SORT_BY_RULE_NAME)) {
builder.addSort(RuleDocument.FIELD_NAME + ".raw", sortOrder);
} else if (query.sort().equals(ProfileRuleQuery.SORT_BY_CREATION_DATE)) {
builder.addSort(RuleDocument.FIELD_CREATED_AT, sortOrder);
}
}
public static SearchRequest randomSearch(Client client, String[] indices) {
final SearchRequestBuilder builder = new SearchRequestBuilder(client);
builder.setIndices(indices);
builder.setTypes(INDEX_TYPE);
builder.setQuery(randomFrom(TestQueryType.values()).getQuery());
return builder.request();
}
@Override
public <T> List<String> queryForIds(SearchQuery query) {
SearchRequestBuilder request = prepareSearch(query).setQuery(query.getQuery()).setNoFields();
if (query.getFilter() != null) {
request.setPostFilter(query.getFilter());
}
SearchResponse response = request.execute().actionGet();
return extractIds(response);
}
@Override
public Map<String, SharedStatusInfo> searchStatus(
final String domain, final String query, int page, int size) {
Assert.notNull(query);
Assert.notNull(domain);
if (page < 0) {
page = 0; // Default value
}
if (size <= 0) {
size = SearchService.DEFAULT_PAGE_SIZE;
}
String name = ALL_FIELDS;
QueryBuilder qb = QueryBuilders.matchQuery(name, query);
String dataType = Status.class.getSimpleName().toLowerCase();
FilterBuilder domainFilter = new TermFilterBuilder("domain", domain);
SearchResponse searchResponse = null;
try {
SearchRequestBuilder builder =
this.client
.prepareSearch(this.indexName)
.setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
.setQuery(qb)
.setFilter(domainFilter)
.setTypes(dataType)
.setFrom(page * size)
.setSize(size)
.setExplain(false);
builder.addSort("statusDate", SortOrder.DESC);
searchResponse = builder.execute().actionGet();
} catch (IndexMissingException e) {
log.warn("The index was not found in the cluster.");
return new HashMap<String, SharedStatusInfo>(0);
}
final SearchHits searchHits = searchResponse.getHits();
final Long hitsNumber = searchHits.getTotalHits();
if (hitsNumber == 0) {
return new HashMap<String, SharedStatusInfo>(0);
}
final SearchHit[] searchHitsArray = searchHits.getHits();
final Map<String, SharedStatusInfo> items =
new LinkedHashMap<String, SharedStatusInfo>(hitsNumber.intValue());
for (int i = 0; i < searchHitsArray.length; i++) {
items.put(searchHitsArray[i].getId(), null);
}
return items;
}
// #10067: create-bwc-index.py deleted any doc with long_sort:[10-20]
void assertDeleteByQueryWorked(String indexName, Version version) throws Exception {
if (version.onOrBefore(Version.V_1_0_0_Beta2)) {
// TODO: remove this once #10262 is fixed
return;
}
// these documents are supposed to be deleted by a delete by query operation in the translog
SearchRequestBuilder searchReq =
client()
.prepareSearch(indexName)
.setQuery(QueryBuilders.queryStringQuery("long_sort:[10 TO 20]"));
assertEquals(0, searchReq.get().getHits().getTotalHits());
}
private SearchRequestBuilder standardSearchRequest(
String query, Set<String> indices, int limit, int offset, TimeRange range, Sorting sort)
throws IndexHelper.InvalidRangeFormatException {
if (query == null || query.trim().isEmpty()) {
query = "*";
}
SearchRequestBuilder srb = c.prepareSearch();
srb.setIndices(indices.toArray(new String[] {}));
if (query.trim().equals("*")) {
srb.setQuery(matchAllQuery());
} else {
QueryStringQueryBuilder qs = queryString(query);
qs.allowLeadingWildcard(server.getConfiguration().isAllowLeadingWildcardSearches());
srb.setQuery(qs);
}
srb.setFrom(offset);
if (limit > 0) {
srb.setSize(limit);
}
if (range != null) {
srb.setFilter(IndexHelper.getTimestampRangeFilter(range));
}
if (sort != null) {
srb.addSort(sort.getField(), sort.asElastic());
}
return srb;
}
public SearchResponse searchSimpleQuery(String queryString) {
QueryBuilder query = QueryBuilders.queryStringQuery(queryString);
SearchRequestBuilder searchBuilder =
mClient
.prepareSearch()
.setQuery(query)
.setIndices(idxName) // restrict the search to our indice
.addField("title")
.addHighlightedField("file");
SearchResponse response = searchBuilder.execute().actionGet();
return response;
}
/**
* Perform filter query into one ES index and type to get records with any value stored in one
* field.
*
* @param indexName to search in
* @param indexType to search
* @param fieldName name of field to search for any value in it
* @return
*/
public SearchResponse performQueryByOneFieldAnyValue(
String indexName, String indexType, String fieldName) throws SearchIndexMissingException {
try {
SearchRequestBuilder searchBuilder = getClient().prepareSearch(indexName).setTypes(indexType);
searchBuilder.setFilter(FilterBuilders.notFilter(FilterBuilders.missingFilter(fieldName)));
searchBuilder.setQuery(QueryBuilders.matchAllQuery());
final SearchResponse response = searchBuilder.execute().actionGet();
return response;
} catch (IndexMissingException e) {
log.log(Level.WARNING, e.getMessage());
throw new SearchIndexMissingException(e);
}
}
private SearchRequestBuilder buildRequest(String script, Object... params) {
ensureGreen("test");
Map<String, Object> paramsMap = new HashMap<>();
assert (params.length % 2 == 0);
for (int i = 0; i < params.length; i += 2) {
paramsMap.put(params[i].toString(), params[i + 1]);
}
SearchRequestBuilder req = client().prepareSearch().setIndices("test");
req.setQuery(QueryBuilders.matchAllQuery())
.addSort(SortBuilders.fieldSort("_uid").order(SortOrder.ASC))
.addScriptField("foo", new Script(script, ScriptType.INLINE, "expression", paramsMap));
return req;
}
public final ImmutableList<ModelEntryT> getModels(
final Logger logger,
final Marker logMarker,
final ModelFactory<ModelEntryT> modelFactory,
final SearchRequestBuilder requestBuilder)
throws IoExceptionT {
checkNotNull(requestBuilder);
try {
logger.debug(logMarker, "searching index {} with {}", indexName, requestBuilder);
final SearchResponse response = requestBuilder.execute().actionGet();
final ImmutableList.Builder<ModelEntryT> modelsBuilder = ImmutableList.builder();
for (final SearchHit searchHit : response.getHits()) {
try {
modelsBuilder.add(
modelFactory.createModelEntry(searchHit.getId(), searchHit.getSourceRef()));
} catch (final InvalidModelException e) {
logger.error(logMarker, "invalid model {} from index, ignoring: ", searchHit.getId(), e);
continue;
}
}
return modelsBuilder.build();
} catch (final IndexNotFoundException e) {
throw e;
} catch (final ElasticsearchException e) {
logger.error(
logMarker, "error searching models on index {} with {}: ", indexName, requestBuilder, e);
throw exceptionFactory.newIoException(
e,
String.format("error searching models on index %s with '%s'", indexName, requestBuilder));
}
}
private static SearchResponse searchByAddress(
final String aIndex,
final Client aClient,
final String aType,
SearchType aSearchType,
QueryBuilder aQuery) {
SearchRequestBuilder searchRequest = aClient.prepareSearch(aIndex);
if (aType != null) {
searchRequest.setTypes(aType);
}
if (aSearchType != null) {
searchRequest.setSearchType(aSearchType);
}
searchRequest.setQuery(aQuery);
return searchRequest.execute().actionGet();
}
private SearchHits searchRules(
ProfileRuleQuery query, Paging paging, FilterBuilder filterBuilder, String... fields) {
SearchRequestBuilder builder =
index
.client()
.prepareSearch(INDEX_RULES)
.setTypes(TYPE_RULE)
.setPostFilter(filterBuilder)
.setSize(paging.pageSize())
.setFrom(paging.offset());
if (fields.length > 0) {
builder.addFields(fields);
}
addOrder(query, builder);
return index.executeRequest(builder);
}
private SearchRequestBuilder filteredSearchRequest(
String query,
String filter,
Set<String> indices,
int limit,
int offset,
TimeRange range,
Sorting sort)
throws IndexHelper.InvalidRangeFormatException {
SearchRequestBuilder srb = standardSearchRequest(query, indices, limit, offset, range, sort);
if (range != null && filter != null) {
srb.setFilter(standardFilters(range, filter));
}
return srb;
}
@Test
public void testThatCustomSuggestersCanBeRegisteredAndWork() throws Exception {
createIndex("test");
client()
.prepareIndex("test", "test", "1")
.setSource(jsonBuilder().startObject().field("name", "arbitrary content").endObject())
.setRefresh(true)
.execute()
.actionGet();
ensureYellow();
String randomText = randomAsciiOfLength(10);
String randomField = randomAsciiOfLength(10);
String randomSuffix = randomAsciiOfLength(10);
SearchRequestBuilder searchRequestBuilder =
client().prepareSearch("test").setTypes("test").setFrom(0).setSize(1);
XContentBuilder query =
jsonBuilder()
.startObject()
.startObject("suggest")
.startObject("someName")
.field("text", randomText)
.startObject("custom")
.field("field", randomField)
.field("suffix", randomSuffix)
.endObject()
.endObject()
.endObject()
.endObject();
searchRequestBuilder.setExtraSource(query.bytes());
SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
// TODO: infer type once JI-9019884 is fixed
List<Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>> suggestions =
Lists.<Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>newArrayList(
searchResponse.getSuggest().getSuggestion("someName").iterator());
assertThat(suggestions, hasSize(2));
assertThat(
suggestions.get(0).getText().string(),
is(String.format(Locale.ROOT, "%s-%s-%s-12", randomText, randomField, randomSuffix)));
assertThat(
suggestions.get(1).getText().string(),
is(String.format(Locale.ROOT, "%s-%s-%s-123", randomText, randomField, randomSuffix)));
}
private void explanFields(
SearchRequestBuilder request, List<Field> fields, AggregationBuilder<?> groupByAgg)
throws SqlParseException {
for (Field field : fields) {
if (field instanceof MethodField) {
AbstractAggregationBuilder makeAgg = aggMaker.makeFieldAgg((MethodField) field, groupByAgg);
if (groupByAgg != null) {
groupByAgg.subAggregation(makeAgg);
} else {
request.addAggregation(makeAgg);
}
} else if (field instanceof Field) {
request.addField(field.getName());
} else {
throw new SqlParseException("it did not support this field method " + field);
}
}
}
public List<ActiveRule> findByProfile(QualityProfileKey key) {
SearchRequestBuilder request =
getClient()
.prepareSearch(getIndexName())
.setQuery(
QueryBuilders.termQuery(
ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field(), key.toString()))
.setRouting(key.toString())
// TODO replace by scrolling
.setSize(Integer.MAX_VALUE);
SearchResponse response = request.get();
List<ActiveRule> activeRules = new ArrayList<ActiveRule>();
for (SearchHit hit : response.getHits()) {
activeRules.add(toDoc(hit.getSource()));
}
return activeRules;
}
private SearchHits searchActiveRules(
ProfileRuleQuery query, List<Integer> ruleIds, String... fields) {
SearchRequestBuilder activeRuleBuilder =
index
.client()
.prepareSearch(INDEX_RULES)
.setTypes(ESActiveRule.TYPE_ACTIVE_RULE)
.setPostFilter(
boolFilter()
.must(
termFilter(ActiveRuleDocument.FIELD_PROFILE_ID, query.profileId()),
hasParentFilter(TYPE_RULE, termsFilter(RuleDocument.FIELD_ID, ruleIds))))
.setSize(ruleIds.size());
if (fields.length > 0) {
activeRuleBuilder.addFields(fields);
}
return index.executeRequest(activeRuleBuilder);
}
public long count(Date startTime, Date endTime) {
QueryBuilder queryBuilder =
QueryBuilders.boolQuery()
.must(
QueryBuilders.rangeQuery("lastModifyTime")
.gt(startTime.getTime())
.lt(endTime.getTime()));
SearchRequestBuilder requestBuilder =
esClient
.prepareSearch("hermes")
.setTypes(Trade.class.getSimpleName())
.setQuery(queryBuilder)
.setSearchType(SearchType.COUNT);
SearchResponse response = requestBuilder.execute().actionGet();
return response.getHits().getTotalHits();
}