void assertBasicSearchWorks(String indexName) {
logger.info("--> testing basic search");
SearchRequestBuilder searchReq =
client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery());
SearchResponse searchRsp = searchReq.get();
ElasticsearchAssertions.assertNoFailures(searchRsp);
long numDocs = searchRsp.getHits().getTotalHits();
logger.info("Found " + numDocs + " in old index");
logger.info("--> testing basic search with sort");
searchReq.addSort("long_sort", SortOrder.ASC);
ElasticsearchAssertions.assertNoFailures(searchReq.get());
logger.info("--> testing exists filter");
searchReq = client().prepareSearch(indexName).setQuery(QueryBuilders.existsQuery("string"));
searchRsp = searchReq.get();
ElasticsearchAssertions.assertNoFailures(searchRsp);
assertEquals(numDocs, searchRsp.getHits().getTotalHits());
logger.info("--> testing missing filter");
// the field for the missing filter here needs to be different than the exists filter above, to
// avoid being found in the cache
searchReq = client().prepareSearch(indexName).setQuery(QueryBuilders.missingQuery("long_sort"));
searchRsp = searchReq.get();
ElasticsearchAssertions.assertNoFailures(searchRsp);
assertEquals(0, searchRsp.getHits().getTotalHits());
}
void assertBasicSearchWorks(String indexName) {
logger.info("--> testing basic search");
SearchRequestBuilder searchReq =
client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery());
SearchResponse searchRsp = searchReq.get();
ElasticsearchAssertions.assertNoFailures(searchRsp);
long numDocs = searchRsp.getHits().getTotalHits();
logger.info("Found {} in old index", numDocs);
logger.info("--> testing basic search with sort");
searchReq.addSort("long_sort", SortOrder.ASC);
ElasticsearchAssertions.assertNoFailures(searchReq.get());
logger.info("--> testing exists filter");
searchReq = client().prepareSearch(indexName).setQuery(QueryBuilders.existsQuery("string"));
searchRsp = searchReq.get();
ElasticsearchAssertions.assertNoFailures(searchRsp);
assertEquals(numDocs, searchRsp.getHits().getTotalHits());
GetSettingsResponse getSettingsResponse =
client().admin().indices().prepareGetSettings(indexName).get();
Version versionCreated =
Version.fromId(
Integer.parseInt(getSettingsResponse.getSetting(indexName, "index.version.created")));
if (versionCreated.onOrAfter(Version.V_2_4_0)) {
searchReq =
client().prepareSearch(indexName).setQuery(QueryBuilders.existsQuery("field.with.dots"));
searchRsp = searchReq.get();
ElasticsearchAssertions.assertNoFailures(searchRsp);
assertEquals(numDocs, searchRsp.getHits().getTotalHits());
}
}
public Map<String, Long> countByField(IndexField indexField, FilterBuilder filter) {
Map<String, Long> counts = new HashMap<>();
SearchRequestBuilder request =
client
.prepareSearch(this.getIndexName())
.setTypes(this.getIndexType())
.setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), filter))
.setSize(0)
.addAggregation(
AggregationBuilders.terms(indexField.field())
.field(indexField.field())
.order(Terms.Order.count(false))
.size(Integer.MAX_VALUE)
.minDocCount(0));
SearchResponse response = request.get();
Terms values = response.getAggregations().get(indexField.field());
for (Terms.Bucket value : values.getBuckets()) {
counts.put(value.getKey(), value.getDocCount());
}
return counts;
}
@Override
@CheckForNull
public Date getLastSynchronization(Map<String, String> params) {
SearchRequestBuilder request =
client
.prepareSearch(this.getIndexName())
.setTypes(this.getIndexType())
.setQuery(
QueryBuilders.filteredQuery(
QueryBuilders.matchAllQuery(), getLastSynchronizationBuilder(params)))
.setSize(0)
.addAggregation(
AggregationBuilders.max("latest").field(BaseNormalizer.UPDATED_AT_FIELD));
SearchResponse response = request.get();
Max max = response.getAggregations().get("latest");
if (max.getValue() > 0) {
Date date = new DateTime((long) max.getValue()).toDate();
LOG.debug(
"Index {}:{} has last update of {}", this.getIndexName(), this.getIndexType(), date);
return date;
} else {
LOG.debug("Index {}:{} has no last update date", this.getIndexName(), this.getIndexType());
return null;
}
}
public void addDeletion(SearchRequestBuilder searchRequest) {
searchRequest
.addSort("_doc", SortOrder.ASC)
.setScroll(TimeValue.timeValueMinutes(5))
.setSize(100)
// load only doc ids, not _source fields
.setFetchSource(false);
// this search is synchronous. An optimization would be to be non-blocking,
// but it requires to tracking pending requests in close().
// Same semaphore can't be reused because of potential deadlock (requires to acquire
// two locks)
SearchResponse searchResponse = searchRequest.get();
while (true) {
SearchHit[] hits = searchResponse.getHits().getHits();
for (SearchHit hit : hits) {
DeleteRequestBuilder deleteRequestBuilder =
client.prepareDelete(hit.index(), hit.type(), hit.getId());
SearchHitField routing = hit.field("_routing");
if (routing != null) {
deleteRequestBuilder.setRouting(routing.getValue());
}
add(deleteRequestBuilder.request());
}
String scrollId = searchResponse.getScrollId();
searchResponse =
client.prepareSearchScroll(scrollId).setScroll(TimeValue.timeValueMinutes(5)).get();
if (hits.length == 0) {
client.nativeClient().prepareClearScroll().addScrollId(scrollId).get();
break;
}
}
}
void assertRealtimeGetWorks(String indexName) {
assertAcked(
client()
.admin()
.indices()
.prepareUpdateSettings(indexName)
.setSettings(Settings.builder().put("refresh_interval", -1).build()));
SearchRequestBuilder searchReq =
client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery());
SearchHit hit = searchReq.get().getHits().getAt(0);
String docId = hit.getId();
// foo is new, it is not a field in the generated index
client().prepareUpdate(indexName, "doc", docId).setDoc("foo", "bar").get();
GetResponse getRsp = client().prepareGet(indexName, "doc", docId).get();
Map<String, Object> source = getRsp.getSourceAsMap();
assertThat(source, Matchers.hasKey("foo"));
assertAcked(
client()
.admin()
.indices()
.prepareUpdateSettings(indexName)
.setSettings(
Settings.builder()
.put("refresh_interval", EngineConfig.DEFAULT_REFRESH_INTERVAL)
.build()));
}
public SearchResponse executeQuery(
MetricConfig conf, String type, List<String> fields, BuilderCustomizer customizer) {
SearchRequestBuilder builder =
client
.prepareSearch(MetricUtils.getIndexes(conf.getStart(), conf.getStop(), client))
.setTypes(type)
.setSearchType(SearchType.QUERY_AND_FETCH)
.setSize((int) (conf.getStop() - conf.getStart()) / 10);
for (String next : fields) {
builder.addField(next);
}
BoolQueryBuilder query =
QueryBuilders.boolQuery()
.must(
QueryBuilders.rangeQuery("timestamp")
.from(conf.getStart() - conf.getStep())
.to(conf.getStop())
.includeLower(false)
.includeUpper(true));
for (String tag : conf.getTags()) {
query = query.must(QueryBuilders.termQuery("tags", tag));
}
builder.setQuery(query);
if (customizer != null) {
customizer.customize(builder);
}
return builder.get();
}
@Test
public void testMultipleRescores() throws Exception {
int numDocs = indexRandomNumbers("keyword", 1);
QueryRescorer eightIsGreat =
RescoreBuilder.queryRescorer(
QueryBuilders.functionScoreQuery(
QueryBuilders.termQuery("field1", English.intToEnglish(8)))
.boostMode(CombineFunction.REPLACE)
.add(ScoreFunctionBuilders.scriptFunction("1000.0f")))
.setScoreMode("total");
QueryRescorer sevenIsBetter =
RescoreBuilder.queryRescorer(
QueryBuilders.functionScoreQuery(
QueryBuilders.termQuery("field1", English.intToEnglish(7)))
.boostMode(CombineFunction.REPLACE)
.add(ScoreFunctionBuilders.scriptFunction("10000.0f")))
.setScoreMode("total");
// First set the rescore window large enough that both rescores take effect
SearchRequestBuilder request = client().prepareSearch().setRescoreWindow(numDocs);
request.addRescorer(eightIsGreat).addRescorer(sevenIsBetter);
SearchResponse response = request.get();
assertFirstHit(response, hasId("7"));
assertSecondHit(response, hasId("8"));
// Now squash the second rescore window so it never gets to see a seven
response =
request
.setSize(1)
.clearRescorers()
.addRescorer(eightIsGreat)
.addRescorer(sevenIsBetter, 1)
.get();
assertFirstHit(response, hasId("8"));
// We have no idea what the second hit will be because we didn't get a chance to look for seven
// Now use one rescore to drag the number we're looking for into the window of another
QueryRescorer ninetyIsGood =
RescoreBuilder.queryRescorer(
QueryBuilders.functionScoreQuery(QueryBuilders.queryString("*ninety*"))
.boostMode(CombineFunction.REPLACE)
.add(ScoreFunctionBuilders.scriptFunction("1000.0f")))
.setScoreMode("total");
QueryRescorer oneToo =
RescoreBuilder.queryRescorer(
QueryBuilders.functionScoreQuery(QueryBuilders.queryString("*one*"))
.boostMode(CombineFunction.REPLACE)
.add(ScoreFunctionBuilders.scriptFunction("1000.0f")))
.setScoreMode("total");
request.clearRescorers().addRescorer(ninetyIsGood).addRescorer(oneToo, 10);
response = request.setSize(2).get();
assertFirstHit(response, hasId("91"));
assertFirstHit(response, hasScore(2001.0f));
assertSecondHit(
response, hasScore(1001.0f)); // Not sure which one it is but it is ninety something
}
// #10067: create-bwc-index.py deleted any doc with long_sort:[10-20]
void assertDeleteByQueryWorked(String indexName, Version version) throws Exception {
if (version.onOrBefore(Version.V_1_0_0_Beta2)) {
// TODO: remove this once #10262 is fixed
return;
}
// these documents are supposed to be deleted by a delete by query operation in the translog
SearchRequestBuilder searchReq =
client()
.prepareSearch(indexName)
.setQuery(QueryBuilders.queryStringQuery("long_sort:[10 TO 20]"));
assertEquals(0, searchReq.get().getHits().getTotalHits());
}
public SearchResponse doSearch(ActivityQuery query, SearchOptions options) {
SearchRequestBuilder requestBuilder =
getClient()
.prepareSearch(ActivityIndexDefinition.INDEX)
.setTypes(ActivityIndexDefinition.TYPE);
requestBuilder.setFrom(options.getOffset());
requestBuilder.setSize(options.getLimit());
requestBuilder.addSort(ActivityIndexDefinition.FIELD_CREATED_AT, SortOrder.DESC);
AndFilterBuilder filter = FilterBuilders.andFilter();
if (!query.getTypes().isEmpty()) {
OrFilterBuilder typeFilter = FilterBuilders.orFilter();
for (String type : query.getTypes()) {
typeFilter.add(FilterBuilders.termFilter(ActivityIndexDefinition.FIELD_TYPE, type));
}
filter.add(typeFilter);
}
if (!query.getDataOrFilters().isEmpty()) {
for (Map.Entry<String, Object> entry : query.getDataOrFilters().entrySet()) {
OrFilterBuilder orFilter = FilterBuilders.orFilter();
orFilter.add(
FilterBuilders.nestedFilter(
ActivityIndexDefinition.FIELD_DETAILS,
FilterBuilders.termFilter(
ActivityIndexDefinition.FIELD_DETAILS + "." + entry.getKey(),
entry.getValue())));
filter.add(orFilter);
}
}
Date since = query.getSince();
if (since != null) {
filter.add(
FilterBuilders.rangeFilter(ActivityIndexDefinition.FIELD_CREATED_AT)
.gt(since)
.cache(false));
}
Date to = query.getTo();
if (to != null) {
filter.add(
FilterBuilders.rangeFilter(ActivityIndexDefinition.FIELD_CREATED_AT).lt(to).cache(false));
}
requestBuilder.setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), filter));
return requestBuilder.get();
}
public List<ActiveRule> findByProfile(QualityProfileKey key) {
SearchRequestBuilder request =
getClient()
.prepareSearch(getIndexName())
.setQuery(
QueryBuilders.termQuery(
ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field(), key.toString()))
.setRouting(key.toString())
// TODO replace by scrolling
.setSize(Integer.MAX_VALUE);
SearchResponse response = request.get();
List<ActiveRule> activeRules = new ArrayList<ActiveRule>();
for (SearchHit hit : response.getHits()) {
activeRules.add(toDoc(hit.getSource()));
}
return activeRules;
}
/** finder methods */
public List<ActiveRule> findByRule(RuleKey key) {
SearchRequestBuilder request =
getClient()
.prepareSearch(this.getIndexName())
.setQuery(
QueryBuilders.hasParentQuery(
this.getParentType(),
QueryBuilders.idsQuery(this.getParentType()).addIds(key.toString())))
.setRouting(key.toString())
// TODO replace by scrolling
.setSize(Integer.MAX_VALUE);
SearchResponse response = request.get();
List<ActiveRule> activeRules = new ArrayList<ActiveRule>();
for (SearchHit hit : response.getHits()) {
activeRules.add(toDoc(hit.getSource()));
}
return activeRules;
}
public void testStringSpecialValueVariable() throws Exception {
// i.e. expression script for term aggregations, which is not allowed
createIndex("test");
ensureGreen("test");
indexRandom(
true,
client().prepareIndex("test", "doc", "1").setSource("text", "hello"),
client().prepareIndex("test", "doc", "2").setSource("text", "goodbye"),
client().prepareIndex("test", "doc", "3").setSource("text", "hello"));
SearchRequestBuilder req = client().prepareSearch().setIndices("test");
req.setQuery(QueryBuilders.matchAllQuery())
.addAggregation(
AggregationBuilders.terms("term_agg")
.field("text")
.script(
new Script(
"_value", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null)));
String message;
try {
// shards that don't have docs with the "text" field will not fail,
// so we may or may not get a total failure
SearchResponse rsp = req.get();
assertThat(
rsp.getShardFailures().length,
greaterThan(0)); // at least the shards containing the docs should have failed
message = rsp.getShardFailures()[0].reason();
} catch (SearchPhaseExecutionException e) {
message = e.toString();
}
assertThat(
message + "should have contained ExpressionScriptExecutionException",
message.contains("ExpressionScriptExecutionException"),
equalTo(true));
assertThat(
message + "should have contained text variable error",
message.contains("text variable"),
equalTo(true));
}
public void testSpecialValueVariable() throws Exception {
// i.e. _value for aggregations
createIndex("test");
ensureGreen("test");
indexRandom(
true,
client().prepareIndex("test", "doc", "1").setSource("x", 5, "y", 1.2),
client().prepareIndex("test", "doc", "2").setSource("x", 10, "y", 1.4),
client().prepareIndex("test", "doc", "3").setSource("x", 13, "y", 1.8));
SearchRequestBuilder req = client().prepareSearch().setIndices("test");
req.setQuery(QueryBuilders.matchAllQuery())
.addAggregation(
AggregationBuilders.stats("int_agg")
.field("x")
.script(
new Script(
"_value * 3", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null)))
.addAggregation(
AggregationBuilders.stats("double_agg")
.field("y")
.script(
new Script(
"_value - 1.1",
ScriptType.INLINE,
ExpressionScriptEngineService.NAME,
null)));
SearchResponse rsp = req.get();
assertEquals(3, rsp.getHits().getTotalHits());
Stats stats = rsp.getAggregations().get("int_agg");
assertEquals(39.0, stats.getMax(), 0.0001);
assertEquals(15.0, stats.getMin(), 0.0001);
stats = rsp.getAggregations().get("double_agg");
assertEquals(0.7, stats.getMax(), 0.0001);
assertEquals(0.1, stats.getMin(), 0.0001);
}
public void testScroll() throws ExecutionException, InterruptedException {
createIndex("test");
ensureYellow("test");
int numDocs = iterations(10, 100);
IndexRequestBuilder[] indexRequestBuilders = new IndexRequestBuilder[numDocs];
for (int i = 0; i < numDocs; i++) {
indexRequestBuilders[i] =
client()
.prepareIndex("test", "type", Integer.toString(i))
.setSource("field", "value" + Integer.toString(i));
}
indexRandom(true, indexRequestBuilders);
int size = randomIntBetween(1, 10);
SearchRequestBuilder searchRequestBuilder =
client().prepareSearch("test").setScroll("1m").setSize(size);
SearchResponse searchResponse = searchRequestBuilder.get();
assertThat(searchResponse.getScrollId(), notNullValue());
assertHitCount(searchResponse, numDocs);
int hits = 0;
assertThat(searchResponse.getHits().getHits().length, greaterThan(0));
hits += searchResponse.getHits().getHits().length;
try {
do {
searchResponse =
client().prepareSearchScroll(searchResponse.getScrollId()).setScroll("1m").get();
assertThat(searchResponse.getScrollId(), notNullValue());
assertHitCount(searchResponse, numDocs);
hits += searchResponse.getHits().getHits().length;
} while (searchResponse.getHits().getHits().length > 0);
assertThat(hits, equalTo(numDocs));
} finally {
clearScroll(searchResponse.getScrollId());
}
}
public void testScore() throws Exception {
createIndex("test");
ensureGreen("test");
indexRandom(
true,
client().prepareIndex("test", "doc", "1").setSource("text", "hello goodbye"),
client().prepareIndex("test", "doc", "2").setSource("text", "hello hello hello goodbye"),
client().prepareIndex("test", "doc", "3").setSource("text", "hello hello goodebye"));
ScoreFunctionBuilder score =
ScoreFunctionBuilders.scriptFunction(
new Script("1 / _score", ScriptType.INLINE, "expression", null));
SearchRequestBuilder req = client().prepareSearch().setIndices("test");
req.setQuery(
QueryBuilders.functionScoreQuery(QueryBuilders.termQuery("text", "hello"), score)
.boostMode("replace"));
req.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); // make sure DF is consistent
SearchResponse rsp = req.get();
assertSearchResponse(rsp);
SearchHits hits = rsp.getHits();
assertEquals(3, hits.getTotalHits());
assertEquals("1", hits.getAt(0).getId());
assertEquals("3", hits.getAt(1).getId());
assertEquals("2", hits.getAt(2).getId());
}
@Test
public void update_existing_document_when_indexing_one_project() throws Exception {
String uuid = "PROJECT-UUID";
esTester.putDocuments(
INDEX_PROJECT_MEASURES,
TYPE_PROJECT_MEASURES,
new ProjectMeasuresDoc()
.setId(uuid)
.setKey("Old Key")
.setName("Old Name")
.setAnalysedAt(new Date(1_000_000L)));
ComponentDto project = newProjectDto(uuid).setKey("New key").setName("New name");
SnapshotDto analysis = componentDbTester.insertProjectAndSnapshot(project);
underTest.index(project.uuid());
assertThat(esTester.getIds(INDEX_PROJECT_MEASURES, TYPE_PROJECT_MEASURES)).containsOnly(uuid);
SearchRequestBuilder request =
esTester
.client()
.prepareSearch(INDEX_PROJECT_MEASURES)
.setTypes(TYPE_PROJECT_MEASURES)
.setQuery(
boolQuery()
.must(matchAllQuery())
.filter(
boolQuery()
.must(termQuery("_id", uuid))
.must(termQuery(ProjectMeasuresIndexDefinition.FIELD_KEY, "New key"))
.must(termQuery(ProjectMeasuresIndexDefinition.FIELD_NAME, "New name"))
.must(
termQuery(
ProjectMeasuresIndexDefinition.FIELD_ANALYSED_AT,
new Date(analysis.getCreatedAt())))));
assertThat(request.get().getHits()).hasSize(1);
}
private SearchHits query(String query)
throws SqlParseException, SQLFeatureNotSupportedException, SQLFeatureNotSupportedException {
SearchDao searchDao = MainTestSuite.getSearchDao();
SearchRequestBuilder select = (SearchRequestBuilder) searchDao.explain(query);
return select.get().getHits();
}