@SuppressWarnings("unchecked")
@Override
public Collection<String> searchUserByPrefix(String domain, String prefix) {
QueryBuilder qb = QueryBuilders.prefixQuery("username", prefix);
String dataType = User.class.getSimpleName().toLowerCase();
FilterBuilder domainFilter = new TermFilterBuilder("domain", domain);
SearchResponse searchResponse;
try {
searchResponse =
this.client
.prepareSearch(this.indexName)
.setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
.setQuery(qb)
.setFilter(domainFilter)
.setTypes(dataType)
.setFrom(0)
.setSize(DEFAULT_TOP_N_SEARCH_USER)
.setExplain(false)
.addSort(SortBuilders.fieldSort("username").order(SortOrder.ASC))
.execute()
.actionGet();
} catch (IndexMissingException e) {
log.warn("The index was not found in the cluster.");
return new ArrayList<String>(0);
}
final SearchHits searchHits = searchResponse.getHits();
final Long hitsNumber = searchHits.getTotalHits();
if (hitsNumber == 0) {
return new ArrayList<String>(0);
}
final SearchHit[] searchHitsArray = searchHits.getHits();
final List<String> logins = new ArrayList<String>(hitsNumber.intValue());
Map<String, Object> user = null;
try {
String username = null;
String login = null;
for (int i = 0; i < searchHitsArray.length; i++) {
user = this.mapper.readValue(searchHitsArray[i].source(), Map.class);
username = (String) user.get("username");
login = DomainUtil.getLoginFromUsernameAndDomain(username, domain);
logins.add(login);
}
} catch (JsonParseException e) {
log.error("Json parse exception", e);
} catch (JsonMappingException e) {
log.error("Json mapping exception", e);
} catch (IOException e) {
log.error("IO exception", e);
}
return logins;
}
@Test
public void testIgnoreUnmapped() throws Exception {
client.admin().indices().prepareDelete().execute().actionGet();
client
.prepareIndex("test", "type1", "1")
.setSource(
jsonBuilder()
.startObject()
.field("id", "1")
.field("i_value", -1)
.field("d_value", -1.1)
.endObject())
.execute()
.actionGet();
client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
logger.info("--> sort with an unmapped field, verify it fails");
try {
SearchResponse searchResponse =
client
.prepareSearch()
.setQuery(matchAllQuery())
.addSort(SortBuilders.fieldSort("kkk"))
.execute()
.actionGet();
assert false;
} catch (SearchPhaseExecutionException e) {
}
SearchResponse searchResponse =
client
.prepareSearch()
.setQuery(matchAllQuery())
.addSort(SortBuilders.fieldSort("kkk").ignoreUnmapped(true))
.execute()
.actionGet();
assertThat(searchResponse.getFailedShards(), equalTo(0));
}
private SearchRequestBuilder buildRequest(String script, Object... params) {
ensureGreen("test");
Map<String, Object> paramsMap = new HashMap<>();
assert (params.length % 2 == 0);
for (int i = 0; i < params.length; i += 2) {
paramsMap.put(params[i].toString(), params[i + 1]);
}
SearchRequestBuilder req = client().prepareSearch().setIndices("test");
req.setQuery(QueryBuilders.matchAllQuery())
.addSort(SortBuilders.fieldSort("_uid").order(SortOrder.ASC))
.addScriptField("foo", new Script(script, ScriptType.INLINE, "expression", paramsMap));
return req;
}
@Test
public void testSortMVField() throws Exception {
try {
client.admin().indices().prepareDelete("test").execute().actionGet();
} catch (Exception e) {
// ignore
}
client
.admin()
.indices()
.prepareCreate("test")
.setSettings(
ImmutableSettings.settingsBuilder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0))
.addMapping(
"type1",
XContentFactory.jsonBuilder()
.startObject()
.startObject("type1")
.startObject("properties")
.startObject("long_values")
.field("type", "long")
.endObject()
.startObject("int_values")
.field("type", "integer")
.endObject()
.startObject("short_values")
.field("type", "short")
.endObject()
.startObject("byte_values")
.field("type", "byte")
.endObject()
.startObject("float_values")
.field("type", "float")
.endObject()
.startObject("double_values")
.field("type", "double")
.endObject()
.startObject("string_values")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
.endObject()
.endObject()
.endObject())
.execute()
.actionGet();
client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
client
.prepareIndex("test", "type1", Integer.toString(1))
.setSource(
jsonBuilder()
.startObject()
.array("long_values", 1l, 5l, 10l, 8l)
.array("int_values", 1, 5, 10, 8)
.array("short_values", 1, 5, 10, 8)
.array("byte_values", 1, 5, 10, 8)
.array("float_values", 1f, 5f, 10f, 8f)
.array("double_values", 1d, 5d, 10d, 8d)
.array("string_values", "01", "05", "10", "08")
.endObject())
.execute()
.actionGet();
client
.prepareIndex("test", "type1", Integer.toString(2))
.setSource(
jsonBuilder()
.startObject()
.array("long_values", 11l, 15l, 20l, 7l)
.array("int_values", 11, 15, 20, 7)
.array("short_values", 11, 15, 20, 7)
.array("byte_values", 11, 15, 20, 7)
.array("float_values", 11f, 15f, 20f, 7f)
.array("double_values", 11d, 15d, 20d, 7d)
.array("string_values", "11", "15", "20", "07")
.endObject())
.execute()
.actionGet();
client
.prepareIndex("test", "type1", Integer.toString(3))
.setSource(
jsonBuilder()
.startObject()
.array("long_values", 2l, 1l, 3l, -4l)
.array("int_values", 2, 1, 3, -4)
.array("short_values", 2, 1, 3, -4)
.array("byte_values", 2, 1, 3, -4)
.array("float_values", 2f, 1f, 3f, -4f)
.array("double_values", 2d, 1d, 3d, -4d)
.array("string_values", "02", "01", "03", "!4")
.endObject())
.execute()
.actionGet();
client.admin().indices().prepareRefresh().execute().actionGet();
SearchResponse searchResponse =
client
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(10)
.addSort("long_values", SortOrder.ASC)
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
assertThat(searchResponse.getHits().hits().length, equalTo(3));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3)));
assertThat(
((Number) searchResponse.getHits().getAt(0).sortValues()[0]).longValue(), equalTo(-4l));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
assertThat(
((Number) searchResponse.getHits().getAt(1).sortValues()[0]).longValue(), equalTo(1l));
assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2)));
assertThat(
((Number) searchResponse.getHits().getAt(2).sortValues()[0]).longValue(), equalTo(7l));
searchResponse =
client
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(10)
.addSort("long_values", SortOrder.DESC)
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
assertThat(searchResponse.getHits().hits().length, equalTo(3));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2)));
assertThat(
((Number) searchResponse.getHits().getAt(0).sortValues()[0]).longValue(), equalTo(20l));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
assertThat(
((Number) searchResponse.getHits().getAt(1).sortValues()[0]).longValue(), equalTo(10l));
assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3)));
assertThat(
((Number) searchResponse.getHits().getAt(2).sortValues()[0]).longValue(), equalTo(3l));
searchResponse =
client
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(10)
.addSort(SortBuilders.fieldSort("long_values").order(SortOrder.DESC).sortMode("sum"))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
assertThat(searchResponse.getHits().hits().length, equalTo(3));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2)));
assertThat(
((Number) searchResponse.getHits().getAt(0).sortValues()[0]).longValue(), equalTo(53l));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
assertThat(
((Number) searchResponse.getHits().getAt(1).sortValues()[0]).longValue(), equalTo(24l));
assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3)));
assertThat(
((Number) searchResponse.getHits().getAt(2).sortValues()[0]).longValue(), equalTo(2l));
searchResponse =
client
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(10)
.addSort("int_values", SortOrder.ASC)
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
assertThat(searchResponse.getHits().hits().length, equalTo(3));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3)));
assertThat(
((Number) searchResponse.getHits().getAt(0).sortValues()[0]).intValue(), equalTo(-4));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).intValue(), equalTo(1));
assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2)));
assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).intValue(), equalTo(7));
searchResponse =
client
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(10)
.addSort("int_values", SortOrder.DESC)
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
assertThat(searchResponse.getHits().hits().length, equalTo(3));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2)));
assertThat(
((Number) searchResponse.getHits().getAt(0).sortValues()[0]).intValue(), equalTo(20));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
assertThat(
((Number) searchResponse.getHits().getAt(1).sortValues()[0]).intValue(), equalTo(10));
assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3)));
assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).intValue(), equalTo(3));
searchResponse =
client
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(10)
.addSort("short_values", SortOrder.ASC)
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
assertThat(searchResponse.getHits().hits().length, equalTo(3));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3)));
assertThat(
((Number) searchResponse.getHits().getAt(0).sortValues()[0]).intValue(), equalTo(-4));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).intValue(), equalTo(1));
assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2)));
assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).intValue(), equalTo(7));
searchResponse =
client
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(10)
.addSort("short_values", SortOrder.DESC)
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
assertThat(searchResponse.getHits().hits().length, equalTo(3));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2)));
assertThat(
((Number) searchResponse.getHits().getAt(0).sortValues()[0]).intValue(), equalTo(20));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
assertThat(
((Number) searchResponse.getHits().getAt(1).sortValues()[0]).intValue(), equalTo(10));
assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3)));
assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).intValue(), equalTo(3));
searchResponse =
client
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(10)
.addSort("byte_values", SortOrder.ASC)
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
assertThat(searchResponse.getHits().hits().length, equalTo(3));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3)));
assertThat(
((Number) searchResponse.getHits().getAt(0).sortValues()[0]).intValue(), equalTo(-4));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).intValue(), equalTo(1));
assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2)));
assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).intValue(), equalTo(7));
searchResponse =
client
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(10)
.addSort("byte_values", SortOrder.DESC)
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
assertThat(searchResponse.getHits().hits().length, equalTo(3));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2)));
assertThat(
((Number) searchResponse.getHits().getAt(0).sortValues()[0]).intValue(), equalTo(20));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
assertThat(
((Number) searchResponse.getHits().getAt(1).sortValues()[0]).intValue(), equalTo(10));
assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3)));
assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).intValue(), equalTo(3));
searchResponse =
client
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(10)
.addSort("float_values", SortOrder.ASC)
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
assertThat(searchResponse.getHits().hits().length, equalTo(3));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3)));
assertThat(
((Number) searchResponse.getHits().getAt(0).sortValues()[0]).floatValue(), equalTo(-4f));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
assertThat(
((Number) searchResponse.getHits().getAt(1).sortValues()[0]).floatValue(), equalTo(1f));
assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2)));
assertThat(
((Number) searchResponse.getHits().getAt(2).sortValues()[0]).floatValue(), equalTo(7f));
searchResponse =
client
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(10)
.addSort("float_values", SortOrder.DESC)
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
assertThat(searchResponse.getHits().hits().length, equalTo(3));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2)));
assertThat(
((Number) searchResponse.getHits().getAt(0).sortValues()[0]).floatValue(), equalTo(20f));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
assertThat(
((Number) searchResponse.getHits().getAt(1).sortValues()[0]).floatValue(), equalTo(10f));
assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3)));
assertThat(
((Number) searchResponse.getHits().getAt(2).sortValues()[0]).floatValue(), equalTo(3f));
searchResponse =
client
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(10)
.addSort("double_values", SortOrder.ASC)
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
assertThat(searchResponse.getHits().hits().length, equalTo(3));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3)));
assertThat(
((Number) searchResponse.getHits().getAt(0).sortValues()[0]).doubleValue(), equalTo(-4d));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
assertThat(
((Number) searchResponse.getHits().getAt(1).sortValues()[0]).doubleValue(), equalTo(1d));
assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2)));
assertThat(
((Number) searchResponse.getHits().getAt(2).sortValues()[0]).doubleValue(), equalTo(7d));
searchResponse =
client
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(10)
.addSort("double_values", SortOrder.DESC)
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
assertThat(searchResponse.getHits().hits().length, equalTo(3));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2)));
assertThat(
((Number) searchResponse.getHits().getAt(0).sortValues()[0]).doubleValue(), equalTo(20d));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
assertThat(
((Number) searchResponse.getHits().getAt(1).sortValues()[0]).doubleValue(), equalTo(10d));
assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3)));
assertThat(
((Number) searchResponse.getHits().getAt(2).sortValues()[0]).doubleValue(), equalTo(3d));
searchResponse =
client
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(10)
.addSort("string_values", SortOrder.ASC)
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
assertThat(searchResponse.getHits().hits().length, equalTo(3));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3)));
assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("!4"));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
assertThat(((Text) searchResponse.getHits().getAt(1).sortValues()[0]).string(), equalTo("01"));
assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2)));
assertThat(((Text) searchResponse.getHits().getAt(2).sortValues()[0]).string(), equalTo("07"));
searchResponse =
client
.prepareSearch()
.setQuery(matchAllQuery())
.setSize(10)
.addSort("string_values", SortOrder.DESC)
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
assertThat(searchResponse.getHits().hits().length, equalTo(3));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2)));
assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("20"));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
assertThat(((Text) searchResponse.getHits().getAt(1).sortValues()[0]).string(), equalTo("10"));
assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3)));
assertThat(((Text) searchResponse.getHits().getAt(2).sortValues()[0]).string(), equalTo("03"));
}
@Test
public void testSortMissing() throws Exception {
try {
client.admin().indices().prepareDelete("test").execute().actionGet();
} catch (Exception e) {
// ignore
}
client.admin().indices().prepareCreate("test").execute().actionGet();
client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
client
.prepareIndex("test", "type1", "1")
.setSource(
jsonBuilder()
.startObject()
.field("id", "1")
.field("i_value", -1)
.field("d_value", -1.1)
.endObject())
.execute()
.actionGet();
client
.prepareIndex("test", "type1", "2")
.setSource(jsonBuilder().startObject().field("id", "2").endObject())
.execute()
.actionGet();
client
.prepareIndex("test", "type1", "3")
.setSource(
jsonBuilder()
.startObject()
.field("id", "1")
.field("i_value", 2)
.field("d_value", 2.2)
.endObject())
.execute()
.actionGet();
client.admin().indices().prepareFlush().setRefresh(true).execute().actionGet();
logger.info("--> sort with no missing (same as missing _last)");
SearchResponse searchResponse =
client
.prepareSearch()
.setQuery(matchAllQuery())
.addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC))
.execute()
.actionGet();
assertThat(
Arrays.toString(searchResponse.getShardFailures()),
searchResponse.getFailedShards(),
equalTo(0));
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo("3"));
assertThat(searchResponse.getHits().getAt(2).id(), equalTo("2"));
logger.info("--> sort with missing _last");
searchResponse =
client
.prepareSearch()
.setQuery(matchAllQuery())
.addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC).missing("_last"))
.execute()
.actionGet();
assertThat(
Arrays.toString(searchResponse.getShardFailures()),
searchResponse.getFailedShards(),
equalTo(0));
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo("3"));
assertThat(searchResponse.getHits().getAt(2).id(), equalTo("2"));
logger.info("--> sort with missing _first");
searchResponse =
client
.prepareSearch()
.setQuery(matchAllQuery())
.addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC).missing("_first"))
.execute()
.actionGet();
assertThat(
Arrays.toString(searchResponse.getShardFailures()),
searchResponse.getFailedShards(),
equalTo(0));
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3l));
assertThat(searchResponse.getHits().getAt(0).id(), equalTo("2"));
assertThat(searchResponse.getHits().getAt(1).id(), equalTo("1"));
assertThat(searchResponse.getHits().getAt(2).id(), equalTo("3"));
}
/**
* Add a sort against the given field name.
*
* @param name The name of the field to sort by
*/
public SearchSourceBuilder sort(String name) {
return sort(SortBuilders.fieldSort(name));
}
/**
* Adds a sort against the given field name and the sort ordering.
*
* @param name The name of the field
* @param order The sort ordering
*/
public SearchSourceBuilder sort(String name, SortOrder order) {
return sort(SortBuilders.fieldSort(name).order(order));
}
@Test
public void simpleDistanceTests() throws Exception {
try {
client.admin().indices().prepareDelete("test").execute().actionGet();
} catch (Exception e) {
// ignore
}
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type1")
.startObject("properties")
.startObject("location")
.field("type", "geo_point")
.field("lat_lon", true)
.endObject()
.endObject()
.endObject()
.endObject()
.string();
client
.admin()
.indices()
.prepareCreate("test")
.addMapping("type1", mapping)
.execute()
.actionGet();
client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
client
.prepareIndex("test", "type1", "1")
.setSource(
jsonBuilder()
.startObject()
.field("name", "New York")
.startObject("location")
.field("lat", 40.7143528)
.field("lon", -74.0059731)
.endObject()
.endObject())
.execute()
.actionGet();
// to NY: 5.286 km
client
.prepareIndex("test", "type1", "2")
.setSource(
jsonBuilder()
.startObject()
.field("name", "Times Square")
.startObject("location")
.field("lat", 40.759011)
.field("lon", -73.9844722)
.endObject()
.endObject())
.execute()
.actionGet();
// to NY: 0.4621 km
client
.prepareIndex("test", "type1", "3")
.setSource(
jsonBuilder()
.startObject()
.field("name", "Tribeca")
.startObject("location")
.field("lat", 40.718266)
.field("lon", -74.007819)
.endObject()
.endObject())
.execute()
.actionGet();
// to NY: 1.055 km
client
.prepareIndex("test", "type1", "4")
.setSource(
jsonBuilder()
.startObject()
.field("name", "Wall Street")
.startObject("location")
.field("lat", 40.7051157)
.field("lon", -74.0088305)
.endObject()
.endObject())
.execute()
.actionGet();
// to NY: 1.258 km
client
.prepareIndex("test", "type1", "5")
.setSource(
jsonBuilder()
.startObject()
.field("name", "Soho")
.startObject("location")
.field("lat", 40.7247222)
.field("lon", -74)
.endObject()
.endObject())
.execute()
.actionGet();
// to NY: 2.029 km
client
.prepareIndex("test", "type1", "6")
.setSource(
jsonBuilder()
.startObject()
.field("name", "Greenwich Village")
.startObject("location")
.field("lat", 40.731033)
.field("lon", -73.9962255)
.endObject()
.endObject())
.execute()
.actionGet();
// to NY: 8.572 km
client
.prepareIndex("test", "type1", "7")
.setSource(
jsonBuilder()
.startObject()
.field("name", "Brooklyn")
.startObject("location")
.field("lat", 40.65)
.field("lon", -73.95)
.endObject()
.endObject())
.execute()
.actionGet();
client.admin().indices().prepareRefresh().execute().actionGet();
SearchResponse searchResponse =
client
.prepareSearch() // from NY
.setQuery(
filteredQuery(
matchAllQuery(),
geoDistanceFilter("location").distance("3km").point(40.7143528, -74.0059731)))
.execute()
.actionGet();
assertThat(searchResponse.hits().getTotalHits(), equalTo(5l));
assertThat(searchResponse.hits().hits().length, equalTo(5));
for (SearchHit hit : searchResponse.hits()) {
assertThat(
hit.id(), anyOf(equalTo("1"), equalTo("3"), equalTo("4"), equalTo("5"), equalTo("6")));
}
searchResponse =
client
.prepareSearch() // from NY
.setQuery(
filteredQuery(
matchAllQuery(),
geoDistanceFilter("location")
.distance("3km")
.point(40.7143528, -74.0059731)
.optimizeBbox("indexed")))
.execute()
.actionGet();
assertThat(searchResponse.hits().getTotalHits(), equalTo(5l));
assertThat(searchResponse.hits().hits().length, equalTo(5));
for (SearchHit hit : searchResponse.hits()) {
assertThat(
hit.id(), anyOf(equalTo("1"), equalTo("3"), equalTo("4"), equalTo("5"), equalTo("6")));
}
// now with a PLANE type
searchResponse =
client
.prepareSearch() // from NY
.setQuery(
filteredQuery(
matchAllQuery(),
geoDistanceFilter("location")
.distance("3km")
.geoDistance(GeoDistance.PLANE)
.point(40.7143528, -74.0059731)))
.execute()
.actionGet();
assertThat(searchResponse.hits().getTotalHits(), equalTo(5l));
assertThat(searchResponse.hits().hits().length, equalTo(5));
for (SearchHit hit : searchResponse.hits()) {
assertThat(
hit.id(), anyOf(equalTo("1"), equalTo("3"), equalTo("4"), equalTo("5"), equalTo("6")));
}
// factor type is really too small for this resolution
searchResponse =
client
.prepareSearch() // from NY
.setQuery(
filteredQuery(
matchAllQuery(),
geoDistanceFilter("location").distance("2km").point(40.7143528, -74.0059731)))
.execute()
.actionGet();
assertThat(searchResponse.hits().getTotalHits(), equalTo(4l));
assertThat(searchResponse.hits().hits().length, equalTo(4));
for (SearchHit hit : searchResponse.hits()) {
assertThat(hit.id(), anyOf(equalTo("1"), equalTo("3"), equalTo("4"), equalTo("5")));
}
searchResponse =
client
.prepareSearch() // from NY
.setQuery(
filteredQuery(
matchAllQuery(),
geoDistanceFilter("location")
.distance("2km")
.point(40.7143528, -74.0059731)
.optimizeBbox("indexed")))
.execute()
.actionGet();
assertThat(searchResponse.hits().getTotalHits(), equalTo(4l));
assertThat(searchResponse.hits().hits().length, equalTo(4));
for (SearchHit hit : searchResponse.hits()) {
assertThat(hit.id(), anyOf(equalTo("1"), equalTo("3"), equalTo("4"), equalTo("5")));
}
searchResponse =
client
.prepareSearch() // from NY
.setQuery(
filteredQuery(
matchAllQuery(),
geoDistanceFilter("location")
.distance("1.242mi")
.point(40.7143528, -74.0059731)))
.execute()
.actionGet();
assertThat(searchResponse.hits().getTotalHits(), equalTo(4l));
assertThat(searchResponse.hits().hits().length, equalTo(4));
for (SearchHit hit : searchResponse.hits()) {
assertThat(hit.id(), anyOf(equalTo("1"), equalTo("3"), equalTo("4"), equalTo("5")));
}
searchResponse =
client
.prepareSearch() // from NY
.setQuery(
filteredQuery(
matchAllQuery(),
geoDistanceFilter("location")
.distance("1.242mi")
.point(40.7143528, -74.0059731)
.optimizeBbox("indexed")))
.execute()
.actionGet();
assertThat(searchResponse.hits().getTotalHits(), equalTo(4l));
assertThat(searchResponse.hits().hits().length, equalTo(4));
for (SearchHit hit : searchResponse.hits()) {
assertThat(hit.id(), anyOf(equalTo("1"), equalTo("3"), equalTo("4"), equalTo("5")));
}
searchResponse =
client
.prepareSearch() // from NY
.setQuery(
filteredQuery(
matchAllQuery(),
geoDistanceRangeFilter("location")
.from("1.0km")
.to("2.0km")
.point(40.7143528, -74.0059731)))
.execute()
.actionGet();
assertThat(searchResponse.hits().getTotalHits(), equalTo(2l));
assertThat(searchResponse.hits().hits().length, equalTo(2));
for (SearchHit hit : searchResponse.hits()) {
assertThat(hit.id(), anyOf(equalTo("4"), equalTo("5")));
}
searchResponse =
client
.prepareSearch() // from NY
.setQuery(
filteredQuery(
matchAllQuery(),
geoDistanceRangeFilter("location")
.from("1.0km")
.to("2.0km")
.point(40.7143528, -74.0059731)
.optimizeBbox("indexed")))
.execute()
.actionGet();
assertThat(searchResponse.hits().getTotalHits(), equalTo(2l));
assertThat(searchResponse.hits().hits().length, equalTo(2));
for (SearchHit hit : searchResponse.hits()) {
assertThat(hit.id(), anyOf(equalTo("4"), equalTo("5")));
}
// SORTING
searchResponse =
client
.prepareSearch()
.setQuery(matchAllQuery())
.addSort(
SortBuilders.geoDistanceSort("location")
.point(40.7143528, -74.0059731)
.order(SortOrder.ASC))
.execute()
.actionGet();
assertThat(searchResponse.hits().getTotalHits(), equalTo(7l));
assertThat(searchResponse.hits().hits().length, equalTo(7));
assertThat(searchResponse.hits().getAt(0).id(), equalTo("1"));
assertThat(searchResponse.hits().getAt(1).id(), equalTo("3"));
assertThat(searchResponse.hits().getAt(2).id(), equalTo("4"));
assertThat(searchResponse.hits().getAt(3).id(), equalTo("5"));
assertThat(searchResponse.hits().getAt(4).id(), equalTo("6"));
assertThat(searchResponse.hits().getAt(5).id(), equalTo("2"));
assertThat(searchResponse.hits().getAt(6).id(), equalTo("7"));
searchResponse =
client
.prepareSearch()
.setQuery(matchAllQuery())
.addSort(
SortBuilders.geoDistanceSort("location")
.point(40.7143528, -74.0059731)
.order(SortOrder.DESC))
.execute()
.actionGet();
assertThat(searchResponse.hits().getTotalHits(), equalTo(7l));
assertThat(searchResponse.hits().hits().length, equalTo(7));
assertThat(searchResponse.hits().getAt(6).id(), equalTo("1"));
assertThat(searchResponse.hits().getAt(5).id(), equalTo("3"));
assertThat(searchResponse.hits().getAt(4).id(), equalTo("4"));
assertThat(searchResponse.hits().getAt(3).id(), equalTo("5"));
assertThat(searchResponse.hits().getAt(2).id(), equalTo("6"));
assertThat(searchResponse.hits().getAt(1).id(), equalTo("2"));
assertThat(searchResponse.hits().getAt(0).id(), equalTo("7"));
}
/**
* Pull action thread
*
* @param request request
* @param state state
* @param transportClient bulk client for remote cluster access
* @param nodeClient bulk client for local cluster access
*/
final void performPull(
final KnapsackPullRequest request,
final KnapsackState state,
final BulkTransportClient transportClient,
final BulkNodeClient nodeClient) {
try {
logger.info("start of pull: {}", state);
long count = 0L;
Map<String, Set<String>> indices = new HashMap<>();
for (String s : Strings.commaDelimitedListToSet(request.getIndex())) {
indices.put(s, Strings.commaDelimitedListToSet(request.getType()));
}
if (request.withMetadata()) {
// renaming indices/types
if (request.getIndexTypeNames() != null) {
for (Object spec : request.getIndexTypeNames().keySet()) {
if (spec == null) {
continue;
}
String[] s = spec.toString().split("/");
String index = s[0];
String type = s.length > 1 ? s[1] : null;
if (!"_all".equals(index)) {
Set<String> types = indices.get(index);
if (types == null) {
types = new HashSet<>();
}
if (type != null) {
types.add(type);
}
indices.put(index, types);
}
}
}
// get settings for all indices
logger.info("getting settings for indices {}", indices.keySet());
Set<String> settingsIndices = new HashSet<>(indices.keySet());
settingsIndices.remove("_all");
Map<String, String> settings =
getSettings(
transportClient.client(),
settingsIndices.toArray(new String[settingsIndices.size()]));
logger.info("found indices: {}", settings.keySet());
// we resolved the specs in indices to the real indices in the settings
// get mapping and alias per index and create index if copy mode is enabled
for (String index : settings.keySet()) {
CreateIndexRequest createIndexRequest = createIndexRequest(mapIndex(request, index));
Set<String> types = indices.get(index);
createIndexRequest.settings(settings.get(index));
logger.info("getting mappings for index {} and types {}", index, types);
Map<String, String> mappings =
getMapping(
transportClient.client(), index, types != null ? new HashSet<>(types) : null);
logger.info("found mappings: {}", mappings.keySet());
for (String type : mappings.keySet()) {
logger.info("adding mapping: {}", mapType(request, index, type));
createIndexRequest.mapping(mapType(request, index, type), mappings.get(type));
}
// create index
logger.info("creating index: {}", mapIndex(request, index));
nodeClient.client().execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
logger.info("index created: {}", mapIndex(request, index));
logger.info("getting aliases for index {}", index);
Map<String, String> aliases = getAliases(client, index);
logger.info("found {} aliases", aliases.size());
if (!aliases.isEmpty()) {
IndicesAliasesRequestBuilder requestBuilder =
new IndicesAliasesRequestBuilder(
nodeClient.client(), IndicesAliasesAction.INSTANCE);
for (String alias : aliases.keySet()) {
if (aliases.get(alias).isEmpty()) {
requestBuilder.addAlias(index, alias);
} else {
requestBuilder.addAlias(index, alias, aliases.get(alias)); // with filter
}
}
requestBuilder.execute().actionGet();
logger.info("aliases created", aliases.size());
}
}
}
SearchRequest searchRequest = request.getSearchRequest();
if (searchRequest == null) {
searchRequest =
new SearchRequestBuilder(transportClient.client(), SearchAction.INSTANCE)
.setQuery(QueryBuilders.matchAllQuery())
.addSort(SortBuilders.fieldSort("_doc"))
.request();
}
long total = 0L;
for (String index : indices.keySet()) {
if (!"_all".equals(index)) {
searchRequest.indices(index);
}
Set<String> types = indices.get(index);
if (types != null) {
searchRequest.types(types.toArray(new String[types.size()]));
}
searchRequest.scroll(request.getTimeout());
SearchResponse searchResponse =
transportClient.client().execute(SearchAction.INSTANCE, searchRequest).actionGet();
do {
total += searchResponse.getHits().getHits().length;
logger.debug(
"total={} hits={} took={}",
total,
searchResponse.getHits().getHits().length,
searchResponse.getTookInMillis());
for (SearchHit hit : searchResponse.getHits()) {
indexSearchHit(nodeClient, request, hit);
count++;
}
searchResponse =
transportClient
.client()
.execute(
SearchScrollAction.INSTANCE,
new SearchScrollRequest(searchResponse.getScrollId())
.scroll(request.getTimeout()))
.actionGet();
} while (searchResponse.getHits().getHits().length > 0 && !Thread.interrupted());
}
nodeClient.flushIngest();
nodeClient.waitForResponses(TimeValue.timeValueSeconds(60));
for (String index : indices.keySet()) {
nodeClient.refreshIndex(index);
}
nodeClient.shutdown();
transportClient.shutdown();
logger.info("end of pull: {}, docs = {}, count = {}", state, total, count);
} catch (Throwable e) {
logger.error(e.getMessage(), e);
} finally {
try {
knapsack.removeImport(state);
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
}
}
public static SearchSourceBuilder randomSearchSourceBuilder(
Supplier<HighlightBuilder> randomHighlightBuilder,
Supplier<SuggestBuilder> randomSuggestBuilder,
Supplier<RescoreBuilder<?>> randomRescoreBuilder,
Supplier<List<SearchExtBuilder>> randomExtBuilders) {
SearchSourceBuilder builder = new SearchSourceBuilder();
if (randomBoolean()) {
builder.from(randomIntBetween(0, 10000));
}
if (randomBoolean()) {
builder.size(randomIntBetween(0, 10000));
}
if (randomBoolean()) {
builder.explain(randomBoolean());
}
if (randomBoolean()) {
builder.version(randomBoolean());
}
if (randomBoolean()) {
builder.trackScores(randomBoolean());
}
if (randomBoolean()) {
builder.minScore(randomFloat() * 1000);
}
if (randomBoolean()) {
builder.timeout(TimeValue.parseTimeValue(randomTimeValue(), null, "timeout"));
}
if (randomBoolean()) {
builder.terminateAfter(randomIntBetween(1, 100000));
}
switch (randomInt(2)) {
case 0:
builder.storedFields();
break;
case 1:
builder.storedField("_none_");
break;
case 2:
int fieldsSize = randomInt(25);
List<String> fields = new ArrayList<>(fieldsSize);
for (int i = 0; i < fieldsSize; i++) {
fields.add(randomAsciiOfLengthBetween(5, 50));
}
builder.storedFields(fields);
break;
default:
throw new IllegalStateException();
}
if (randomBoolean()) {
int scriptFieldsSize = randomInt(25);
for (int i = 0; i < scriptFieldsSize; i++) {
if (randomBoolean()) {
builder.scriptField(
randomAsciiOfLengthBetween(5, 50), new Script("foo"), randomBoolean());
} else {
builder.scriptField(randomAsciiOfLengthBetween(5, 50), new Script("foo"));
}
}
}
if (randomBoolean()) {
FetchSourceContext fetchSourceContext;
int branch = randomInt(5);
String[] includes = new String[randomIntBetween(0, 20)];
for (int i = 0; i < includes.length; i++) {
includes[i] = randomAsciiOfLengthBetween(5, 20);
}
String[] excludes = new String[randomIntBetween(0, 20)];
for (int i = 0; i < excludes.length; i++) {
excludes[i] = randomAsciiOfLengthBetween(5, 20);
}
switch (branch) {
case 0:
fetchSourceContext = new FetchSourceContext(randomBoolean());
break;
case 1:
fetchSourceContext = new FetchSourceContext(true, includes, excludes);
break;
case 2:
fetchSourceContext =
new FetchSourceContext(
true,
new String[] {randomAsciiOfLengthBetween(5, 20)},
new String[] {randomAsciiOfLengthBetween(5, 20)});
break;
case 3:
fetchSourceContext = new FetchSourceContext(true, includes, excludes);
break;
case 4:
fetchSourceContext = new FetchSourceContext(true, includes, null);
break;
case 5:
fetchSourceContext =
new FetchSourceContext(true, new String[] {randomAsciiOfLengthBetween(5, 20)}, null);
break;
default:
throw new IllegalStateException();
}
builder.fetchSource(fetchSourceContext);
}
if (randomBoolean()) {
int size = randomIntBetween(0, 20);
List<String> statsGroups = new ArrayList<>(size);
for (int i = 0; i < size; i++) {
statsGroups.add(randomAsciiOfLengthBetween(5, 20));
}
builder.stats(statsGroups);
}
if (randomBoolean()) {
int indexBoostSize = randomIntBetween(1, 10);
for (int i = 0; i < indexBoostSize; i++) {
builder.indexBoost(randomAsciiOfLengthBetween(5, 20), randomFloat() * 10);
}
}
if (randomBoolean()) {
builder.query(
QueryBuilders.termQuery(
randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20)));
}
if (randomBoolean()) {
builder.postFilter(
QueryBuilders.termQuery(
randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20)));
}
if (randomBoolean()) {
int numSorts = randomIntBetween(1, 5);
for (int i = 0; i < numSorts; i++) {
int branch = randomInt(5);
switch (branch) {
case 0:
builder.sort(
SortBuilders.fieldSort(randomAsciiOfLengthBetween(5, 20))
.order(randomFrom(SortOrder.values())));
break;
case 1:
builder.sort(
SortBuilders.geoDistanceSort(
randomAsciiOfLengthBetween(5, 20),
AbstractQueryTestCase.randomGeohash(1, 12))
.order(randomFrom(SortOrder.values())));
break;
case 2:
builder.sort(SortBuilders.scoreSort().order(randomFrom(SortOrder.values())));
break;
case 3:
builder.sort(
SortBuilders.scriptSort(new Script("foo"), ScriptSortBuilder.ScriptSortType.NUMBER)
.order(randomFrom(SortOrder.values())));
break;
case 4:
builder.sort(randomAsciiOfLengthBetween(5, 20));
break;
case 5:
builder.sort(randomAsciiOfLengthBetween(5, 20), randomFrom(SortOrder.values()));
break;
}
}
}
if (randomBoolean()) {
int numSearchFrom = randomIntBetween(1, 5);
try {
// We build a json version of the search_from first in order to
// ensure that every number type remain the same before/after xcontent (de)serialization.
// This is not a problem because the final type of each field value is extracted from
// associated sort field.
// This little trick ensure that equals and hashcode are the same when using the xcontent
// serialization.
XContentBuilder jsonBuilder = XContentFactory.jsonBuilder();
jsonBuilder.startObject();
jsonBuilder.startArray("search_from");
for (int i = 0; i < numSearchFrom; i++) {
int branch = randomInt(8);
switch (branch) {
case 0:
jsonBuilder.value(randomInt());
break;
case 1:
jsonBuilder.value(randomFloat());
break;
case 2:
jsonBuilder.value(randomLong());
break;
case 3:
jsonBuilder.value(randomDouble());
break;
case 4:
jsonBuilder.value(randomAsciiOfLengthBetween(5, 20));
break;
case 5:
jsonBuilder.value(randomBoolean());
break;
case 6:
jsonBuilder.value(randomByte());
break;
case 7:
jsonBuilder.value(randomShort());
break;
case 8:
jsonBuilder.value(new Text(randomAsciiOfLengthBetween(5, 20)));
break;
}
}
jsonBuilder.endArray();
jsonBuilder.endObject();
XContentParser parser =
XContentFactory.xContent(XContentType.JSON).createParser(jsonBuilder.bytes());
parser.nextToken();
parser.nextToken();
parser.nextToken();
builder.searchAfter(SearchAfterBuilder.fromXContent(parser, null).getSortValues());
} catch (IOException e) {
throw new RuntimeException("Error building search_from", e);
}
}
if (randomBoolean()) {
builder.highlighter(randomHighlightBuilder.get());
}
if (randomBoolean()) {
builder.suggest(randomSuggestBuilder.get());
}
if (randomBoolean()) {
int numRescores = randomIntBetween(1, 5);
for (int i = 0; i < numRescores; i++) {
builder.addRescorer(randomRescoreBuilder.get());
}
}
if (randomBoolean()) {
builder.aggregation(AggregationBuilders.avg(randomAsciiOfLengthBetween(5, 20)));
}
if (randomBoolean()) {
builder.ext(randomExtBuilders.get());
}
if (randomBoolean()) {
String field = randomBoolean() ? null : randomAsciiOfLengthBetween(5, 20);
int max = between(2, 1000);
int id = randomInt(max - 1);
if (field == null) {
builder.slice(new SliceBuilder(id, max));
} else {
builder.slice(new SliceBuilder(field, id, max));
}
}
return builder;
}
