@Override
public List<Article> query(String queryString, int size) {
Assert.hasText(queryString, "queryString must not be empty!");
SearchResponse searchResponse =
transportClient
.prepareSearch(INDICE)
.setTypes(TYPE)
.setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
.setQuery(QueryBuilders.queryString(queryString))
.setFrom(0)
.setSize(size)
.setExplain(true)
.execute()
.actionGet();
SearchHits hits = searchResponse.getHits();
long total = hits.getTotalHits();
logger.debug("search articles result total:{}", total);
List<Article> list = new ArrayList<Article>();
Article article = null;
for (SearchHit hit : hits) {
Long id = (Long) hit.getSource().get("id");
String title = (String) hit.getSource().get(TITLE);
String summary = (String) hit.getSource().get(SUMMARY);
logger.debug("id:{},title:{},summary:{}", id, title, summary);
article = new Article();
article.setId(id);
article.setTitle(title);
article.setSummary(summary);
list.add(article);
}
return list;
}
@Test
public void andTest() throws IOException, SqlParseException, SQLFeatureNotSupportedException {
SearchHits response =
query(String.format("SELECT * FROM %s WHERE age=32 AND gender='M' LIMIT 1000", TEST_INDEX));
SearchHit[] hits = response.getHits();
for (SearchHit hit : hits) {
Assert.assertEquals(32, hit.getSource().get("age"));
Assert.assertEquals("M", hit.getSource().get("gender"));
}
}
public SearchResult(SearchResponse resp) {
SearchHits hits = resp.getHits();
this.total = hits.getTotalHits();
results = new ArrayList<>(hits.getHits().length);
for (SearchHit searchHit : hits.getHits()) {
if (searchHit.getSource() != null) {
results.add(searchHit.getSource());
} else if (searchHit.getFields() != null) {
Map<String, SearchHitField> fields = searchHit.getFields();
results.add(toFieldsMap(fields));
}
}
}
@Test
public void dateBetweenSearch()
throws IOException, SqlParseException, SQLFeatureNotSupportedException {
DateTimeFormatter formatter = DateTimeFormat.forPattern(DATE_FORMAT);
DateTime dateLimit1 = new DateTime(2014, 8, 18, 0, 0, 0);
DateTime dateLimit2 = new DateTime(2014, 8, 21, 0, 0, 0);
SearchHits response =
query(
String.format(
"SELECT insert_time FROM %s/online WHERE insert_time BETWEEN '2014-08-18' AND '2014-08-21' LIMIT 3",
TEST_INDEX));
SearchHit[] hits = response.getHits();
for (SearchHit hit : hits) {
Map<String, Object> source = hit.getSource();
DateTime insertTime = formatter.parseDateTime((String) source.get("insert_time"));
boolean isBetween =
(insertTime.isAfter(dateLimit1) || insertTime.isEqual(dateLimit1))
&& (insertTime.isBefore(dateLimit2) || insertTime.isEqual(dateLimit2));
Assert.assertTrue("insert_time must be between 2014-08-18 and 2014-08-21", isBetween);
}
}
@Test
public void complexConditionQuery()
throws IOException, SqlParseException, SQLFeatureNotSupportedException {
String errorMessage =
"Result does not exist to the condition (gender='m' AND (age> 25 OR account_number>5)) OR (gender='f' AND (age>30 OR account_number < 8)";
SearchHits response =
query(
String.format(
"SELECT * FROM %s/account WHERE (gender='m' AND (age> 25 OR account_number>5)) OR (gender='f' AND (age>30 OR account_number < 8))",
TEST_INDEX));
SearchHit[] hits = response.getHits();
for (SearchHit hit : hits) {
Map<String, Object> source = hit.getSource();
String gender = ((String) source.get("gender")).toLowerCase();
int age = (int) source.get("age");
int account_number = (int) source.get("account_number");
Assert.assertTrue(
errorMessage,
(gender.equals("m") && (age > 25 || account_number > 5))
|| (gender.equals("f") && (age > 30 || account_number < 8)));
}
}
public void assertSimpleSort(String... numericFields) {
for (String field : numericFields) {
SearchResponse searchResponse = client().prepareSearch().addSort(field, SortOrder.ASC).get();
SearchHit[] hits = searchResponse.getHits().getHits();
assertThat(hits.length, greaterThan(0));
Number previous = null;
for (SearchHit hit : hits) {
assertNotNull(hit.getSource().get(field));
if (previous != null) {
assertThat(
previous.doubleValue(),
lessThanOrEqualTo(((Number) hit.getSource().get(field)).doubleValue()));
}
previous = (Number) hit.getSource().get(field);
}
}
}
public static <D extends BaseDoc> List<D> convertToDocs(
SearchHits hits, Function<Map<String, Object>, D> converter) {
List<D> docs = new ArrayList<>();
for (SearchHit hit : hits.getHits()) {
docs.add(converter.apply(hit.getSource()));
}
return docs;
}
public static List<Map<String, Object>> doSearchResultEs(String value, String fq) {
System.out.println("fq: " + fq);
SearchHit[] shs = getFromEs(value, fq);
Map<String, Object> itemContent;
List<Map<String, Object>> rcdStr = new LinkedList<Map<String, Object>>();
// id:3341326TAX_CODE
// score:2.1178803
// source:
// {
// "NUMBER_OF_CLICKS":0,
// "KEY":"TAX_CODE",
// "VALUE":"Tax Code",
//
// "FILE_PATH":"ipm/src/imaging-ui/src/oracle/imaging/axf/resources/user/ApplicationStrings.java",
// "FID":41326,
// "LAST_MODIFIED":"2012-03-08T21:38:33.627-08:00",
// "FTID":263,
// "CN":"IPM",
// "FE":".java",
// "DID":3,
// "DN":"drop30",
// "RID":3,
// "RN":"11.1.1.6",
// "PID":3,
// "PN":"ias",
// "BID":97
// }
// {"NUMBER_OF_CLICKS":0,"KEY":"TAX_CODE","VALUE":"Tax
// Code","FILE_PATH":"ipm/src/imaging-ui/src/oracle/imaging/axf/resources/user/ApplicationStrings.java","FID":41326,"LAST_MODIFIED":"2012-03-08T21:38:33.627-08:00","FTID":263,"CN":"IPM","FE":".java","DID":3,"DN":"drop30","RID":3,"RN":"11.1.1.6","PID":3,"PN":"ias","BID":97}
String resource = null;
for (SearchHit hit : shs) {
itemContent = new HashMap<String, Object>();
itemContent.put("id", hit.getId());
itemContent.put("score", hit.getScore());
for (Map.Entry<String, Object> entry : hit.getSource().entrySet()) {
// System.out.println("key: "+entry.getKey() + ":" + entry.getValue());
if (entry.getKey().equals("FILE_PATH")) itemContent.put("fP", entry.getValue());
else if (entry.getKey().equals("VALUE")) itemContent.put("vL", entry.getValue());
else if (entry.getKey().equals("DN")) itemContent.put("dN", entry.getValue());
else if (entry.getKey().equals("RN")) itemContent.put("rN", entry.getValue());
else if (entry.getKey().equals("PN")) itemContent.put("pN", entry.getValue());
else if (entry.getKey().equals("CN")) itemContent.put("sP", entry.getValue());
else if (entry.getKey().equals("BID")) itemContent.put("bid", entry.getValue());
else if (entry.getKey().equals("KEY")) itemContent.put("key", entry.getValue());
else if (entry.getKey().equals("FID")) itemContent.put("fid", entry.getValue());
}
if (itemContent != null) {
rcdStr.add(itemContent);
}
}
return rcdStr;
}
private <T> T createSavedObjectWithHit(SearchHit hit, Query<T> query) {
T object = createSavedObject(hit.getType(), hit.getId(), query);
State objectState = State.getInstance(object);
if (!objectState.isReferenceOnly()) {
objectState.setValues(hit.getSource());
}
return swapObjectType(query, object);
}
@Test
public void selectFieldWithSpace()
throws IOException, SqlParseException, SQLFeatureNotSupportedException {
String[] arr = new String[] {"test field"};
Set expectedSource = new HashSet(Arrays.asList(arr));
SearchHits response = query(String.format("SELECT `test field` FROM %s/phrase_2", TEST_INDEX));
SearchHit[] hits = response.getHits();
for (SearchHit hit : hits) {
Assert.assertEquals(expectedSource, hit.getSource().keySet());
}
}
@Test
public void inTest() throws IOException, SqlParseException, SQLFeatureNotSupportedException {
SearchHits response =
query(
String.format(
"SELECT age FROM %s/phrase WHERE age IN (20, 22) LIMIT 1000", TEST_INDEX));
SearchHit[] hits = response.getHits();
for (SearchHit hit : hits) {
int age = (int) hit.getSource().get("age");
assertThat(age, isOneOf(20, 22));
}
}
@Test
public void lessThanTest()
throws IOException, SqlParseException, SQLFeatureNotSupportedException {
int someAge = 25;
SearchHits response =
query(String.format("SELECT * FROM %s WHERE age < %s LIMIT 1000", TEST_INDEX, someAge));
SearchHit[] hits = response.getHits();
for (SearchHit hit : hits) {
int age = (int) hit.getSource().get("age");
assertThat(age, lessThan(someAge));
}
}
/*
TODO when using not between on some field, documents that not contains this
field will return as well, That may considered a Wrong behaivor.
*/
@Test
public void notBetweenTest()
throws IOException, SqlParseException, SQLFeatureNotSupportedException {
int min = 20;
int max = 37;
SearchHits response =
query(
String.format(
"SELECT * FROM %s WHERE age NOT BETWEEN %s AND %s LIMIT 1000",
TEST_INDEX, min, max));
SearchHit[] hits = response.getHits();
for (SearchHit hit : hits) {
Map<String, Object> source = hit.getSource();
// ignore document which not contains the age field.
if (source.containsKey("age")) {
int age = (int) hit.getSource().get("age");
assertThat(age, not(allOf(greaterThanOrEqualTo(min), lessThanOrEqualTo(max))));
}
}
}
@Test
public void missFilterSearch()
throws IOException, SqlParseException, SQLFeatureNotSupportedException {
SearchHits response =
query(String.format("SELECT * FROM %s/phrase WHERE insert_time IS missing", TEST_INDEX));
SearchHit[] hits = response.getHits();
// should be 2 according to the data.
Assert.assertEquals(response.getTotalHits(), 2);
for (SearchHit hit : hits) {
assertThat(hit.getSource(), not(hasKey("insert_time")));
}
}
@Test
public void selectSpecificFields()
throws IOException, SqlParseException, SQLFeatureNotSupportedException {
String[] arr = new String[] {"age", "account_number"};
Set expectedSource = new HashSet(Arrays.asList(arr));
SearchHits response =
query(String.format("SELECT age, account_number FROM %s/account", TEST_INDEX));
SearchHit[] hits = response.getHits();
for (SearchHit hit : hits) {
Assert.assertEquals(expectedSource, hit.getSource().keySet());
}
}
@Test
public void betweenTest() throws IOException, SqlParseException, SQLFeatureNotSupportedException {
int min = 27;
int max = 30;
SearchHits response =
query(
String.format(
"SELECT * FROM %s WHERE age BETWEEN %s AND %s LIMIT 1000", TEST_INDEX, min, max));
SearchHit[] hits = response.getHits();
for (SearchHit hit : hits) {
int age = (int) hit.getSource().get("age");
assertThat(age, allOf(greaterThanOrEqualTo(min), lessThanOrEqualTo(max)));
}
}
@Override
public Page<Article> queryPage(Page<Article> page, String queryString) {
Assert.notNull(page, "page must not null");
Assert.hasText(queryString, "queryString must not be empty!");
SearchResponse searchResponse =
transportClient
.prepareSearch(INDICE)
.setTypes(TYPE)
.setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
.setQuery(QueryBuilders.queryString(queryString))
.setFrom(page.getFirst() - 1)
.setSize(page.getPageSize())
.setExplain(true)
.execute()
.actionGet();
SearchHits hits = searchResponse.getHits();
long total = hits.getTotalHits();
logger.debug("search articles result total:{}", total);
List<Article> list = new ArrayList<Article>();
Article article = null;
for (SearchHit hit : hits) {
long id = Long.parseLong(hit.getId());
String title = (String) hit.getSource().get(TITLE);
String summary = (String) hit.getSource().get(SUMMARY);
logger.debug("id:{},title:{},summary:{}", id, title, summary);
article = new Article();
article.setId(id);
article.setTitle(title);
article.setSummary(summary);
list.add(article);
}
page.setTotalCount(total);
page.setResult(list);
return page;
}
@Test
public void inTestWithStrings()
throws IOException, SqlParseException, SQLFeatureNotSupportedException {
SearchHits response =
query(
String.format(
"SELECT phrase FROM %s/phrase WHERE phrase IN ('quick fox here', 'fox brown') LIMIT 1000",
TEST_INDEX));
SearchHit[] hits = response.getHits();
Assert.assertEquals(2, response.getTotalHits());
for (SearchHit hit : hits) {
String phrase = (String) hit.getSource().get("phrase");
assertThat(phrase, isOneOf("quick fox here", "fox brown"));
}
}
/* TODO when using not in on some field, documents that not contains this
field will return as well, That may considered a Wrong behaivor.
*/
@Test
public void notInTest() throws IOException, SqlParseException, SQLFeatureNotSupportedException {
SearchHits response =
query(String.format("SELECT age FROM %s WHERE age NOT IN (20, 22) LIMIT 1000", TEST_INDEX));
SearchHit[] hits = response.getHits();
for (SearchHit hit : hits) {
Map<String, Object> source = hit.getSource();
// ignore document which not contains the age field.
if (source.containsKey("age")) {
int age = (int) source.get("age");
assertThat(age, not(isOneOf(20, 22)));
}
}
}
@Test
public void orderByDescTest()
throws IOException, SqlParseException, SQLFeatureNotSupportedException {
SearchHits response =
query(String.format("SELECT age FROM %s/account ORDER BY age DESC LIMIT 1000", TEST_INDEX));
SearchHit[] hits = response.getHits();
ArrayList<Integer> ages = new ArrayList<Integer>();
for (SearchHit hit : hits) {
ages.add((int) hit.getSource().get("age"));
}
ArrayList<Integer> sortedAges = (ArrayList<Integer>) ages.clone();
Collections.sort(sortedAges, Collections.reverseOrder());
Assert.assertTrue("The list is not ordered descending", sortedAges.equals(ages));
}
@Test
public void orderByAscFieldWithSpaceTest()
throws IOException, SqlParseException, SQLFeatureNotSupportedException {
SearchHits response =
query(
String.format(
"SELECT * FROM %s/phrase_2 ORDER BY `test field` ASC LIMIT 1000", TEST_INDEX));
SearchHit[] hits = response.getHits();
ArrayList<Integer> testFields = new ArrayList<Integer>();
for (SearchHit hit : hits) {
testFields.add((int) hit.getSource().get("test field"));
}
ArrayList<Integer> sortedTestFields = (ArrayList<Integer>) testFields.clone();
Collections.sort(sortedTestFields);
Assert.assertTrue("The list is not ordered ascending", sortedTestFields.equals(testFields));
}
@Override
public List<T> collectObjects(SearchResponse rsp) {
SearchHits docs = rsp.getHits();
List<T> list = new ArrayList<T>(docs.hits().length);
for (SearchHit sd : docs) {
if (sd.getExplanation() != null) {
String res = "";
for (Explanation str : sd.getExplanation().getDetails()) {
res += str.toString();
}
logger.info(sd.getId() + " " + res);
}
T o = readDoc(sd.getId(), sd.getVersion(), sd.getSource());
if (o != null) list.add(o);
}
return list;
}
public List<ActiveRule> findByProfile(QualityProfileKey key) {
SearchRequestBuilder request =
getClient()
.prepareSearch(getIndexName())
.setQuery(
QueryBuilders.termQuery(
ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field(), key.toString()))
.setRouting(key.toString())
// TODO replace by scrolling
.setSize(Integer.MAX_VALUE);
SearchResponse response = request.get();
List<ActiveRule> activeRules = new ArrayList<ActiveRule>();
for (SearchHit hit : response.getHits()) {
activeRules.add(toDoc(hit.getSource()));
}
return activeRules;
}
public List<String> autocomplete(String prefix, int limit) {
SearchRequestBuilder builder = new SearchRequestBuilder(factory.client());
builder.setIndices(Constants.INDEX).setTypes(Constants.TYPE);
builder.setSearchType(SearchType.DEFAULT);
builder.setFrom(0).setSize(limit).setExplain(IS_EXPLAIN);
QueryBuilder queryBuilder = QueryBuilders.matchPhraseQuery("title", prefix);
builder.setQuery(queryBuilder.toString());
logger.info("Search qeury");
logger.info(builder.toString());
SearchResponse sr = builder.execute().actionGet();
logger.info("Search response");
logger.info(sr.toString());
Iterator<SearchHit> it = sr.getHits().iterator();
while (it.hasNext()) {
SearchHit hit = it.next();
hit.getSource();
}
return Arrays.asList("a", "b", "c");
}
@Override
public List<Map<String, Object>> search(String term, List<Class<? extends Entity>> entityTypes)
throws SearchEngineException {
SearchResponse response =
client
.prepareSearch("entities")
.setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
.setQuery(queryString(term))
.setFrom(0)
.setSize(10)
.setExplain(false)
.execute()
.actionGet();
List<Map<String, Object>> result = new ArrayList<Map<String, Object>>();
for (SearchHit hit : response.getHits()) {
result.add(hit.getSource());
}
return result;
}
@Test
public void dateSearch()
throws IOException, SqlParseException, SQLFeatureNotSupportedException, ParseException {
DateTimeFormatter formatter = DateTimeFormat.forPattern(DATE_FORMAT);
DateTime dateToCompare = new DateTime(2014, 8, 18, 0, 0, 0);
SearchHits response =
query(
String.format(
"SELECT insert_time FROM %s/online WHERE insert_time < '2014-08-18'", TEST_INDEX));
SearchHit[] hits = response.getHits();
for (SearchHit hit : hits) {
Map<String, Object> source = hit.getSource();
DateTime insertTime = formatter.parseDateTime((String) source.get("insert_time"));
String errorMessage =
String.format("insert_time must be smaller then 2014-08-18. found: %s", insertTime);
Assert.assertTrue(errorMessage, insertTime.isBefore(dateToCompare));
}
}
@Test
public void lessThanOrEqualTest()
throws IOException, SqlParseException, SQLFeatureNotSupportedException {
int someAge = 25;
SearchHits response =
query(String.format("SELECT * FROM %s WHERE age <= %s LIMIT 1000", TEST_INDEX, someAge));
SearchHit[] hits = response.getHits();
boolean isEqualFound = false;
for (SearchHit hit : hits) {
int age = (int) hit.getSource().get("age");
assertThat(age, lessThanOrEqualTo(someAge));
if (age == someAge) isEqualFound = true;
}
Assert.assertTrue(
String.format("at least one of the documents need to contains age equal to %s", someAge),
isEqualFound);
}
/** finder methods */
public List<ActiveRule> findByRule(RuleKey key) {
SearchRequestBuilder request =
getClient()
.prepareSearch(this.getIndexName())
.setQuery(
QueryBuilders.hasParentQuery(
this.getParentType(),
QueryBuilders.idsQuery(this.getParentType()).addIds(key.toString())))
.setRouting(key.toString())
// TODO replace by scrolling
.setSize(Integer.MAX_VALUE);
SearchResponse response = request.get();
List<ActiveRule> activeRules = new ArrayList<ActiveRule>();
for (SearchHit hit : response.getHits()) {
activeRules.add(toDoc(hit.getSource()));
}
return activeRules;
}
private void assertCorrectResponse(int count, Event event, SearchResponse response) {
SearchHits hits = response.getHits();
assertEquals(count, hits.getTotalHits());
SearchHit hit = hits.getAt(0);
Map<String, Object> source = hit.getSource();
assertEquals(event.getHost(), source.get("host"));
assertEquals("1970-01-01T00:00:00.000Z", source.get("timestamp"));
assertEquals(event.getPriority().name(), source.get("priority"));
@SuppressWarnings("unchecked")
Map<String, Object> message = (Map<String, Object>) source.get("message");
assertEquals(new String(event.getBody()), message.get("text"));
@SuppressWarnings("unchecked")
Map<String, Object> fields = (Map<String, Object>) source.get("fields");
assertEquals(new String(event.getAttrs().get("attr1")), fields.get("attr1"));
assertEquals(new String(event.getAttrs().get("attr2")), fields.get("attr2"));
}
private Result<QProfileActivity> searchActivities(
QProfileActivityQuery query, SearchOptions options) {
DbSession session = dbClient.openSession(false);
try {
SearchResponse response = activityIndex.doSearch(query, options);
Result<QProfileActivity> result = new Result<>(response);
for (SearchHit hit : response.getHits().getHits()) {
QProfileActivity profileActivity = new QProfileActivity(hit.getSource());
RuleDto ruleDto = dbClient.ruleDao().getNullableByKey(session, profileActivity.ruleKey());
profileActivity.ruleName(ruleDto != null ? ruleDto.getName() : null);
String login = profileActivity.getLogin();
if (login != null) {
UserDto user = dbClient.userDao().selectActiveUserByLogin(session, login);
profileActivity.authorName(user != null ? user.getName() : null);
}
result.getHits().add(profileActivity);
}
return result;
} finally {
session.close();
}
}
