@Override
  public List<Article> query(String queryString, int size) {
    Assert.hasText(queryString, "queryString must not be empty!");
    SearchResponse searchResponse =
        transportClient
            .prepareSearch(INDICE)
            .setTypes(TYPE)
            .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
            .setQuery(QueryBuilders.queryString(queryString))
            .setFrom(0)
            .setSize(size)
            .setExplain(true)
            .execute()
            .actionGet();

    SearchHits hits = searchResponse.getHits();
    long total = hits.getTotalHits();
    logger.debug("search articles result total:{}", total);
    List<Article> list = new ArrayList<Article>();
    Article article = null;
    for (SearchHit hit : hits) {

      Long id = (Long) hit.getSource().get("id");
      String title = (String) hit.getSource().get(TITLE);
      String summary = (String) hit.getSource().get(SUMMARY);
      logger.debug("id:{},title:{},summary:{}", id, title, summary);

      article = new Article();
      article.setId(id);
      article.setTitle(title);
      article.setSummary(summary);
      list.add(article);
    }
    return list;
  }
Esempio n. 2
0
 @Test
 public void andTest() throws IOException, SqlParseException, SQLFeatureNotSupportedException {
   SearchHits response =
       query(String.format("SELECT * FROM %s WHERE age=32 AND gender='M' LIMIT 1000", TEST_INDEX));
   SearchHit[] hits = response.getHits();
   for (SearchHit hit : hits) {
     Assert.assertEquals(32, hit.getSource().get("age"));
     Assert.assertEquals("M", hit.getSource().get("gender"));
   }
 }
Esempio n. 3
0
 public SearchResult(SearchResponse resp) {
   SearchHits hits = resp.getHits();
   this.total = hits.getTotalHits();
   results = new ArrayList<>(hits.getHits().length);
   for (SearchHit searchHit : hits.getHits()) {
     if (searchHit.getSource() != null) {
       results.add(searchHit.getSource());
     } else if (searchHit.getFields() != null) {
       Map<String, SearchHitField> fields = searchHit.getFields();
       results.add(toFieldsMap(fields));
     }
   }
 }
Esempio n. 4
0
  @Test
  public void dateBetweenSearch()
      throws IOException, SqlParseException, SQLFeatureNotSupportedException {
    DateTimeFormatter formatter = DateTimeFormat.forPattern(DATE_FORMAT);

    DateTime dateLimit1 = new DateTime(2014, 8, 18, 0, 0, 0);
    DateTime dateLimit2 = new DateTime(2014, 8, 21, 0, 0, 0);

    SearchHits response =
        query(
            String.format(
                "SELECT insert_time FROM %s/online WHERE insert_time BETWEEN '2014-08-18' AND '2014-08-21' LIMIT 3",
                TEST_INDEX));
    SearchHit[] hits = response.getHits();
    for (SearchHit hit : hits) {
      Map<String, Object> source = hit.getSource();
      DateTime insertTime = formatter.parseDateTime((String) source.get("insert_time"));

      boolean isBetween =
          (insertTime.isAfter(dateLimit1) || insertTime.isEqual(dateLimit1))
              && (insertTime.isBefore(dateLimit2) || insertTime.isEqual(dateLimit2));

      Assert.assertTrue("insert_time must be between 2014-08-18 and 2014-08-21", isBetween);
    }
  }
Esempio n. 5
0
  @Test
  public void complexConditionQuery()
      throws IOException, SqlParseException, SQLFeatureNotSupportedException {
    String errorMessage =
        "Result does not exist to the condition (gender='m' AND (age> 25 OR account_number>5)) OR (gender='f' AND (age>30 OR account_number < 8)";

    SearchHits response =
        query(
            String.format(
                "SELECT * FROM %s/account WHERE (gender='m' AND (age> 25 OR account_number>5)) OR (gender='f' AND (age>30 OR account_number < 8))",
                TEST_INDEX));
    SearchHit[] hits = response.getHits();

    for (SearchHit hit : hits) {
      Map<String, Object> source = hit.getSource();
      String gender = ((String) source.get("gender")).toLowerCase();
      int age = (int) source.get("age");
      int account_number = (int) source.get("account_number");

      Assert.assertTrue(
          errorMessage,
          (gender.equals("m") && (age > 25 || account_number > 5))
              || (gender.equals("f") && (age > 30 || account_number < 8)));
    }
  }
 public void assertSimpleSort(String... numericFields) {
   for (String field : numericFields) {
     SearchResponse searchResponse = client().prepareSearch().addSort(field, SortOrder.ASC).get();
     SearchHit[] hits = searchResponse.getHits().getHits();
     assertThat(hits.length, greaterThan(0));
     Number previous = null;
     for (SearchHit hit : hits) {
       assertNotNull(hit.getSource().get(field));
       if (previous != null) {
         assertThat(
             previous.doubleValue(),
             lessThanOrEqualTo(((Number) hit.getSource().get(field)).doubleValue()));
       }
       previous = (Number) hit.getSource().get(field);
     }
   }
 }
Esempio n. 7
0
 public static <D extends BaseDoc> List<D> convertToDocs(
     SearchHits hits, Function<Map<String, Object>, D> converter) {
   List<D> docs = new ArrayList<>();
   for (SearchHit hit : hits.getHits()) {
     docs.add(converter.apply(hit.getSource()));
   }
   return docs;
 }
  public static List<Map<String, Object>> doSearchResultEs(String value, String fq) {

    System.out.println("fq: " + fq);
    SearchHit[] shs = getFromEs(value, fq);

    Map<String, Object> itemContent;
    List<Map<String, Object>> rcdStr = new LinkedList<Map<String, Object>>();
    //       id:3341326TAX_CODE
    //       score:2.1178803
    //       source:
    //       {
    //       "NUMBER_OF_CLICKS":0,
    //       "KEY":"TAX_CODE",
    //       "VALUE":"Tax Code",
    //
    // "FILE_PATH":"ipm/src/imaging-ui/src/oracle/imaging/axf/resources/user/ApplicationStrings.java",
    //       "FID":41326,
    //       "LAST_MODIFIED":"2012-03-08T21:38:33.627-08:00",
    //       "FTID":263,
    //       "CN":"IPM",
    //       "FE":".java",
    //       "DID":3,
    //       "DN":"drop30",
    //       "RID":3,
    //       "RN":"11.1.1.6",
    //       "PID":3,
    //       "PN":"ias",
    //       "BID":97
    //       }

    // {"NUMBER_OF_CLICKS":0,"KEY":"TAX_CODE","VALUE":"Tax
    // Code","FILE_PATH":"ipm/src/imaging-ui/src/oracle/imaging/axf/resources/user/ApplicationStrings.java","FID":41326,"LAST_MODIFIED":"2012-03-08T21:38:33.627-08:00","FTID":263,"CN":"IPM","FE":".java","DID":3,"DN":"drop30","RID":3,"RN":"11.1.1.6","PID":3,"PN":"ias","BID":97}

    String resource = null;

    for (SearchHit hit : shs) {
      itemContent = new HashMap<String, Object>();
      itemContent.put("id", hit.getId());
      itemContent.put("score", hit.getScore());
      for (Map.Entry<String, Object> entry : hit.getSource().entrySet()) {
        // System.out.println("key: "+entry.getKey() + ":" + entry.getValue());
        if (entry.getKey().equals("FILE_PATH")) itemContent.put("fP", entry.getValue());
        else if (entry.getKey().equals("VALUE")) itemContent.put("vL", entry.getValue());
        else if (entry.getKey().equals("DN")) itemContent.put("dN", entry.getValue());
        else if (entry.getKey().equals("RN")) itemContent.put("rN", entry.getValue());
        else if (entry.getKey().equals("PN")) itemContent.put("pN", entry.getValue());
        else if (entry.getKey().equals("CN")) itemContent.put("sP", entry.getValue());
        else if (entry.getKey().equals("BID")) itemContent.put("bid", entry.getValue());
        else if (entry.getKey().equals("KEY")) itemContent.put("key", entry.getValue());
        else if (entry.getKey().equals("FID")) itemContent.put("fid", entry.getValue());
      }
      if (itemContent != null) {
        rcdStr.add(itemContent);
      }
    }
    return rcdStr;
  }
  private <T> T createSavedObjectWithHit(SearchHit hit, Query<T> query) {
    T object = createSavedObject(hit.getType(), hit.getId(), query);
    State objectState = State.getInstance(object);

    if (!objectState.isReferenceOnly()) {
      objectState.setValues(hit.getSource());
    }

    return swapObjectType(query, object);
  }
Esempio n. 10
0
  @Test
  public void selectFieldWithSpace()
      throws IOException, SqlParseException, SQLFeatureNotSupportedException {
    String[] arr = new String[] {"test field"};
    Set expectedSource = new HashSet(Arrays.asList(arr));

    SearchHits response = query(String.format("SELECT `test field` FROM %s/phrase_2", TEST_INDEX));
    SearchHit[] hits = response.getHits();
    for (SearchHit hit : hits) {
      Assert.assertEquals(expectedSource, hit.getSource().keySet());
    }
  }
Esempio n. 11
0
 @Test
 public void inTest() throws IOException, SqlParseException, SQLFeatureNotSupportedException {
   SearchHits response =
       query(
           String.format(
               "SELECT age FROM %s/phrase WHERE age IN (20, 22) LIMIT 1000", TEST_INDEX));
   SearchHit[] hits = response.getHits();
   for (SearchHit hit : hits) {
     int age = (int) hit.getSource().get("age");
     assertThat(age, isOneOf(20, 22));
   }
 }
Esempio n. 12
0
 @Test
 public void lessThanTest()
     throws IOException, SqlParseException, SQLFeatureNotSupportedException {
   int someAge = 25;
   SearchHits response =
       query(String.format("SELECT * FROM %s WHERE age < %s LIMIT 1000", TEST_INDEX, someAge));
   SearchHit[] hits = response.getHits();
   for (SearchHit hit : hits) {
     int age = (int) hit.getSource().get("age");
     assertThat(age, lessThan(someAge));
   }
 }
Esempio n. 13
0
  /*
  TODO when using not between on some field, documents that not contains this
   field will return as well, That may considered a Wrong behaivor.
   */
  @Test
  public void notBetweenTest()
      throws IOException, SqlParseException, SQLFeatureNotSupportedException {
    int min = 20;
    int max = 37;
    SearchHits response =
        query(
            String.format(
                "SELECT * FROM %s WHERE age NOT BETWEEN %s AND %s LIMIT 1000",
                TEST_INDEX, min, max));
    SearchHit[] hits = response.getHits();
    for (SearchHit hit : hits) {
      Map<String, Object> source = hit.getSource();

      // ignore document which not contains the age field.
      if (source.containsKey("age")) {
        int age = (int) hit.getSource().get("age");
        assertThat(age, not(allOf(greaterThanOrEqualTo(min), lessThanOrEqualTo(max))));
      }
    }
  }
Esempio n. 14
0
  @Test
  public void missFilterSearch()
      throws IOException, SqlParseException, SQLFeatureNotSupportedException {
    SearchHits response =
        query(String.format("SELECT * FROM %s/phrase WHERE insert_time IS missing", TEST_INDEX));
    SearchHit[] hits = response.getHits();

    // should be 2 according to the data.
    Assert.assertEquals(response.getTotalHits(), 2);
    for (SearchHit hit : hits) {
      assertThat(hit.getSource(), not(hasKey("insert_time")));
    }
  }
Esempio n. 15
0
  @Test
  public void selectSpecificFields()
      throws IOException, SqlParseException, SQLFeatureNotSupportedException {
    String[] arr = new String[] {"age", "account_number"};
    Set expectedSource = new HashSet(Arrays.asList(arr));

    SearchHits response =
        query(String.format("SELECT age, account_number FROM %s/account", TEST_INDEX));
    SearchHit[] hits = response.getHits();
    for (SearchHit hit : hits) {
      Assert.assertEquals(expectedSource, hit.getSource().keySet());
    }
  }
Esempio n. 16
0
 @Test
 public void betweenTest() throws IOException, SqlParseException, SQLFeatureNotSupportedException {
   int min = 27;
   int max = 30;
   SearchHits response =
       query(
           String.format(
               "SELECT * FROM %s WHERE age BETWEEN %s AND %s LIMIT 1000", TEST_INDEX, min, max));
   SearchHit[] hits = response.getHits();
   for (SearchHit hit : hits) {
     int age = (int) hit.getSource().get("age");
     assertThat(age, allOf(greaterThanOrEqualTo(min), lessThanOrEqualTo(max)));
   }
 }
  @Override
  public Page<Article> queryPage(Page<Article> page, String queryString) {
    Assert.notNull(page, "page must not null");
    Assert.hasText(queryString, "queryString must not be empty!");
    SearchResponse searchResponse =
        transportClient
            .prepareSearch(INDICE)
            .setTypes(TYPE)
            .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
            .setQuery(QueryBuilders.queryString(queryString))
            .setFrom(page.getFirst() - 1)
            .setSize(page.getPageSize())
            .setExplain(true)
            .execute()
            .actionGet();

    SearchHits hits = searchResponse.getHits();
    long total = hits.getTotalHits();
    logger.debug("search articles result total:{}", total);
    List<Article> list = new ArrayList<Article>();
    Article article = null;
    for (SearchHit hit : hits) {
      long id = Long.parseLong(hit.getId());
      String title = (String) hit.getSource().get(TITLE);
      String summary = (String) hit.getSource().get(SUMMARY);
      logger.debug("id:{},title:{},summary:{}", id, title, summary);

      article = new Article();
      article.setId(id);
      article.setTitle(title);
      article.setSummary(summary);
      list.add(article);
    }
    page.setTotalCount(total);
    page.setResult(list);
    return page;
  }
Esempio n. 18
0
 @Test
 public void inTestWithStrings()
     throws IOException, SqlParseException, SQLFeatureNotSupportedException {
   SearchHits response =
       query(
           String.format(
               "SELECT phrase FROM %s/phrase WHERE phrase IN ('quick fox here', 'fox brown') LIMIT 1000",
               TEST_INDEX));
   SearchHit[] hits = response.getHits();
   Assert.assertEquals(2, response.getTotalHits());
   for (SearchHit hit : hits) {
     String phrase = (String) hit.getSource().get("phrase");
     assertThat(phrase, isOneOf("quick fox here", "fox brown"));
   }
 }
Esempio n. 19
0
  /* TODO when using not in on some field, documents that not contains this
  field will return as well, That may considered a Wrong behaivor.
  */
  @Test
  public void notInTest() throws IOException, SqlParseException, SQLFeatureNotSupportedException {
    SearchHits response =
        query(String.format("SELECT age FROM %s WHERE age NOT IN (20, 22) LIMIT 1000", TEST_INDEX));
    SearchHit[] hits = response.getHits();
    for (SearchHit hit : hits) {
      Map<String, Object> source = hit.getSource();

      // ignore document which not contains the age field.
      if (source.containsKey("age")) {
        int age = (int) source.get("age");
        assertThat(age, not(isOneOf(20, 22)));
      }
    }
  }
Esempio n. 20
0
  @Test
  public void orderByDescTest()
      throws IOException, SqlParseException, SQLFeatureNotSupportedException {
    SearchHits response =
        query(String.format("SELECT age FROM %s/account ORDER BY age DESC LIMIT 1000", TEST_INDEX));
    SearchHit[] hits = response.getHits();

    ArrayList<Integer> ages = new ArrayList<Integer>();
    for (SearchHit hit : hits) {
      ages.add((int) hit.getSource().get("age"));
    }

    ArrayList<Integer> sortedAges = (ArrayList<Integer>) ages.clone();
    Collections.sort(sortedAges, Collections.reverseOrder());
    Assert.assertTrue("The list is not ordered descending", sortedAges.equals(ages));
  }
Esempio n. 21
0
  @Test
  public void orderByAscFieldWithSpaceTest()
      throws IOException, SqlParseException, SQLFeatureNotSupportedException {
    SearchHits response =
        query(
            String.format(
                "SELECT * FROM %s/phrase_2 ORDER BY `test field` ASC LIMIT 1000", TEST_INDEX));
    SearchHit[] hits = response.getHits();

    ArrayList<Integer> testFields = new ArrayList<Integer>();
    for (SearchHit hit : hits) {
      testFields.add((int) hit.getSource().get("test field"));
    }

    ArrayList<Integer> sortedTestFields = (ArrayList<Integer>) testFields.clone();
    Collections.sort(sortedTestFields);
    Assert.assertTrue("The list is not ordered ascending", sortedTestFields.equals(testFields));
  }
  @Override
  public List<T> collectObjects(SearchResponse rsp) {
    SearchHits docs = rsp.getHits();
    List<T> list = new ArrayList<T>(docs.hits().length);
    for (SearchHit sd : docs) {
      if (sd.getExplanation() != null) {
        String res = "";
        for (Explanation str : sd.getExplanation().getDetails()) {
          res += str.toString();
        }
        logger.info(sd.getId() + " " + res);
      }
      T o = readDoc(sd.getId(), sd.getVersion(), sd.getSource());
      if (o != null) list.add(o);
    }

    return list;
  }
Esempio n. 23
0
  public List<ActiveRule> findByProfile(QualityProfileKey key) {
    SearchRequestBuilder request =
        getClient()
            .prepareSearch(getIndexName())
            .setQuery(
                QueryBuilders.termQuery(
                    ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field(), key.toString()))
            .setRouting(key.toString())
            // TODO replace by scrolling
            .setSize(Integer.MAX_VALUE);
    SearchResponse response = request.get();

    List<ActiveRule> activeRules = new ArrayList<ActiveRule>();
    for (SearchHit hit : response.getHits()) {
      activeRules.add(toDoc(hit.getSource()));
    }
    return activeRules;
  }
 public List<String> autocomplete(String prefix, int limit) {
   SearchRequestBuilder builder = new SearchRequestBuilder(factory.client());
   builder.setIndices(Constants.INDEX).setTypes(Constants.TYPE);
   builder.setSearchType(SearchType.DEFAULT);
   builder.setFrom(0).setSize(limit).setExplain(IS_EXPLAIN);
   QueryBuilder queryBuilder = QueryBuilders.matchPhraseQuery("title", prefix);
   builder.setQuery(queryBuilder.toString());
   logger.info("Search qeury");
   logger.info(builder.toString());
   SearchResponse sr = builder.execute().actionGet();
   logger.info("Search response");
   logger.info(sr.toString());
   Iterator<SearchHit> it = sr.getHits().iterator();
   while (it.hasNext()) {
     SearchHit hit = it.next();
     hit.getSource();
   }
   return Arrays.asList("a", "b", "c");
 }
  @Override
  public List<Map<String, Object>> search(String term, List<Class<? extends Entity>> entityTypes)
      throws SearchEngineException {
    SearchResponse response =
        client
            .prepareSearch("entities")
            .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
            .setQuery(queryString(term))
            .setFrom(0)
            .setSize(10)
            .setExplain(false)
            .execute()
            .actionGet();

    List<Map<String, Object>> result = new ArrayList<Map<String, Object>>();
    for (SearchHit hit : response.getHits()) {
      result.add(hit.getSource());
    }
    return result;
  }
Esempio n. 26
0
  @Test
  public void dateSearch()
      throws IOException, SqlParseException, SQLFeatureNotSupportedException, ParseException {
    DateTimeFormatter formatter = DateTimeFormat.forPattern(DATE_FORMAT);
    DateTime dateToCompare = new DateTime(2014, 8, 18, 0, 0, 0);

    SearchHits response =
        query(
            String.format(
                "SELECT insert_time FROM %s/online WHERE insert_time < '2014-08-18'", TEST_INDEX));
    SearchHit[] hits = response.getHits();
    for (SearchHit hit : hits) {
      Map<String, Object> source = hit.getSource();
      DateTime insertTime = formatter.parseDateTime((String) source.get("insert_time"));

      String errorMessage =
          String.format("insert_time must be smaller then 2014-08-18. found: %s", insertTime);
      Assert.assertTrue(errorMessage, insertTime.isBefore(dateToCompare));
    }
  }
Esempio n. 27
0
  @Test
  public void lessThanOrEqualTest()
      throws IOException, SqlParseException, SQLFeatureNotSupportedException {
    int someAge = 25;
    SearchHits response =
        query(String.format("SELECT * FROM %s WHERE age <= %s LIMIT 1000", TEST_INDEX, someAge));
    SearchHit[] hits = response.getHits();

    boolean isEqualFound = false;
    for (SearchHit hit : hits) {
      int age = (int) hit.getSource().get("age");
      assertThat(age, lessThanOrEqualTo(someAge));

      if (age == someAge) isEqualFound = true;
    }

    Assert.assertTrue(
        String.format("at least one of the documents need to contains age equal to %s", someAge),
        isEqualFound);
  }
Esempio n. 28
0
  /** finder methods */
  public List<ActiveRule> findByRule(RuleKey key) {
    SearchRequestBuilder request =
        getClient()
            .prepareSearch(this.getIndexName())
            .setQuery(
                QueryBuilders.hasParentQuery(
                    this.getParentType(),
                    QueryBuilders.idsQuery(this.getParentType()).addIds(key.toString())))
            .setRouting(key.toString())
            // TODO replace by scrolling
            .setSize(Integer.MAX_VALUE);

    SearchResponse response = request.get();

    List<ActiveRule> activeRules = new ArrayList<ActiveRule>();
    for (SearchHit hit : response.getHits()) {
      activeRules.add(toDoc(hit.getSource()));
    }
    return activeRules;
  }
  private void assertCorrectResponse(int count, Event event, SearchResponse response) {
    SearchHits hits = response.getHits();

    assertEquals(count, hits.getTotalHits());

    SearchHit hit = hits.getAt(0);

    Map<String, Object> source = hit.getSource();

    assertEquals(event.getHost(), source.get("host"));
    assertEquals("1970-01-01T00:00:00.000Z", source.get("timestamp"));
    assertEquals(event.getPriority().name(), source.get("priority"));

    @SuppressWarnings("unchecked")
    Map<String, Object> message = (Map<String, Object>) source.get("message");
    assertEquals(new String(event.getBody()), message.get("text"));

    @SuppressWarnings("unchecked")
    Map<String, Object> fields = (Map<String, Object>) source.get("fields");

    assertEquals(new String(event.getAttrs().get("attr1")), fields.get("attr1"));
    assertEquals(new String(event.getAttrs().get("attr2")), fields.get("attr2"));
  }
Esempio n. 30
0
  private Result<QProfileActivity> searchActivities(
      QProfileActivityQuery query, SearchOptions options) {
    DbSession session = dbClient.openSession(false);
    try {
      SearchResponse response = activityIndex.doSearch(query, options);
      Result<QProfileActivity> result = new Result<>(response);
      for (SearchHit hit : response.getHits().getHits()) {
        QProfileActivity profileActivity = new QProfileActivity(hit.getSource());
        RuleDto ruleDto = dbClient.ruleDao().getNullableByKey(session, profileActivity.ruleKey());
        profileActivity.ruleName(ruleDto != null ? ruleDto.getName() : null);

        String login = profileActivity.getLogin();
        if (login != null) {
          UserDto user = dbClient.userDao().selectActiveUserByLogin(session, login);
          profileActivity.authorName(user != null ? user.getName() : null);
        }
        result.getHits().add(profileActivity);
      }
      return result;
    } finally {
      session.close();
    }
  }