void assertBasicSearchWorks(String indexName) {
    logger.info("--> testing basic search");
    SearchRequestBuilder searchReq =
        client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery());
    SearchResponse searchRsp = searchReq.get();
    ElasticsearchAssertions.assertNoFailures(searchRsp);
    long numDocs = searchRsp.getHits().getTotalHits();
    logger.info("Found " + numDocs + " in old index");

    logger.info("--> testing basic search with sort");
    searchReq.addSort("long_sort", SortOrder.ASC);
    ElasticsearchAssertions.assertNoFailures(searchReq.get());

    logger.info("--> testing exists filter");
    searchReq = client().prepareSearch(indexName).setQuery(QueryBuilders.existsQuery("string"));
    searchRsp = searchReq.get();
    ElasticsearchAssertions.assertNoFailures(searchRsp);
    assertEquals(numDocs, searchRsp.getHits().getTotalHits());

    logger.info("--> testing missing filter");
    // the field for the missing filter here needs to be different than the exists filter above, to
    // avoid being found in the cache
    searchReq = client().prepareSearch(indexName).setQuery(QueryBuilders.missingQuery("long_sort"));
    searchRsp = searchReq.get();
    ElasticsearchAssertions.assertNoFailures(searchRsp);
    assertEquals(0, searchRsp.getHits().getTotalHits());
  }
Пример #2
0
  void assertBasicSearchWorks(String indexName) {
    logger.info("--> testing basic search");
    SearchRequestBuilder searchReq =
        client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery());
    SearchResponse searchRsp = searchReq.get();
    ElasticsearchAssertions.assertNoFailures(searchRsp);
    long numDocs = searchRsp.getHits().getTotalHits();
    logger.info("Found {} in old index", numDocs);

    logger.info("--> testing basic search with sort");
    searchReq.addSort("long_sort", SortOrder.ASC);
    ElasticsearchAssertions.assertNoFailures(searchReq.get());

    logger.info("--> testing exists filter");
    searchReq = client().prepareSearch(indexName).setQuery(QueryBuilders.existsQuery("string"));
    searchRsp = searchReq.get();
    ElasticsearchAssertions.assertNoFailures(searchRsp);
    assertEquals(numDocs, searchRsp.getHits().getTotalHits());
    GetSettingsResponse getSettingsResponse =
        client().admin().indices().prepareGetSettings(indexName).get();
    Version versionCreated =
        Version.fromId(
            Integer.parseInt(getSettingsResponse.getSetting(indexName, "index.version.created")));
    if (versionCreated.onOrAfter(Version.V_2_4_0)) {
      searchReq =
          client().prepareSearch(indexName).setQuery(QueryBuilders.existsQuery("field.with.dots"));
      searchRsp = searchReq.get();
      ElasticsearchAssertions.assertNoFailures(searchRsp);
      assertEquals(numDocs, searchRsp.getHits().getTotalHits());
    }
  }
Пример #3
0
  public Map<String, Long> countByField(IndexField indexField, FilterBuilder filter) {
    Map<String, Long> counts = new HashMap<>();

    SearchRequestBuilder request =
        client
            .prepareSearch(this.getIndexName())
            .setTypes(this.getIndexType())
            .setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), filter))
            .setSize(0)
            .addAggregation(
                AggregationBuilders.terms(indexField.field())
                    .field(indexField.field())
                    .order(Terms.Order.count(false))
                    .size(Integer.MAX_VALUE)
                    .minDocCount(0));

    SearchResponse response = request.get();

    Terms values = response.getAggregations().get(indexField.field());

    for (Terms.Bucket value : values.getBuckets()) {
      counts.put(value.getKey(), value.getDocCount());
    }
    return counts;
  }
Пример #4
0
  @Override
  @CheckForNull
  public Date getLastSynchronization(Map<String, String> params) {
    SearchRequestBuilder request =
        client
            .prepareSearch(this.getIndexName())
            .setTypes(this.getIndexType())
            .setQuery(
                QueryBuilders.filteredQuery(
                    QueryBuilders.matchAllQuery(), getLastSynchronizationBuilder(params)))
            .setSize(0)
            .addAggregation(
                AggregationBuilders.max("latest").field(BaseNormalizer.UPDATED_AT_FIELD));

    SearchResponse response = request.get();

    Max max = response.getAggregations().get("latest");
    if (max.getValue() > 0) {
      Date date = new DateTime((long) max.getValue()).toDate();
      LOG.debug(
          "Index {}:{} has last update of {}", this.getIndexName(), this.getIndexType(), date);
      return date;
    } else {
      LOG.debug("Index {}:{} has no last update date", this.getIndexName(), this.getIndexType());
      return null;
    }
  }
Пример #5
0
  public void addDeletion(SearchRequestBuilder searchRequest) {
    searchRequest
        .addSort("_doc", SortOrder.ASC)
        .setScroll(TimeValue.timeValueMinutes(5))
        .setSize(100)
        // load only doc ids, not _source fields
        .setFetchSource(false);

    // this search is synchronous. An optimization would be to be non-blocking,
    // but it requires to tracking pending requests in close().
    // Same semaphore can't be reused because of potential deadlock (requires to acquire
    // two locks)
    SearchResponse searchResponse = searchRequest.get();

    while (true) {
      SearchHit[] hits = searchResponse.getHits().getHits();
      for (SearchHit hit : hits) {
        DeleteRequestBuilder deleteRequestBuilder =
            client.prepareDelete(hit.index(), hit.type(), hit.getId());
        SearchHitField routing = hit.field("_routing");
        if (routing != null) {
          deleteRequestBuilder.setRouting(routing.getValue());
        }
        add(deleteRequestBuilder.request());
      }

      String scrollId = searchResponse.getScrollId();
      searchResponse =
          client.prepareSearchScroll(scrollId).setScroll(TimeValue.timeValueMinutes(5)).get();
      if (hits.length == 0) {
        client.nativeClient().prepareClearScroll().addScrollId(scrollId).get();
        break;
      }
    }
  }
  void assertRealtimeGetWorks(String indexName) {
    assertAcked(
        client()
            .admin()
            .indices()
            .prepareUpdateSettings(indexName)
            .setSettings(Settings.builder().put("refresh_interval", -1).build()));
    SearchRequestBuilder searchReq =
        client().prepareSearch(indexName).setQuery(QueryBuilders.matchAllQuery());
    SearchHit hit = searchReq.get().getHits().getAt(0);
    String docId = hit.getId();
    // foo is new, it is not a field in the generated index
    client().prepareUpdate(indexName, "doc", docId).setDoc("foo", "bar").get();
    GetResponse getRsp = client().prepareGet(indexName, "doc", docId).get();
    Map<String, Object> source = getRsp.getSourceAsMap();
    assertThat(source, Matchers.hasKey("foo"));

    assertAcked(
        client()
            .admin()
            .indices()
            .prepareUpdateSettings(indexName)
            .setSettings(
                Settings.builder()
                    .put("refresh_interval", EngineConfig.DEFAULT_REFRESH_INTERVAL)
                    .build()));
  }
Пример #7
0
 public SearchResponse executeQuery(
     MetricConfig conf, String type, List<String> fields, BuilderCustomizer customizer) {
   SearchRequestBuilder builder =
       client
           .prepareSearch(MetricUtils.getIndexes(conf.getStart(), conf.getStop(), client))
           .setTypes(type)
           .setSearchType(SearchType.QUERY_AND_FETCH)
           .setSize((int) (conf.getStop() - conf.getStart()) / 10);
   for (String next : fields) {
     builder.addField(next);
   }
   BoolQueryBuilder query =
       QueryBuilders.boolQuery()
           .must(
               QueryBuilders.rangeQuery("timestamp")
                   .from(conf.getStart() - conf.getStep())
                   .to(conf.getStop())
                   .includeLower(false)
                   .includeUpper(true));
   for (String tag : conf.getTags()) {
     query = query.must(QueryBuilders.termQuery("tags", tag));
   }
   builder.setQuery(query);
   if (customizer != null) {
     customizer.customize(builder);
   }
   return builder.get();
 }
Пример #8
0
  @Test
  public void testMultipleRescores() throws Exception {
    int numDocs = indexRandomNumbers("keyword", 1);
    QueryRescorer eightIsGreat =
        RescoreBuilder.queryRescorer(
                QueryBuilders.functionScoreQuery(
                        QueryBuilders.termQuery("field1", English.intToEnglish(8)))
                    .boostMode(CombineFunction.REPLACE)
                    .add(ScoreFunctionBuilders.scriptFunction("1000.0f")))
            .setScoreMode("total");
    QueryRescorer sevenIsBetter =
        RescoreBuilder.queryRescorer(
                QueryBuilders.functionScoreQuery(
                        QueryBuilders.termQuery("field1", English.intToEnglish(7)))
                    .boostMode(CombineFunction.REPLACE)
                    .add(ScoreFunctionBuilders.scriptFunction("10000.0f")))
            .setScoreMode("total");

    // First set the rescore window large enough that both rescores take effect
    SearchRequestBuilder request = client().prepareSearch().setRescoreWindow(numDocs);
    request.addRescorer(eightIsGreat).addRescorer(sevenIsBetter);
    SearchResponse response = request.get();
    assertFirstHit(response, hasId("7"));
    assertSecondHit(response, hasId("8"));

    // Now squash the second rescore window so it never gets to see a seven
    response =
        request
            .setSize(1)
            .clearRescorers()
            .addRescorer(eightIsGreat)
            .addRescorer(sevenIsBetter, 1)
            .get();
    assertFirstHit(response, hasId("8"));
    // We have no idea what the second hit will be because we didn't get a chance to look for seven

    // Now use one rescore to drag the number we're looking for into the window of another
    QueryRescorer ninetyIsGood =
        RescoreBuilder.queryRescorer(
                QueryBuilders.functionScoreQuery(QueryBuilders.queryString("*ninety*"))
                    .boostMode(CombineFunction.REPLACE)
                    .add(ScoreFunctionBuilders.scriptFunction("1000.0f")))
            .setScoreMode("total");
    QueryRescorer oneToo =
        RescoreBuilder.queryRescorer(
                QueryBuilders.functionScoreQuery(QueryBuilders.queryString("*one*"))
                    .boostMode(CombineFunction.REPLACE)
                    .add(ScoreFunctionBuilders.scriptFunction("1000.0f")))
            .setScoreMode("total");
    request.clearRescorers().addRescorer(ninetyIsGood).addRescorer(oneToo, 10);
    response = request.setSize(2).get();
    assertFirstHit(response, hasId("91"));
    assertFirstHit(response, hasScore(2001.0f));
    assertSecondHit(
        response, hasScore(1001.0f)); // Not sure which one it is but it is ninety something
  }
 // #10067: create-bwc-index.py deleted any doc with long_sort:[10-20]
 void assertDeleteByQueryWorked(String indexName, Version version) throws Exception {
   if (version.onOrBefore(Version.V_1_0_0_Beta2)) {
     // TODO: remove this once #10262 is fixed
     return;
   }
   // these documents are supposed to be deleted by a delete by query operation in the translog
   SearchRequestBuilder searchReq =
       client()
           .prepareSearch(indexName)
           .setQuery(QueryBuilders.queryStringQuery("long_sort:[10 TO 20]"));
   assertEquals(0, searchReq.get().getHits().getTotalHits());
 }
Пример #10
0
  public SearchResponse doSearch(ActivityQuery query, SearchOptions options) {
    SearchRequestBuilder requestBuilder =
        getClient()
            .prepareSearch(ActivityIndexDefinition.INDEX)
            .setTypes(ActivityIndexDefinition.TYPE);

    requestBuilder.setFrom(options.getOffset());
    requestBuilder.setSize(options.getLimit());
    requestBuilder.addSort(ActivityIndexDefinition.FIELD_CREATED_AT, SortOrder.DESC);

    AndFilterBuilder filter = FilterBuilders.andFilter();
    if (!query.getTypes().isEmpty()) {
      OrFilterBuilder typeFilter = FilterBuilders.orFilter();
      for (String type : query.getTypes()) {
        typeFilter.add(FilterBuilders.termFilter(ActivityIndexDefinition.FIELD_TYPE, type));
      }
      filter.add(typeFilter);
    }

    if (!query.getDataOrFilters().isEmpty()) {
      for (Map.Entry<String, Object> entry : query.getDataOrFilters().entrySet()) {
        OrFilterBuilder orFilter = FilterBuilders.orFilter();
        orFilter.add(
            FilterBuilders.nestedFilter(
                ActivityIndexDefinition.FIELD_DETAILS,
                FilterBuilders.termFilter(
                    ActivityIndexDefinition.FIELD_DETAILS + "." + entry.getKey(),
                    entry.getValue())));
        filter.add(orFilter);
      }
    }

    Date since = query.getSince();
    if (since != null) {
      filter.add(
          FilterBuilders.rangeFilter(ActivityIndexDefinition.FIELD_CREATED_AT)
              .gt(since)
              .cache(false));
    }
    Date to = query.getTo();
    if (to != null) {
      filter.add(
          FilterBuilders.rangeFilter(ActivityIndexDefinition.FIELD_CREATED_AT).lt(to).cache(false));
    }

    requestBuilder.setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), filter));
    return requestBuilder.get();
  }
Пример #11
0
  public List<ActiveRule> findByProfile(QualityProfileKey key) {
    SearchRequestBuilder request =
        getClient()
            .prepareSearch(getIndexName())
            .setQuery(
                QueryBuilders.termQuery(
                    ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field(), key.toString()))
            .setRouting(key.toString())
            // TODO replace by scrolling
            .setSize(Integer.MAX_VALUE);
    SearchResponse response = request.get();

    List<ActiveRule> activeRules = new ArrayList<ActiveRule>();
    for (SearchHit hit : response.getHits()) {
      activeRules.add(toDoc(hit.getSource()));
    }
    return activeRules;
  }
Пример #12
0
  /** finder methods */
  public List<ActiveRule> findByRule(RuleKey key) {
    SearchRequestBuilder request =
        getClient()
            .prepareSearch(this.getIndexName())
            .setQuery(
                QueryBuilders.hasParentQuery(
                    this.getParentType(),
                    QueryBuilders.idsQuery(this.getParentType()).addIds(key.toString())))
            .setRouting(key.toString())
            // TODO replace by scrolling
            .setSize(Integer.MAX_VALUE);

    SearchResponse response = request.get();

    List<ActiveRule> activeRules = new ArrayList<ActiveRule>();
    for (SearchHit hit : response.getHits()) {
      activeRules.add(toDoc(hit.getSource()));
    }
    return activeRules;
  }
  public void testStringSpecialValueVariable() throws Exception {
    // i.e. expression script for term aggregations, which is not allowed
    createIndex("test");
    ensureGreen("test");
    indexRandom(
        true,
        client().prepareIndex("test", "doc", "1").setSource("text", "hello"),
        client().prepareIndex("test", "doc", "2").setSource("text", "goodbye"),
        client().prepareIndex("test", "doc", "3").setSource("text", "hello"));

    SearchRequestBuilder req = client().prepareSearch().setIndices("test");
    req.setQuery(QueryBuilders.matchAllQuery())
        .addAggregation(
            AggregationBuilders.terms("term_agg")
                .field("text")
                .script(
                    new Script(
                        "_value", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null)));

    String message;
    try {
      // shards that don't have docs with the "text" field will not fail,
      // so we may or may not get a total failure
      SearchResponse rsp = req.get();
      assertThat(
          rsp.getShardFailures().length,
          greaterThan(0)); // at least the shards containing the docs should have failed
      message = rsp.getShardFailures()[0].reason();
    } catch (SearchPhaseExecutionException e) {
      message = e.toString();
    }
    assertThat(
        message + "should have contained ExpressionScriptExecutionException",
        message.contains("ExpressionScriptExecutionException"),
        equalTo(true));
    assertThat(
        message + "should have contained text variable error",
        message.contains("text variable"),
        equalTo(true));
  }
  public void testSpecialValueVariable() throws Exception {
    // i.e. _value for aggregations
    createIndex("test");
    ensureGreen("test");
    indexRandom(
        true,
        client().prepareIndex("test", "doc", "1").setSource("x", 5, "y", 1.2),
        client().prepareIndex("test", "doc", "2").setSource("x", 10, "y", 1.4),
        client().prepareIndex("test", "doc", "3").setSource("x", 13, "y", 1.8));

    SearchRequestBuilder req = client().prepareSearch().setIndices("test");
    req.setQuery(QueryBuilders.matchAllQuery())
        .addAggregation(
            AggregationBuilders.stats("int_agg")
                .field("x")
                .script(
                    new Script(
                        "_value * 3", ScriptType.INLINE, ExpressionScriptEngineService.NAME, null)))
        .addAggregation(
            AggregationBuilders.stats("double_agg")
                .field("y")
                .script(
                    new Script(
                        "_value - 1.1",
                        ScriptType.INLINE,
                        ExpressionScriptEngineService.NAME,
                        null)));

    SearchResponse rsp = req.get();
    assertEquals(3, rsp.getHits().getTotalHits());

    Stats stats = rsp.getAggregations().get("int_agg");
    assertEquals(39.0, stats.getMax(), 0.0001);
    assertEquals(15.0, stats.getMin(), 0.0001);

    stats = rsp.getAggregations().get("double_agg");
    assertEquals(0.7, stats.getMax(), 0.0001);
    assertEquals(0.1, stats.getMin(), 0.0001);
  }
  public void testScroll() throws ExecutionException, InterruptedException {
    createIndex("test");
    ensureYellow("test");

    int numDocs = iterations(10, 100);
    IndexRequestBuilder[] indexRequestBuilders = new IndexRequestBuilder[numDocs];
    for (int i = 0; i < numDocs; i++) {
      indexRequestBuilders[i] =
          client()
              .prepareIndex("test", "type", Integer.toString(i))
              .setSource("field", "value" + Integer.toString(i));
    }
    indexRandom(true, indexRequestBuilders);

    int size = randomIntBetween(1, 10);
    SearchRequestBuilder searchRequestBuilder =
        client().prepareSearch("test").setScroll("1m").setSize(size);

    SearchResponse searchResponse = searchRequestBuilder.get();
    assertThat(searchResponse.getScrollId(), notNullValue());
    assertHitCount(searchResponse, numDocs);
    int hits = 0;
    assertThat(searchResponse.getHits().getHits().length, greaterThan(0));
    hits += searchResponse.getHits().getHits().length;

    try {
      do {
        searchResponse =
            client().prepareSearchScroll(searchResponse.getScrollId()).setScroll("1m").get();
        assertThat(searchResponse.getScrollId(), notNullValue());
        assertHitCount(searchResponse, numDocs);
        hits += searchResponse.getHits().getHits().length;
      } while (searchResponse.getHits().getHits().length > 0);
      assertThat(hits, equalTo(numDocs));
    } finally {
      clearScroll(searchResponse.getScrollId());
    }
  }
 public void testScore() throws Exception {
   createIndex("test");
   ensureGreen("test");
   indexRandom(
       true,
       client().prepareIndex("test", "doc", "1").setSource("text", "hello goodbye"),
       client().prepareIndex("test", "doc", "2").setSource("text", "hello hello hello goodbye"),
       client().prepareIndex("test", "doc", "3").setSource("text", "hello hello goodebye"));
   ScoreFunctionBuilder score =
       ScoreFunctionBuilders.scriptFunction(
           new Script("1 / _score", ScriptType.INLINE, "expression", null));
   SearchRequestBuilder req = client().prepareSearch().setIndices("test");
   req.setQuery(
       QueryBuilders.functionScoreQuery(QueryBuilders.termQuery("text", "hello"), score)
           .boostMode("replace"));
   req.setSearchType(SearchType.DFS_QUERY_THEN_FETCH); // make sure DF is consistent
   SearchResponse rsp = req.get();
   assertSearchResponse(rsp);
   SearchHits hits = rsp.getHits();
   assertEquals(3, hits.getTotalHits());
   assertEquals("1", hits.getAt(0).getId());
   assertEquals("3", hits.getAt(1).getId());
   assertEquals("2", hits.getAt(2).getId());
 }
  @Test
  public void update_existing_document_when_indexing_one_project() throws Exception {
    String uuid = "PROJECT-UUID";
    esTester.putDocuments(
        INDEX_PROJECT_MEASURES,
        TYPE_PROJECT_MEASURES,
        new ProjectMeasuresDoc()
            .setId(uuid)
            .setKey("Old Key")
            .setName("Old Name")
            .setAnalysedAt(new Date(1_000_000L)));
    ComponentDto project = newProjectDto(uuid).setKey("New key").setName("New name");
    SnapshotDto analysis = componentDbTester.insertProjectAndSnapshot(project);

    underTest.index(project.uuid());

    assertThat(esTester.getIds(INDEX_PROJECT_MEASURES, TYPE_PROJECT_MEASURES)).containsOnly(uuid);
    SearchRequestBuilder request =
        esTester
            .client()
            .prepareSearch(INDEX_PROJECT_MEASURES)
            .setTypes(TYPE_PROJECT_MEASURES)
            .setQuery(
                boolQuery()
                    .must(matchAllQuery())
                    .filter(
                        boolQuery()
                            .must(termQuery("_id", uuid))
                            .must(termQuery(ProjectMeasuresIndexDefinition.FIELD_KEY, "New key"))
                            .must(termQuery(ProjectMeasuresIndexDefinition.FIELD_NAME, "New name"))
                            .must(
                                termQuery(
                                    ProjectMeasuresIndexDefinition.FIELD_ANALYSED_AT,
                                    new Date(analysis.getCreatedAt())))));
    assertThat(request.get().getHits()).hasSize(1);
  }
Пример #18
0
 private SearchHits query(String query)
     throws SqlParseException, SQLFeatureNotSupportedException, SQLFeatureNotSupportedException {
   SearchDao searchDao = MainTestSuite.getSearchDao();
   SearchRequestBuilder select = (SearchRequestBuilder) searchDao.explain(query);
   return select.get().getHits();
 }