private FilterBuilder[] makeFilters(ZonedDateTime currentDate) {
   List<FilterBuilder> builders = new ArrayList<>();
   boolean minRange = down(TimeUnit.HOURS, currentDate).isBefore(config.getFrom());
   boolean maxRange = up(TimeUnit.HOURS, currentDate).isAfter(config.getTo());
   if (minRange || maxRange) {
     RangeFilterBuilder rangeFilterBuilder = FilterBuilders.rangeFilter("received_at");
     if (minRange)
       rangeFilterBuilder = rangeFilterBuilder.gte(DATE_FORMAT.format(config.getFrom()));
     if (maxRange) rangeFilterBuilder = rangeFilterBuilder.lte(DATE_FORMAT.format(config.getTo()));
     builders.add(rangeFilterBuilder);
   }
   return builders.toArray(new FilterBuilder[builders.size()]);
 }
  public String createQuery(
      boolean includeFacets, int simpleSelections, int rangeSelections, int pathSelections) {
    SenseiClientRequest clientRequest =
        senseiQueryProducer.createQuery(
            includeFacets, simpleSelections, rangeSelections, pathSelections);

    FilterBuilder[] filters = new FilterBuilder[clientRequest.getSelections().size()];
    for (int i = 0; i < filters.length; i++) {
      Selection selection = clientRequest.getSelections().get(i);
      if (selection instanceof Terms) {
        filters[i] = createFilterFromTerm((Terms) selection);
      }
      if (selection instanceof Range) {
        Range range = (Range) selection;
        RangeFilterBuilder rangeFilter = FilterBuilders.rangeFilter(range.getField());
        filters[i] = rangeFilter;
        if (!"*".equals(range.getFrom())) rangeFilter.from(range.getFrom());
        if (!"*".equals(range.getTo())) rangeFilter.to(range.getTo());
      }
      if (selection instanceof Path) {
        Path path = (Path) selection;
        filters[i] = FilterBuilders.prefixFilter(path.getField(), path.getValue());
      }
    }
    FilterBuilder selections = FilterBuilders.orFilter(filters);
    SearchRequestBuilder requestBuilder = new SearchRequestBuilder(new MockClient());
    requestBuilder.setFilter(selections);
    try {
      JSONObject request = new JSONObject(requestBuilder.toString());
      JSONObject facets = new JSONObject();
      for (Selection selection : clientRequest.getSelections()) {
        if (selection instanceof Terms) {
          String field = selection.getField();
          facets.put(
              field, new JSONObject("{\"terms\": {\"field\":\"" + field + "\", \"size\" : 300}}"));
        }
      }
      request.put("facets", facets);
      return request.toString();
    } catch (Exception ex) {
      throw new RuntimeException(ex);
    }
  }
Example #3
0
  // Make sure that unordered, reversed, disjoint and/or overlapping ranges are supported
  // Duel with filters
  public void testRandomRanges() throws Exception {
    final int numDocs = scaledRandomIntBetween(500, 5000);
    final double[][] docs = new double[numDocs][];
    for (int i = 0; i < numDocs; ++i) {
      final int numValues = randomInt(5);
      docs[i] = new double[numValues];
      for (int j = 0; j < numValues; ++j) {
        docs[i][j] = randomDouble() * 100;
      }
    }

    createIndex("idx");
    for (int i = 0; i < docs.length; ++i) {
      XContentBuilder source = jsonBuilder().startObject().startArray("values");
      for (int j = 0; j < docs[i].length; ++j) {
        source = source.value(docs[i][j]);
      }
      source = source.endArray().endObject();
      client().prepareIndex("idx", "type").setSource(source).execute().actionGet();
    }
    assertNoFailures(
        client()
            .admin()
            .indices()
            .prepareRefresh("idx")
            .setIndicesOptions(IndicesOptions.lenientExpandOpen())
            .execute()
            .get());

    final int numRanges = randomIntBetween(1, 20);
    final double[][] ranges = new double[numRanges][];
    for (int i = 0; i < ranges.length; ++i) {
      switch (randomInt(2)) {
        case 0:
          ranges[i] = new double[] {Double.NEGATIVE_INFINITY, randomInt(100)};
          break;
        case 1:
          ranges[i] = new double[] {randomInt(100), Double.POSITIVE_INFINITY};
          break;
        case 2:
          ranges[i] = new double[] {randomInt(100), randomInt(100)};
          break;
        default:
          throw new AssertionError();
      }
    }

    RangeBuilder query = range("range").field("values");
    for (int i = 0; i < ranges.length; ++i) {
      String key = Integer.toString(i);
      if (ranges[i][0] == Double.NEGATIVE_INFINITY) {
        query.addUnboundedTo(key, ranges[i][1]);
      } else if (ranges[i][1] == Double.POSITIVE_INFINITY) {
        query.addUnboundedFrom(key, ranges[i][0]);
      } else {
        query.addRange(key, ranges[i][0], ranges[i][1]);
      }
    }

    SearchRequestBuilder reqBuilder = client().prepareSearch("idx").addAggregation(query);
    for (int i = 0; i < ranges.length; ++i) {
      RangeFilterBuilder filter = FilterBuilders.rangeFilter("values");
      if (ranges[i][0] != Double.NEGATIVE_INFINITY) {
        filter = filter.from(ranges[i][0]);
      }
      if (ranges[i][1] != Double.POSITIVE_INFINITY) {
        filter = filter.to(ranges[i][1]);
      }
      reqBuilder = reqBuilder.addAggregation(filter("filter" + i).filter(filter));
    }

    SearchResponse resp = reqBuilder.execute().actionGet();
    Range range = resp.getAggregations().get("range");

    for (int i = 0; i < ranges.length; ++i) {

      long count = 0;
      for (double[] values : docs) {
        for (double value : values) {
          if (value >= ranges[i][0] && value < ranges[i][1]) {
            ++count;
            break;
          }
        }
      }

      final Range.Bucket bucket = range.getBucketByKey(Integer.toString(i));
      assertEquals(bucket.getKey(), count, bucket.getDocCount());

      final Filter filter = resp.getAggregations().get("filter" + i);
      assertThat(filter.getDocCount(), equalTo(count));
    }
  }
  /**
   * Prepares cache of parsed filter configurations and search filters valid for actual request.
   * This method should be called as soon as {@link QuerySettings.Filters} is available.
   *
   * @param filters to use to prepare relevant parsed filter configurations into request scope
   * @throws java.lang.ReflectiveOperationException if filter field configuration file can not be
   *     parsed correctly
   */
  protected void prepareFiltersForRequest(QuerySettings.Filters filters)
      throws ReflectiveOperationException {

    semiParsedFilters = new LinkedHashMap<>();
    searchFilters = new LinkedHashMap<>();
    rangeFiltersIntervals = new LinkedHashMap<>();

    if (filters != null && !filters.getFilterCandidatesKeys().isEmpty()) {
      Map<String, Object> filtersConfig =
          configService.get(ConfigService.CFGNAME_SEARCH_FULLTEXT_FILTER_FIELDS);

      if (filtersConfig == null || filtersConfig.isEmpty()) {
        if (log.isLoggable(Level.FINEST)) {
          log.log(
              Level.FINEST,
              "Configuration document ["
                  + ConfigService.CFGNAME_SEARCH_FULLTEXT_FILTER_FIELDS
                  + "] not found or is empty! This might be a bug.");
        }
        return;
      }

      // collect parsed filter configurations that are relevant to filters required by client
      for (String filterCandidateKey : filters.getFilterCandidatesKeys()) {
        if (filtersConfig.containsKey(filterCandidateKey)) {
          // get filter types for filterCandidateKey and check all types are the same
          Object filterConfig = filtersConfig.get(filterCandidateKey);
          // TODO search filter configuration - cache it
          SemiParsedFilterConfig parsedFilterConfig =
              ConfigParseUtil.parseFilterType(filterConfig, filterCandidateKey);
          semiParsedFilters.put(filterCandidateKey, parsedFilterConfig);
        }
      }

      // iterate over all collected filters and drop those that are suppressed
      for (String filterName :
          semiParsedFilters.keySet().toArray(new String[semiParsedFilters.size()])) {
        // parsed filters could have been removed in the meantime so we check if it is still present
        if (semiParsedFilters.containsKey(filterName)) {
          SemiParsedFilterConfig parsedFilterConfig = semiParsedFilters.get(filterName);
          if (parsedFilterConfig instanceof SemiParsedFilterConfigSupportSuppressed) {
            List<String> suppressed =
                ((SemiParsedFilterConfigSupportSuppressed) parsedFilterConfig).getSuppressed();
            if (suppressed != null) {
              for (String suppress : suppressed) {
                if (semiParsedFilters.containsKey(suppress)) {
                  semiParsedFilters.remove(suppress);
                }
              }
            }
          }
        }
      }

      // iterate over filters
      for (SemiParsedFilterConfig filterConfig : semiParsedFilters.values()) {
        // terms filter
        if (filterConfig instanceof SemiParsedTermsFilterConfig) {
          SemiParsedTermsFilterConfig conf = (SemiParsedTermsFilterConfig) filterConfig;
          Set<String> fn = this.getFilterNamesForDocumentField(conf.getFieldName());
          final List<String> filterValues = new ArrayList<>(filters.getFilterCandidateValues(fn));
          if (!filterValues.isEmpty()) {
            // handle <_lowercase> if specified
            if (conf.isLowercase()) {
              for (int i = 0; i < filterValues.size(); i++) {
                filterValues.set(i, filterValues.get(i).toLowerCase(Locale.ENGLISH));
              }
            }
            TermsFilterBuilder tfb = new TermsFilterBuilder(conf.getFieldName(), filterValues);

            // handle terms filter <optional_settings>
            if (conf.getName() != null) {
              tfb.filterName(conf.getName());
            }
            if (conf.isCache() != null) {
              tfb.cache(conf.isCache());
            }
            if (conf.isCache() != null && conf.isCache() && conf.getCacheKey() != null) {
              tfb.cacheKey(conf.getCacheKey());
            }
            // TODO handle tfb.execution()

            searchFilters.put(conf.getFieldName(), tfb);
          }

          // range filter
        } else if (filterConfig instanceof SemiParsedRangeFilterConfig) {
          SemiParsedRangeFilterConfig conf = (SemiParsedRangeFilterConfig) filterConfig;

          RangeFilterBuilder rfb;
          // check if there is already range filter for this document field
          if (searchFilters.containsKey(conf.getFieldName())
              && searchFilters.get(conf.getFieldName()) instanceof RangeFilterBuilder) {
            // in this case we will be adding (or overriding) settings of existing filter
            rfb = (RangeFilterBuilder) searchFilters.get(conf.getFieldName());
          } else {
            rfb = new RangeFilterBuilder(conf.getFieldName());
          }

          final String filterValue = filters.getFirstValueForFilterCandidate(conf.getFilterName());

          ParsableIntervalConfig interval = null;
          if (filterValue != null) {

            IntervalRange intervalRange = rangeFiltersIntervals.get(conf.getFieldName());
            if (intervalRange == null) {
              intervalRange = new IntervalRange();
              rangeFiltersIntervals.put(conf.getFieldName(), intervalRange);
            }

            // handle <_processor> if specified
            if (conf.getProcessor() != null) {
              Class<?> processorClass = Class.forName(conf.getProcessor());
              if (!processorClass.isEnum()) {
                throw new RuntimeException(
                    "Class [" + conf.getProcessor() + "] is not an enum type.");
              }
              // TODO: improve ParsableIntervalConfig design to make sure this method has to be
              // implemented
              Method m = processorClass.getMethod("parseRequestParameterValue", String.class);
              interval = (ParsableIntervalConfig) m.invoke(processorClass, filterValue);
            }
            if (conf.definesGte()) {
              if (interval != null) {
                DateTime gte = new DateTime(interval.getGteValue(System.currentTimeMillis()));
                rfb.gte(gte.toString(DATE_TIME_FORMATTER_UTC));
                intervalRange.setGte(gte);
              } else {
                rfb.gte(filterValue);
                intervalRange.setGte(DATE_TIME_FORMATTER_UTC.parseDateTime(filterValue));
              }
            } else if (conf.definesLte()) {
              if (interval != null) {
                DateTime lte = new DateTime(interval.getLteValue(System.currentTimeMillis()));
                rfb.lte(lte.toString(DATE_TIME_FORMATTER_UTC));
                intervalRange.setLte(lte);
              } else {
                rfb.lte(filterValue);
                intervalRange.setLte(DATE_TIME_FORMATTER_UTC.parseDateTime(filterValue));
              }
            }
          }

          // handle range filter <optional_settings>
          if (conf.getName() != null) {
            rfb.filterName(conf.getName());
          }
          if (conf.isCache() != null) {
            rfb.cache(conf.isCache());
          }
          if (conf.isCache() != null && conf.isCache() && conf.getCacheKey() != null) {
            rfb.cacheKey(conf.getCacheKey());
          }

          searchFilters.put(conf.getFieldName(), rfb);

        } else {
          if (log.isLoggable(Level.FINE)) {
            log.log(
                Level.FINE,
                "Unsupported SemiParsedFilterConfig type: " + filterConfig.getClass().getName());
          }
        }
      }
    }
  }