private FilterBuilder[] makeFilters(ZonedDateTime currentDate) {
List<FilterBuilder> builders = new ArrayList<>();
boolean minRange = down(TimeUnit.HOURS, currentDate).isBefore(config.getFrom());
boolean maxRange = up(TimeUnit.HOURS, currentDate).isAfter(config.getTo());
if (minRange || maxRange) {
RangeFilterBuilder rangeFilterBuilder = FilterBuilders.rangeFilter("received_at");
if (minRange)
rangeFilterBuilder = rangeFilterBuilder.gte(DATE_FORMAT.format(config.getFrom()));
if (maxRange) rangeFilterBuilder = rangeFilterBuilder.lte(DATE_FORMAT.format(config.getTo()));
builders.add(rangeFilterBuilder);
}
return builders.toArray(new FilterBuilder[builders.size()]);
}
public String createQuery(
boolean includeFacets, int simpleSelections, int rangeSelections, int pathSelections) {
SenseiClientRequest clientRequest =
senseiQueryProducer.createQuery(
includeFacets, simpleSelections, rangeSelections, pathSelections);
FilterBuilder[] filters = new FilterBuilder[clientRequest.getSelections().size()];
for (int i = 0; i < filters.length; i++) {
Selection selection = clientRequest.getSelections().get(i);
if (selection instanceof Terms) {
filters[i] = createFilterFromTerm((Terms) selection);
}
if (selection instanceof Range) {
Range range = (Range) selection;
RangeFilterBuilder rangeFilter = FilterBuilders.rangeFilter(range.getField());
filters[i] = rangeFilter;
if (!"*".equals(range.getFrom())) rangeFilter.from(range.getFrom());
if (!"*".equals(range.getTo())) rangeFilter.to(range.getTo());
}
if (selection instanceof Path) {
Path path = (Path) selection;
filters[i] = FilterBuilders.prefixFilter(path.getField(), path.getValue());
}
}
FilterBuilder selections = FilterBuilders.orFilter(filters);
SearchRequestBuilder requestBuilder = new SearchRequestBuilder(new MockClient());
requestBuilder.setFilter(selections);
try {
JSONObject request = new JSONObject(requestBuilder.toString());
JSONObject facets = new JSONObject();
for (Selection selection : clientRequest.getSelections()) {
if (selection instanceof Terms) {
String field = selection.getField();
facets.put(
field, new JSONObject("{\"terms\": {\"field\":\"" + field + "\", \"size\" : 300}}"));
}
}
request.put("facets", facets);
return request.toString();
} catch (Exception ex) {
throw new RuntimeException(ex);
}
}
// Make sure that unordered, reversed, disjoint and/or overlapping ranges are supported
// Duel with filters
public void testRandomRanges() throws Exception {
final int numDocs = scaledRandomIntBetween(500, 5000);
final double[][] docs = new double[numDocs][];
for (int i = 0; i < numDocs; ++i) {
final int numValues = randomInt(5);
docs[i] = new double[numValues];
for (int j = 0; j < numValues; ++j) {
docs[i][j] = randomDouble() * 100;
}
}
createIndex("idx");
for (int i = 0; i < docs.length; ++i) {
XContentBuilder source = jsonBuilder().startObject().startArray("values");
for (int j = 0; j < docs[i].length; ++j) {
source = source.value(docs[i][j]);
}
source = source.endArray().endObject();
client().prepareIndex("idx", "type").setSource(source).execute().actionGet();
}
assertNoFailures(
client()
.admin()
.indices()
.prepareRefresh("idx")
.setIndicesOptions(IndicesOptions.lenientExpandOpen())
.execute()
.get());
final int numRanges = randomIntBetween(1, 20);
final double[][] ranges = new double[numRanges][];
for (int i = 0; i < ranges.length; ++i) {
switch (randomInt(2)) {
case 0:
ranges[i] = new double[] {Double.NEGATIVE_INFINITY, randomInt(100)};
break;
case 1:
ranges[i] = new double[] {randomInt(100), Double.POSITIVE_INFINITY};
break;
case 2:
ranges[i] = new double[] {randomInt(100), randomInt(100)};
break;
default:
throw new AssertionError();
}
}
RangeBuilder query = range("range").field("values");
for (int i = 0; i < ranges.length; ++i) {
String key = Integer.toString(i);
if (ranges[i][0] == Double.NEGATIVE_INFINITY) {
query.addUnboundedTo(key, ranges[i][1]);
} else if (ranges[i][1] == Double.POSITIVE_INFINITY) {
query.addUnboundedFrom(key, ranges[i][0]);
} else {
query.addRange(key, ranges[i][0], ranges[i][1]);
}
}
SearchRequestBuilder reqBuilder = client().prepareSearch("idx").addAggregation(query);
for (int i = 0; i < ranges.length; ++i) {
RangeFilterBuilder filter = FilterBuilders.rangeFilter("values");
if (ranges[i][0] != Double.NEGATIVE_INFINITY) {
filter = filter.from(ranges[i][0]);
}
if (ranges[i][1] != Double.POSITIVE_INFINITY) {
filter = filter.to(ranges[i][1]);
}
reqBuilder = reqBuilder.addAggregation(filter("filter" + i).filter(filter));
}
SearchResponse resp = reqBuilder.execute().actionGet();
Range range = resp.getAggregations().get("range");
for (int i = 0; i < ranges.length; ++i) {
long count = 0;
for (double[] values : docs) {
for (double value : values) {
if (value >= ranges[i][0] && value < ranges[i][1]) {
++count;
break;
}
}
}
final Range.Bucket bucket = range.getBucketByKey(Integer.toString(i));
assertEquals(bucket.getKey(), count, bucket.getDocCount());
final Filter filter = resp.getAggregations().get("filter" + i);
assertThat(filter.getDocCount(), equalTo(count));
}
}
/**
* Prepares cache of parsed filter configurations and search filters valid for actual request.
* This method should be called as soon as {@link QuerySettings.Filters} is available.
*
* @param filters to use to prepare relevant parsed filter configurations into request scope
* @throws java.lang.ReflectiveOperationException if filter field configuration file can not be
* parsed correctly
*/
protected void prepareFiltersForRequest(QuerySettings.Filters filters)
throws ReflectiveOperationException {
semiParsedFilters = new LinkedHashMap<>();
searchFilters = new LinkedHashMap<>();
rangeFiltersIntervals = new LinkedHashMap<>();
if (filters != null && !filters.getFilterCandidatesKeys().isEmpty()) {
Map<String, Object> filtersConfig =
configService.get(ConfigService.CFGNAME_SEARCH_FULLTEXT_FILTER_FIELDS);
if (filtersConfig == null || filtersConfig.isEmpty()) {
if (log.isLoggable(Level.FINEST)) {
log.log(
Level.FINEST,
"Configuration document ["
+ ConfigService.CFGNAME_SEARCH_FULLTEXT_FILTER_FIELDS
+ "] not found or is empty! This might be a bug.");
}
return;
}
// collect parsed filter configurations that are relevant to filters required by client
for (String filterCandidateKey : filters.getFilterCandidatesKeys()) {
if (filtersConfig.containsKey(filterCandidateKey)) {
// get filter types for filterCandidateKey and check all types are the same
Object filterConfig = filtersConfig.get(filterCandidateKey);
// TODO search filter configuration - cache it
SemiParsedFilterConfig parsedFilterConfig =
ConfigParseUtil.parseFilterType(filterConfig, filterCandidateKey);
semiParsedFilters.put(filterCandidateKey, parsedFilterConfig);
}
}
// iterate over all collected filters and drop those that are suppressed
for (String filterName :
semiParsedFilters.keySet().toArray(new String[semiParsedFilters.size()])) {
// parsed filters could have been removed in the meantime so we check if it is still present
if (semiParsedFilters.containsKey(filterName)) {
SemiParsedFilterConfig parsedFilterConfig = semiParsedFilters.get(filterName);
if (parsedFilterConfig instanceof SemiParsedFilterConfigSupportSuppressed) {
List<String> suppressed =
((SemiParsedFilterConfigSupportSuppressed) parsedFilterConfig).getSuppressed();
if (suppressed != null) {
for (String suppress : suppressed) {
if (semiParsedFilters.containsKey(suppress)) {
semiParsedFilters.remove(suppress);
}
}
}
}
}
}
// iterate over filters
for (SemiParsedFilterConfig filterConfig : semiParsedFilters.values()) {
// terms filter
if (filterConfig instanceof SemiParsedTermsFilterConfig) {
SemiParsedTermsFilterConfig conf = (SemiParsedTermsFilterConfig) filterConfig;
Set<String> fn = this.getFilterNamesForDocumentField(conf.getFieldName());
final List<String> filterValues = new ArrayList<>(filters.getFilterCandidateValues(fn));
if (!filterValues.isEmpty()) {
// handle <_lowercase> if specified
if (conf.isLowercase()) {
for (int i = 0; i < filterValues.size(); i++) {
filterValues.set(i, filterValues.get(i).toLowerCase(Locale.ENGLISH));
}
}
TermsFilterBuilder tfb = new TermsFilterBuilder(conf.getFieldName(), filterValues);
// handle terms filter <optional_settings>
if (conf.getName() != null) {
tfb.filterName(conf.getName());
}
if (conf.isCache() != null) {
tfb.cache(conf.isCache());
}
if (conf.isCache() != null && conf.isCache() && conf.getCacheKey() != null) {
tfb.cacheKey(conf.getCacheKey());
}
// TODO handle tfb.execution()
searchFilters.put(conf.getFieldName(), tfb);
}
// range filter
} else if (filterConfig instanceof SemiParsedRangeFilterConfig) {
SemiParsedRangeFilterConfig conf = (SemiParsedRangeFilterConfig) filterConfig;
RangeFilterBuilder rfb;
// check if there is already range filter for this document field
if (searchFilters.containsKey(conf.getFieldName())
&& searchFilters.get(conf.getFieldName()) instanceof RangeFilterBuilder) {
// in this case we will be adding (or overriding) settings of existing filter
rfb = (RangeFilterBuilder) searchFilters.get(conf.getFieldName());
} else {
rfb = new RangeFilterBuilder(conf.getFieldName());
}
final String filterValue = filters.getFirstValueForFilterCandidate(conf.getFilterName());
ParsableIntervalConfig interval = null;
if (filterValue != null) {
IntervalRange intervalRange = rangeFiltersIntervals.get(conf.getFieldName());
if (intervalRange == null) {
intervalRange = new IntervalRange();
rangeFiltersIntervals.put(conf.getFieldName(), intervalRange);
}
// handle <_processor> if specified
if (conf.getProcessor() != null) {
Class<?> processorClass = Class.forName(conf.getProcessor());
if (!processorClass.isEnum()) {
throw new RuntimeException(
"Class [" + conf.getProcessor() + "] is not an enum type.");
}
// TODO: improve ParsableIntervalConfig design to make sure this method has to be
// implemented
Method m = processorClass.getMethod("parseRequestParameterValue", String.class);
interval = (ParsableIntervalConfig) m.invoke(processorClass, filterValue);
}
if (conf.definesGte()) {
if (interval != null) {
DateTime gte = new DateTime(interval.getGteValue(System.currentTimeMillis()));
rfb.gte(gte.toString(DATE_TIME_FORMATTER_UTC));
intervalRange.setGte(gte);
} else {
rfb.gte(filterValue);
intervalRange.setGte(DATE_TIME_FORMATTER_UTC.parseDateTime(filterValue));
}
} else if (conf.definesLte()) {
if (interval != null) {
DateTime lte = new DateTime(interval.getLteValue(System.currentTimeMillis()));
rfb.lte(lte.toString(DATE_TIME_FORMATTER_UTC));
intervalRange.setLte(lte);
} else {
rfb.lte(filterValue);
intervalRange.setLte(DATE_TIME_FORMATTER_UTC.parseDateTime(filterValue));
}
}
}
// handle range filter <optional_settings>
if (conf.getName() != null) {
rfb.filterName(conf.getName());
}
if (conf.isCache() != null) {
rfb.cache(conf.isCache());
}
if (conf.isCache() != null && conf.isCache() && conf.getCacheKey() != null) {
rfb.cacheKey(conf.getCacheKey());
}
searchFilters.put(conf.getFieldName(), rfb);
} else {
if (log.isLoggable(Level.FINE)) {
log.log(
Level.FINE,
"Unsupported SemiParsedFilterConfig type: " + filterConfig.getClass().getName());
}
}
}
}
}