static List<Entry<String, RequestInfo>> initNativeSearchRequests(
int minTerms, int maxTerms, String script, boolean langNative) {
List<Entry<String, RequestInfo>> nativeSearchRequests = new ArrayList<>();
for (int nTerms = minTerms; nTerms < maxTerms; nTerms++) {
Map<String, Object> params = new HashMap<>();
String[] terms = getTerms(nTerms + 1);
params.put("text", terms);
String infoString = "Results for native script with " + (nTerms + 1) + " terms:";
ScriptScoreFunctionBuilder scriptFunction =
(langNative == true)
? scriptFunction(script, "native", params)
: scriptFunction(script, params);
SearchRequest request =
searchRequest()
.searchType(SearchType.QUERY_THEN_FETCH)
.source(
searchSource()
.explain(false)
.size(0)
.query(
functionScoreQuery(
FilterBuilders.termsFilter("text", terms), scriptFunction)
.boostMode(CombineFunction.REPLACE)));
nativeSearchRequests.add(
new AbstractMap.SimpleEntry<>(infoString, new RequestInfo(request, nTerms + 1)));
}
return nativeSearchRequests;
}
public Map<QualityProfileKey, Multimap<String, FacetValue>> getStatsByProfileKey(
List<QualityProfileKey> keys) {
String[] stringKeys = new String[keys.size()];
for (int i = 0; i < keys.size(); i++) {
stringKeys[i] = keys.get(i).toString();
}
SearchResponse response =
getClient()
.prepareSearch(this.getIndexName())
.setQuery(
QueryBuilders.filteredQuery(
QueryBuilders.matchAllQuery(),
FilterBuilders.termsFilter(
ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field(), stringKeys)))
.addAggregation(
AggregationBuilders.terms(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field())
.field(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field())
.subAggregation(
AggregationBuilders.terms(
ActiveRuleNormalizer.ActiveRuleField.INHERITANCE.field())
.field(ActiveRuleNormalizer.ActiveRuleField.INHERITANCE.field()))
.subAggregation(
AggregationBuilders.terms(
ActiveRuleNormalizer.ActiveRuleField.SEVERITY.field())
.field(ActiveRuleNormalizer.ActiveRuleField.SEVERITY.field())))
.setSize(0)
.setTypes(this.getIndexType())
.get();
Map<QualityProfileKey, Multimap<String, FacetValue>> stats =
new HashMap<QualityProfileKey, Multimap<String, FacetValue>>();
Aggregation aggregation =
response.getAggregations().get(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field());
for (Terms.Bucket value : ((Terms) aggregation).getBuckets()) {
stats.put(
QualityProfileKey.parse(value.getKey()),
this.processAggregations(value.getAggregations()));
}
return stats;
}
public static void parseOutputAggregation(
AdvancedQueryPojo.QueryOutputPojo.AggregationOutputPojo aggregation,
AliasLookupTable aliasLookup,
boolean geoLowAccuracy,
String[] entTypeFilterStrings,
String[] assocVerbFilterStrings,
SearchRequestBuilder searchSettings,
BoolFilterBuilder parentFilterObj,
String[] communityIdStrs) {
// 1.] Go through aggregation list
// 1.1] Apply "simple specifications" if necessary
// Geo
if ((null != aggregation)
&& (null != aggregation.geoNumReturn)
&& (aggregation.geoNumReturn > 0)) {
CrossVersionFacetBuilder.TermsFacetBuilder fb =
CrossVersionFacetBuilders.termsFacet("geo")
.field(DocumentPojo.locs_)
.size(aggregation.geoNumReturn);
// Gross raw handling for facets
if (null != parentFilterObj) {
fb = fb.facetFilter(parentFilterObj);
}
searchSettings.addFacet(fb);
} // (TESTED)
// Temporal
if ((null != aggregation) && (null != aggregation.timesInterval)) {
if (aggregation.timesInterval.contains("m")) {
aggregation.timesInterval = "month";
}
CrossVersionFacetBuilder.DateHistogramFacetBuilder fb =
CrossVersionFacetBuilders.dateHistogramFacet("time")
.field(DocumentPojo.publishedDate_)
.interval(aggregation.timesInterval);
// Gross raw handling for facets
if (null != parentFilterObj) {
fb = fb.facetFilter(parentFilterObj);
}
searchSettings.addFacet(fb);
// TODO (INF-2688): if using certain types of moments then don't want this?
} // (TESTED)
// Temporal Moments
if ((null != aggregation) && (null != aggregation.moments)) {
if (null == aggregation.moments.timesInterval) {
if (null != aggregation.timesInterval) {
aggregation.moments.timesInterval = aggregation.timesInterval;
} else {
aggregation.moments.timesInterval = "m";
}
}
if (aggregation.moments.timesInterval.contains("m")) {
aggregation.moments.timesInterval = "month";
}
// TODO (INF-2688): Other cross filter type things
if (!geoLowAccuracy
&& (null != aggregation.moments.geoNumReturn)
&& (aggregation.moments.geoNumReturn > 0)) {
DateHistogramBuilder timeAgg =
AggregationBuilders.dateHistogram("moments")
.field(DocumentPojo.publishedDate_)
.interval(new Interval(aggregation.moments.timesInterval));
TermsBuilder geoAgg =
AggregationBuilders.terms("geo")
.field(DocumentPojo.locs_)
.size(aggregation.moments.geoNumReturn);
timeAgg.subAggregation(geoAgg);
searchSettings.addAggregation(timeAgg);
}
// TODO (CORE-89)
if (null != aggregation.moments.associationsNumReturn
&& aggregation.moments.associationsNumReturn >= 0) {
// TODO need to check if indexes mapping use doc.associations.assoc_index == docValue
// fail out or don't include those communities if they don't
if (validateAssociationMapping(communityIdStrs)) {
DateHistogramBuilder assocTimeAgg =
AggregationBuilders.dateHistogram("moments.assoc")
.field(DocumentPojo.publishedDate_)
.interval(new Interval(aggregation.moments.timesInterval));
TermsBuilder assocAgg =
AggregationBuilders.terms("assoc")
.field(AssociationPojo.assoc_index_)
.size(aggregation.moments.associationsNumReturn);
NestedBuilder nested =
AggregationBuilders.nested("moments.assoc.nested")
.path(DocumentPojo.associations_)
.subAggregation(assocAgg);
assocTimeAgg.subAggregation(nested);
searchSettings.addAggregation(assocTimeAgg);
}
}
if (null != aggregation.moments.entityList) {
for (String entIndex : aggregation.moments.entityList) {
CrossVersionFacetBuilder.DateHistogramFacetBuilder fb =
CrossVersionFacetBuilders.dateHistogramFacet("moments." + entIndex)
.field(DocumentPojo.publishedDate_)
.interval(aggregation.moments.timesInterval);
EntityFeaturePojo alias = null;
if (null != aliasLookup) {
alias = aliasLookup.getAliases(entIndex);
}
if (null == alias) { // no alias
fb =
fb.facetFilter(
FilterBuilders.nestedFilter(
DocumentPojo.entities_,
FilterBuilders.termFilter(EntityPojo.index_, entIndex)));
} // TESTED
else {
QueryFilterBuilder qfb = null;
if ((null != alias.getSemanticLinks()) && !alias.getSemanticLinks().isEmpty()) {
BoolQueryBuilder qb = QueryBuilders.boolQuery();
for (String textAlias : alias.getSemanticLinks()) {
qb =
qb.should(
CrossVersionQueryBuilders.matchPhraseQuery(
DocumentPojo.fullText_, textAlias));
}
qfb = FilterBuilders.queryFilter(qb);
} // TESTED
if (!alias.getAlias().isEmpty()) {
NestedFilterBuilder nfb =
FilterBuilders.nestedFilter(
DocumentPojo.entities_,
FilterBuilders.termsFilter(EntityPojo.index_, entIndex, alias.getAlias()));
if (null == qfb) {
fb = fb.facetFilter(nfb);
} // TESTED
else {
BoolFilterBuilder bfb = FilterBuilders.boolFilter().should(nfb).should(qfb);
fb = fb.facetFilter(bfb);
} // TESTED
} else if (null != qfb) {
fb = fb.facetFilter(qfb);
} // TESTED
} // TESTED
// Gross raw handling for facets
if (null != parentFilterObj) {
fb = fb.facetFilter(parentFilterObj);
}
searchSettings.addFacet(fb);
}
} // (end list over entities)
} // TESTED
// Entities - due to problems with significance, handled on a document by document basis, see
// Significance helper class
// Associations (Events/Facts)
// Association verb category filter
StringBuilder verbCatRegex = null;
StringBuilder entTypeRegex = null;
if (((null != aggregation)
&& (null != aggregation.eventsNumReturn)
&& (aggregation.eventsNumReturn > 0))
|| ((null != aggregation)
&& (null != aggregation.factsNumReturn)
&& (aggregation.factsNumReturn > 0))) {
if (null != entTypeFilterStrings) {
boolean bNegative = false;
if ('-' != entTypeFilterStrings[0].charAt(0)) { // positive filtering
entTypeRegex = new StringBuilder("(?:");
} else {
bNegative = true;
entTypeRegex = new StringBuilder("(?!");
// (this is a lookahead but will be fine because of the .*/ in front of it)
}
for (String entType : entTypeFilterStrings) {
if (bNegative && ('-' == entType.charAt(0))) {
entType = entType.substring(1);
}
entType = entType.replace("|", "%7C");
entTypeRegex.append(".*?/").append(Pattern.quote(entType.toLowerCase())).append('|');
// (can't match greedily because of the 2nd instance of entity type)
}
entTypeRegex.setLength(entTypeRegex.length() - 1); // (remove trailing |)
entTypeRegex.append(")");
if (bNegative) {
entTypeRegex.append("[^|]*"); // (now the actual verb, if a -ve lookahead)
}
} // TESTED
if (null != assocVerbFilterStrings) {
boolean bNegative = false;
if ('-' != assocVerbFilterStrings[0].charAt(0)) { // positive filtering
verbCatRegex = new StringBuilder("\\|(?:");
} else {
bNegative = true;
verbCatRegex = new StringBuilder("\\|(?!");
// (this is a lookahead but will be fine because of the "^[^|]*\\" in front of it)
// eg say I have -VERB then subject|VERB|object will match because if the
}
for (String assocVerbFilterString : assocVerbFilterStrings) {
if (bNegative && ('-' == assocVerbFilterString.charAt(0))) {
assocVerbFilterString = assocVerbFilterString.substring(1);
}
assocVerbFilterString = assocVerbFilterString.replace("|", "%7C");
verbCatRegex.append(Pattern.quote(assocVerbFilterString)).append('|');
}
verbCatRegex.setLength(verbCatRegex.length() - 1); // (remove trailing |)
verbCatRegex.append(")");
if (bNegative) {
verbCatRegex.append("[^|]*"); // (now the actual verb, if a -ve lookahead)
}
} // TESTED
}
// TESTED (all combinations of 1/2 people, 1/2 verbs)
if ((null != aggregation)
&& (null != aggregation.eventsNumReturn)
&& (aggregation.eventsNumReturn > 0)) {
StringBuffer regex = new StringBuffer("^Event\\|");
if (null != entTypeRegex) {
regex.append(entTypeRegex);
} else {
regex.append("[^|]*");
}
if (null != verbCatRegex) {
regex.append(verbCatRegex);
} else if (null != entTypeRegex) {
regex.append("\\|[^|]*");
} else {
regex.append(".*");
}
if (null != entTypeRegex) {
regex.append("\\|").append(entTypeRegex);
regex.append(".*");
} else {
regex.append("\\|.*");
}
// DEBUG
// System.out.println("REGEX==" + regex.toString());
// TESTED (all combinations of 1/2 people, 1/2 verbs)
CrossVersionFacetBuilder.TermsFacetBuilder fb =
CrossVersionFacetBuilders.termsFacet("events")
.field(AssociationPojo.assoc_index_)
.size(aggregation.eventsNumReturn)
.nested(DocumentPojo.associations_);
fb.regex(regex.toString());
// Gross raw handling for facets
if (null != parentFilterObj) {
fb = fb.facetFilter(parentFilterObj);
}
searchSettings.addFacet(fb);
}
if ((null != aggregation)
&& (null != aggregation.factsNumReturn)
&& (aggregation.factsNumReturn > 0)) {
StringBuffer regex = new StringBuffer("^Fact\\|");
if (null != entTypeRegex) {
regex.append(entTypeRegex);
} else {
regex.append("[^|]*");
}
if (null != verbCatRegex) {
regex.append(verbCatRegex);
} else if (null != entTypeRegex) {
regex.append("\\|[^|]*");
} else {
regex.append(".*");
}
if (null != entTypeRegex) {
regex.append("\\|").append(entTypeRegex);
regex.append(".*");
} else {
regex.append("\\|.*");
}
// DEBUG
// System.out.println("REGEX==" + regex.toString());
// TESTED (all combinations of 1/2 people, 1/2 verbs)
CrossVersionFacetBuilder.TermsFacetBuilder fb =
CrossVersionFacetBuilders.termsFacet("facts")
.field(AssociationPojo.assoc_index_)
.size(aggregation.factsNumReturn)
.nested(DocumentPojo.associations_);
fb.regex(regex.toString());
// Gross raw handling for facets
if (null != parentFilterObj) {
fb = fb.facetFilter(parentFilterObj);
}
searchSettings.addFacet(fb);
}
// Source management/monitoring
if ((null != aggregation)
&& (null != aggregation.sourceMetadata)
&& (aggregation.sourceMetadata > 0)) {
CrossVersionFacetBuilder.TermsFacetBuilder fb =
CrossVersionFacetBuilders.termsFacet("sourceTags")
.field(DocumentPojo.tags_)
.size(aggregation.sourceMetadata)
.facetFilter(parentFilterObj);
CrossVersionFacetBuilder.TermsFacetBuilder fb1 =
CrossVersionFacetBuilders.termsFacet("sourceTypes")
.field(DocumentPojo.mediaType_)
.size(aggregation.sourceMetadata)
.facetFilter(parentFilterObj);
// Gross raw handling for facets
if (null != parentFilterObj) {
fb = fb.facetFilter(parentFilterObj);
fb1 = fb1.facetFilter(parentFilterObj);
}
searchSettings.addFacet(fb);
searchSettings.addFacet(fb1);
}
if ((null != aggregation) && (null != aggregation.sources) && (aggregation.sources > 0)) {
CrossVersionFacetBuilder.TermsFacetBuilder fb =
CrossVersionFacetBuilders.termsFacet("sourceKeys")
.field(DocumentPojo.sourceKey_)
.size(aggregation.sources);
// Gross raw handling for facets
if (null != parentFilterObj) {
fb = fb.facetFilter(parentFilterObj);
}
searchSettings.addFacet(fb);
}
} // TESTED
public void testIndexTemplateWithAliases() throws Exception {
client()
.admin()
.indices()
.preparePutTemplate("template_with_aliases")
.setTemplate("te*")
.addMapping(
"type1", "{\"type1\" : {\"properties\" : {\"value\" : {\"type\" : \"string\"}}}}")
.addAlias(new Alias("simple_alias"))
.addAlias(new Alias("templated_alias-{index}"))
.addAlias(new Alias("filtered_alias").filter("{\"type\":{\"value\":\"type2\"}}"))
.addAlias(
new Alias("complex_filtered_alias")
.filter(
FilterBuilders.termsFilter("_type", "typeX", "typeY", "typeZ")
.execution("bool")
.cache(true)))
.get();
assertAcked(
prepareCreate("test_index")
.addMapping("type1")
.addMapping("type2")
.addMapping("typeX")
.addMapping("typeY")
.addMapping("typeZ"));
ensureGreen();
client().prepareIndex("test_index", "type1", "1").setSource("field", "A value").get();
client().prepareIndex("test_index", "type2", "2").setSource("field", "B value").get();
client().prepareIndex("test_index", "typeX", "3").setSource("field", "C value").get();
client().prepareIndex("test_index", "typeY", "4").setSource("field", "D value").get();
client().prepareIndex("test_index", "typeZ", "5").setSource("field", "E value").get();
GetAliasesResponse getAliasesResponse =
client().admin().indices().prepareGetAliases().setIndices("test_index").get();
assertThat(getAliasesResponse.getAliases().size(), equalTo(1));
assertThat(getAliasesResponse.getAliases().get("test_index").size(), equalTo(4));
refresh();
SearchResponse searchResponse = client().prepareSearch("test_index").get();
assertHitCount(searchResponse, 5l);
searchResponse = client().prepareSearch("simple_alias").get();
assertHitCount(searchResponse, 5l);
searchResponse = client().prepareSearch("templated_alias-test_index").get();
assertHitCount(searchResponse, 5l);
searchResponse = client().prepareSearch("filtered_alias").get();
assertHitCount(searchResponse, 1l);
assertThat(searchResponse.getHits().getAt(0).type(), equalTo("type2"));
// Search the complex filter alias
searchResponse = client().prepareSearch("complex_filtered_alias").get();
assertHitCount(searchResponse, 3l);
Set<String> types = Sets.newHashSet();
for (SearchHit searchHit : searchResponse.getHits().getHits()) {
types.add(searchHit.getType());
}
assertThat(types.size(), equalTo(3));
assertThat(types, containsInAnyOrder("typeX", "typeY", "typeZ"));
}
