@Override
public SearchRequestBuilder initRequestBuilder(SearchRequestBuilder srb) {
// the dateFilter should not apply to the date facets!
dateFilter = null;
srb = super.initRequestBuilder(srb);
if (dateFilter != null) srb.setFilter(dateFilter);
if (isDateFacets()) {
// too much work to convert the generic case with all the date math
// so cheat for our case:
String name = ElasticTweetSearch.DATE_FACET;
RangeFacetBuilder rfb = FacetBuilders.rangeFacet(name).field(ElasticTweetSearch.DATE);
MyDate date = new MyDate();
// latest
rfb.addUnboundedTo(Helper.toLocalDateTime(date.clone().minusHours(8).castToHour().toDate()));
// first day
rfb.addUnboundedTo(Helper.toLocalDateTime(date.castToDay().toDate()));
for (int i = 0; i < 7; i++) {
// 'from' must be smaller than 'to'!
Date oldDate = date.toDate();
rfb.addRange(
Helper.toLocalDateTime(date.minusDays(1).toDate()), Helper.toLocalDateTime(oldDate));
}
// oldest
rfb.addUnboundedFrom(Helper.toLocalDateTime(date.toDate()));
srb.addFacet(rfb);
}
return srb;
}
public String createQuery(
boolean includeFacets, int simpleSelections, int rangeSelections, int pathSelections) {
SenseiClientRequest clientRequest =
senseiQueryProducer.createQuery(
includeFacets, simpleSelections, rangeSelections, pathSelections);
FilterBuilder[] filters = new FilterBuilder[clientRequest.getSelections().size()];
for (int i = 0; i < filters.length; i++) {
Selection selection = clientRequest.getSelections().get(i);
if (selection instanceof Terms) {
filters[i] = createFilterFromTerm((Terms) selection);
}
if (selection instanceof Range) {
Range range = (Range) selection;
RangeFilterBuilder rangeFilter = FilterBuilders.rangeFilter(range.getField());
filters[i] = rangeFilter;
if (!"*".equals(range.getFrom())) rangeFilter.from(range.getFrom());
if (!"*".equals(range.getTo())) rangeFilter.to(range.getTo());
}
if (selection instanceof Path) {
Path path = (Path) selection;
filters[i] = FilterBuilders.prefixFilter(path.getField(), path.getValue());
}
}
FilterBuilder selections = FilterBuilders.orFilter(filters);
SearchRequestBuilder requestBuilder = new SearchRequestBuilder(new MockClient());
requestBuilder.setFilter(selections);
try {
JSONObject request = new JSONObject(requestBuilder.toString());
JSONObject facets = new JSONObject();
for (Selection selection : clientRequest.getSelections()) {
if (selection instanceof Terms) {
String field = selection.getField();
facets.put(
field, new JSONObject("{\"terms\": {\"field\":\"" + field + "\", \"size\" : 300}}"));
}
}
request.put("facets", facets);
return request.toString();
} catch (Exception ex) {
throw new RuntimeException(ex);
}
}
/** All indices has to be created before! */
public void mergeIndices(
Collection<String> indexList,
String intoIndex,
int hitsPerPage,
boolean forceRefresh,
CreateObjectsInterface<T> createObj,
FilterBuilder additionalFilter) {
if (forceRefresh) {
refresh(indexList);
refresh(intoIndex);
}
int keepTime = 100;
for (String fromIndex : indexList) {
SearchRequestBuilder srb =
client
.prepareSearch(fromIndex)
.setVersion(true)
.setQuery(QueryBuilders.matchAllQuery())
.setSize(hitsPerPage)
.setSearchType(SearchType.SCAN)
.setScroll(TimeValue.timeValueMinutes(keepTime));
if (additionalFilter != null) srb.setFilter(additionalFilter);
SearchResponse rsp = srb.execute().actionGet();
try {
long total = rsp.hits().totalHits();
int collectedResults = 0;
while (true) {
StopWatch queryWatch = new StopWatch().start();
rsp =
client
.prepareSearchScroll(rsp.scrollId())
.setScroll(TimeValue.timeValueMinutes(keepTime))
.execute()
.actionGet();
long currentResults = rsp.hits().hits().length;
if (currentResults == 0) break;
queryWatch.stop();
Collection<T> objs = createObj.collectObjects(rsp);
StopWatch updateWatch = new StopWatch().start();
int failed = bulkUpdate(objs, intoIndex, false, false).size();
// trying to enable flushing to avoid memory issues on the server side?
flush(intoIndex);
updateWatch.stop();
collectedResults += currentResults;
logger.info(
"Progress "
+ collectedResults
+ "/"
+ total
+ " fromIndex="
+ fromIndex
+ " update:"
+ updateWatch.totalTime().getSeconds()
+ " query:"
+ queryWatch.totalTime().getSeconds()
+ " failed:"
+ failed);
}
logger.info(
"Finished copying of index:"
+ fromIndex
+ ". Total:"
+ total
+ " collected:"
+ collectedResults);
} catch (Exception ex) {
// throw new RuntimeException(ex);
logger.error(
"Failed to copy data from index " + fromIndex + " into " + intoIndex + ".", ex);
}
}
if (forceRefresh) refresh(intoIndex);
}