public void addDeletion(SearchRequestBuilder searchRequest) {
searchRequest
.addSort("_doc", SortOrder.ASC)
.setScroll(TimeValue.timeValueMinutes(5))
.setSize(100)
// load only doc ids, not _source fields
.setFetchSource(false);
// this search is synchronous. An optimization would be to be non-blocking,
// but it requires to tracking pending requests in close().
// Same semaphore can't be reused because of potential deadlock (requires to acquire
// two locks)
SearchResponse searchResponse = searchRequest.get();
while (true) {
SearchHit[] hits = searchResponse.getHits().getHits();
for (SearchHit hit : hits) {
DeleteRequestBuilder deleteRequestBuilder =
client.prepareDelete(hit.index(), hit.type(), hit.getId());
SearchHitField routing = hit.field("_routing");
if (routing != null) {
deleteRequestBuilder.setRouting(routing.getValue());
}
add(deleteRequestBuilder.request());
}
String scrollId = searchResponse.getScrollId();
searchResponse =
client.prepareSearchScroll(scrollId).setScroll(TimeValue.timeValueMinutes(5)).get();
if (hits.length == 0) {
client.nativeClient().prepareClearScroll().addScrollId(scrollId).get();
break;
}
}
}
private static long doIndex(BulkIndexer bulk, Iterator<FileSourcesUpdaterHelper.Row> dbRows) {
long maxUpdatedAt = 0L;
bulk.start();
while (dbRows.hasNext()) {
FileSourcesUpdaterHelper.Row row = dbRows.next();
for (UpdateRequest updateRequest : row.getUpdateRequests()) {
bulk.add(updateRequest);
}
maxUpdatedAt = Math.max(maxUpdatedAt, row.getUpdatedAt());
}
bulk.stop();
return maxUpdatedAt;
}
private static long doIndex(BulkIndexer bulk, Iterator<ActiveRuleDoc> activeRules) {
bulk.start();
long maxDate = 0L;
while (activeRules.hasNext()) {
ActiveRuleDoc activeRule = activeRules.next();
bulk.add(newIndexRequest(activeRule));
// it's more efficient to sort programmatically than in SQL on some databases (MySQL for
// instance)
maxDate = Math.max(maxDate, activeRule.updatedAt());
}
bulk.stop();
return maxDate;
}