@Override
public void stop() {
if (bulkRequest.numberOfActions() > 0) {
executeBulk();
}
try {
if (semaphore.tryAcquire(concurrentRequests, 10, TimeUnit.MINUTES)) {
semaphore.release(concurrentRequests);
}
} catch (InterruptedException e) {
throw new IllegalStateException(
"Elasticsearch bulk requests still being executed after 10 minutes", e);
}
progress.stop();
if (!disableRefresh) {
client.prepareRefresh(indexName).get();
}
if (large) {
// optimize lucene segments and revert index settings
// Optimization must be done before re-applying replicas:
// http://www.elasticsearch.org/blog/performance-considerations-elasticsearch-indexing/
client.prepareForceMerge(indexName).get();
updateSettings(largeInitialSettings);
}
bulkRequest = null;
}
public void addDeletion(SearchRequestBuilder searchRequest) {
searchRequest
.addSort("_doc", SortOrder.ASC)
.setScroll(TimeValue.timeValueMinutes(5))
.setSize(100)
// load only doc ids, not _source fields
.setFetchSource(false);
// this search is synchronous. An optimization would be to be non-blocking,
// but it requires to tracking pending requests in close().
// Same semaphore can't be reused because of potential deadlock (requires to acquire
// two locks)
SearchResponse searchResponse = searchRequest.get();
while (true) {
SearchHit[] hits = searchResponse.getHits().getHits();
for (SearchHit hit : hits) {
DeleteRequestBuilder deleteRequestBuilder =
client.prepareDelete(hit.index(), hit.type(), hit.getId());
SearchHitField routing = hit.field("_routing");
if (routing != null) {
deleteRequestBuilder.setRouting(routing.getValue());
}
add(deleteRequestBuilder.request());
}
String scrollId = searchResponse.getScrollId();
searchResponse =
client.prepareSearchScroll(scrollId).setScroll(TimeValue.timeValueMinutes(5)).get();
if (hits.length == 0) {
client.nativeClient().prepareClearScroll().addScrollId(scrollId).get();
break;
}
}
}
@Override
public void start() {
Preconditions.checkState(bulkRequest == null, ALREADY_STARTED_MESSAGE);
if (large) {
largeInitialSettings = Maps.newHashMap();
Map<String, Object> bulkSettings = Maps.newHashMap();
GetSettingsResponse settingsResp =
client.nativeClient().admin().indices().prepareGetSettings(indexName).get();
// deactivate replicas
int initialReplicas =
Integer.parseInt(
settingsResp.getSetting(indexName, IndexMetaData.SETTING_NUMBER_OF_REPLICAS));
if (initialReplicas > 0) {
largeInitialSettings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, initialReplicas);
bulkSettings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0);
}
// deactivate periodical refresh
String refreshInterval = settingsResp.getSetting(indexName, REFRESH_INTERVAL_SETTING);
largeInitialSettings.put(REFRESH_INTERVAL_SETTING, refreshInterval);
bulkSettings.put(REFRESH_INTERVAL_SETTING, "-1");
updateSettings(bulkSettings);
}
bulkRequest = client.prepareBulk().setRefresh(false);
counter.set(0L);
progress.start();
}
@Override
public boolean hasNext() {
if (hits.isEmpty()) {
SearchScrollRequestBuilder esRequest =
esClient
.prepareSearchScroll(scrollId)
.setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES));
Collections.addAll(hits, esRequest.get().getHits().getHits());
}
return !hits.isEmpty();
}
private void executeBulk() {
final BulkRequestBuilder req = this.bulkRequest;
this.bulkRequest = client.prepareBulk().setRefresh(false);
semaphore.acquireUninterruptibly();
req.execute(new BulkResponseActionListener(req));
}
private void updateSettings(Map<String, Object> settings) {
UpdateSettingsRequestBuilder req =
client.nativeClient().admin().indices().prepareUpdateSettings(indexName);
req.setSettings(settings);
req.get();
}