Example #1
0
  @Override
  public void stop() {
    if (bulkRequest.numberOfActions() > 0) {
      executeBulk();
    }
    try {
      if (semaphore.tryAcquire(concurrentRequests, 10, TimeUnit.MINUTES)) {
        semaphore.release(concurrentRequests);
      }
    } catch (InterruptedException e) {
      throw new IllegalStateException(
          "Elasticsearch bulk requests still being executed after 10 minutes", e);
    }
    progress.stop();

    if (!disableRefresh) {
      client.prepareRefresh(indexName).get();
    }
    if (large) {
      // optimize lucene segments and revert index settings
      // Optimization must be done before re-applying replicas:
      // http://www.elasticsearch.org/blog/performance-considerations-elasticsearch-indexing/
      client.prepareForceMerge(indexName).get();

      updateSettings(largeInitialSettings);
    }
    bulkRequest = null;
  }
Example #2
0
  public void addDeletion(SearchRequestBuilder searchRequest) {
    searchRequest
        .addSort("_doc", SortOrder.ASC)
        .setScroll(TimeValue.timeValueMinutes(5))
        .setSize(100)
        // load only doc ids, not _source fields
        .setFetchSource(false);

    // this search is synchronous. An optimization would be to be non-blocking,
    // but it requires to tracking pending requests in close().
    // Same semaphore can't be reused because of potential deadlock (requires to acquire
    // two locks)
    SearchResponse searchResponse = searchRequest.get();

    while (true) {
      SearchHit[] hits = searchResponse.getHits().getHits();
      for (SearchHit hit : hits) {
        DeleteRequestBuilder deleteRequestBuilder =
            client.prepareDelete(hit.index(), hit.type(), hit.getId());
        SearchHitField routing = hit.field("_routing");
        if (routing != null) {
          deleteRequestBuilder.setRouting(routing.getValue());
        }
        add(deleteRequestBuilder.request());
      }

      String scrollId = searchResponse.getScrollId();
      searchResponse =
          client.prepareSearchScroll(scrollId).setScroll(TimeValue.timeValueMinutes(5)).get();
      if (hits.length == 0) {
        client.nativeClient().prepareClearScroll().addScrollId(scrollId).get();
        break;
      }
    }
  }
Example #3
0
  @Override
  public void start() {
    Preconditions.checkState(bulkRequest == null, ALREADY_STARTED_MESSAGE);
    if (large) {
      largeInitialSettings = Maps.newHashMap();
      Map<String, Object> bulkSettings = Maps.newHashMap();
      GetSettingsResponse settingsResp =
          client.nativeClient().admin().indices().prepareGetSettings(indexName).get();

      // deactivate replicas
      int initialReplicas =
          Integer.parseInt(
              settingsResp.getSetting(indexName, IndexMetaData.SETTING_NUMBER_OF_REPLICAS));
      if (initialReplicas > 0) {
        largeInitialSettings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, initialReplicas);
        bulkSettings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0);
      }

      // deactivate periodical refresh
      String refreshInterval = settingsResp.getSetting(indexName, REFRESH_INTERVAL_SETTING);
      largeInitialSettings.put(REFRESH_INTERVAL_SETTING, refreshInterval);
      bulkSettings.put(REFRESH_INTERVAL_SETTING, "-1");

      updateSettings(bulkSettings);
    }
    bulkRequest = client.prepareBulk().setRefresh(false);
    counter.set(0L);
    progress.start();
  }
Example #4
0
 @Override
 public boolean hasNext() {
   if (hits.isEmpty()) {
     SearchScrollRequestBuilder esRequest =
         esClient
             .prepareSearchScroll(scrollId)
             .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES));
     Collections.addAll(hits, esRequest.get().getHits().getHits());
   }
   return !hits.isEmpty();
 }
Example #5
0
 private void executeBulk() {
   final BulkRequestBuilder req = this.bulkRequest;
   this.bulkRequest = client.prepareBulk().setRefresh(false);
   semaphore.acquireUninterruptibly();
   req.execute(new BulkResponseActionListener(req));
 }
Example #6
0
 private void updateSettings(Map<String, Object> settings) {
   UpdateSettingsRequestBuilder req =
       client.nativeClient().admin().indices().prepareUpdateSettings(indexName);
   req.setSettings(settings);
   req.get();
 }