Example #1
0
  public void addDeletion(SearchRequestBuilder searchRequest) {
    searchRequest
        .addSort("_doc", SortOrder.ASC)
        .setScroll(TimeValue.timeValueMinutes(5))
        .setSize(100)
        // load only doc ids, not _source fields
        .setFetchSource(false);

    // this search is synchronous. An optimization would be to be non-blocking,
    // but it requires to tracking pending requests in close().
    // Same semaphore can't be reused because of potential deadlock (requires to acquire
    // two locks)
    SearchResponse searchResponse = searchRequest.get();

    while (true) {
      SearchHit[] hits = searchResponse.getHits().getHits();
      for (SearchHit hit : hits) {
        DeleteRequestBuilder deleteRequestBuilder =
            client.prepareDelete(hit.index(), hit.type(), hit.getId());
        SearchHitField routing = hit.field("_routing");
        if (routing != null) {
          deleteRequestBuilder.setRouting(routing.getValue());
        }
        add(deleteRequestBuilder.request());
      }

      String scrollId = searchResponse.getScrollId();
      searchResponse =
          client.prepareSearchScroll(scrollId).setScroll(TimeValue.timeValueMinutes(5)).get();
      if (hits.length == 0) {
        client.nativeClient().prepareClearScroll().addScrollId(scrollId).get();
        break;
      }
    }
  }
 private static long doIndex(BulkIndexer bulk, Iterator<FileSourcesUpdaterHelper.Row> dbRows) {
   long maxUpdatedAt = 0L;
   bulk.start();
   while (dbRows.hasNext()) {
     FileSourcesUpdaterHelper.Row row = dbRows.next();
     for (UpdateRequest updateRequest : row.getUpdateRequests()) {
       bulk.add(updateRequest);
     }
     maxUpdatedAt = Math.max(maxUpdatedAt, row.getUpdatedAt());
   }
   bulk.stop();
   return maxUpdatedAt;
 }
  private static long doIndex(BulkIndexer bulk, Iterator<ActiveRuleDoc> activeRules) {
    bulk.start();
    long maxDate = 0L;
    while (activeRules.hasNext()) {
      ActiveRuleDoc activeRule = activeRules.next();
      bulk.add(newIndexRequest(activeRule));

      // it's more efficient to sort programmatically than in SQL on some databases (MySQL for
      // instance)
      maxDate = Math.max(maxDate, activeRule.updatedAt());
    }
    bulk.stop();
    return maxDate;
  }