@Override
 public void executeESBulkRequest(BulkRequestBuilder esBulk)
     throws ElasticsearchException, BulkUpdatePartialFailureException {
   BulkResponse response = esBulk.execute().actionGet();
   if (response.hasFailures()) {
     boolean containsSuccess = false;
     int numOfFailures = 0;
     for (BulkItemResponse bir : response.getItems()) {
       if (!bir.isFailed()) {
         containsSuccess = true;
       } else {
         numOfFailures++;
       }
     }
     if (containsSuccess) {
       throw new BulkUpdatePartialFailureException(response.buildFailureMessage(), numOfFailures);
     } else {
       throw new ElasticsearchException(
           "Failed to completely execute ES index bulk update for "
               + numOfFailures
               + " commands: "
               + response.buildFailureMessage());
     }
   }
 }
Esempio n. 2
0
  @Test
  public void testBulk_withIndexAndDeleteItems() throws Exception {
    prepareIndex(1);
    BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
    for (int i = 0; i < 10; i++) {
      bulkRequestBuilder.add(client().prepareIndex("idx", "type").setSource("{}"));
    }

    BulkResponse bulkResponse = bulkRequestBuilder.get();
    bulkRequestBuilder = client().prepareBulk();
    for (BulkItemResponse item : bulkResponse) {
      assertThat(item.isFailed(), equalTo(false));
      assertShardInfo(item.getResponse());
      bulkRequestBuilder.add(client().prepareDelete("idx", "type", item.getId()));
    }

    bulkResponse = bulkRequestBuilder.get();
    for (BulkItemResponse item : bulkResponse) {
      assertThat(item.isFailed(), equalTo(false));
      assertShardInfo(item.getResponse());
    }
  }
Esempio n. 3
0
    private void index(final QueuedTaskExecutor executor) {
      final Esi4JBatchedEntityResolver entityResolver = executor.getEntityResolver();

      if (entityResolver != null) {
        entityResolver.resolveEntities(_tasks);
      }

      final BulkResponseWrapper response =
          executor
              .getTaskProcessor()
              .getIndex()
              .executeBulk(
                  new Esi4JOperation<ListenableActionFuture<BulkResponse>>() {

                    @Override
                    public ListenableActionFuture<BulkResponse> execute(
                        final Client client,
                        final String indexName,
                        final OperationContext helper) {
                      final BulkRequestBuilder bulk = client.prepareBulk();

                      for (final Esi4JEntityTask _task : _tasks) {
                        if (_task != null) {
                          _task.addToBulk(client, bulk, indexName, helper);
                        }
                      }

                      final ListenableActionFuture<BulkResponse> response = bulk.execute();

                      return response;
                    }
                  })
              .actionGet();

      int failed = 0;
      for (final BulkItemResponse item : response.getBulkResponse()) {
        if (item.isFailed()) {
          failed++;
        }
      }

      if (failed > 0) {
        log.warn("failed to index " + failed + " items. index might be out of sync");
      }

      if (log.isDebugEnabled()) {
        final int indexed = response.getBulkResponse().getItems().length - failed;
        log.debug("finished bulk indexing " + indexed + " items");
      }
    }
 protected void executeBulkRequest(BulkRequestBuilder bulkRequest) {
   if (bulkRequest.numberOfActions() == 0) return;
   BulkResponse bulkResponse = bulkRequest.execute().actionGet();
   if (!bulkResponse.hasFailures()) return;
   for (BulkItemResponse response : bulkResponse) {
     if (!response.isFailed()) continue;
     LOG.warning(
         String.format(
             "Unable to save Entity %s in %s/%s, cause: %s",
             response.getId(),
             response.getIndex(),
             response.getType(),
             response.getFailureMessage()));
   }
 }
Esempio n. 5
0
    @Override
    public void onResponse(BulkResponse response) {
      semaphore.release();
      counter.addAndGet(response.getItems().length);

      for (BulkItemResponse item : response.getItems()) {
        if (item.isFailed()) {
          LOGGER.error(
              "index [{}], type [{}], id [{}], message [{}]",
              item.getIndex(),
              item.getType(),
              item.getId(),
              item.getFailureMessage());
        }
      }
    }
  /**
   * This method processes failures by iterating through each bulk response item
   *
   * @param response, a BulkResponse
   */
  private void processBulkResponseFailure(BulkResponse response) {
    logger.warn("There was failures when executing bulk : " + response.buildFailureMessage());

    if (!logger.isDebugEnabled()) return;

    for (BulkItemResponse item : response.getItems()) {
      if (item.isFailed()) {
        logger.debug(
            "Error {} occurred on index {}, type {}, id {} for {} operation ",
            item.getFailureMessage(),
            item.getIndex(),
            item.getType(),
            item.getId(),
            item.getOpType());
      }
    }
  }
Esempio n. 7
0
  @Test
  public void testBulk_withUpdateItems() throws Exception {
    prepareIndex(1);
    BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
    for (int i = 0; i < 10; i++) {
      bulkRequestBuilder.add(
          client()
              .prepareUpdate("idx", "type", Integer.toString(i))
              .setDoc("{}")
              .setDocAsUpsert(true));
    }

    BulkResponse bulkResponse = bulkRequestBuilder.get();
    for (BulkItemResponse item : bulkResponse) {
      assertThat(item.isFailed(), equalTo(false));
      assertShardInfo(item.getResponse());
    }
  }
 @Override
 public void bulkIndex(List<IndexQuery> queries) {
   BulkRequestBuilder bulkRequest = client.prepareBulk();
   for (IndexQuery query : queries) {
     bulkRequest.add(prepareIndex(query));
   }
   BulkResponse bulkResponse = bulkRequest.execute().actionGet();
   if (bulkResponse.hasFailures()) {
     Map<String, String> failedDocuments = new HashMap<String, String>();
     for (BulkItemResponse item : bulkResponse.getItems()) {
       if (item.isFailed()) failedDocuments.put(item.getId(), item.getFailureMessage());
     }
     throw new ElasticsearchException(
         "Bulk indexing has failures. Use ElasticsearchException.getFailedDocuments() for detailed messages ["
             + failedDocuments
             + "]",
         failedDocuments);
   }
 }
  /**
   * Updates the specified objects
   *
   * @return the id's of the failed objects (e.g. due to versioning)
   */
  public Collection<Integer> bulkUpdate(
      Collection<T> objects, String indexName, boolean refresh, boolean enableVersioning) {
    // now using bulk API instead of feeding each doc separate with feedDoc
    BulkRequestBuilder brb = client.prepareBulk();
    // this works differently then the direct call to refresh!? maybe refresh is not async?
    //        brb.setRefresh(refresh);
    for (T o : objects) {
      if (o.getId() == null) {
        logger.warn("Skipped object without id when bulkUpdate:" + o);
        continue;
      }

      try {
        XContentBuilder source = createDoc(o);
        IndexRequest indexReq =
            Requests.indexRequest(indexName).type(getIndexType()).id(o.getId()).source(source);

        if (enableVersioning) indexReq.version(o.getVersion());

        brb.add(indexReq);
      } catch (IOException ex) {
        logger.warn("Cannot add object:" + o + " to bulkIndexing action." + ex.getMessage());
      }
    }
    if (brb.numberOfActions() > 0) {
      BulkResponse rsp = brb.execute().actionGet();
      if (rsp.hasFailures()) {
        List<Integer> list = new ArrayList<Integer>(rsp.items().length);
        for (BulkItemResponse br : rsp.items()) {
          if (br.isFailed()) {
            //                        logger.info("Error:" + br.failureMessage());
            list.add(br.itemId());
          }
        }
        return list;
      }
      if (refresh) refresh(indexName);
    }

    return Collections.emptyList();
  }