예제 #1
0
 @Override
 public List<ActionRequest> doCall(Index index) {
   List<ActionRequest> inserts = new ArrayList<ActionRequest>();
   List<UpdateRequest> updates = index.getNormalizer().normalize(dto);
   for (UpdateRequest update : updates) {
     if (update.doc() != null) {
       inserts.add(
           update
               .upsertRequest()
               .index(index.getIndexName())
               .type(index.getIndexType())
               .id(update.id())
               .routing(update.routing()));
     } else {
       inserts.add(update.index(index.getIndexName()).type(index.getIndexType()));
     }
   }
   return inserts;
 }
  @Override
  public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client)
      throws IOException {
    UpdateRequest updateRequest =
        new UpdateRequest(request.param("index"), request.param("type"), request.param("id"));
    updateRequest.routing(request.param("routing"));
    updateRequest.parent(request.param("parent"));
    updateRequest.timeout(request.paramAsTime("timeout", updateRequest.timeout()));
    updateRequest.setRefreshPolicy(request.param("refresh"));
    String waitForActiveShards = request.param("wait_for_active_shards");
    if (waitForActiveShards != null) {
      updateRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards));
    }
    updateRequest.docAsUpsert(request.paramAsBoolean("doc_as_upsert", updateRequest.docAsUpsert()));
    FetchSourceContext fetchSourceContext = FetchSourceContext.parseFromRestRequest(request);
    String sField = request.param("fields");
    if (sField != null && fetchSourceContext != null) {
      throw new IllegalArgumentException(
          "[fields] and [_source] cannot be used in the same request");
    }
    if (sField != null) {
      DEPRECATION_LOGGER.deprecated("Deprecated field [fields] used, expected [_source] instead");
      String[] sFields = Strings.splitStringByCommaToArray(sField);
      updateRequest.fields(sFields);
    } else if (fetchSourceContext != null) {
      updateRequest.fetchSource(fetchSourceContext);
    }

    updateRequest.retryOnConflict(
        request.paramAsInt("retry_on_conflict", updateRequest.retryOnConflict()));
    updateRequest.version(RestActions.parseVersion(request));
    updateRequest.versionType(
        VersionType.fromString(request.param("version_type"), updateRequest.versionType()));

    // see if we have it in the body
    if (request.hasContent()) {
      updateRequest.fromXContent(request.content());
      IndexRequest upsertRequest = updateRequest.upsertRequest();
      if (upsertRequest != null) {
        upsertRequest.routing(request.param("routing"));
        upsertRequest.parent(
            request.param(
                "parent")); // order is important, set it after routing, so it will set the routing
        upsertRequest.timestamp(request.param("timestamp"));
        if (request.hasParam("ttl")) {
          upsertRequest.ttl(request.param("ttl"));
        }
        upsertRequest.version(RestActions.parseVersion(request));
        upsertRequest.versionType(
            VersionType.fromString(request.param("version_type"), upsertRequest.versionType()));
      }
      IndexRequest doc = updateRequest.doc();
      if (doc != null) {
        doc.routing(request.param("routing"));
        doc.parent(
            request.param(
                "parent")); // order is important, set it after routing, so it will set the routing
        doc.timestamp(request.param("timestamp"));
        if (request.hasParam("ttl")) {
          doc.ttl(request.param("ttl"));
        }
        doc.version(RestActions.parseVersion(request));
        doc.versionType(VersionType.fromString(request.param("version_type"), doc.versionType()));
      }
    }

    return channel ->
        client.update(
            updateRequest,
            new RestStatusToXContentListener<>(
                channel, r -> r.getLocation(updateRequest.routing())));
  }
예제 #3
0
  @Override
  public void handleRequest(final RestRequest request, final RestChannel channel) {
    UpdateRequest updateRequest =
        new UpdateRequest(request.param("index"), request.param("type"), request.param("id"));
    updateRequest.listenerThreaded(false);
    updateRequest.routing(request.param("routing"));
    updateRequest.parent(
        request.param(
            "parent")); // order is important, set it after routing, so it will set the routing
    updateRequest.timeout(request.paramAsTime("timeout", updateRequest.timeout()));
    updateRequest.refresh(request.paramAsBoolean("refresh", updateRequest.refresh()));
    String replicationType = request.param("replication");
    if (replicationType != null) {
      updateRequest.replicationType(ReplicationType.fromString(replicationType));
    }
    String consistencyLevel = request.param("consistency");
    if (consistencyLevel != null) {
      updateRequest.consistencyLevel(WriteConsistencyLevel.fromString(consistencyLevel));
    }
    updateRequest.percolate(request.param("percolate", null));
    updateRequest.script(request.param("script"));
    updateRequest.scriptLang(request.param("lang"));
    for (Map.Entry<String, String> entry : request.params().entrySet()) {
      if (entry.getKey().startsWith("sp_")) {
        updateRequest.addScriptParam(entry.getKey().substring(3), entry.getValue());
      }
    }
    String sField = request.param("fields");
    if (sField != null) {
      String[] sFields = Strings.splitStringByCommaToArray(sField);
      if (sFields != null) {
        updateRequest.fields(sFields);
      }
    }
    updateRequest.retryOnConflict(
        request.paramAsInt("retry_on_conflict", updateRequest.retryOnConflict()));

    // see if we have it in the body
    if (request.hasContent()) {
      try {
        updateRequest.source(
            request.contentByteArray(), request.contentByteArrayOffset(), request.contentLength());
        IndexRequest upsertRequest = updateRequest.upsertRequest();
        if (upsertRequest != null) {
          upsertRequest.routing(request.param("routing"));
          upsertRequest.parent(
              request.param(
                  "parent")); // order is important, set it after routing, so it will set the
          // routing
          upsertRequest.timestamp(request.param("timestamp"));
          if (request.hasParam("ttl")) {
            upsertRequest.ttl(request.paramAsTime("ttl", null).millis());
          }
          upsertRequest.version(RestActions.parseVersion(request));
          upsertRequest.versionType(
              VersionType.fromString(request.param("version_type"), upsertRequest.versionType()));
        }
        IndexRequest doc = updateRequest.doc();
        if (doc != null) {
          doc.routing(request.param("routing"));
          doc.parent(
              request.param(
                  "parent")); // order is important, set it after routing, so it will set the
          // routing
          doc.timestamp(request.param("timestamp"));
          if (request.hasParam("ttl")) {
            doc.ttl(request.paramAsTime("ttl", null).millis());
          }
          doc.version(RestActions.parseVersion(request));
          doc.versionType(VersionType.fromString(request.param("version_type"), doc.versionType()));
        }
      } catch (Exception e) {
        try {
          channel.sendResponse(new XContentThrowableRestResponse(request, e));
        } catch (IOException e1) {
          logger.warn("Failed to send response", e1);
        }
        return;
      }
    }

    client.update(
        updateRequest,
        new ActionListener<UpdateResponse>() {
          @Override
          public void onResponse(UpdateResponse response) {
            try {
              XContentBuilder builder = RestXContentBuilder.restContentBuilder(request);
              builder
                  .startObject()
                  .field(Fields.OK, true)
                  .field(Fields._INDEX, response.index())
                  .field(Fields._TYPE, response.type())
                  .field(Fields._ID, response.id())
                  .field(Fields._VERSION, response.version());

              if (response.getResult() != null) {
                builder.startObject(Fields.GET);
                response.getResult().toXContentEmbedded(builder, request);
                builder.endObject();
              }

              if (response.matches() != null) {
                builder.startArray(Fields.MATCHES);
                for (String match : response.matches()) {
                  builder.value(match);
                }
                builder.endArray();
              }
              builder.endObject();
              RestStatus status = OK;
              if (response.version() == 1) {
                status = CREATED;
              }
              channel.sendResponse(new XContentRestResponse(request, status, builder));
            } catch (Exception e) {
              onFailure(e);
            }
          }

          @Override
          public void onFailure(Throwable e) {
            try {
              channel.sendResponse(new XContentThrowableRestResponse(request, e));
            } catch (IOException e1) {
              logger.error("Failed to send failure response", e1);
            }
          }
        });
  }
  private void executeBulk(
      final BulkRequest bulkRequest,
      final long startTime,
      final ActionListener<BulkResponse> listener,
      final AtomicArray<BulkItemResponse> responses) {
    final ClusterState clusterState = clusterService.state();
    // TODO use timeout to wait here if its blocked...
    clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.WRITE);

    final ConcreteIndices concreteIndices =
        new ConcreteIndices(clusterState, indexNameExpressionResolver);
    MetaData metaData = clusterState.metaData();
    for (int i = 0; i < bulkRequest.requests.size(); i++) {
      ActionRequest request = bulkRequest.requests.get(i);
      if (request instanceof DocumentRequest) {
        DocumentRequest req = (DocumentRequest) request;

        if (addFailureIfIndexIsUnavailable(
            req, bulkRequest, responses, i, concreteIndices, metaData)) {
          continue;
        }

        String concreteIndex = concreteIndices.resolveIfAbsent(req);
        if (request instanceof IndexRequest) {
          IndexRequest indexRequest = (IndexRequest) request;
          MappingMetaData mappingMd = null;
          if (metaData.hasIndex(concreteIndex)) {
            mappingMd = metaData.index(concreteIndex).mappingOrDefault(indexRequest.type());
          }
          try {
            indexRequest.process(metaData, mappingMd, allowIdGeneration, concreteIndex);
          } catch (ElasticsearchParseException | RoutingMissingException e) {
            BulkItemResponse.Failure failure =
                new BulkItemResponse.Failure(
                    concreteIndex, indexRequest.type(), indexRequest.id(), e);
            BulkItemResponse bulkItemResponse = new BulkItemResponse(i, "index", failure);
            responses.set(i, bulkItemResponse);
            // make sure the request gets never processed again
            bulkRequest.requests.set(i, null);
          }
        } else {
          concreteIndices.resolveIfAbsent(req);
          req.routing(
              clusterState
                  .metaData()
                  .resolveIndexRouting(req.parent(), req.routing(), req.index()));
        }
      }
    }

    // first, go over all the requests and create a ShardId -> Operations mapping
    Map<ShardId, List<BulkItemRequest>> requestsByShard = new HashMap<>();

    for (int i = 0; i < bulkRequest.requests.size(); i++) {
      ActionRequest request = bulkRequest.requests.get(i);
      if (request instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) request;
        String concreteIndex = concreteIndices.getConcreteIndex(indexRequest.index());
        ShardId shardId =
            clusterService
                .operationRouting()
                .indexShards(
                    clusterState,
                    concreteIndex,
                    indexRequest.type(),
                    indexRequest.id(),
                    indexRequest.routing())
                .shardId();
        List<BulkItemRequest> list = requestsByShard.get(shardId);
        if (list == null) {
          list = new ArrayList<>();
          requestsByShard.put(shardId, list);
        }
        list.add(new BulkItemRequest(i, request));
      } else if (request instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) request;
        String concreteIndex = concreteIndices.getConcreteIndex(deleteRequest.index());
        MappingMetaData mappingMd =
            clusterState.metaData().index(concreteIndex).mappingOrDefault(deleteRequest.type());
        if (mappingMd != null
            && mappingMd.routing().required()
            && deleteRequest.routing() == null) {
          // if routing is required, and no routing on the delete request, we need to broadcast
          // it....
          GroupShardsIterator groupShards =
              clusterService.operationRouting().broadcastDeleteShards(clusterState, concreteIndex);
          for (ShardIterator shardIt : groupShards) {
            List<BulkItemRequest> list = requestsByShard.get(shardIt.shardId());
            if (list == null) {
              list = new ArrayList<>();
              requestsByShard.put(shardIt.shardId(), list);
            }
            list.add(new BulkItemRequest(i, deleteRequest));
          }
        } else {
          ShardId shardId =
              clusterService
                  .operationRouting()
                  .indexShards(
                      clusterState,
                      concreteIndex,
                      deleteRequest.type(),
                      deleteRequest.id(),
                      deleteRequest.routing())
                  .shardId();
          List<BulkItemRequest> list = requestsByShard.get(shardId);
          if (list == null) {
            list = new ArrayList<>();
            requestsByShard.put(shardId, list);
          }
          list.add(new BulkItemRequest(i, request));
        }
      } else if (request instanceof UpdateRequest) {
        UpdateRequest updateRequest = (UpdateRequest) request;
        String concreteIndex = concreteIndices.getConcreteIndex(updateRequest.index());
        MappingMetaData mappingMd =
            clusterState.metaData().index(concreteIndex).mappingOrDefault(updateRequest.type());
        if (mappingMd != null
            && mappingMd.routing().required()
            && updateRequest.routing() == null) {
          BulkItemResponse.Failure failure =
              new BulkItemResponse.Failure(
                  updateRequest.index(),
                  updateRequest.type(),
                  updateRequest.id(),
                  new IllegalArgumentException("routing is required for this item"));
          responses.set(i, new BulkItemResponse(i, updateRequest.type(), failure));
          continue;
        }
        ShardId shardId =
            clusterService
                .operationRouting()
                .indexShards(
                    clusterState,
                    concreteIndex,
                    updateRequest.type(),
                    updateRequest.id(),
                    updateRequest.routing())
                .shardId();
        List<BulkItemRequest> list = requestsByShard.get(shardId);
        if (list == null) {
          list = new ArrayList<>();
          requestsByShard.put(shardId, list);
        }
        list.add(new BulkItemRequest(i, request));
      }
    }

    if (requestsByShard.isEmpty()) {
      listener.onResponse(
          new BulkResponse(
              responses.toArray(new BulkItemResponse[responses.length()]),
              buildTookInMillis(startTime)));
      return;
    }

    final AtomicInteger counter = new AtomicInteger(requestsByShard.size());
    for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) {
      final ShardId shardId = entry.getKey();
      final List<BulkItemRequest> requests = entry.getValue();
      BulkShardRequest bulkShardRequest =
          new BulkShardRequest(
              bulkRequest,
              shardId,
              bulkRequest.refresh(),
              requests.toArray(new BulkItemRequest[requests.size()]));
      bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel());
      bulkShardRequest.timeout(bulkRequest.timeout());
      shardBulkAction.execute(
          bulkShardRequest,
          new ActionListener<BulkShardResponse>() {
            @Override
            public void onResponse(BulkShardResponse bulkShardResponse) {
              for (BulkItemResponse bulkItemResponse : bulkShardResponse.getResponses()) {
                // we may have no response if item failed
                if (bulkItemResponse.getResponse() != null) {
                  bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo());
                }
                responses.set(bulkItemResponse.getItemId(), bulkItemResponse);
              }
              if (counter.decrementAndGet() == 0) {
                finishHim();
              }
            }

            @Override
            public void onFailure(Throwable e) {
              // create failures for all relevant requests
              for (BulkItemRequest request : requests) {
                if (request.request() instanceof IndexRequest) {
                  IndexRequest indexRequest = (IndexRequest) request.request();
                  responses.set(
                      request.id(),
                      new BulkItemResponse(
                          request.id(),
                          indexRequest.opType().toString().toLowerCase(Locale.ENGLISH),
                          new BulkItemResponse.Failure(
                              concreteIndices.getConcreteIndex(indexRequest.index()),
                              indexRequest.type(),
                              indexRequest.id(),
                              e)));
                } else if (request.request() instanceof DeleteRequest) {
                  DeleteRequest deleteRequest = (DeleteRequest) request.request();
                  responses.set(
                      request.id(),
                      new BulkItemResponse(
                          request.id(),
                          "delete",
                          new BulkItemResponse.Failure(
                              concreteIndices.getConcreteIndex(deleteRequest.index()),
                              deleteRequest.type(),
                              deleteRequest.id(),
                              e)));
                } else if (request.request() instanceof UpdateRequest) {
                  UpdateRequest updateRequest = (UpdateRequest) request.request();
                  responses.set(
                      request.id(),
                      new BulkItemResponse(
                          request.id(),
                          "update",
                          new BulkItemResponse.Failure(
                              concreteIndices.getConcreteIndex(updateRequest.index()),
                              updateRequest.type(),
                              updateRequest.id(),
                              e)));
                }
              }
              if (counter.decrementAndGet() == 0) {
                finishHim();
              }
            }

            private void finishHim() {
              listener.onResponse(
                  new BulkResponse(
                      responses.toArray(new BulkItemResponse[responses.length()]),
                      buildTookInMillis(startTime)));
            }
          });
    }
  }