private Fields addGeneratedTermVectors(
      Engine.GetResult get,
      Fields termVectorsByField,
      TermVectorRequest request,
      Set<String> selectedFields)
      throws IOException {
    /* only keep valid fields */
    Set<String> validFields = new HashSet<>();
    for (String field : selectedFields) {
      FieldMapper fieldMapper = indexShard.mapperService().smartNameFieldMapper(field);
      if (!isValidField(fieldMapper)) {
        continue;
      }
      // already retrieved, only if the analyzer hasn't been overridden at the field
      if (fieldMapper.fieldType().storeTermVectors()
          && (request.perFieldAnalyzer() == null
              || !request.perFieldAnalyzer().containsKey(field))) {
        continue;
      }
      validFields.add(field);
    }

    if (validFields.isEmpty()) {
      return termVectorsByField;
    }

    /* generate term vectors from fetched document fields */
    GetResult getResult =
        indexShard
            .getService()
            .get(
                get,
                request.id(),
                request.type(),
                validFields.toArray(Strings.EMPTY_ARRAY),
                null,
                false);
    Fields generatedTermVectors =
        generateTermVectors(
            getResult.getFields().values(), request.offsets(), request.perFieldAnalyzer());

    /* merge with existing Fields */
    if (termVectorsByField == null) {
      return generatedTermVectors;
    } else {
      return mergeFields(termVectorsByField, generatedTermVectors);
    }
  }
Ejemplo n.º 2
0
 public static GetResult readGetResult(StreamInput in) throws IOException {
   GetResult result = new GetResult();
   result.readFrom(in);
   return result;
 }
  protected void shardOperation(
      final UpdateRequest request,
      final ActionListener<UpdateResponse> listener,
      final int retryCount)
      throws ElasticSearchException {
    IndexService indexService = indicesService.indexServiceSafe(request.index());
    IndexShard indexShard = indexService.shardSafe(request.shardId());

    long getDate = System.currentTimeMillis();
    final GetResult getResult =
        indexShard
            .getService()
            .get(
                request.type(),
                request.id(),
                new String[] {
                  SourceFieldMapper.NAME,
                  RoutingFieldMapper.NAME,
                  ParentFieldMapper.NAME,
                  TTLFieldMapper.NAME
                },
                true);

    // no doc, what to do, what to do...
    if (!getResult.isExists()) {
      if (request.upsertRequest() == null) {
        listener.onFailure(
            new DocumentMissingException(
                new ShardId(request.index(), request.shardId()), request.type(), request.id()));
        return;
      }
      final IndexRequest indexRequest = request.upsertRequest();
      indexRequest
          .index(request.index())
          .type(request.type())
          .id(request.id())
          // it has to be a "create!"
          .create(true)
          .routing(request.routing())
          .percolate(request.percolate())
          .refresh(request.refresh())
          .replicationType(request.replicationType())
          .consistencyLevel(request.consistencyLevel());
      indexRequest.operationThreaded(false);
      // we fetch it from the index request so we don't generate the bytes twice, its already done
      // in the index request
      final BytesReference updateSourceBytes = indexRequest.source();
      indexAction.execute(
          indexRequest,
          new ActionListener<IndexResponse>() {
            @Override
            public void onResponse(IndexResponse response) {
              UpdateResponse update =
                  new UpdateResponse(
                      response.getIndex(),
                      response.getType(),
                      response.getId(),
                      response.getVersion());
              update.setMatches(response.getMatches());
              if (request.fields() != null && request.fields().length > 0) {
                Tuple<XContentType, Map<String, Object>> sourceAndContent =
                    XContentHelper.convertToMap(updateSourceBytes, true);
                update.setGetResult(
                    extractGetResult(
                        request,
                        response.getVersion(),
                        sourceAndContent.v2(),
                        sourceAndContent.v1(),
                        updateSourceBytes));
              } else {
                update.setGetResult(null);
              }
              listener.onResponse(update);
            }

            @Override
            public void onFailure(Throwable e) {
              e = ExceptionsHelper.unwrapCause(e);
              if (e instanceof VersionConflictEngineException
                  || e instanceof DocumentAlreadyExistsException) {
                if (retryCount < request.retryOnConflict()) {
                  threadPool
                      .executor(executor())
                      .execute(
                          new Runnable() {
                            @Override
                            public void run() {
                              shardOperation(request, listener, retryCount + 1);
                            }
                          });
                  return;
                }
              }
              listener.onFailure(e);
            }
          });
      return;
    }

    if (getResult.internalSourceRef() == null) {
      // no source, we can't do nothing, through a failure...
      listener.onFailure(
          new DocumentSourceMissingException(
              new ShardId(request.index(), request.shardId()), request.type(), request.id()));
      return;
    }

    Tuple<XContentType, Map<String, Object>> sourceAndContent =
        XContentHelper.convertToMap(getResult.internalSourceRef(), true);
    String operation = null;
    String timestamp = null;
    Long ttl = null;
    Object fetchedTTL = null;
    final Map<String, Object> updatedSourceAsMap;
    final XContentType updateSourceContentType = sourceAndContent.v1();
    String routing =
        getResult.getFields().containsKey(RoutingFieldMapper.NAME)
            ? getResult.field(RoutingFieldMapper.NAME).getValue().toString()
            : null;
    String parent =
        getResult.getFields().containsKey(ParentFieldMapper.NAME)
            ? getResult.field(ParentFieldMapper.NAME).getValue().toString()
            : null;

    if (request.script() == null && request.doc() != null) {
      IndexRequest indexRequest = request.doc();
      updatedSourceAsMap = sourceAndContent.v2();
      if (indexRequest.ttl() > 0) {
        ttl = indexRequest.ttl();
      }
      timestamp = indexRequest.timestamp();
      if (indexRequest.routing() != null) {
        routing = indexRequest.routing();
      }
      if (indexRequest.parent() != null) {
        parent = indexRequest.parent();
      }
      XContentHelper.update(updatedSourceAsMap, indexRequest.sourceAsMap());
    } else {
      Map<String, Object> ctx = new HashMap<String, Object>(2);
      ctx.put("_source", sourceAndContent.v2());

      try {
        ExecutableScript script =
            scriptService.executable(request.scriptLang, request.script, request.scriptParams);
        script.setNextVar("ctx", ctx);
        script.run();
        // we need to unwrap the ctx...
        ctx = (Map<String, Object>) script.unwrap(ctx);
      } catch (Exception e) {
        throw new ElasticSearchIllegalArgumentException("failed to execute script", e);
      }

      operation = (String) ctx.get("op");
      timestamp = (String) ctx.get("_timestamp");
      fetchedTTL = ctx.get("_ttl");
      if (fetchedTTL != null) {
        if (fetchedTTL instanceof Number) {
          ttl = ((Number) fetchedTTL).longValue();
        } else {
          ttl = TimeValue.parseTimeValue((String) fetchedTTL, null).millis();
        }
      }

      updatedSourceAsMap = (Map<String, Object>) ctx.get("_source");
    }

    // apply script to update the source
    // No TTL has been given in the update script so we keep previous TTL value if there is one
    if (ttl == null) {
      ttl =
          getResult.getFields().containsKey(TTLFieldMapper.NAME)
              ? (Long) getResult.field(TTLFieldMapper.NAME).getValue()
              : null;
      if (ttl != null) {
        ttl =
            ttl
                - (System.currentTimeMillis()
                    - getDate); // It is an approximation of exact TTL value, could be improved
      }
    }

    // TODO: external version type, does it make sense here? does not seem like it...

    if (operation == null || "index".equals(operation)) {
      final IndexRequest indexRequest =
          Requests.indexRequest(request.index())
              .type(request.type())
              .id(request.id())
              .routing(routing)
              .parent(parent)
              .source(updatedSourceAsMap, updateSourceContentType)
              .version(getResult.getVersion())
              .replicationType(request.replicationType())
              .consistencyLevel(request.consistencyLevel())
              .timestamp(timestamp)
              .ttl(ttl)
              .percolate(request.percolate())
              .refresh(request.refresh());
      indexRequest.operationThreaded(false);
      // we fetch it from the index request so we don't generate the bytes twice, its already done
      // in the index request
      final BytesReference updateSourceBytes = indexRequest.source();
      indexAction.execute(
          indexRequest,
          new ActionListener<IndexResponse>() {
            @Override
            public void onResponse(IndexResponse response) {
              UpdateResponse update =
                  new UpdateResponse(
                      response.getIndex(),
                      response.getType(),
                      response.getId(),
                      response.getVersion());
              update.setMatches(response.getMatches());
              update.setGetResult(
                  extractGetResult(
                      request,
                      response.getVersion(),
                      updatedSourceAsMap,
                      updateSourceContentType,
                      updateSourceBytes));
              listener.onResponse(update);
            }

            @Override
            public void onFailure(Throwable e) {
              e = ExceptionsHelper.unwrapCause(e);
              if (e instanceof VersionConflictEngineException) {
                if (retryCount < request.retryOnConflict()) {
                  threadPool
                      .executor(executor())
                      .execute(
                          new Runnable() {
                            @Override
                            public void run() {
                              shardOperation(request, listener, retryCount + 1);
                            }
                          });
                  return;
                }
              }
              listener.onFailure(e);
            }
          });
    } else if ("delete".equals(operation)) {
      DeleteRequest deleteRequest =
          Requests.deleteRequest(request.index())
              .type(request.type())
              .id(request.id())
              .routing(routing)
              .parent(parent)
              .version(getResult.getVersion())
              .replicationType(request.replicationType())
              .consistencyLevel(request.consistencyLevel());
      deleteRequest.operationThreaded(false);
      deleteAction.execute(
          deleteRequest,
          new ActionListener<DeleteResponse>() {
            @Override
            public void onResponse(DeleteResponse response) {
              UpdateResponse update =
                  new UpdateResponse(
                      response.getIndex(),
                      response.getType(),
                      response.getId(),
                      response.getVersion());
              update.setGetResult(
                  extractGetResult(
                      request,
                      response.getVersion(),
                      updatedSourceAsMap,
                      updateSourceContentType,
                      null));
              listener.onResponse(update);
            }

            @Override
            public void onFailure(Throwable e) {
              e = ExceptionsHelper.unwrapCause(e);
              if (e instanceof VersionConflictEngineException) {
                if (retryCount < request.retryOnConflict()) {
                  threadPool
                      .executor(executor())
                      .execute(
                          new Runnable() {
                            @Override
                            public void run() {
                              shardOperation(request, listener, retryCount + 1);
                            }
                          });
                  return;
                }
              }
              listener.onFailure(e);
            }
          });
    } else if ("none".equals(operation)) {
      UpdateResponse update =
          new UpdateResponse(
              getResult.getIndex(), getResult.getType(), getResult.getId(), getResult.getVersion());
      update.setGetResult(
          extractGetResult(
              request, getResult.getVersion(), updatedSourceAsMap, updateSourceContentType, null));
      listener.onResponse(update);
    } else {
      logger.warn(
          "Used update operation [{}] for script [{}], doing nothing...",
          operation,
          request.script);
      listener.onResponse(
          new UpdateResponse(
              getResult.getIndex(),
              getResult.getType(),
              getResult.getId(),
              getResult.getVersion()));
    }
  }
Ejemplo n.º 4
0
  /**
   * Prepares an update request by converting it into an index or delete request or an update
   * response (no action).
   */
  @SuppressWarnings("unchecked")
  protected Result prepare(ShardId shardId, UpdateRequest request, final GetResult getResult) {
    long getDateNS = System.nanoTime();
    if (!getResult.isExists()) {
      if (request.upsertRequest() == null && !request.docAsUpsert()) {
        throw new DocumentMissingException(shardId, request.type(), request.id());
      }
      IndexRequest indexRequest = request.docAsUpsert() ? request.doc() : request.upsertRequest();
      TimeValue ttl = indexRequest.ttl();
      if (request.scriptedUpsert() && request.script() != null) {
        // Run the script to perform the create logic
        IndexRequest upsert = request.upsertRequest();
        Map<String, Object> upsertDoc = upsert.sourceAsMap();
        Map<String, Object> ctx = new HashMap<>(2);
        // Tell the script that this is a create and not an update
        ctx.put("op", "create");
        ctx.put("_source", upsertDoc);
        ctx = executeScript(request.script, ctx);
        // Allow the script to set TTL using ctx._ttl
        if (ttl == null) {
          ttl = getTTLFromScriptContext(ctx);
        }

        // Allow the script to abort the create by setting "op" to "none"
        String scriptOpChoice = (String) ctx.get("op");

        // Only valid options for an upsert script are "create"
        // (the default) or "none", meaning abort upsert
        if (!"create".equals(scriptOpChoice)) {
          if (!"none".equals(scriptOpChoice)) {
            logger.warn(
                "Used upsert operation [{}] for script [{}], doing nothing...",
                scriptOpChoice,
                request.script.getScript());
          }
          UpdateResponse update =
              new UpdateResponse(
                  shardId, getResult.getType(), getResult.getId(), getResult.getVersion(), false);
          update.setGetResult(getResult);
          return new Result(update, Operation.NONE, upsertDoc, XContentType.JSON);
        }
        indexRequest.source((Map) ctx.get("_source"));
      }

      indexRequest
          .index(request.index())
          .type(request.type())
          .id(request.id())
          // it has to be a "create!"
          .create(true)
          .ttl(ttl)
          .refresh(request.refresh())
          .routing(request.routing())
          .parent(request.parent())
          .consistencyLevel(request.consistencyLevel());
      if (request.versionType() != VersionType.INTERNAL) {
        // in all but the internal versioning mode, we want to create the new document using the
        // given version.
        indexRequest.version(request.version()).versionType(request.versionType());
      }
      return new Result(indexRequest, Operation.UPSERT, null, null);
    }

    long updateVersion = getResult.getVersion();

    if (request.versionType() != VersionType.INTERNAL) {
      assert request.versionType() == VersionType.FORCE;
      updateVersion = request.version(); // remember, match_any is excluded by the conflict test
    }

    if (getResult.internalSourceRef() == null) {
      // no source, we can't do nothing, through a failure...
      throw new DocumentSourceMissingException(shardId, request.type(), request.id());
    }

    Tuple<XContentType, Map<String, Object>> sourceAndContent =
        XContentHelper.convertToMap(getResult.internalSourceRef(), true);
    String operation = null;
    String timestamp = null;
    TimeValue ttl = null;
    final Map<String, Object> updatedSourceAsMap;
    final XContentType updateSourceContentType = sourceAndContent.v1();
    String routing =
        getResult.getFields().containsKey(RoutingFieldMapper.NAME)
            ? getResult.field(RoutingFieldMapper.NAME).getValue().toString()
            : null;
    String parent =
        getResult.getFields().containsKey(ParentFieldMapper.NAME)
            ? getResult.field(ParentFieldMapper.NAME).getValue().toString()
            : null;

    if (request.script() == null && request.doc() != null) {
      IndexRequest indexRequest = request.doc();
      updatedSourceAsMap = sourceAndContent.v2();
      if (indexRequest.ttl() != null) {
        ttl = indexRequest.ttl();
      }
      timestamp = indexRequest.timestamp();
      if (indexRequest.routing() != null) {
        routing = indexRequest.routing();
      }
      if (indexRequest.parent() != null) {
        parent = indexRequest.parent();
      }
      boolean noop =
          !XContentHelper.update(
              updatedSourceAsMap, indexRequest.sourceAsMap(), request.detectNoop());
      // noop could still be true even if detectNoop isn't because update detects empty maps as
      // noops.  BUT we can only
      // actually turn the update into a noop if detectNoop is true to preserve backwards
      // compatibility and to handle
      // cases where users repopulating multi-fields or adding synonyms, etc.
      if (request.detectNoop() && noop) {
        operation = "none";
      }
    } else {
      Map<String, Object> ctx = new HashMap<>(16);
      Long originalTtl =
          getResult.getFields().containsKey(TTLFieldMapper.NAME)
              ? (Long) getResult.field(TTLFieldMapper.NAME).getValue()
              : null;
      Long originalTimestamp =
          getResult.getFields().containsKey(TimestampFieldMapper.NAME)
              ? (Long) getResult.field(TimestampFieldMapper.NAME).getValue()
              : null;
      ctx.put("_index", getResult.getIndex());
      ctx.put("_type", getResult.getType());
      ctx.put("_id", getResult.getId());
      ctx.put("_version", getResult.getVersion());
      ctx.put("_routing", routing);
      ctx.put("_parent", parent);
      ctx.put("_timestamp", originalTimestamp);
      ctx.put("_ttl", originalTtl);
      ctx.put("_source", sourceAndContent.v2());

      ctx = executeScript(request.script, ctx);

      operation = (String) ctx.get("op");

      Object fetchedTimestamp = ctx.get("_timestamp");
      if (fetchedTimestamp != null) {
        timestamp = fetchedTimestamp.toString();
      } else if (originalTimestamp != null) {
        // No timestamp has been given in the update script, so we keep the previous timestamp if
        // there is one
        timestamp = originalTimestamp.toString();
      }

      ttl = getTTLFromScriptContext(ctx);

      updatedSourceAsMap = (Map<String, Object>) ctx.get("_source");
    }

    // apply script to update the source
    // No TTL has been given in the update script so we keep previous TTL value if there is one
    if (ttl == null) {
      Long ttlAsLong =
          getResult.getFields().containsKey(TTLFieldMapper.NAME)
              ? (Long) getResult.field(TTLFieldMapper.NAME).getValue()
              : null;
      if (ttlAsLong != null) {
        ttl =
            new TimeValue(
                ttlAsLong
                    - TimeValue.nsecToMSec(
                        System.nanoTime()
                            - getDateNS)); // It is an approximation of exact TTL value, could be
                                           // improved
      }
    }

    if (operation == null || "index".equals(operation)) {
      final IndexRequest indexRequest =
          Requests.indexRequest(request.index())
              .type(request.type())
              .id(request.id())
              .routing(routing)
              .parent(parent)
              .source(updatedSourceAsMap, updateSourceContentType)
              .version(updateVersion)
              .versionType(request.versionType())
              .consistencyLevel(request.consistencyLevel())
              .timestamp(timestamp)
              .ttl(ttl)
              .refresh(request.refresh());
      return new Result(indexRequest, Operation.INDEX, updatedSourceAsMap, updateSourceContentType);
    } else if ("delete".equals(operation)) {
      DeleteRequest deleteRequest =
          Requests.deleteRequest(request.index())
              .type(request.type())
              .id(request.id())
              .routing(routing)
              .parent(parent)
              .version(updateVersion)
              .versionType(request.versionType())
              .consistencyLevel(request.consistencyLevel());
      return new Result(
          deleteRequest, Operation.DELETE, updatedSourceAsMap, updateSourceContentType);
    } else if ("none".equals(operation)) {
      UpdateResponse update =
          new UpdateResponse(
              shardId, getResult.getType(), getResult.getId(), getResult.getVersion(), false);
      update.setGetResult(
          extractGetResult(
              request,
              request.index(),
              getResult.getVersion(),
              updatedSourceAsMap,
              updateSourceContentType,
              getResult.internalSourceRef()));
      return new Result(update, Operation.NONE, updatedSourceAsMap, updateSourceContentType);
    } else {
      logger.warn(
          "Used update operation [{}] for script [{}], doing nothing...",
          operation,
          request.script.getScript());
      UpdateResponse update =
          new UpdateResponse(
              shardId, getResult.getType(), getResult.getId(), getResult.getVersion(), false);
      return new Result(update, Operation.NONE, updatedSourceAsMap, updateSourceContentType);
    }
  }
Ejemplo n.º 5
0
  /**
   * Prepares an update request by converting it into an index request.
   *
   * <p>TODO: detect a NOOP and return an update response if true
   */
  @SuppressWarnings("unchecked")
  public IndexRequest prepareUpdate(
      DocTableInfo tableInfo,
      ShardUpsertRequest request,
      ShardUpsertRequest.Item item,
      ShardId shardId)
      throws ElasticsearchException {
    IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
    IndexShard indexShard = indexService.shardSafe(shardId.id());
    final GetResult getResult =
        indexShard
            .getService()
            .get(
                request.type(),
                item.id(),
                new String[] {RoutingFieldMapper.NAME, ParentFieldMapper.NAME, TTLFieldMapper.NAME},
                true,
                item.version(),
                VersionType.INTERNAL,
                FetchSourceContext.FETCH_SOURCE,
                false);

    if (!getResult.isExists()) {
      throw new DocumentMissingException(
          new ShardId(request.index(), request.shardId()), request.type(), item.id());
    }

    if (getResult.internalSourceRef() == null) {
      // no source, we can't do nothing, through a failure...
      throw new DocumentSourceMissingException(
          new ShardId(request.index(), request.shardId()), request.type(), item.id());
    }

    Tuple<XContentType, Map<String, Object>> sourceAndContent =
        XContentHelper.convertToMap(getResult.internalSourceRef(), true);
    final Map<String, Object> updatedSourceAsMap;
    final XContentType updateSourceContentType = sourceAndContent.v1();
    String routing =
        getResult.getFields().containsKey(RoutingFieldMapper.NAME)
            ? getResult.field(RoutingFieldMapper.NAME).getValue().toString()
            : null;
    String parent =
        getResult.getFields().containsKey(ParentFieldMapper.NAME)
            ? getResult.field(ParentFieldMapper.NAME).getValue().toString()
            : null;

    updatedSourceAsMap = sourceAndContent.v2();

    SymbolToFieldExtractorContext ctx =
        new SymbolToFieldExtractorContext(functions, item.insertValues());

    Map<String, Object> pathsToUpdate = new LinkedHashMap<>();
    Map<String, Object> updatedGeneratedColumns = new LinkedHashMap<>();
    for (int i = 0; i < request.updateColumns().length; i++) {
      /**
       * NOTE: mapping isn't applied. So if an Insert was done using the ES Rest Endpoint the data
       * might be returned in the wrong format (date as string instead of long)
       */
      String columnPath = request.updateColumns()[i];
      Object value =
          SYMBOL_TO_FIELD_EXTRACTOR.convert(item.updateAssignments()[i], ctx).extract(getResult);
      ReferenceInfo referenceInfo = tableInfo.getReferenceInfo(ColumnIdent.fromPath(columnPath));
      if (referenceInfo instanceof GeneratedReferenceInfo) {
        updatedGeneratedColumns.put(columnPath, value);

      } else {
        pathsToUpdate.put(columnPath, value);
      }
    }

    processGeneratedColumns(
        tableInfo,
        pathsToUpdate,
        updatedGeneratedColumns,
        request.validateGeneratedColumns(),
        getResult);

    updateSourceByPaths(updatedSourceAsMap, pathsToUpdate);

    final IndexRequest indexRequest =
        Requests.indexRequest(request.index())
            .type(request.type())
            .id(item.id())
            .routing(routing)
            .parent(parent)
            .source(updatedSourceAsMap, updateSourceContentType)
            .version(getResult.getVersion());
    indexRequest.operationThreaded(false);
    return indexRequest;
  }