@Test
  public void testEnabled() throws Exception {
    String mapping =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("type")
            .startObject("_timestamp")
            .field("enabled", "yes")
            .field("store", "yes")
            .endObject()
            .endObject()
            .endObject()
            .string();
    DocumentMapper docMapper = MapperTests.newParser().parse(mapping);
    byte[] source =
        XContentFactory.jsonBuilder()
            .startObject()
            .field("field", "value")
            .endObject()
            .copiedBytes();
    ParsedDocument doc =
        docMapper.parse(SourceToParse.source(source).type("type").id("1").timestamp(1));

    assertThat(doc.rootDoc().getFieldable("_timestamp").isStored(), equalTo(true));
    assertThat(doc.rootDoc().getFieldable("_timestamp").isIndexed(), equalTo(true));
    assertThat(doc.rootDoc().getFieldable("_timestamp").tokenStreamValue(), notNullValue());
  }
  @Override
  protected void shardOperationOnReplica(ReplicaOperationRequest shardRequest) {
    IndexShard indexShard =
        indicesService
            .indexServiceSafe(shardRequest.request.index())
            .shardSafe(shardRequest.shardId);
    final IngestShardRequest request = shardRequest.request;
    int size = request.items().size();
    for (int i = 0; i < size; i++) {
      IngestItemRequest item = request.items().get(i);
      if (item == null) {
        continue;
      }
      if (item.request() instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) item.request();
        try {
          SourceToParse sourceToParse =
              SourceToParse.source(SourceToParse.Origin.REPLICA, indexRequest.source())
                  .type(indexRequest.type())
                  .id(indexRequest.id())
                  .routing(indexRequest.routing())
                  .parent(indexRequest.parent())
                  .timestamp(indexRequest.timestamp())
                  .ttl(indexRequest.ttl());

          if (indexRequest.opType() == IndexRequest.OpType.INDEX) {
            Engine.Index index =
                indexShard
                    .prepareIndex(sourceToParse)
                    .version(indexRequest.version())
                    .origin(Engine.Operation.Origin.REPLICA);
            indexShard.index(index);
          } else {
            Engine.Create create =
                indexShard
                    .prepareCreate(sourceToParse)
                    .version(indexRequest.version())
                    .origin(Engine.Operation.Origin.REPLICA);
            indexShard.create(create);
          }
        } catch (Exception e) {
          // ignore, we are on backup
        }
      } else if (item.request() instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) item.request();
        try {
          Engine.Delete delete =
              indexShard
                  .prepareDelete(deleteRequest.type(), deleteRequest.id(), deleteRequest.version())
                  .origin(Engine.Operation.Origin.REPLICA);
          indexShard.delete(delete);
        } catch (Exception e) {
          // ignore, we are on backup
        }
      }
    }
  }
 /**
  * Utility method to create either an index or a create operation depending on the {@link OpType}
  * of the request.
  */
 private final Engine.Index prepareIndexOperationOnPrimary(
     BulkShardRequest shardRequest, IndexRequest request, IndexShard indexShard) {
   SourceToParse sourceToParse =
       SourceToParse.source(SourceToParse.Origin.PRIMARY, request.source())
           .index(request.index())
           .type(request.type())
           .id(request.id())
           .routing(request.routing())
           .parent(request.parent())
           .timestamp(request.timestamp())
           .ttl(request.ttl());
   return indexShard.prepareIndex(
       sourceToParse, request.version(), request.versionType(), Engine.Operation.Origin.PRIMARY);
 }
  @Test
  public void testSimpleDisabled() throws Exception {
    String mapping =
        XContentFactory.jsonBuilder().startObject().startObject("type").endObject().string();
    DocumentMapper docMapper = MapperTests.newParser().parse(mapping);
    byte[] source =
        XContentFactory.jsonBuilder()
            .startObject()
            .field("field", "value")
            .endObject()
            .copiedBytes();
    ParsedDocument doc =
        docMapper.parse(SourceToParse.source(source).type("type").id("1").timestamp(1));

    assertThat(doc.rootDoc().getFieldable("_timestamp"), equalTo(null));
  }
  @Test
  public void testSizeNotSet() throws Exception {
    String mapping =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("type")
            .endObject()
            .endObject()
            .string();
    DocumentMapper docMapper = MapperTests.newParser().parse(mapping);

    BytesReference source =
        XContentFactory.jsonBuilder().startObject().field("field", "value").endObject().bytes();
    ParsedDocument doc = docMapper.parse(SourceToParse.source(source).type("type").id("1"));

    assertThat(doc.rootDoc().getField("_size"), nullValue());
  }
 @Override
 protected void shardOperationOnReplica(ReplicaOperationRequest shardRequest) {
   IndexShard indexShard =
       indicesService
           .indexServiceSafe(shardRequest.shardId.getIndex())
           .shardSafe(shardRequest.shardId.id());
   IndexRequest request = shardRequest.request;
   SourceToParse sourceToParse =
       SourceToParse.source(SourceToParse.Origin.REPLICA, request.source())
           .type(request.type())
           .id(request.id())
           .routing(request.routing())
           .parent(request.parent())
           .timestamp(request.timestamp())
           .ttl(request.ttl());
   if (request.opType() == IndexRequest.OpType.INDEX) {
     Engine.Index index =
         indexShard.prepareIndex(
             sourceToParse,
             request.version(),
             request.versionType(),
             Engine.Operation.Origin.REPLICA,
             request.canHaveDuplicates());
     indexShard.index(index);
   } else {
     Engine.Create create =
         indexShard.prepareCreate(
             sourceToParse,
             request.version(),
             request.versionType(),
             Engine.Operation.Origin.REPLICA,
             request.canHaveDuplicates(),
             request.autoGeneratedId());
     indexShard.create(create);
   }
   if (request.refresh()) {
     try {
       indexShard.refresh(new Engine.Refresh("refresh_flag_index").force(false));
     } catch (Exception e) {
       // ignore
     }
   }
 }
  @Test
  public void testSizeEnabled() throws Exception {
    String mapping =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("type")
            .startObject("_size")
            .field("enabled", true)
            .endObject()
            .endObject()
            .endObject()
            .string();
    DocumentMapper docMapper = MapperTests.newParser().parse(mapping);

    BytesReference source =
        XContentFactory.jsonBuilder().startObject().field("field", "value").endObject().bytes();
    ParsedDocument doc = docMapper.parse(SourceToParse.source(source).type("type").id("1"));

    assertThat(doc.rootDoc().getField("_size").fieldType().stored(), equalTo(false));
    assertThat(
        doc.rootDoc().getField("_size").tokenStream(docMapper.indexAnalyzer()), notNullValue());
  }
  @Override
  protected void shardOperationOnReplica(ShardId shardId, IndexRequest request) {
    IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
    IndexShard indexShard = indexService.shardSafe(shardId.id());
    SourceToParse sourceToParse =
        SourceToParse.source(SourceToParse.Origin.REPLICA, request.source())
            .index(shardId.getIndex())
            .type(request.type())
            .id(request.id())
            .routing(request.routing())
            .parent(request.parent())
            .timestamp(request.timestamp())
            .ttl(request.ttl());

    final Engine.IndexingOperation operation;
    if (request.opType() == IndexRequest.OpType.INDEX) {
      operation =
          indexShard.prepareIndex(
              sourceToParse,
              request.version(),
              request.versionType(),
              Engine.Operation.Origin.REPLICA);
    } else {
      assert request.opType() == IndexRequest.OpType.CREATE : request.opType();
      operation =
          indexShard.prepareCreate(
              sourceToParse,
              request.version(),
              request.versionType(),
              Engine.Operation.Origin.REPLICA);
    }
    Mapping update = operation.parsedDoc().dynamicMappingsUpdate();
    if (update != null) {
      throw new RetryOnReplicaException(
          shardId, "Mappings are not available on the replica yet, triggered update: " + update);
    }
    operation.execute(indexShard);
    processAfter(request.refresh(), indexShard, operation.getTranslogLocation());
  }
  @Override
  protected PrimaryResponse<IndexResponse, IndexRequest> shardOperationOnPrimary(
      ClusterState clusterState, PrimaryOperationRequest shardRequest) {
    final IndexRequest request = shardRequest.request;

    // validate, if routing is required, that we got routing
    IndexMetaData indexMetaData = clusterState.metaData().index(shardRequest.shardId.getIndex());
    MappingMetaData mappingMd = indexMetaData.mappingOrDefault(request.type());
    if (mappingMd != null && mappingMd.routing().required()) {
      if (request.routing() == null) {
        throw new RoutingMissingException(
            shardRequest.shardId.getIndex(), request.type(), request.id());
      }
    }

    IndexService indexService = indicesService.indexServiceSafe(shardRequest.shardId.getIndex());
    IndexShard indexShard = indexService.shardSafe(shardRequest.shardId.id());
    SourceToParse sourceToParse =
        SourceToParse.source(SourceToParse.Origin.PRIMARY, request.source())
            .type(request.type())
            .id(request.id())
            .routing(request.routing())
            .parent(request.parent())
            .timestamp(request.timestamp())
            .ttl(request.ttl());
    long version;
    boolean created;
    Engine.IndexingOperation op;
    if (request.opType() == IndexRequest.OpType.INDEX) {
      Engine.Index index =
          indexShard.prepareIndex(
              sourceToParse,
              request.version(),
              request.versionType(),
              Engine.Operation.Origin.PRIMARY,
              request.canHaveDuplicates());
      if (index.parsedDoc().mappingsModified()) {
        mappingUpdatedAction.updateMappingOnMaster(
            shardRequest.shardId.getIndex(), index.docMapper(), indexService.indexUUID());
      }
      indexShard.index(index);
      version = index.version();
      op = index;
      created = index.created();
    } else {
      Engine.Create create =
          indexShard.prepareCreate(
              sourceToParse,
              request.version(),
              request.versionType(),
              Engine.Operation.Origin.PRIMARY,
              request.canHaveDuplicates(),
              request.autoGeneratedId());
      if (create.parsedDoc().mappingsModified()) {
        mappingUpdatedAction.updateMappingOnMaster(
            shardRequest.shardId.getIndex(), create.docMapper(), indexService.indexUUID());
      }
      indexShard.create(create);
      version = create.version();
      op = create;
      created = true;
    }
    if (request.refresh()) {
      try {
        indexShard.refresh(new Engine.Refresh("refresh_flag_index").force(false));
      } catch (Throwable e) {
        // ignore
      }
    }

    // update the version on the request, so it will be used for the replicas
    request.version(version);
    request.versionType(request.versionType().versionTypeForReplicationAndRecovery());

    assert request.versionType().validateVersionForWrites(request.version());

    IndexResponse response =
        new IndexResponse(
            shardRequest.shardId.getIndex(), request.type(), request.id(), version, created);
    return new PrimaryResponse<>(shardRequest.request, response, op);
  }
 @Override
 protected PrimaryResponse<IngestShardResponse, IngestShardRequest> shardOperationOnPrimary(
     ClusterState clusterState, PrimaryOperationRequest shardRequest) {
   final IngestShardRequest request = shardRequest.request;
   IndexShard indexShard =
       indicesService
           .indexServiceSafe(shardRequest.request.index())
           .shardSafe(shardRequest.shardId);
   int successSize = 0;
   List<IngestItemFailure> failure = newLinkedList();
   int size = request.items().size();
   long[] versions = new long[size];
   Set<Tuple<String, String>> mappingsToUpdate = newHashSet();
   for (int i = 0; i < size; i++) {
     IngestItemRequest item = request.items().get(i);
     if (item.request() instanceof IndexRequest) {
       IndexRequest indexRequest = (IndexRequest) item.request();
       Engine.IndexingOperation op = null;
       try {
         // validate, if routing is required, that we got routing
         MappingMetaData mappingMd =
             clusterState.metaData().index(request.index()).mappingOrDefault(indexRequest.type());
         if (mappingMd != null && mappingMd.routing().required()) {
           if (indexRequest.routing() == null) {
             throw new RoutingMissingException(
                 indexRequest.index(), indexRequest.type(), indexRequest.id());
           }
         }
         SourceToParse sourceToParse =
             SourceToParse.source(SourceToParse.Origin.PRIMARY, indexRequest.source())
                 .type(indexRequest.type())
                 .id(indexRequest.id())
                 .routing(indexRequest.routing())
                 .parent(indexRequest.parent())
                 .timestamp(indexRequest.timestamp())
                 .ttl(indexRequest.ttl());
         long version;
         if (indexRequest.opType() == IndexRequest.OpType.INDEX) {
           Engine.Index index =
               indexShard
                   .prepareIndex(sourceToParse)
                   .version(indexRequest.version())
                   .versionType(indexRequest.versionType())
                   .origin(Engine.Operation.Origin.PRIMARY);
           op = index;
           indexShard.index(index);
           version = index.version();
         } else {
           Engine.Create create =
               indexShard
                   .prepareCreate(sourceToParse)
                   .version(indexRequest.version())
                   .versionType(indexRequest.versionType())
                   .origin(Engine.Operation.Origin.PRIMARY);
           op = create;
           indexShard.create(create);
           version = create.version();
         }
         versions[i] = indexRequest.version();
         // update the version on request so it will happen on the replicas
         indexRequest.version(version);
         successSize++;
       } catch (Throwable e) {
         // rethrow the failure if we are going to retry on primary and let parent failure to
         // handle it
         if (retryPrimaryException(e)) {
           // restore updated versions...
           for (int j = 0; j < i; j++) {
             applyVersion(request.items().get(j), versions[j]);
           }
           logger.error(e.getMessage(), e);
           throw new ElasticsearchException(e.getMessage());
         }
         if (e instanceof ElasticsearchException
             && ((ElasticsearchException) e).status() == RestStatus.CONFLICT) {
           logger.error(
               "[{}][{}] failed to execute bulk item (index) {}",
               e,
               shardRequest.request.index(),
               shardRequest.shardId,
               indexRequest);
         } else {
           logger.error(
               "[{}][{}] failed to execute bulk item (index) {}",
               e,
               shardRequest.request.index(),
               shardRequest.shardId,
               indexRequest);
         }
         failure.add(new IngestItemFailure(item.id(), ExceptionsHelper.detailedMessage(e)));
         // nullify the request so it won't execute on the replicas
         request.items().set(i, null);
       } finally {
         // update mapping on master if needed, we won't update changes to the same type, since
         // once its changed, it won't have mappers added
         if (op != null && op.parsedDoc().mappingsModified()) {
           mappingsToUpdate.add(Tuple.tuple(indexRequest.index(), indexRequest.type()));
         }
       }
     } else if (item.request() instanceof DeleteRequest) {
       DeleteRequest deleteRequest = (DeleteRequest) item.request();
       try {
         Engine.Delete delete =
             indexShard
                 .prepareDelete(deleteRequest.type(), deleteRequest.id(), deleteRequest.version())
                 .versionType(deleteRequest.versionType())
                 .origin(Engine.Operation.Origin.PRIMARY);
         indexShard.delete(delete);
         // update the request with teh version so it will go to the replicas
         deleteRequest.version(delete.version());
         successSize++;
       } catch (Throwable e) {
         // rethrow the failure if we are going to retry on primary and let parent failure to
         // handle it
         if (retryPrimaryException(e)) {
           // restore updated versions...
           for (int j = 0; j < i; j++) {
             applyVersion(request.items().get(j), versions[j]);
           }
           logger.error(e.getMessage(), e);
           throw new ElasticsearchException(e.getMessage());
         }
         if (e instanceof ElasticsearchException
             && ((ElasticsearchException) e).status() == RestStatus.CONFLICT) {
           logger.trace(
               "[{}][{}] failed to execute bulk item (delete) {}",
               e,
               shardRequest.request.index(),
               shardRequest.shardId,
               deleteRequest);
         } else {
           logger.debug(
               "[{}][{}] failed to execute bulk item (delete) {}",
               e,
               shardRequest.request.index(),
               shardRequest.shardId,
               deleteRequest);
         }
         failure.add(new IngestItemFailure(item.id(), ExceptionsHelper.detailedMessage(e)));
         // nullify the request so it won't execute on the replicas
         request.items().set(i, null);
       }
     }
   }
   if (!mappingsToUpdate.isEmpty()) {
     for (Tuple<String, String> mappingToUpdate : mappingsToUpdate) {
       logger.info("mapping update {} {}", mappingToUpdate.v1(), mappingToUpdate.v2());
       updateMappingOnMaster(mappingToUpdate.v1(), mappingToUpdate.v2());
     }
   }
   IngestShardResponse response =
       new IngestShardResponse(
           new ShardId(request.index(), request.shardId()), successSize, failure);
   return new PrimaryResponse<IngestShardResponse, IngestShardRequest>(
       shardRequest.request, response, null);
 }
  @Override
  protected void shardOperationOnReplica(ShardId shardId, BulkShardRequest request) {
    IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
    IndexShard indexShard = indexService.shardSafe(shardId.id());
    Translog.Location location = null;
    for (int i = 0; i < request.items().length; i++) {
      BulkItemRequest item = request.items()[i];
      if (item == null || item.isIgnoreOnReplica()) {
        continue;
      }
      if (item.request() instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) item.request();
        try {
          SourceToParse sourceToParse =
              SourceToParse.source(SourceToParse.Origin.REPLICA, indexRequest.source())
                  .index(shardId.getIndex())
                  .type(indexRequest.type())
                  .id(indexRequest.id())
                  .routing(indexRequest.routing())
                  .parent(indexRequest.parent())
                  .timestamp(indexRequest.timestamp())
                  .ttl(indexRequest.ttl());

          final Engine.IndexingOperation operation;
          if (indexRequest.opType() == IndexRequest.OpType.INDEX) {
            operation =
                indexShard.prepareIndex(
                    sourceToParse,
                    indexRequest.version(),
                    indexRequest.versionType(),
                    Engine.Operation.Origin.REPLICA,
                    request.canHaveDuplicates() || indexRequest.canHaveDuplicates());
          } else {
            assert indexRequest.opType() == IndexRequest.OpType.CREATE : indexRequest.opType();
            operation =
                indexShard.prepareCreate(
                    sourceToParse,
                    indexRequest.version(),
                    indexRequest.versionType(),
                    Engine.Operation.Origin.REPLICA,
                    request.canHaveDuplicates() || indexRequest.canHaveDuplicates(),
                    indexRequest.autoGeneratedId());
          }
          Mapping update = operation.parsedDoc().dynamicMappingsUpdate();
          if (update != null) {
            throw new RetryOnReplicaException(
                shardId,
                "Mappings are not available on the replica yet, triggered update: " + update);
          }
          operation.execute(indexShard);
          location = locationToSync(location, operation.getTranslogLocation());
        } catch (Throwable e) {
          // if its not an ignore replica failure, we need to make sure to bubble up the failure
          // so we will fail the shard
          if (!ignoreReplicaException(e)) {
            throw e;
          }
        }
      } else if (item.request() instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) item.request();
        try {
          Engine.Delete delete =
              indexShard.prepareDelete(
                  deleteRequest.type(),
                  deleteRequest.id(),
                  deleteRequest.version(),
                  deleteRequest.versionType(),
                  Engine.Operation.Origin.REPLICA);
          indexShard.delete(delete);
          location = locationToSync(location, delete.getTranslogLocation());
        } catch (Throwable e) {
          // if its not an ignore replica failure, we need to make sure to bubble up the failure
          // so we will fail the shard
          if (!ignoreReplicaException(e)) {
            throw e;
          }
        }
      } else {
        throw new IllegalStateException("Unexpected index operation: " + item.request());
      }
    }

    processAfter(request.refresh(), indexShard, location);
  }