@Test
public void testEnabled() throws Exception {
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("_timestamp")
.field("enabled", "yes")
.field("store", "yes")
.endObject()
.endObject()
.endObject()
.string();
DocumentMapper docMapper = MapperTests.newParser().parse(mapping);
byte[] source =
XContentFactory.jsonBuilder()
.startObject()
.field("field", "value")
.endObject()
.copiedBytes();
ParsedDocument doc =
docMapper.parse(SourceToParse.source(source).type("type").id("1").timestamp(1));
assertThat(doc.rootDoc().getFieldable("_timestamp").isStored(), equalTo(true));
assertThat(doc.rootDoc().getFieldable("_timestamp").isIndexed(), equalTo(true));
assertThat(doc.rootDoc().getFieldable("_timestamp").tokenStreamValue(), notNullValue());
}
@Override
protected void shardOperationOnReplica(ReplicaOperationRequest shardRequest) {
IndexShard indexShard =
indicesService
.indexServiceSafe(shardRequest.request.index())
.shardSafe(shardRequest.shardId);
final IngestShardRequest request = shardRequest.request;
int size = request.items().size();
for (int i = 0; i < size; i++) {
IngestItemRequest item = request.items().get(i);
if (item == null) {
continue;
}
if (item.request() instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) item.request();
try {
SourceToParse sourceToParse =
SourceToParse.source(SourceToParse.Origin.REPLICA, indexRequest.source())
.type(indexRequest.type())
.id(indexRequest.id())
.routing(indexRequest.routing())
.parent(indexRequest.parent())
.timestamp(indexRequest.timestamp())
.ttl(indexRequest.ttl());
if (indexRequest.opType() == IndexRequest.OpType.INDEX) {
Engine.Index index =
indexShard
.prepareIndex(sourceToParse)
.version(indexRequest.version())
.origin(Engine.Operation.Origin.REPLICA);
indexShard.index(index);
} else {
Engine.Create create =
indexShard
.prepareCreate(sourceToParse)
.version(indexRequest.version())
.origin(Engine.Operation.Origin.REPLICA);
indexShard.create(create);
}
} catch (Exception e) {
// ignore, we are on backup
}
} else if (item.request() instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) item.request();
try {
Engine.Delete delete =
indexShard
.prepareDelete(deleteRequest.type(), deleteRequest.id(), deleteRequest.version())
.origin(Engine.Operation.Origin.REPLICA);
indexShard.delete(delete);
} catch (Exception e) {
// ignore, we are on backup
}
}
}
}
/**
* Utility method to create either an index or a create operation depending on the {@link OpType}
* of the request.
*/
private final Engine.Index prepareIndexOperationOnPrimary(
BulkShardRequest shardRequest, IndexRequest request, IndexShard indexShard) {
SourceToParse sourceToParse =
SourceToParse.source(SourceToParse.Origin.PRIMARY, request.source())
.index(request.index())
.type(request.type())
.id(request.id())
.routing(request.routing())
.parent(request.parent())
.timestamp(request.timestamp())
.ttl(request.ttl());
return indexShard.prepareIndex(
sourceToParse, request.version(), request.versionType(), Engine.Operation.Origin.PRIMARY);
}
@Test
public void testSimpleDisabled() throws Exception {
String mapping =
XContentFactory.jsonBuilder().startObject().startObject("type").endObject().string();
DocumentMapper docMapper = MapperTests.newParser().parse(mapping);
byte[] source =
XContentFactory.jsonBuilder()
.startObject()
.field("field", "value")
.endObject()
.copiedBytes();
ParsedDocument doc =
docMapper.parse(SourceToParse.source(source).type("type").id("1").timestamp(1));
assertThat(doc.rootDoc().getFieldable("_timestamp"), equalTo(null));
}
@Test
public void testSizeNotSet() throws Exception {
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.endObject()
.endObject()
.string();
DocumentMapper docMapper = MapperTests.newParser().parse(mapping);
BytesReference source =
XContentFactory.jsonBuilder().startObject().field("field", "value").endObject().bytes();
ParsedDocument doc = docMapper.parse(SourceToParse.source(source).type("type").id("1"));
assertThat(doc.rootDoc().getField("_size"), nullValue());
}
@Override
protected void shardOperationOnReplica(ReplicaOperationRequest shardRequest) {
IndexShard indexShard =
indicesService
.indexServiceSafe(shardRequest.shardId.getIndex())
.shardSafe(shardRequest.shardId.id());
IndexRequest request = shardRequest.request;
SourceToParse sourceToParse =
SourceToParse.source(SourceToParse.Origin.REPLICA, request.source())
.type(request.type())
.id(request.id())
.routing(request.routing())
.parent(request.parent())
.timestamp(request.timestamp())
.ttl(request.ttl());
if (request.opType() == IndexRequest.OpType.INDEX) {
Engine.Index index =
indexShard.prepareIndex(
sourceToParse,
request.version(),
request.versionType(),
Engine.Operation.Origin.REPLICA,
request.canHaveDuplicates());
indexShard.index(index);
} else {
Engine.Create create =
indexShard.prepareCreate(
sourceToParse,
request.version(),
request.versionType(),
Engine.Operation.Origin.REPLICA,
request.canHaveDuplicates(),
request.autoGeneratedId());
indexShard.create(create);
}
if (request.refresh()) {
try {
indexShard.refresh(new Engine.Refresh("refresh_flag_index").force(false));
} catch (Exception e) {
// ignore
}
}
}
@Test
public void testSizeEnabled() throws Exception {
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("_size")
.field("enabled", true)
.endObject()
.endObject()
.endObject()
.string();
DocumentMapper docMapper = MapperTests.newParser().parse(mapping);
BytesReference source =
XContentFactory.jsonBuilder().startObject().field("field", "value").endObject().bytes();
ParsedDocument doc = docMapper.parse(SourceToParse.source(source).type("type").id("1"));
assertThat(doc.rootDoc().getField("_size").fieldType().stored(), equalTo(false));
assertThat(
doc.rootDoc().getField("_size").tokenStream(docMapper.indexAnalyzer()), notNullValue());
}
@Override
protected void shardOperationOnReplica(ShardId shardId, IndexRequest request) {
IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
IndexShard indexShard = indexService.shardSafe(shardId.id());
SourceToParse sourceToParse =
SourceToParse.source(SourceToParse.Origin.REPLICA, request.source())
.index(shardId.getIndex())
.type(request.type())
.id(request.id())
.routing(request.routing())
.parent(request.parent())
.timestamp(request.timestamp())
.ttl(request.ttl());
final Engine.IndexingOperation operation;
if (request.opType() == IndexRequest.OpType.INDEX) {
operation =
indexShard.prepareIndex(
sourceToParse,
request.version(),
request.versionType(),
Engine.Operation.Origin.REPLICA);
} else {
assert request.opType() == IndexRequest.OpType.CREATE : request.opType();
operation =
indexShard.prepareCreate(
sourceToParse,
request.version(),
request.versionType(),
Engine.Operation.Origin.REPLICA);
}
Mapping update = operation.parsedDoc().dynamicMappingsUpdate();
if (update != null) {
throw new RetryOnReplicaException(
shardId, "Mappings are not available on the replica yet, triggered update: " + update);
}
operation.execute(indexShard);
processAfter(request.refresh(), indexShard, operation.getTranslogLocation());
}
@Override
protected PrimaryResponse<IndexResponse, IndexRequest> shardOperationOnPrimary(
ClusterState clusterState, PrimaryOperationRequest shardRequest) {
final IndexRequest request = shardRequest.request;
// validate, if routing is required, that we got routing
IndexMetaData indexMetaData = clusterState.metaData().index(shardRequest.shardId.getIndex());
MappingMetaData mappingMd = indexMetaData.mappingOrDefault(request.type());
if (mappingMd != null && mappingMd.routing().required()) {
if (request.routing() == null) {
throw new RoutingMissingException(
shardRequest.shardId.getIndex(), request.type(), request.id());
}
}
IndexService indexService = indicesService.indexServiceSafe(shardRequest.shardId.getIndex());
IndexShard indexShard = indexService.shardSafe(shardRequest.shardId.id());
SourceToParse sourceToParse =
SourceToParse.source(SourceToParse.Origin.PRIMARY, request.source())
.type(request.type())
.id(request.id())
.routing(request.routing())
.parent(request.parent())
.timestamp(request.timestamp())
.ttl(request.ttl());
long version;
boolean created;
Engine.IndexingOperation op;
if (request.opType() == IndexRequest.OpType.INDEX) {
Engine.Index index =
indexShard.prepareIndex(
sourceToParse,
request.version(),
request.versionType(),
Engine.Operation.Origin.PRIMARY,
request.canHaveDuplicates());
if (index.parsedDoc().mappingsModified()) {
mappingUpdatedAction.updateMappingOnMaster(
shardRequest.shardId.getIndex(), index.docMapper(), indexService.indexUUID());
}
indexShard.index(index);
version = index.version();
op = index;
created = index.created();
} else {
Engine.Create create =
indexShard.prepareCreate(
sourceToParse,
request.version(),
request.versionType(),
Engine.Operation.Origin.PRIMARY,
request.canHaveDuplicates(),
request.autoGeneratedId());
if (create.parsedDoc().mappingsModified()) {
mappingUpdatedAction.updateMappingOnMaster(
shardRequest.shardId.getIndex(), create.docMapper(), indexService.indexUUID());
}
indexShard.create(create);
version = create.version();
op = create;
created = true;
}
if (request.refresh()) {
try {
indexShard.refresh(new Engine.Refresh("refresh_flag_index").force(false));
} catch (Throwable e) {
// ignore
}
}
// update the version on the request, so it will be used for the replicas
request.version(version);
request.versionType(request.versionType().versionTypeForReplicationAndRecovery());
assert request.versionType().validateVersionForWrites(request.version());
IndexResponse response =
new IndexResponse(
shardRequest.shardId.getIndex(), request.type(), request.id(), version, created);
return new PrimaryResponse<>(shardRequest.request, response, op);
}
@Override
protected PrimaryResponse<IngestShardResponse, IngestShardRequest> shardOperationOnPrimary(
ClusterState clusterState, PrimaryOperationRequest shardRequest) {
final IngestShardRequest request = shardRequest.request;
IndexShard indexShard =
indicesService
.indexServiceSafe(shardRequest.request.index())
.shardSafe(shardRequest.shardId);
int successSize = 0;
List<IngestItemFailure> failure = newLinkedList();
int size = request.items().size();
long[] versions = new long[size];
Set<Tuple<String, String>> mappingsToUpdate = newHashSet();
for (int i = 0; i < size; i++) {
IngestItemRequest item = request.items().get(i);
if (item.request() instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) item.request();
Engine.IndexingOperation op = null;
try {
// validate, if routing is required, that we got routing
MappingMetaData mappingMd =
clusterState.metaData().index(request.index()).mappingOrDefault(indexRequest.type());
if (mappingMd != null && mappingMd.routing().required()) {
if (indexRequest.routing() == null) {
throw new RoutingMissingException(
indexRequest.index(), indexRequest.type(), indexRequest.id());
}
}
SourceToParse sourceToParse =
SourceToParse.source(SourceToParse.Origin.PRIMARY, indexRequest.source())
.type(indexRequest.type())
.id(indexRequest.id())
.routing(indexRequest.routing())
.parent(indexRequest.parent())
.timestamp(indexRequest.timestamp())
.ttl(indexRequest.ttl());
long version;
if (indexRequest.opType() == IndexRequest.OpType.INDEX) {
Engine.Index index =
indexShard
.prepareIndex(sourceToParse)
.version(indexRequest.version())
.versionType(indexRequest.versionType())
.origin(Engine.Operation.Origin.PRIMARY);
op = index;
indexShard.index(index);
version = index.version();
} else {
Engine.Create create =
indexShard
.prepareCreate(sourceToParse)
.version(indexRequest.version())
.versionType(indexRequest.versionType())
.origin(Engine.Operation.Origin.PRIMARY);
op = create;
indexShard.create(create);
version = create.version();
}
versions[i] = indexRequest.version();
// update the version on request so it will happen on the replicas
indexRequest.version(version);
successSize++;
} catch (Throwable e) {
// rethrow the failure if we are going to retry on primary and let parent failure to
// handle it
if (retryPrimaryException(e)) {
// restore updated versions...
for (int j = 0; j < i; j++) {
applyVersion(request.items().get(j), versions[j]);
}
logger.error(e.getMessage(), e);
throw new ElasticsearchException(e.getMessage());
}
if (e instanceof ElasticsearchException
&& ((ElasticsearchException) e).status() == RestStatus.CONFLICT) {
logger.error(
"[{}][{}] failed to execute bulk item (index) {}",
e,
shardRequest.request.index(),
shardRequest.shardId,
indexRequest);
} else {
logger.error(
"[{}][{}] failed to execute bulk item (index) {}",
e,
shardRequest.request.index(),
shardRequest.shardId,
indexRequest);
}
failure.add(new IngestItemFailure(item.id(), ExceptionsHelper.detailedMessage(e)));
// nullify the request so it won't execute on the replicas
request.items().set(i, null);
} finally {
// update mapping on master if needed, we won't update changes to the same type, since
// once its changed, it won't have mappers added
if (op != null && op.parsedDoc().mappingsModified()) {
mappingsToUpdate.add(Tuple.tuple(indexRequest.index(), indexRequest.type()));
}
}
} else if (item.request() instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) item.request();
try {
Engine.Delete delete =
indexShard
.prepareDelete(deleteRequest.type(), deleteRequest.id(), deleteRequest.version())
.versionType(deleteRequest.versionType())
.origin(Engine.Operation.Origin.PRIMARY);
indexShard.delete(delete);
// update the request with teh version so it will go to the replicas
deleteRequest.version(delete.version());
successSize++;
} catch (Throwable e) {
// rethrow the failure if we are going to retry on primary and let parent failure to
// handle it
if (retryPrimaryException(e)) {
// restore updated versions...
for (int j = 0; j < i; j++) {
applyVersion(request.items().get(j), versions[j]);
}
logger.error(e.getMessage(), e);
throw new ElasticsearchException(e.getMessage());
}
if (e instanceof ElasticsearchException
&& ((ElasticsearchException) e).status() == RestStatus.CONFLICT) {
logger.trace(
"[{}][{}] failed to execute bulk item (delete) {}",
e,
shardRequest.request.index(),
shardRequest.shardId,
deleteRequest);
} else {
logger.debug(
"[{}][{}] failed to execute bulk item (delete) {}",
e,
shardRequest.request.index(),
shardRequest.shardId,
deleteRequest);
}
failure.add(new IngestItemFailure(item.id(), ExceptionsHelper.detailedMessage(e)));
// nullify the request so it won't execute on the replicas
request.items().set(i, null);
}
}
}
if (!mappingsToUpdate.isEmpty()) {
for (Tuple<String, String> mappingToUpdate : mappingsToUpdate) {
logger.info("mapping update {} {}", mappingToUpdate.v1(), mappingToUpdate.v2());
updateMappingOnMaster(mappingToUpdate.v1(), mappingToUpdate.v2());
}
}
IngestShardResponse response =
new IngestShardResponse(
new ShardId(request.index(), request.shardId()), successSize, failure);
return new PrimaryResponse<IngestShardResponse, IngestShardRequest>(
shardRequest.request, response, null);
}
@Override
protected void shardOperationOnReplica(ShardId shardId, BulkShardRequest request) {
IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
IndexShard indexShard = indexService.shardSafe(shardId.id());
Translog.Location location = null;
for (int i = 0; i < request.items().length; i++) {
BulkItemRequest item = request.items()[i];
if (item == null || item.isIgnoreOnReplica()) {
continue;
}
if (item.request() instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) item.request();
try {
SourceToParse sourceToParse =
SourceToParse.source(SourceToParse.Origin.REPLICA, indexRequest.source())
.index(shardId.getIndex())
.type(indexRequest.type())
.id(indexRequest.id())
.routing(indexRequest.routing())
.parent(indexRequest.parent())
.timestamp(indexRequest.timestamp())
.ttl(indexRequest.ttl());
final Engine.IndexingOperation operation;
if (indexRequest.opType() == IndexRequest.OpType.INDEX) {
operation =
indexShard.prepareIndex(
sourceToParse,
indexRequest.version(),
indexRequest.versionType(),
Engine.Operation.Origin.REPLICA,
request.canHaveDuplicates() || indexRequest.canHaveDuplicates());
} else {
assert indexRequest.opType() == IndexRequest.OpType.CREATE : indexRequest.opType();
operation =
indexShard.prepareCreate(
sourceToParse,
indexRequest.version(),
indexRequest.versionType(),
Engine.Operation.Origin.REPLICA,
request.canHaveDuplicates() || indexRequest.canHaveDuplicates(),
indexRequest.autoGeneratedId());
}
Mapping update = operation.parsedDoc().dynamicMappingsUpdate();
if (update != null) {
throw new RetryOnReplicaException(
shardId,
"Mappings are not available on the replica yet, triggered update: " + update);
}
operation.execute(indexShard);
location = locationToSync(location, operation.getTranslogLocation());
} catch (Throwable e) {
// if its not an ignore replica failure, we need to make sure to bubble up the failure
// so we will fail the shard
if (!ignoreReplicaException(e)) {
throw e;
}
}
} else if (item.request() instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) item.request();
try {
Engine.Delete delete =
indexShard.prepareDelete(
deleteRequest.type(),
deleteRequest.id(),
deleteRequest.version(),
deleteRequest.versionType(),
Engine.Operation.Origin.REPLICA);
indexShard.delete(delete);
location = locationToSync(location, delete.getTranslogLocation());
} catch (Throwable e) {
// if its not an ignore replica failure, we need to make sure to bubble up the failure
// so we will fail the shard
if (!ignoreReplicaException(e)) {
throw e;
}
}
} else {
throw new IllegalStateException("Unexpected index operation: " + item.request());
}
}
processAfter(request.refresh(), indexShard, location);
}