コード例 #1
0
 @Override
 public void postDeleteUnderLock(Engine.Delete delete) {
   // remove the query under a lock
   if (PercolatorService.TYPE_NAME.equals(delete.type())) {
     removePercolateQuery(delete.id());
   }
 }
コード例 #2
0
 @Override
 public Engine.Delete beforeDelete(Engine.Delete delete) {
   if (delete.type().equals(index().name())) {
     percolator.removeQuery(delete.id());
   }
   return delete;
 }
コード例 #3
0
  @Override
  protected PrimaryResponse<ShardDeleteResponse, ShardDeleteRequest> shardOperationOnPrimary(
      ClusterState clusterState, PrimaryOperationRequest shardRequest) {
    ShardDeleteRequest request = shardRequest.request;
    IndexShard indexShard =
        indicesService
            .indexServiceSafe(shardRequest.request.index())
            .shardSafe(shardRequest.shardId);
    Engine.Delete delete =
        indexShard
            .prepareDelete(request.type(), request.id(), request.version())
            .origin(Engine.Operation.Origin.PRIMARY);
    indexShard.delete(delete);
    // update the version to happen on the replicas
    request.version(delete.version());

    if (request.refresh()) {
      try {
        indexShard.refresh(new Engine.Refresh("refresh_flag_delete").force(false));
      } catch (Exception e) {
        // ignore
      }
    }

    ShardDeleteResponse response = new ShardDeleteResponse(delete.version(), delete.found());
    return new PrimaryResponse<ShardDeleteResponse, ShardDeleteRequest>(
        shardRequest.request, response, null);
  }
コード例 #4
0
  @Override
  protected void shardOperationOnReplica(ReplicaOperationRequest shardRequest) {
    ShardDeleteRequest request = shardRequest.request;
    IndexShard indexShard =
        indicesService
            .indexServiceSafe(shardRequest.request.index())
            .shardSafe(shardRequest.shardId);
    Engine.Delete delete =
        indexShard
            .prepareDelete(request.type(), request.id(), request.version())
            .origin(Engine.Operation.Origin.REPLICA);

    // IndexDeleteAction doesn't support version type at the moment. Hard coded for the INTERNAL
    // version
    delete.versionType(VersionType.INTERNAL.versionTypeForReplicationAndRecovery());

    assert delete.versionType().validateVersion(delete.version());

    indexShard.delete(delete);

    if (request.refresh()) {
      try {
        indexShard.refresh(new Engine.Refresh("refresh_flag_delete").force(false));
      } catch (Exception e) {
        // ignore
      }
    }
  }
コード例 #5
0
 @Override
 public void postDelete(Engine.Delete delete, Throwable ex) {
   if (delete.origin() != Engine.Operation.Origin.RECOVERY) {
     totalStats.deleteCurrent.dec();
     typeStats(delete.type()).deleteCurrent.dec();
   }
 }
コード例 #6
0
 @Override
 public Engine.Delete preDelete(Engine.Delete delete) {
   if (delete.origin() != Engine.Operation.Origin.RECOVERY) {
     totalStats.deleteCurrent.inc();
     typeStats(delete.type()).deleteCurrent.inc();
   }
   return delete;
 }
コード例 #7
0
  @Test
  public void testVersioningDeleteConflictWithFlush() {
    ParsedDocument doc =
        testParsedDocument(
            "1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
    Engine.Index index = new Engine.Index(null, newUid("1"), doc);
    engine.index(index);
    assertThat(index.version(), equalTo(1l));

    index = new Engine.Index(null, newUid("1"), doc);
    engine.index(index);
    assertThat(index.version(), equalTo(2l));

    engine.flush(new Engine.Flush());

    Engine.Delete delete = new Engine.Delete("test", "1", newUid("1")).version(1l);
    try {
      engine.delete(delete);
      assert false;
    } catch (VersionConflictEngineException e) {
      // all is well
    }

    // future versions should not work as well
    delete = new Engine.Delete("test", "1", newUid("1")).version(3l);
    try {
      engine.delete(delete);
      assert false;
    } catch (VersionConflictEngineException e) {
      // all is well
    }

    engine.flush(new Engine.Flush());

    // now actually delete
    delete = new Engine.Delete("test", "1", newUid("1")).version(2l);
    engine.delete(delete);
    assertThat(delete.version(), equalTo(3l));

    engine.flush(new Engine.Flush());

    // now check if we can index to a delete doc with version
    index = new Engine.Index(null, newUid("1"), doc).version(2l);
    try {
      engine.index(index);
      assert false;
    } catch (VersionConflictEngineException e) {
      // all is well
    }

    // we shouldn't be able to create as well
    Engine.Create create = new Engine.Create(null, newUid("1"), doc).version(2l);
    try {
      engine.create(create);
    } catch (VersionConflictEngineException e) {
      // all is well
    }
  }
コード例 #8
0
 @Override
 public void postDelete(Engine.Delete delete) {
   if (delete.origin() != Engine.Operation.Origin.RECOVERY) {
     long took = delete.endTime() - delete.startTime();
     totalStats.deleteMetric.inc(took);
     totalStats.deleteCurrent.dec();
     StatsHolder typeStats = typeStats(delete.type());
     typeStats.deleteMetric.inc(took);
     typeStats.deleteCurrent.dec();
   }
 }
コード例 #9
0
  @Test
  public void testVersioningReplicaConflict2() {
    ParsedDocument doc =
        testParsedDocument(
            "1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
    Engine.Index index = new Engine.Index(null, newUid("1"), doc);
    engine.index(index);
    assertThat(index.version(), equalTo(1l));

    // apply the first index to the replica, should work fine
    index = new Engine.Index(null, newUid("1"), doc).version(1l).origin(REPLICA);
    replicaEngine.index(index);
    assertThat(index.version(), equalTo(1l));

    // index it again
    index = new Engine.Index(null, newUid("1"), doc);
    engine.index(index);
    assertThat(index.version(), equalTo(2l));

    // now delete it
    Engine.Delete delete = new Engine.Delete("test", "1", newUid("1"));
    engine.delete(delete);
    assertThat(delete.version(), equalTo(3l));

    // apply the delete on the replica (skipping the second index)
    delete = new Engine.Delete("test", "1", newUid("1")).version(3l).origin(REPLICA);
    replicaEngine.delete(delete);
    assertThat(delete.version(), equalTo(3l));

    // second time delete with same version should fail
    try {
      delete = new Engine.Delete("test", "1", newUid("1")).version(3l).origin(REPLICA);
      replicaEngine.delete(delete);
      assertThat(delete.version(), equalTo(3l));
    } catch (VersionConflictEngineException e) {
      // all is well
    }

    // now do the second index on the replica, it should fail
    try {
      index = new Engine.Index(null, newUid("1"), doc).version(2l).origin(REPLICA);
      replicaEngine.index(index);
      assertThat(index.version(), equalTo(2l));
    } catch (VersionConflictEngineException e) {
      // all is well
    }
  }
コード例 #10
0
  private WriteResult<DeleteResponse> shardDeleteOperation(
      BulkShardRequest request, DeleteRequest deleteRequest, IndexShard indexShard) {
    Engine.Delete delete =
        indexShard.prepareDelete(
            deleteRequest.type(),
            deleteRequest.id(),
            deleteRequest.version(),
            deleteRequest.versionType(),
            Engine.Operation.Origin.PRIMARY);
    indexShard.delete(delete);
    // update the request with the version so it will go to the replicas
    deleteRequest.versionType(delete.versionType().versionTypeForReplicationAndRecovery());
    deleteRequest.version(delete.version());

    assert deleteRequest.versionType().validateVersionForWrites(deleteRequest.version());

    DeleteResponse deleteResponse =
        new DeleteResponse(
            request.index(),
            deleteRequest.type(),
            deleteRequest.id(),
            delete.version(),
            delete.found());
    return new WriteResult(deleteResponse, delete.getTranslogLocation());
  }
コード例 #11
0
 public Delete(Engine.Delete delete) {
   this(delete.uid());
   this.version = delete.version();
 }
コード例 #12
0
 @Override
 protected PrimaryResponse<IngestShardResponse, IngestShardRequest> shardOperationOnPrimary(
     ClusterState clusterState, PrimaryOperationRequest shardRequest) {
   final IngestShardRequest request = shardRequest.request;
   IndexShard indexShard =
       indicesService
           .indexServiceSafe(shardRequest.request.index())
           .shardSafe(shardRequest.shardId);
   int successSize = 0;
   List<IngestItemFailure> failure = newLinkedList();
   int size = request.items().size();
   long[] versions = new long[size];
   Set<Tuple<String, String>> mappingsToUpdate = newHashSet();
   for (int i = 0; i < size; i++) {
     IngestItemRequest item = request.items().get(i);
     if (item.request() instanceof IndexRequest) {
       IndexRequest indexRequest = (IndexRequest) item.request();
       Engine.IndexingOperation op = null;
       try {
         // validate, if routing is required, that we got routing
         MappingMetaData mappingMd =
             clusterState.metaData().index(request.index()).mappingOrDefault(indexRequest.type());
         if (mappingMd != null && mappingMd.routing().required()) {
           if (indexRequest.routing() == null) {
             throw new RoutingMissingException(
                 indexRequest.index(), indexRequest.type(), indexRequest.id());
           }
         }
         SourceToParse sourceToParse =
             SourceToParse.source(SourceToParse.Origin.PRIMARY, indexRequest.source())
                 .type(indexRequest.type())
                 .id(indexRequest.id())
                 .routing(indexRequest.routing())
                 .parent(indexRequest.parent())
                 .timestamp(indexRequest.timestamp())
                 .ttl(indexRequest.ttl());
         long version;
         if (indexRequest.opType() == IndexRequest.OpType.INDEX) {
           Engine.Index index =
               indexShard
                   .prepareIndex(sourceToParse)
                   .version(indexRequest.version())
                   .versionType(indexRequest.versionType())
                   .origin(Engine.Operation.Origin.PRIMARY);
           op = index;
           indexShard.index(index);
           version = index.version();
         } else {
           Engine.Create create =
               indexShard
                   .prepareCreate(sourceToParse)
                   .version(indexRequest.version())
                   .versionType(indexRequest.versionType())
                   .origin(Engine.Operation.Origin.PRIMARY);
           op = create;
           indexShard.create(create);
           version = create.version();
         }
         versions[i] = indexRequest.version();
         // update the version on request so it will happen on the replicas
         indexRequest.version(version);
         successSize++;
       } catch (Throwable e) {
         // rethrow the failure if we are going to retry on primary and let parent failure to
         // handle it
         if (retryPrimaryException(e)) {
           // restore updated versions...
           for (int j = 0; j < i; j++) {
             applyVersion(request.items().get(j), versions[j]);
           }
           logger.error(e.getMessage(), e);
           throw new ElasticsearchException(e.getMessage());
         }
         if (e instanceof ElasticsearchException
             && ((ElasticsearchException) e).status() == RestStatus.CONFLICT) {
           logger.error(
               "[{}][{}] failed to execute bulk item (index) {}",
               e,
               shardRequest.request.index(),
               shardRequest.shardId,
               indexRequest);
         } else {
           logger.error(
               "[{}][{}] failed to execute bulk item (index) {}",
               e,
               shardRequest.request.index(),
               shardRequest.shardId,
               indexRequest);
         }
         failure.add(new IngestItemFailure(item.id(), ExceptionsHelper.detailedMessage(e)));
         // nullify the request so it won't execute on the replicas
         request.items().set(i, null);
       } finally {
         // update mapping on master if needed, we won't update changes to the same type, since
         // once its changed, it won't have mappers added
         if (op != null && op.parsedDoc().mappingsModified()) {
           mappingsToUpdate.add(Tuple.tuple(indexRequest.index(), indexRequest.type()));
         }
       }
     } else if (item.request() instanceof DeleteRequest) {
       DeleteRequest deleteRequest = (DeleteRequest) item.request();
       try {
         Engine.Delete delete =
             indexShard
                 .prepareDelete(deleteRequest.type(), deleteRequest.id(), deleteRequest.version())
                 .versionType(deleteRequest.versionType())
                 .origin(Engine.Operation.Origin.PRIMARY);
         indexShard.delete(delete);
         // update the request with teh version so it will go to the replicas
         deleteRequest.version(delete.version());
         successSize++;
       } catch (Throwable e) {
         // rethrow the failure if we are going to retry on primary and let parent failure to
         // handle it
         if (retryPrimaryException(e)) {
           // restore updated versions...
           for (int j = 0; j < i; j++) {
             applyVersion(request.items().get(j), versions[j]);
           }
           logger.error(e.getMessage(), e);
           throw new ElasticsearchException(e.getMessage());
         }
         if (e instanceof ElasticsearchException
             && ((ElasticsearchException) e).status() == RestStatus.CONFLICT) {
           logger.trace(
               "[{}][{}] failed to execute bulk item (delete) {}",
               e,
               shardRequest.request.index(),
               shardRequest.shardId,
               deleteRequest);
         } else {
           logger.debug(
               "[{}][{}] failed to execute bulk item (delete) {}",
               e,
               shardRequest.request.index(),
               shardRequest.shardId,
               deleteRequest);
         }
         failure.add(new IngestItemFailure(item.id(), ExceptionsHelper.detailedMessage(e)));
         // nullify the request so it won't execute on the replicas
         request.items().set(i, null);
       }
     }
   }
   if (!mappingsToUpdate.isEmpty()) {
     for (Tuple<String, String> mappingToUpdate : mappingsToUpdate) {
       logger.info("mapping update {} {}", mappingToUpdate.v1(), mappingToUpdate.v2());
       updateMappingOnMaster(mappingToUpdate.v1(), mappingToUpdate.v2());
     }
   }
   IngestShardResponse response =
       new IngestShardResponse(
           new ShardId(request.index(), request.shardId()), successSize, failure);
   return new PrimaryResponse<IngestShardResponse, IngestShardRequest>(
       shardRequest.request, response, null);
 }
コード例 #13
0
  @Override
  protected void shardOperationOnReplica(ShardId shardId, BulkShardRequest request) {
    IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
    IndexShard indexShard = indexService.shardSafe(shardId.id());
    Translog.Location location = null;
    for (int i = 0; i < request.items().length; i++) {
      BulkItemRequest item = request.items()[i];
      if (item == null || item.isIgnoreOnReplica()) {
        continue;
      }
      if (item.request() instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) item.request();
        try {
          SourceToParse sourceToParse =
              SourceToParse.source(SourceToParse.Origin.REPLICA, indexRequest.source())
                  .index(shardId.getIndex())
                  .type(indexRequest.type())
                  .id(indexRequest.id())
                  .routing(indexRequest.routing())
                  .parent(indexRequest.parent())
                  .timestamp(indexRequest.timestamp())
                  .ttl(indexRequest.ttl());

          final Engine.IndexingOperation operation;
          if (indexRequest.opType() == IndexRequest.OpType.INDEX) {
            operation =
                indexShard.prepareIndex(
                    sourceToParse,
                    indexRequest.version(),
                    indexRequest.versionType(),
                    Engine.Operation.Origin.REPLICA,
                    request.canHaveDuplicates() || indexRequest.canHaveDuplicates());
          } else {
            assert indexRequest.opType() == IndexRequest.OpType.CREATE : indexRequest.opType();
            operation =
                indexShard.prepareCreate(
                    sourceToParse,
                    indexRequest.version(),
                    indexRequest.versionType(),
                    Engine.Operation.Origin.REPLICA,
                    request.canHaveDuplicates() || indexRequest.canHaveDuplicates(),
                    indexRequest.autoGeneratedId());
          }
          Mapping update = operation.parsedDoc().dynamicMappingsUpdate();
          if (update != null) {
            throw new RetryOnReplicaException(
                shardId,
                "Mappings are not available on the replica yet, triggered update: " + update);
          }
          operation.execute(indexShard);
          location = locationToSync(location, operation.getTranslogLocation());
        } catch (Throwable e) {
          // if its not an ignore replica failure, we need to make sure to bubble up the failure
          // so we will fail the shard
          if (!ignoreReplicaException(e)) {
            throw e;
          }
        }
      } else if (item.request() instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) item.request();
        try {
          Engine.Delete delete =
              indexShard.prepareDelete(
                  deleteRequest.type(),
                  deleteRequest.id(),
                  deleteRequest.version(),
                  deleteRequest.versionType(),
                  Engine.Operation.Origin.REPLICA);
          indexShard.delete(delete);
          location = locationToSync(location, delete.getTranslogLocation());
        } catch (Throwable e) {
          // if its not an ignore replica failure, we need to make sure to bubble up the failure
          // so we will fail the shard
          if (!ignoreReplicaException(e)) {
            throw e;
          }
        }
      } else {
        throw new IllegalStateException("Unexpected index operation: " + item.request());
      }
    }

    processAfter(request.refresh(), indexShard, location);
  }