Ejemplo n.º 1
0
 /** Clears the post allocation flag for the specified shard */
 public Builder clearPostAllocationFlag(ShardId shardId) {
   assert this.index.equals(shardId.index().name());
   IndexShardRoutingTable indexShard = shards.get(shardId.id());
   shards.put(
       indexShard.shardId().id(),
       new IndexShardRoutingTable(indexShard.shardId(), indexShard.shards(), false));
   return this;
 }
  public static void main(String[] args) throws Exception {
    ShardId shardId = new ShardId(new Index("index"), 1);
    Settings settings = EMPTY_SETTINGS;

    //        Store store = new RamStore(shardId, settings);
    Store store = new ByteBufferStore(shardId, settings, null, new ByteBufferCache(settings));
    //        Store store = new NioFsStore(shardId, settings);

    store.deleteContent();

    ThreadPool threadPool = new ScalingThreadPool();
    SnapshotDeletionPolicy deletionPolicy =
        new SnapshotDeletionPolicy(new KeepOnlyLastDeletionPolicy(shardId, settings));
    Engine engine =
        new RobinEngine(
            shardId,
            settings,
            store,
            deletionPolicy,
            new MemoryTranslog(shardId, settings),
            new LogByteSizeMergePolicyProvider(store),
            new ConcurrentMergeSchedulerProvider(shardId, settings),
            new AnalysisService(shardId.index()),
            new SimilarityService(shardId.index()));
    engine.start();

    SimpleEngineBenchmark benchmark =
        new SimpleEngineBenchmark(store, engine)
            .numberOfContentItems(1000)
            .searcherThreads(50)
            .searcherIterations(10000)
            .writerThreads(10)
            .writerIterations(10000)
            .refreshSchedule(new TimeValue(1, TimeUnit.SECONDS))
            .flushSchedule(new TimeValue(1, TimeUnit.MINUTES))
            .create(false)
            .build();

    benchmark.run();

    engine.close();
    store.close();
    threadPool.shutdown();
  }
 private StoreFilesMetaData listStoreMetaData(ShardId shardId) throws IOException {
   logger.trace("listing store meta data for {}", shardId);
   long startTimeNS = System.nanoTime();
   boolean exists = false;
   try {
     IndexService indexService = indicesService.indexService(shardId.index().name());
     if (indexService != null) {
       IndexShard indexShard = indexService.shard(shardId.id());
       if (indexShard != null) {
         final Store store = indexShard.store();
         store.incRef();
         try {
           exists = true;
           return new StoreFilesMetaData(true, shardId, store.getMetadataOrEmpty());
         } finally {
           store.decRef();
         }
       }
     }
     // try and see if we an list unallocated
     IndexMetaData metaData = clusterService.state().metaData().index(shardId.index().name());
     if (metaData == null) {
       return new StoreFilesMetaData(false, shardId, Store.MetadataSnapshot.EMPTY);
     }
     String storeType = metaData.getSettings().get(IndexStoreModule.STORE_TYPE, "fs");
     if (!storeType.contains("fs")) {
       return new StoreFilesMetaData(false, shardId, Store.MetadataSnapshot.EMPTY);
     }
     final ShardPath shardPath =
         ShardPath.loadShardPath(logger, nodeEnv, shardId, metaData.getSettings());
     if (shardPath == null) {
       return new StoreFilesMetaData(false, shardId, Store.MetadataSnapshot.EMPTY);
     }
     return new StoreFilesMetaData(
         false, shardId, Store.readMetadataSnapshot(shardPath.resolveIndex(), logger));
   } finally {
     TimeValue took = new TimeValue(System.nanoTime() - startTimeNS, TimeUnit.NANOSECONDS);
     if (exists) {
       logger.debug("{} loaded store meta data (took [{}])", shardId, took);
     } else {
       logger.trace("{} didn't find any store meta data to load (took [{}])", shardId, took);
     }
   }
 }
 @Before
 public void setUp() throws Exception {
   super.setUp();
   defaultSettings =
       ImmutableSettings.builder()
           .put(RobinEngine.INDEX_COMPOUND_ON_FLUSH, getRandom().nextBoolean())
           .build(); // TODO randomize more settings
   threadPool = new ThreadPool();
   store = createStore();
   store.deleteContent();
   storeReplica = createStoreReplica();
   storeReplica.deleteContent();
   engineSettingsService = new IndexSettingsService(shardId.index(), EMPTY_SETTINGS);
   engine = createEngine(engineSettingsService, store, createTranslog());
   engine.start();
   replicaSettingsService = new IndexSettingsService(shardId.index(), EMPTY_SETTINGS);
   replicaEngine = createEngine(replicaSettingsService, storeReplica, createTranslogReplica());
   replicaEngine.start();
 }
 protected Engine createEngine(
     IndexSettingsService indexSettingsService, Store store, Translog translog) {
   return new RobinEngine(
       shardId,
       defaultSettings,
       threadPool,
       indexSettingsService,
       new ShardIndexingService(
           shardId,
           EMPTY_SETTINGS,
           new ShardSlowLogIndexingService(shardId, EMPTY_SETTINGS, indexSettingsService)),
       null,
       store,
       createSnapshotDeletionPolicy(),
       translog,
       createMergePolicy(),
       createMergeScheduler(),
       new AnalysisService(shardId.index()),
       new SimilarityService(shardId.index()),
       new CodecService(shardId.index()));
 }
 @Override
 public void onFailedEngine(
     final ShardId shardId, final String reason, final @Nullable Throwable failure) {
   ShardRouting shardRouting = null;
   final IndexService indexService = indicesService.indexService(shardId.index().name());
   if (indexService != null) {
     IndexShard indexShard = indexService.shard(shardId.id());
     if (indexShard != null) {
       shardRouting = indexShard.routingEntry();
     }
   }
   if (shardRouting == null) {
     logger.warn(
         "[{}][{}] engine failed, but can't find index shard. failure reason: [{}]",
         failure,
         shardId.index().name(),
         shardId.id(),
         reason);
     return;
   }
   final ShardRouting fShardRouting = shardRouting;
   threadPool
       .generic()
       .execute(
           new Runnable() {
             @Override
             public void run() {
               synchronized (mutex) {
                 failAndRemoveShard(
                     fShardRouting,
                     indexService,
                     true,
                     "engine failure, reason [" + reason + "]",
                     failure);
               }
             }
           });
 }
  @Override
  protected GetFieldMappingsResponse shardOperation(
      final GetFieldMappingsIndexRequest request, ShardId shardId) throws ElasticsearchException {
    assert shardId != null;
    IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
    Collection<String> typeIntersection;
    if (request.types().length == 0) {
      typeIntersection = indexService.mapperService().types();

    } else {
      typeIntersection =
          Collections2.filter(
              indexService.mapperService().types(),
              new Predicate<String>() {

                @Override
                public boolean apply(String type) {
                  return Regex.simpleMatch(request.types(), type);
                }
              });
      if (typeIntersection.isEmpty()) {
        throw new TypeMissingException(shardId.index(), request.types());
      }
    }

    MapBuilder<String, ImmutableMap<String, FieldMappingMetaData>> typeMappings =
        new MapBuilder<>();
    for (String type : typeIntersection) {
      DocumentMapper documentMapper = indexService.mapperService().documentMapper(type);
      ImmutableMap<String, FieldMappingMetaData> fieldMapping =
          findFieldMappingsByType(documentMapper, request);
      if (!fieldMapping.isEmpty()) {
        typeMappings.put(type, fieldMapping);
      }
    }

    return new GetFieldMappingsResponse(
        ImmutableMap.of(shardId.getIndex(), typeMappings.immutableMap()));
  }
Ejemplo n.º 8
0
 /** Resolves the given shards directory against this NodePath */
 public Path resolve(ShardId shardId) {
   return resolve(shardId.index()).resolve(Integer.toString(shardId.id()));
 }
Ejemplo n.º 9
0
 /**
  * Resolve the custom path for a index's shard. Uses the {@code IndexMetaData.SETTING_DATA_PATH}
  * setting to determine the root path for the index.
  *
  * @param indexSettings settings for the index
  * @param shardId shard to resolve the path to
  */
 public Path resolveCustomLocation(@IndexSettings Settings indexSettings, final ShardId shardId) {
   return resolveCustomLocation(indexSettings, shardId.index().name())
       .resolve(Integer.toString(shardId.id()));
 }
Ejemplo n.º 10
0
  @Override
  protected void doExecute(
      final MultiGetRequest request, final ActionListener<MultiGetResponse> listener) {
    ClusterState clusterState = clusterService.state();
    Map<ShardId, MultiGetShardRequest> shardRequests = new HashMap<ShardId, MultiGetShardRequest>();
    for (int i = 0; i < request.items.size(); i++) {
      MultiGetRequest.Item item = request.items.get(i);
      ShardId shardId =
          clusterService
              .operationRouting()
              .getShards(clusterState, item.index(), item.type(), item.id(), item.routing(), null)
              .shardId();
      MultiGetShardRequest shardRequest = shardRequests.get(shardId);
      if (shardRequest == null) {
        shardRequest = new MultiGetShardRequest(shardId.index().name(), shardId.id());
        shardRequest.preference(request.preference);
        shardRequest.realtime(request.realtime);
        shardRequest.refresh(request.refresh);

        shardRequests.put(shardId, shardRequest);
      }
      shardRequest.add(i, item.type(), item.id(), item.fields());
    }

    final MultiGetItemResponse[] responses = new MultiGetItemResponse[request.items.size()];
    final AtomicInteger counter = new AtomicInteger(shardRequests.size());

    for (final MultiGetShardRequest shardRequest : shardRequests.values()) {
      shardAction.execute(
          shardRequest,
          new ActionListener<MultiGetShardResponse>() {
            @Override
            public void onResponse(MultiGetShardResponse response) {
              synchronized (responses) {
                for (int i = 0; i < response.locations.size(); i++) {
                  responses[response.locations.get(i)] =
                      new MultiGetItemResponse(response.responses.get(i), response.failures.get(i));
                }
              }
              if (counter.decrementAndGet() == 0) {
                finishHim();
              }
            }

            @Override
            public void onFailure(Throwable e) {
              // create failures for all relevant requests
              String message = ExceptionsHelper.detailedMessage(e);
              synchronized (responses) {
                for (int i = 0; i < shardRequest.locations.size(); i++) {
                  responses[shardRequest.locations.get(i)] =
                      new MultiGetItemResponse(
                          null,
                          new MultiGetResponse.Failure(
                              shardRequest.index(),
                              shardRequest.types.get(i),
                              shardRequest.ids.get(i),
                              message));
                }
              }
              if (counter.decrementAndGet() == 0) {
                finishHim();
              }
            }

            private void finishHim() {
              listener.onResponse(new MultiGetResponse(responses));
            }
          });
    }
  }
 protected Releasable getIndexShardOperationsCounter(ShardId shardId) {
   IndexService indexService = indicesService.indexServiceSafe(shardId.index().getName());
   IndexShard indexShard = indexService.getShard(shardId.id());
   return new IndexShardReference(indexShard);
 }
Ejemplo n.º 12
0
  private void executeBulk(
      final BulkRequest bulkRequest,
      final long startTime,
      final ActionListener<BulkResponse> listener) {
    ClusterState clusterState = clusterService.state();
    for (ActionRequest request : bulkRequest.requests) {
      if (request instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) request;
        indexRequest.routing(
            clusterState
                .metaData()
                .resolveIndexRouting(indexRequest.routing(), indexRequest.index()));
        indexRequest.index(clusterState.metaData().concreteIndex(indexRequest.index()));
        if (allowIdGeneration) {
          if (indexRequest.id() == null) {
            indexRequest.id(UUID.randomBase64UUID());
            // since we generate the id, change it to CREATE
            indexRequest.opType(IndexRequest.OpType.CREATE);
          }
        }
      } else if (request instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) request;
        deleteRequest.index(clusterState.metaData().concreteIndex(deleteRequest.index()));
      }
    }
    final BulkItemResponse[] responses = new BulkItemResponse[bulkRequest.requests.size()];

    // first, go over all the requests and create a ShardId -> Operations mapping
    Map<ShardId, List<BulkItemRequest>> requestsByShard = Maps.newHashMap();
    for (int i = 0; i < bulkRequest.requests.size(); i++) {
      ActionRequest request = bulkRequest.requests.get(i);
      if (request instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) request;
        // handle routing
        MappingMetaData mappingMd =
            clusterState.metaData().index(indexRequest.index()).mapping(indexRequest.type());
        if (mappingMd != null) {
          try {
            indexRequest.processRouting(mappingMd);
          } catch (ElasticSearchException e) {
            responses[i] =
                new BulkItemResponse(
                    i,
                    indexRequest.opType().toString().toLowerCase(),
                    new BulkItemResponse.Failure(
                        indexRequest.index(),
                        indexRequest.type(),
                        indexRequest.id(),
                        e.getDetailedMessage()));
            continue;
          }
        }

        ShardId shardId =
            clusterService
                .operationRouting()
                .indexShards(
                    clusterState,
                    indexRequest.index(),
                    indexRequest.type(),
                    indexRequest.id(),
                    indexRequest.routing())
                .shardId();
        List<BulkItemRequest> list = requestsByShard.get(shardId);
        if (list == null) {
          list = Lists.newArrayList();
          requestsByShard.put(shardId, list);
        }
        list.add(new BulkItemRequest(i, request));
      } else if (request instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) request;
        MappingMetaData mappingMd =
            clusterState.metaData().index(deleteRequest.index()).mapping(deleteRequest.type());
        if (mappingMd != null
            && mappingMd.routing().required()
            && deleteRequest.routing() == null) {
          // if routing is required, and no routing on the delete request, we need to broadcast
          // it....
          GroupShardsIterator groupShards =
              clusterService
                  .operationRouting()
                  .broadcastDeleteShards(clusterState, deleteRequest.index());
          for (ShardIterator shardIt : groupShards) {
            List<BulkItemRequest> list = requestsByShard.get(shardIt.shardId());
            if (list == null) {
              list = Lists.newArrayList();
              requestsByShard.put(shardIt.shardId(), list);
            }
            list.add(new BulkItemRequest(i, request));
          }
        } else {
          ShardId shardId =
              clusterService
                  .operationRouting()
                  .deleteShards(
                      clusterState,
                      deleteRequest.index(),
                      deleteRequest.type(),
                      deleteRequest.id(),
                      deleteRequest.routing())
                  .shardId();
          List<BulkItemRequest> list = requestsByShard.get(shardId);
          if (list == null) {
            list = Lists.newArrayList();
            requestsByShard.put(shardId, list);
          }
          list.add(new BulkItemRequest(i, request));
        }
      }
    }

    if (requestsByShard.isEmpty()) {
      listener.onResponse(new BulkResponse(responses, System.currentTimeMillis() - startTime));
      return;
    }

    final AtomicInteger counter = new AtomicInteger(requestsByShard.size());
    for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) {
      final ShardId shardId = entry.getKey();
      final List<BulkItemRequest> requests = entry.getValue();
      BulkShardRequest bulkShardRequest =
          new BulkShardRequest(
              shardId.index().name(),
              shardId.id(),
              bulkRequest.refresh(),
              requests.toArray(new BulkItemRequest[requests.size()]));
      bulkShardRequest.replicationType(bulkRequest.replicationType());
      bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel());
      shardBulkAction.execute(
          bulkShardRequest,
          new ActionListener<BulkShardResponse>() {
            @Override
            public void onResponse(BulkShardResponse bulkShardResponse) {
              synchronized (responses) {
                for (BulkItemResponse bulkItemResponse : bulkShardResponse.responses()) {
                  responses[bulkItemResponse.itemId()] = bulkItemResponse;
                }
              }
              if (counter.decrementAndGet() == 0) {
                finishHim();
              }
            }

            @Override
            public void onFailure(Throwable e) {
              // create failures for all relevant requests
              String message = ExceptionsHelper.detailedMessage(e);
              synchronized (responses) {
                for (BulkItemRequest request : requests) {
                  if (request.request() instanceof IndexRequest) {
                    IndexRequest indexRequest = (IndexRequest) request.request();
                    responses[request.id()] =
                        new BulkItemResponse(
                            request.id(),
                            indexRequest.opType().toString().toLowerCase(),
                            new BulkItemResponse.Failure(
                                indexRequest.index(),
                                indexRequest.type(),
                                indexRequest.id(),
                                message));
                  } else if (request.request() instanceof DeleteRequest) {
                    DeleteRequest deleteRequest = (DeleteRequest) request.request();
                    responses[request.id()] =
                        new BulkItemResponse(
                            request.id(),
                            "delete",
                            new BulkItemResponse.Failure(
                                deleteRequest.index(),
                                deleteRequest.type(),
                                deleteRequest.id(),
                                message));
                  }
                }
              }
              if (counter.decrementAndGet() == 0) {
                finishHim();
              }
            }

            private void finishHim() {
              listener.onResponse(
                  new BulkResponse(responses, System.currentTimeMillis() - startTime));
            }
          });
    }
  }