Exemplo n.º 1
0
 public synchronized void updateMetaData(final IndexMetaData metadata) {
   if (indexSettings.updateIndexMetaData(metadata)) {
     final Settings settings = indexSettings.getSettings();
     for (final IndexShard shard : this.shards.values()) {
       try {
         shard.onRefreshSettings(settings);
       } catch (Exception e) {
         logger.warn("[{}] failed to refresh shard settings", e, shard.shardId().id());
       }
     }
     try {
       indexStore.onRefreshSettings(settings);
     } catch (Exception e) {
       logger.warn("failed to refresh index store settings", e);
     }
     try {
       slowLog.onRefreshSettings(
           settings); // this will be refactored soon anyway so duplication is ok here
     } catch (Exception e) {
       logger.warn("failed to refresh slowlog settings", e);
     }
     if (refreshTask.getInterval().equals(indexSettings.getRefreshInterval()) == false) {
       rescheduleRefreshTasks();
     }
   }
 }
 private FsDirectoryService randomDirectorService(IndexStore indexStore, ShardPath path) {
   final IndexSettings indexSettings = indexStore.getIndexSettings();
   final IndexMetaData build =
       IndexMetaData.builder(indexSettings.getIndexMetaData())
           .settings(
               Settings.builder()
                   .put(indexSettings.getSettings())
                   .put(
                       IndexStoreModule.STORE_TYPE,
                       RandomPicks.randomFrom(random, IndexStoreModule.Type.values())
                           .getSettingsKey()))
           .build();
   final IndexSettings newIndexSettings =
       new IndexSettings(build, indexSettings.getNodeSettings(), Collections.EMPTY_LIST);
   return new FsDirectoryService(newIndexSettings, indexStore, path);
 }
Exemplo n.º 3
0
  public synchronized IndexShard createShard(ShardRouting routing) throws IOException {
    final boolean primary = routing.primary();
    /*
     * TODO: we execute this in parallel but it's a synced method. Yet, we might
     * be able to serialize the execution via the cluster state in the future. for now we just
     * keep it synced.
     */
    if (closed.get()) {
      throw new IllegalStateException("Can't create shard " + routing.shardId() + ", closed");
    }
    final Settings indexSettings = this.indexSettings.getSettings();
    final ShardId shardId = routing.shardId();
    boolean success = false;
    Store store = null;
    IndexShard indexShard = null;
    ShardLock lock = null;
    try {
      lock = nodeEnv.shardLock(shardId, TimeUnit.SECONDS.toMillis(5));
      eventListener.beforeIndexShardCreated(shardId, indexSettings);
      ShardPath path;
      try {
        path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings);
      } catch (IllegalStateException ex) {
        logger.warn("{} failed to load shard path, trying to remove leftover", shardId);
        try {
          ShardPath.deleteLeftoverShardDirectory(logger, nodeEnv, lock, this.indexSettings);
          path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings);
        } catch (Exception inner) {
          ex.addSuppressed(inner);
          throw ex;
        }
      }

      if (path == null) {
        // TODO: we should, instead, hold a "bytes reserved" of how large we anticipate this shard
        // will be, e.g. for a shard
        // that's being relocated/replicated we know how large it will become once it's done
        // copying:
        // Count up how many shards are currently on each data path:
        Map<Path, Integer> dataPathToShardCount = new HashMap<>();
        for (IndexShard shard : this) {
          Path dataPath = shard.shardPath().getRootStatePath();
          Integer curCount = dataPathToShardCount.get(dataPath);
          if (curCount == null) {
            curCount = 0;
          }
          dataPathToShardCount.put(dataPath, curCount + 1);
        }
        path =
            ShardPath.selectNewPathForShard(
                nodeEnv,
                shardId,
                this.indexSettings,
                routing.getExpectedShardSize() == ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE
                    ? getAvgShardSizeInBytes()
                    : routing.getExpectedShardSize(),
                dataPathToShardCount);
        logger.debug("{} creating using a new path [{}]", shardId, path);
      } else {
        logger.debug("{} creating using an existing path [{}]", shardId, path);
      }

      if (shards.containsKey(shardId.id())) {
        throw new IndexShardAlreadyExistsException(shardId + " already exists");
      }

      logger.debug("creating shard_id {}", shardId);
      // if we are on a shared FS we only own the shard (ie. we can safely delete it) if we are the
      // primary.
      final boolean canDeleteShardContent =
          IndexMetaData.isOnSharedFilesystem(indexSettings) == false
              || (primary && IndexMetaData.isOnSharedFilesystem(indexSettings));
      final Engine.Warmer engineWarmer =
          (searcher) -> {
            IndexShard shard = getShardOrNull(shardId.getId());
            if (shard != null) {
              warmer.warm(searcher, shard, IndexService.this.indexSettings);
            }
          };
      store =
          new Store(
              shardId,
              this.indexSettings,
              indexStore.newDirectoryService(path),
              lock,
              new StoreCloseListener(
                  shardId, canDeleteShardContent, () -> eventListener.onStoreClosed(shardId)));
      if (useShadowEngine(primary, indexSettings)) {
        indexShard =
            new ShadowIndexShard(
                routing,
                this.indexSettings,
                path,
                store,
                indexCache,
                mapperService,
                similarityService,
                indexFieldData,
                engineFactory,
                eventListener,
                searcherWrapper,
                threadPool,
                bigArrays,
                engineWarmer,
                searchOperationListeners);
        // no indexing listeners - shadow  engines don't index
      } else {
        indexShard =
            new IndexShard(
                routing,
                this.indexSettings,
                path,
                store,
                indexCache,
                mapperService,
                similarityService,
                indexFieldData,
                engineFactory,
                eventListener,
                searcherWrapper,
                threadPool,
                bigArrays,
                engineWarmer,
                searchOperationListeners,
                indexingOperationListeners);
      }
      eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created");
      eventListener.afterIndexShardCreated(indexShard);
      shards = newMapBuilder(shards).put(shardId.id(), indexShard).immutableMap();
      success = true;
      return indexShard;
    } catch (ShardLockObtainFailedException e) {
      throw new IOException("failed to obtain in-memory shard lock", e);
    } finally {
      if (success == false) {
        if (lock != null) {
          IOUtils.closeWhileHandlingException(lock);
        }
        closeShard("initialization failed", shardId, indexShard, store, eventListener);
      }
    }
  }
  private void deleteShard(
      int shardId, boolean delete, boolean snapshotGateway, boolean deleteGateway)
      throws ElasticSearchException {
    Injector shardInjector;
    IndexShard indexShard;
    synchronized (this) {
      Map<Integer, Injector> tmpShardInjectors = newHashMap(shardsInjectors);
      shardInjector = tmpShardInjectors.remove(shardId);
      if (shardInjector == null) {
        if (!delete) {
          return;
        }
        throw new IndexShardMissingException(new ShardId(index, shardId));
      }
      shardsInjectors = ImmutableMap.copyOf(tmpShardInjectors);
      if (delete) {
        logger.debug("deleting shard_id [{}]", shardId);
      }

      Map<Integer, IndexShard> tmpShardsMap = newHashMap(shards);
      indexShard = tmpShardsMap.remove(shardId);
      shards = ImmutableMap.copyOf(tmpShardsMap);
    }

    ShardId sId = new ShardId(index, shardId);

    indicesLifecycle.beforeIndexShardClosed(sId, indexShard, delete);

    for (Class<? extends CloseableIndexComponent> closeable : pluginsService.shardServices()) {
      try {
        shardInjector.getInstance(closeable).close(delete);
      } catch (Exception e) {
        logger.debug("failed to clean plugin shard service [{}]", e, closeable);
      }
    }

    try {
      // now we can close the translog service, we need to close it before the we close the shard
      shardInjector.getInstance(TranslogService.class).close();
    } catch (Exception e) {
      // ignore
    }

    // close shard actions
    if (indexShard != null) {
      shardInjector.getInstance(IndexShardManagement.class).close();
    }

    // this logic is tricky, we want to close the engine so we rollback the changes done to it
    // and close the shard so no operations are allowed to it
    if (indexShard != null) {
      indexShard.close();
    }
    try {
      shardInjector.getInstance(Engine.class).close();
    } catch (Exception e) {
      // ignore
    }

    try {
      // now, we can snapshot to the gateway, it will be only the translog
      if (snapshotGateway) {
        shardInjector.getInstance(IndexShardGatewayService.class).snapshotOnClose();
      }
    } catch (Exception e) {
      // ignore
    }
    try {
      shardInjector.getInstance(IndexShardGatewayService.class).close(deleteGateway);
    } catch (Exception e) {
      // ignore
    }
    try {
      // now we can close the translog
      shardInjector.getInstance(Translog.class).close(delete);
    } catch (Exception e) {
      // ignore
    }

    // call this before we close the store, so we can release resources for it
    indicesLifecycle.afterIndexShardClosed(sId, delete);

    // if we delete or have no gateway or the store is not persistent, clean the store...
    Store store = shardInjector.getInstance(Store.class);
    if (delete || indexGateway.type().equals(NoneGateway.TYPE) || !indexStore.persistent()) {
      try {
        store.fullDelete();
      } catch (IOException e) {
        logger.warn("failed to clean store on shard deletion", e);
      }
    }
    // and close it
    try {
      store.close();
    } catch (IOException e) {
      logger.warn("failed to close store on shard deletion", e);
    }

    Injectors.close(injector);

    // delete the shard location if needed
    if (delete || indexGateway.type().equals(NoneGateway.TYPE)) {
      FileSystemUtils.deleteRecursively(nodeEnv.shardLocation(sId));
    }
  }