public void testValidCtor() {
   Index index = new Index("foo", "foo");
   final Path path = createTempDir().resolve(index.getUUID()).resolve("0");
   ShardPath shardPath = new ShardPath(false, path, path, new ShardId(index, 0));
   assertFalse(shardPath.isCustomDataPath());
   assertEquals(shardPath.getDataPath(), path);
   assertEquals(shardPath.getShardStatePath(), path);
 }
 public void testIllegalCustomDataPath() {
   Index index = new Index("foo", "foo");
   final Path path = createTempDir().resolve(index.getUUID()).resolve("0");
   Exception e =
       expectThrows(
           IllegalArgumentException.class,
           () -> new ShardPath(true, path, path, new ShardId(index, 0)));
   assertThat(
       e.getMessage(),
       is("shard state path must be different to the data path when using custom data paths"));
 }
Ejemplo n.º 3
0
 @Override
 public void writeTo(StreamOutput out) throws IOException {
   out.writeString(index.getName()); // uuid will come as part of settings
   out.writeLong(version);
   out.writeByte(state.id());
   writeSettingsToStream(settings, out);
   out.writeVInt(mappings.size());
   for (ObjectCursor<MappingMetaData> cursor : mappings.values()) {
     cursor.value.writeTo(out);
   }
   out.writeVInt(aliases.size());
   for (ObjectCursor<AliasMetaData> cursor : aliases.values()) {
     cursor.value.writeTo(out);
   }
   out.writeVInt(customs.size());
   for (ObjectObjectCursor<String, Custom> cursor : customs) {
     out.writeString(cursor.key);
     cursor.value.writeTo(out);
   }
   out.writeVInt(activeAllocationIds.size());
   for (IntObjectCursor<Set<String>> cursor : activeAllocationIds) {
     out.writeVInt(cursor.key);
     DiffableUtils.StringSetValueSerializer.getInstance().write(cursor.value, out);
   }
 }
Ejemplo n.º 4
0
  @Override
  public boolean equals(Object o) {
    if (this == o) {
      return true;
    }
    if (o == null || getClass() != o.getClass()) {
      return false;
    }

    IndexMetaData that = (IndexMetaData) o;

    if (!aliases.equals(that.aliases)) {
      return false;
    }
    if (!index.equals(that.index)) {
      return false;
    }
    if (!mappings.equals(that.mappings)) {
      return false;
    }
    if (!settings.equals(that.settings)) {
      return false;
    }
    if (state != that.state) {
      return false;
    }
    if (!customs.equals(that.customs)) {
      return false;
    }
    if (!activeAllocationIds.equals(that.activeAllocationIds)) {
      return false;
    }
    return true;
  }
Ejemplo n.º 5
0
 private void freeAllContextForIndex(Index index) {
   assert index != null;
   for (SearchContext ctx : activeContexts.values()) {
     if (index.equals(ctx.indexShard().shardId().index())) {
       freeContext(ctx.id());
     }
   }
 }
Ejemplo n.º 6
0
 /** Returns all index paths. */
 public Path[] indexPaths(Index index) {
   assert assertEnvIsLocked();
   Path[] indexPaths = new Path[nodePaths.length];
   for (int i = 0; i < nodePaths.length; i++) {
     indexPaths[i] = nodePaths[i].indicesPath.resolve(index.name());
   }
   return indexPaths;
 }
Ejemplo n.º 7
0
  @Inject
  public MapperService(
      Index index,
      @IndexSettings Settings indexSettings,
      AnalysisService analysisService,
      SimilarityLookupService similarityLookupService,
      ScriptService scriptService) {
    super(index, indexSettings);
    this.analysisService = analysisService;
    this.fieldTypes = new FieldTypeLookup();
    this.documentParser =
        new DocumentMapperParser(
            indexSettings, this, analysisService, similarityLookupService, scriptService);
    this.indexAnalyzer =
        new MapperAnalyzerWrapper(analysisService.defaultIndexAnalyzer(), INDEX_ANALYZER_EXTRACTOR);
    this.searchAnalyzer =
        new MapperAnalyzerWrapper(
            analysisService.defaultSearchAnalyzer(), SEARCH_ANALYZER_EXTRACTOR);
    this.searchQuoteAnalyzer =
        new MapperAnalyzerWrapper(
            analysisService.defaultSearchQuoteAnalyzer(), SEARCH_QUOTE_ANALYZER_EXTRACTOR);

    this.dynamic = indexSettings.getAsBoolean("index.mapper.dynamic", true);
    defaultPercolatorMappingSource =
        "{\n"
            + "\"_default_\":{\n"
            + "\"properties\" : {\n"
            + "\"query\" : {\n"
            + "\"type\" : \"object\",\n"
            + "\"enabled\" : false\n"
            + "}\n"
            + "}\n"
            + "}\n"
            + "}";
    if (index.getName().equals(ScriptService.SCRIPT_INDEX)) {
      defaultMappingSource =
          "{"
              + "\"_default_\": {"
              + "\"properties\": {"
              + "\"script\": { \"enabled\": false },"
              + "\"template\": { \"enabled\": false }"
              + "}"
              + "}"
              + "}";
    } else {
      defaultMappingSource = "{\"_default_\":{}}";
    }

    if (logger.isTraceEnabled()) {
      logger.trace(
          "using dynamic[{}], default mapping source[{}], default percolator mapping source[{}]",
          dynamic,
          defaultMappingSource,
          defaultPercolatorMappingSource);
    } else if (logger.isDebugEnabled()) {
      logger.debug("using dynamic[{}]", dynamic);
    }
  }
Ejemplo n.º 8
0
 @Override
 public int hashCode() {
   int result = index.hashCode();
   result = 31 * result + state.hashCode();
   result = 31 * result + aliases.hashCode();
   result = 31 * result + settings.hashCode();
   result = 31 * result + mappings.hashCode();
   result = 31 * result + activeAllocationIds.hashCode();
   return result;
 }
 private ShardRouting corruptRandomPrimaryFile(final boolean includePerCommitFiles)
     throws IOException {
   ClusterState state = client().admin().cluster().prepareState().get().getState();
   Index test = state.metaData().index("test").getIndex();
   GroupShardsIterator shardIterators =
       state.getRoutingTable().activePrimaryShardsGrouped(new String[] {"test"}, false);
   List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
   ShardIterator shardIterator = RandomPicks.randomFrom(random(), iterators);
   ShardRouting shardRouting = shardIterator.nextOrNull();
   assertNotNull(shardRouting);
   assertTrue(shardRouting.primary());
   assertTrue(shardRouting.assignedToNode());
   String nodeId = shardRouting.currentNodeId();
   NodesStatsResponse nodeStatses =
       client().admin().cluster().prepareNodesStats(nodeId).setFs(true).get();
   Set<Path> files = new TreeSet<>(); // treeset makes sure iteration order is deterministic
   for (FsInfo.Path info : nodeStatses.getNodes().get(0).getFs()) {
     String path = info.getPath();
     Path file =
         PathUtils.get(path)
             .resolve("indices")
             .resolve(test.getUUID())
             .resolve(Integer.toString(shardRouting.getId()))
             .resolve("index");
     if (Files.exists(file)) { // multi data path might only have one path in use
       try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) {
         for (Path item : stream) {
           if (Files.isRegularFile(item)
               && "write.lock".equals(item.getFileName().toString()) == false) {
             if (includePerCommitFiles || isPerSegmentFile(item.getFileName().toString())) {
               files.add(item);
             }
           }
         }
       }
     }
   }
   pruneOldDeleteGenerations(files);
   CorruptionUtils.corruptFile(random(), files.toArray(new Path[0]));
   return shardRouting;
 }
 protected ReplicationGroup createGroup(int replicas) throws IOException {
   final Path homePath = createTempDir();
   Settings build =
       Settings.builder()
           .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
           .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicas)
           .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
           .build();
   IndexMetaData metaData =
       IndexMetaData.builder(index.getName()).settings(build).primaryTerm(0, 1).build();
   return new ReplicationGroup(metaData, homePath);
 }
Ejemplo n.º 11
0
 /**
  * Deletes an indexes data directory recursively. Note: this method assumes that the shard lock is
  * acquired
  *
  * @param index the index to delete
  * @param indexSettings settings for the index being deleted
  */
 public void deleteIndexDirectoryUnderLock(Index index, IndexSettings indexSettings)
     throws IOException {
   final Path[] indexPaths = indexPaths(index);
   logger.trace(
       "deleting index {} directory, paths({}): [{}]", index, indexPaths.length, indexPaths);
   IOUtils.rm(indexPaths);
   if (indexSettings.hasCustomDataPath()) {
     Path customLocation = resolveCustomLocation(indexSettings, index.name());
     logger.trace("deleting custom index {} directory [{}]", index, customLocation);
     IOUtils.rm(customLocation);
   }
 }
Ejemplo n.º 12
0
 /**
  * Deletes an indexes data directory recursively. Note: this method assumes that the shard lock is
  * acquired
  *
  * @param index the index to delete
  * @param indexSettings settings for the index being deleted
  */
 public void deleteIndexDirectoryUnderLock(Index index, @IndexSettings Settings indexSettings)
     throws IOException {
   // This is to ensure someone doesn't use Settings.EMPTY
   assert indexSettings != Settings.EMPTY;
   final Path[] indexPaths = indexPaths(index);
   logger.trace(
       "deleting index {} directory, paths({}): [{}]", index, indexPaths.length, indexPaths);
   IOUtils.rm(indexPaths);
   if (hasCustomDataPath(indexSettings)) {
     Path customLocation = resolveCustomLocation(indexSettings, index.name());
     logger.trace("deleting custom index {} directory [{}]", index, customLocation);
     IOUtils.rm(customLocation);
   }
 }
Ejemplo n.º 13
0
 public List<Path> listShardFiles(ShardRouting routing) throws IOException {
   NodesStatsResponse nodeStatses =
       client().admin().cluster().prepareNodesStats(routing.currentNodeId()).setFs(true).get();
   ClusterState state = client().admin().cluster().prepareState().get().getState();
   final Index test = state.metaData().index("test").getIndex();
   assertThat(routing.toString(), nodeStatses.getNodes().size(), equalTo(1));
   List<Path> files = new ArrayList<>();
   for (FsInfo.Path info : nodeStatses.getNodes().get(0).getFs()) {
     String path = info.getPath();
     Path file =
         PathUtils.get(path)
             .resolve(
                 "indices/" + test.getUUID() + "/" + Integer.toString(routing.getId()) + "/index");
     if (Files.exists(file)) { // multi data path might only have one path in use
       try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) {
         for (Path item : stream) {
           files.add(item);
         }
       }
     }
   }
   return files;
 }
 /**
  * Renames <code>indexFolderName</code> index folders found in node paths and custom path iff
  * {@link #needsUpgrade(Index, String)} is true. Index folder in custom paths are renamed first
  * followed by index folders in each node path.
  */
 void upgrade(final String indexFolderName) throws IOException {
   for (NodeEnvironment.NodePath nodePath : nodeEnv.nodePaths()) {
     final Path indexFolderPath = nodePath.indicesPath.resolve(indexFolderName);
     final IndexMetaData indexMetaData =
         IndexMetaData.FORMAT.loadLatestState(logger, indexFolderPath);
     if (indexMetaData != null) {
       final Index index = indexMetaData.getIndex();
       if (needsUpgrade(index, indexFolderName)) {
         logger.info("{} upgrading [{}] to new naming convention", index, indexFolderPath);
         final IndexSettings indexSettings = new IndexSettings(indexMetaData, settings);
         if (indexSettings.hasCustomDataPath()) {
           // we rename index folder in custom path before renaming them in any node path
           // to have the index state under a not-yet-upgraded index folder, which we use to
           // continue renaming after a incomplete upgrade.
           final Path customLocationSource =
               nodeEnv.resolveBaseCustomLocation(indexSettings).resolve(indexFolderName);
           final Path customLocationTarget = customLocationSource.resolveSibling(index.getUUID());
           // we rename the folder in custom path only the first time we encounter a state
           // in a node path, which needs upgrading, it is a no-op for subsequent node paths
           if (Files.exists(
                   customLocationSource) // might not exist if no data was written for this index
               && Files.exists(customLocationTarget) == false) {
             upgrade(index, customLocationSource, customLocationTarget);
           } else {
             logger.info("[{}] no upgrade needed - already upgraded", customLocationTarget);
           }
         }
         upgrade(index, indexFolderPath, indexFolderPath.resolveSibling(index.getUUID()));
       } else {
         logger.debug("[{}] no upgrade needed - already upgraded", indexFolderPath);
       }
     } else {
       logger.warn("[{}] no index state found - ignoring", indexFolderPath);
     }
   }
 }
  protected BlobStoreIndexGateway(
      Index index, @IndexSettings Settings indexSettings, Gateway gateway) {
    super(index, indexSettings);

    if (gateway.type().equals(NoneGateway.TYPE)) {
      logger.warn(
          "index gateway is configured, but no cluster level gateway configured, cluster level metadata will be lost on full shutdown");
    }

    this.gateway = (BlobStoreGateway) gateway;
    this.blobStore = this.gateway.blobStore();

    this.chunkSize = componentSettings.getAsBytesSize("chunk_size", this.gateway.chunkSize());

    this.indexPath = this.gateway.basePath().add("indices").add(index.name());
  }
Ejemplo n.º 16
0
 private static Set<ShardId> findAllShardsForIndex(Path indexPath, Index index)
     throws IOException {
   assert indexPath.getFileName().toString().equals(index.getUUID());
   Set<ShardId> shardIds = new HashSet<>();
   if (Files.isDirectory(indexPath)) {
     try (DirectoryStream<Path> stream = Files.newDirectoryStream(indexPath)) {
       for (Path shardPath : stream) {
         String fileName = shardPath.getFileName().toString();
         if (Files.isDirectory(shardPath) && fileName.chars().allMatch(Character::isDigit)) {
           int shardId = Integer.parseInt(fileName);
           ShardId id = new ShardId(index, shardId);
           shardIds.add(id);
         }
       }
     }
   }
   return shardIds;
 }
Ejemplo n.º 17
0
 /**
  * Tries to find all allocated shards for the given index on the current node. NOTE: This methods
  * is prone to race-conditions on the filesystem layer since it might not see directories created
  * concurrently or while it's traversing.
  *
  * @param index the index to filter shards
  * @return a set of shard IDs
  * @throws IOException if an IOException occurs
  */
 public Set<ShardId> findAllShardIds(final Index index) throws IOException {
   assert index != null;
   if (nodePaths == null || locks == null) {
     throw new IllegalStateException("node is not configured to store local location");
   }
   assert assertEnvIsLocked();
   final Set<ShardId> shardIds = Sets.newHashSet();
   String indexName = index.name();
   for (final NodePath nodePath : nodePaths) {
     Path location = nodePath.indicesPath;
     if (Files.isDirectory(location)) {
       try (DirectoryStream<Path> indexStream = Files.newDirectoryStream(location)) {
         for (Path indexPath : indexStream) {
           if (indexName.equals(indexPath.getFileName().toString())) {
             shardIds.addAll(findAllShardsForIndex(indexPath));
           }
         }
       }
     }
   }
   return shardIds;
 }
 public void setIndex(Index index) {
   if (index != null) {
     addHeader(INDEX_HEADER_KEY, index.getName());
     addHeader(INDEX_HEADER_KEY_UUID, index.getUUID());
   }
 }
Ejemplo n.º 19
0
 /**
  * Resolves index directory against this NodePath
  * ${data.paths}/nodes/{node.id}/indices/{index.uuid}
  */
 public Path resolve(Index index) {
   return indicesPath.resolve(index.getUUID());
 }
  /** Setup for the whole base test class. */
  @BeforeClass
  public static void init() throws IOException {
    // we have to prefer CURRENT since with the range of versions we support
    // it's rather unlikely to get the current actually.
    Version version =
        randomBoolean()
            ? Version.CURRENT
            : VersionUtils.randomVersionBetween(random(), Version.V_2_0_0_beta1, Version.CURRENT);
    Settings settings =
        Settings.builder()
            .put("node.name", AbstractQueryTestCase.class.toString())
            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
            .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
            .build();

    index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
    Settings indexSettings =
        Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
    final ThreadPool threadPool = new ThreadPool(settings);
    final ClusterService clusterService = createClusterService(threadPool);
    setState(
        clusterService,
        new ClusterState.Builder(clusterService.state())
            .metaData(
                new MetaData.Builder()
                    .put(
                        new IndexMetaData.Builder(index.getName())
                            .settings(indexSettings)
                            .numberOfShards(1)
                            .numberOfReplicas(0))));
    ScriptModule scriptModule = newTestScriptModule();
    List<Setting<?>> scriptSettings = scriptModule.getSettings();
    scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
    SettingsModule settingsModule =
        new SettingsModule(settings, scriptSettings, Collections.emptyList());

    IndicesModule indicesModule =
        new IndicesModule(Collections.emptyList()) {
          @Override
          protected void configure() {
            bindMapperExtension();
          }
        };
    SearchModule searchModule =
        new SearchModule(settings, false, emptyList()) {
          @Override
          protected void configureSearch() {
            // Skip me
          }
        };
    List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
    entries.addAll(indicesModule.getNamedWriteables());
    entries.addAll(searchModule.getNamedWriteables());
    namedWriteableRegistry = new NamedWriteableRegistry(entries);
    injector =
        new ModulesBuilder()
            .add(
                (b) -> {
                  b.bind(Environment.class).toInstance(new Environment(settings));
                  b.bind(ThreadPool.class).toInstance(threadPool);
                  b.bind(ScriptService.class).toInstance(scriptModule.getScriptService());
                },
                settingsModule,
                indicesModule,
                searchModule,
                new IndexSettingsModule(index, settings),
                new AbstractModule() {
                  @Override
                  protected void configure() {
                    bind(ClusterService.class).toInstance(clusterService);
                    bind(CircuitBreakerService.class).toInstance(new NoneCircuitBreakerService());
                    bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry);
                  }
                })
            .createInjector();
    aggParsers = injector.getInstance(SearchRequestParsers.class).aggParsers;
    // create some random type with some default field, those types will
    // stick around for all of the subclasses
    currentTypes = new String[randomIntBetween(0, 5)];
    for (int i = 0; i < currentTypes.length; i++) {
      String type = randomAsciiOfLengthBetween(1, 10);
      currentTypes[i] = type;
    }
    queriesRegistry = injector.getInstance(IndicesQueriesRegistry.class);
    parseFieldMatcher = ParseFieldMatcher.STRICT;
  }
Ejemplo n.º 21
0
  /**
   * This test verifies that if we corrupt a replica, we can still get to green, even though listing
   * its store fails. Note, we need to make sure that replicas are allocated on all data nodes, so
   * that replica won't be sneaky and allocated on a node that doesn't have a corrupted replica.
   */
  public void testReplicaCorruption() throws Exception {
    int numDocs = scaledRandomIntBetween(100, 1000);
    internalCluster().ensureAtLeastNumDataNodes(2);

    assertAcked(
        prepareCreate("test")
            .setSettings(
                Settings.builder()
                    .put(
                        PrimaryShardAllocator.INDEX_RECOVERY_INITIAL_SHARDS_SETTING.getKey(), "one")
                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, cluster().numDataNodes() - 1)
                    .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)
                    .put(
                        MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(),
                        false) // no checkindex - we corrupt shards on purpose
                    .put(
                        IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(),
                        new ByteSizeValue(
                            1,
                            ByteSizeUnit
                                .PB)) // no translog based flush - it might change the .liv /
                // segments.N files
                ));
    ensureGreen();
    IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs];
    for (int i = 0; i < builders.length; i++) {
      builders[i] = client().prepareIndex("test", "type").setSource("field", "value");
    }
    indexRandom(true, builders);
    ensureGreen();
    assertAllSuccessful(
        client()
            .admin()
            .indices()
            .prepareFlush()
            .setForce(true)
            .setWaitIfOngoing(true)
            .execute()
            .actionGet());
    // we have to flush at least once here since we don't corrupt the translog
    SearchResponse countResponse = client().prepareSearch().setSize(0).get();
    assertHitCount(countResponse, numDocs);

    // disable allocations of replicas post restart (the restart will change replicas to primaries,
    // so we have
    // to capture replicas post restart)
    assertAcked(
        client()
            .admin()
            .cluster()
            .prepareUpdateSettings()
            .setPersistentSettings(
                Settings.builder()
                    .put(
                        EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(),
                        "primaries")));

    internalCluster().fullRestart();

    ensureYellow();

    final Index index = resolveIndex("test");

    final IndicesShardStoresResponse stores =
        client().admin().indices().prepareShardStores(index.getName()).get();

    for (IntObjectCursor<List<IndicesShardStoresResponse.StoreStatus>> shards :
        stores.getStoreStatuses().get(index.getName())) {
      for (IndicesShardStoresResponse.StoreStatus store : shards.value) {
        final ShardId shardId = new ShardId(index, shards.key);
        if (store
            .getAllocationStatus()
            .equals(IndicesShardStoresResponse.StoreStatus.AllocationStatus.UNUSED)) {
          for (Path path : findFilesToCorruptOnNode(store.getNode().getName(), shardId)) {
            try (OutputStream os = Files.newOutputStream(path)) {
              os.write(0);
            }
            logger.info("corrupting file {} on node {}", path, store.getNode().getName());
          }
        }
      }
    }

    // enable allocation
    assertAcked(
        client()
            .admin()
            .cluster()
            .prepareUpdateSettings()
            .setPersistentSettings(
                Settings.builder()
                    .putNull(
                        EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING
                            .getKey())));

    ensureGreen();
  }
 static boolean needsUpgrade(Index index, String indexFolderName) {
   return indexFolderName.equals(index.getUUID()) == false;
 }
Ejemplo n.º 23
0
 /** Resolves the given indexes directory against this NodePath */
 public Path resolve(Index index) {
   return indicesPath.resolve(index.name());
 }
Ejemplo n.º 24
0
 public String getIndexUUID() {
   return index.getUUID();
 }
Ejemplo n.º 25
0
 public static ESLogger getLogger(
     Class clazz, Settings settings, Index index, String... prefixes) {
   return getLogger(
       clazz, settings, asArrayList(SPACE, index.getName(), prefixes).toArray(new String[0]));
 }
Ejemplo n.º 26
0
 public IndexMetaData index(Index index) {
   return index(index.getName());
 }