コード例 #1
0
 public void testValidCtor() {
   Index index = new Index("foo", "foo");
   final Path path = createTempDir().resolve(index.getUUID()).resolve("0");
   ShardPath shardPath = new ShardPath(false, path, path, new ShardId(index, 0));
   assertFalse(shardPath.isCustomDataPath());
   assertEquals(shardPath.getDataPath(), path);
   assertEquals(shardPath.getShardStatePath(), path);
 }
コード例 #2
0
 public void testIllegalCustomDataPath() {
   Index index = new Index("foo", "foo");
   final Path path = createTempDir().resolve(index.getUUID()).resolve("0");
   Exception e =
       expectThrows(
           IllegalArgumentException.class,
           () -> new ShardPath(true, path, path, new ShardId(index, 0)));
   assertThat(
       e.getMessage(),
       is("shard state path must be different to the data path when using custom data paths"));
 }
コード例 #3
0
 /**
  * Renames <code>indexFolderName</code> index folders found in node paths and custom path iff
  * {@link #needsUpgrade(Index, String)} is true. Index folder in custom paths are renamed first
  * followed by index folders in each node path.
  */
 void upgrade(final String indexFolderName) throws IOException {
   for (NodeEnvironment.NodePath nodePath : nodeEnv.nodePaths()) {
     final Path indexFolderPath = nodePath.indicesPath.resolve(indexFolderName);
     final IndexMetaData indexMetaData =
         IndexMetaData.FORMAT.loadLatestState(logger, indexFolderPath);
     if (indexMetaData != null) {
       final Index index = indexMetaData.getIndex();
       if (needsUpgrade(index, indexFolderName)) {
         logger.info("{} upgrading [{}] to new naming convention", index, indexFolderPath);
         final IndexSettings indexSettings = new IndexSettings(indexMetaData, settings);
         if (indexSettings.hasCustomDataPath()) {
           // we rename index folder in custom path before renaming them in any node path
           // to have the index state under a not-yet-upgraded index folder, which we use to
           // continue renaming after a incomplete upgrade.
           final Path customLocationSource =
               nodeEnv.resolveBaseCustomLocation(indexSettings).resolve(indexFolderName);
           final Path customLocationTarget = customLocationSource.resolveSibling(index.getUUID());
           // we rename the folder in custom path only the first time we encounter a state
           // in a node path, which needs upgrading, it is a no-op for subsequent node paths
           if (Files.exists(
                   customLocationSource) // might not exist if no data was written for this index
               && Files.exists(customLocationTarget) == false) {
             upgrade(index, customLocationSource, customLocationTarget);
           } else {
             logger.info("[{}] no upgrade needed - already upgraded", customLocationTarget);
           }
         }
         upgrade(index, indexFolderPath, indexFolderPath.resolveSibling(index.getUUID()));
       } else {
         logger.debug("[{}] no upgrade needed - already upgraded", indexFolderPath);
       }
     } else {
       logger.warn("[{}] no index state found - ignoring", indexFolderPath);
     }
   }
 }
コード例 #4
0
 private static Set<ShardId> findAllShardsForIndex(Path indexPath, Index index)
     throws IOException {
   assert indexPath.getFileName().toString().equals(index.getUUID());
   Set<ShardId> shardIds = new HashSet<>();
   if (Files.isDirectory(indexPath)) {
     try (DirectoryStream<Path> stream = Files.newDirectoryStream(indexPath)) {
       for (Path shardPath : stream) {
         String fileName = shardPath.getFileName().toString();
         if (Files.isDirectory(shardPath) && fileName.chars().allMatch(Character::isDigit)) {
           int shardId = Integer.parseInt(fileName);
           ShardId id = new ShardId(index, shardId);
           shardIds.add(id);
         }
       }
     }
   }
   return shardIds;
 }
コード例 #5
0
 private ShardRouting corruptRandomPrimaryFile(final boolean includePerCommitFiles)
     throws IOException {
   ClusterState state = client().admin().cluster().prepareState().get().getState();
   Index test = state.metaData().index("test").getIndex();
   GroupShardsIterator shardIterators =
       state.getRoutingTable().activePrimaryShardsGrouped(new String[] {"test"}, false);
   List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
   ShardIterator shardIterator = RandomPicks.randomFrom(random(), iterators);
   ShardRouting shardRouting = shardIterator.nextOrNull();
   assertNotNull(shardRouting);
   assertTrue(shardRouting.primary());
   assertTrue(shardRouting.assignedToNode());
   String nodeId = shardRouting.currentNodeId();
   NodesStatsResponse nodeStatses =
       client().admin().cluster().prepareNodesStats(nodeId).setFs(true).get();
   Set<Path> files = new TreeSet<>(); // treeset makes sure iteration order is deterministic
   for (FsInfo.Path info : nodeStatses.getNodes().get(0).getFs()) {
     String path = info.getPath();
     Path file =
         PathUtils.get(path)
             .resolve("indices")
             .resolve(test.getUUID())
             .resolve(Integer.toString(shardRouting.getId()))
             .resolve("index");
     if (Files.exists(file)) { // multi data path might only have one path in use
       try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) {
         for (Path item : stream) {
           if (Files.isRegularFile(item)
               && "write.lock".equals(item.getFileName().toString()) == false) {
             if (includePerCommitFiles || isPerSegmentFile(item.getFileName().toString())) {
               files.add(item);
             }
           }
         }
       }
     }
   }
   pruneOldDeleteGenerations(files);
   CorruptionUtils.corruptFile(random(), files.toArray(new Path[0]));
   return shardRouting;
 }
コード例 #6
0
 /**
  * Tries to find all allocated shards for the given index on the current node. NOTE: This methods
  * is prone to race-conditions on the filesystem layer since it might not see directories created
  * concurrently or while it's traversing.
  *
  * @param index the index to filter shards
  * @return a set of shard IDs
  * @throws IOException if an IOException occurs
  */
 public Set<ShardId> findAllShardIds(final Index index) throws IOException {
   assert index != null;
   if (nodePaths == null || locks == null) {
     throw new IllegalStateException("node is not configured to store local location");
   }
   assertEnvIsLocked();
   final Set<ShardId> shardIds = new HashSet<>();
   final String indexUniquePathId = index.getUUID();
   for (final NodePath nodePath : nodePaths) {
     Path location = nodePath.indicesPath;
     if (Files.isDirectory(location)) {
       try (DirectoryStream<Path> indexStream = Files.newDirectoryStream(location)) {
         for (Path indexPath : indexStream) {
           if (indexUniquePathId.equals(indexPath.getFileName().toString())) {
             shardIds.addAll(findAllShardsForIndex(indexPath, index));
           }
         }
       }
     }
   }
   return shardIds;
 }
コード例 #7
0
 public List<Path> listShardFiles(ShardRouting routing) throws IOException {
   NodesStatsResponse nodeStatses =
       client().admin().cluster().prepareNodesStats(routing.currentNodeId()).setFs(true).get();
   ClusterState state = client().admin().cluster().prepareState().get().getState();
   final Index test = state.metaData().index("test").getIndex();
   assertThat(routing.toString(), nodeStatses.getNodes().size(), equalTo(1));
   List<Path> files = new ArrayList<>();
   for (FsInfo.Path info : nodeStatses.getNodes().get(0).getFs()) {
     String path = info.getPath();
     Path file =
         PathUtils.get(path)
             .resolve(
                 "indices/" + test.getUUID() + "/" + Integer.toString(routing.getId()) + "/index");
     if (Files.exists(file)) { // multi data path might only have one path in use
       try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) {
         for (Path item : stream) {
           files.add(item);
         }
       }
     }
   }
   return files;
 }
コード例 #8
0
 static boolean needsUpgrade(Index index, String indexFolderName) {
   return indexFolderName.equals(index.getUUID()) == false;
 }
コード例 #9
0
 public String getIndexUUID() {
   return index.getUUID();
 }
コード例 #10
0
 /**
  * Resolves index directory against this NodePath
  * ${data.paths}/nodes/{node.id}/indices/{index.uuid}
  */
 public Path resolve(Index index) {
   return indicesPath.resolve(index.getUUID());
 }
コード例 #11
0
 public void setIndex(Index index) {
   if (index != null) {
     addHeader(INDEX_HEADER_KEY, index.getName());
     addHeader(INDEX_HEADER_KEY_UUID, index.getUUID());
   }
 }