@Override protected Settings nodeSettings(int nodeOrdinal) { Settings.Builder settings = Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) .extendArray("plugin.types", Ec2DiscoveryPlugin.class.getName()) .put("cloud.aws.test.random", randomInt()) .put("cloud.aws.test.write_failures", 0.1) .put("cloud.aws.test.read_failures", 0.1); // if explicit, just load it and don't load from env try { if (Strings.hasText(System.getProperty("tests.config"))) { settings.loadFromPath(PathUtils.get(System.getProperty("tests.config"))); } else { throw new IllegalStateException( "to run integration tests, you need to set -Dtest.thirdparty=true and -Dtests.config=/path/to/elasticsearch.yml"); } } catch (SettingsException exception) { throw new IllegalStateException( "your test configuration file is incorrect: " + System.getProperty("tests.config"), exception); } return settings.build(); }
List<String> getIndexSettingsValidationErrors(Settings settings) { String customPath = IndexMetaData.INDEX_DATA_PATH_SETTING.get(settings); List<String> validationErrors = new ArrayList<>(); if (Strings.isEmpty(customPath) == false && env.sharedDataFile() == null) { validationErrors.add("path.shared_data must be set in order to use custom data paths"); } else if (Strings.isEmpty(customPath) == false) { Path resolvedPath = PathUtils.get(new Path[] {env.sharedDataFile()}, customPath); if (resolvedPath == null) { validationErrors.add( "custom path [" + customPath + "] is not a sub-path of path.shared_data [" + env.sharedDataFile() + "]"); } } // norelease - this can be removed? Integer number_of_primaries = settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, null); Integer number_of_replicas = settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, null); if (number_of_primaries != null && number_of_primaries <= 0) { validationErrors.add("index must have 1 or more primary shards"); } if (number_of_replicas != null && number_of_replicas < 0) { validationErrors.add("index must have 0 or more replica shards"); } return validationErrors; }
@ParametersFactory public static Iterable<Object[]> parameters() { class Parameter { private final FileSystem fileSystem; private final Function<String, Path> temp; public Parameter(FileSystem fileSystem, String root) { this( fileSystem, s -> { try { return Files.createTempDirectory(fileSystem.getPath(root), s); } catch (IOException e) { throw new RuntimeException(e); } }); } public Parameter(FileSystem fileSystem, Function<String, Path> temp) { this.fileSystem = fileSystem; this.temp = temp; } } List<Parameter> parameters = new ArrayList<>(); parameters.add(new Parameter(Jimfs.newFileSystem(Configuration.windows()), "c:\\")); parameters.add(new Parameter(Jimfs.newFileSystem(toPosix(Configuration.osX())), "/")); parameters.add(new Parameter(Jimfs.newFileSystem(toPosix(Configuration.unix())), "/")); parameters.add(new Parameter(PathUtils.getDefaultFileSystem(), LuceneTestCase::createTempDir)); return parameters .stream() .map(p -> new Object[] {p.fileSystem, p.temp}) .collect(Collectors.toList()); }
public void testSelectNewPathForShard() throws Exception { assumeFalse( "Consistenty fails on windows ('could not remove the following files')", Constants.WINDOWS); Path path = PathUtils.get(createTempDir().toString()); // Use 2 data paths: String[] paths = new String[] {path.resolve("a").toString(), path.resolve("b").toString()}; Settings settings = Settings.builder().put("path.home", path).putArray("path.data", paths).build(); NodeEnvironment nodeEnv = new NodeEnvironment(settings, new Environment(settings)); // Make sure all our mocking above actually worked: NodePath[] nodePaths = nodeEnv.nodePaths(); assertEquals(2, nodePaths.length); assertEquals("mocka", nodePaths[0].fileStore.name()); assertEquals("mockb", nodePaths[1].fileStore.name()); // Path a has lots of free space, but b has little, so new shard should go to a: aFileStore.usableSpace = 100000; bFileStore.usableSpace = 1000; ShardId shardId = new ShardId("index", 0); ShardPath result = ShardPath.selectNewPathForShard( nodeEnv, shardId, Settings.EMPTY, 100, Collections.<Path, Integer>emptyMap()); assertTrue(result.getDataPath().toString().contains(aPathPart)); // Test the reverse: b has lots of free space, but a has little, so new shard should go to b: aFileStore.usableSpace = 1000; bFileStore.usableSpace = 100000; shardId = new ShardId("index", 0); result = ShardPath.selectNewPathForShard( nodeEnv, shardId, Settings.EMPTY, 100, Collections.<Path, Integer>emptyMap()); assertTrue(result.getDataPath().toString().contains(bPathPart)); // Now a and be have equal usable space; we allocate two shards to the node, and each should go // to different paths: aFileStore.usableSpace = 100000; bFileStore.usableSpace = 100000; Map<Path, Integer> dataPathToShardCount = new HashMap<>(); ShardPath result1 = ShardPath.selectNewPathForShard( nodeEnv, shardId, Settings.EMPTY, 100, dataPathToShardCount); dataPathToShardCount.put(NodeEnvironment.shardStatePathToDataPath(result1.getDataPath()), 1); ShardPath result2 = ShardPath.selectNewPathForShard( nodeEnv, shardId, Settings.EMPTY, 100, dataPathToShardCount); // #11122: this was the original failure: on a node with 2 disks that have nearly equal // free space, we would always allocate all N incoming shards to the one path that // had the most free space, never using the other drive unless new shards arrive // after the first shards started using storage: assertNotEquals(result1.getDataPath(), result2.getDataPath()); }
/** * Resolve the custom path for a index's shard. Uses the {@code IndexMetaData.SETTING_DATA_PATH} * setting to determine the root path for the index. * * @param indexSettings settings for the index */ @SuppressForbidden( reason = "Lee is working on it: https://github.com/elastic/elasticsearch/pull/11065") private Path resolveCustomLocation(@IndexSettings Settings indexSettings) { assert indexSettings != Settings.EMPTY; String customDataDir = indexSettings.get(IndexMetaData.SETTING_DATA_PATH); if (customDataDir != null) { // This assert is because this should be caught by MetaDataCreateIndexService assert customPathsEnabled; if (addNodeId) { return PathUtils.get(customDataDir).resolve(Integer.toString(this.localNodeId)); } else { return PathUtils.get(customDataDir); } } else { throw new IllegalArgumentException( "no custom " + IndexMetaData.SETTING_DATA_PATH + " setting available"); } }
@SuppressForbidden(reason = "sets java.io.tmpdir") public InstallPluginCommandTests(FileSystem fs, Function<String, Path> temp) { this.fs = fs; this.temp = temp; this.isPosix = fs.supportedFileAttributeViews().contains("posix"); this.isReal = fs == PathUtils.getDefaultFileSystem(); PathUtilsForTesting.installMock(fs); javaIoTmpdir = System.getProperty("java.io.tmpdir"); System.setProperty("java.io.tmpdir", temp.apply("tmpdir").toString()); }
@ClusterScope(scope = SUITE, numDataNodes = 1) public class SitePluginRelativePathConfigTests extends ElasticsearchIntegrationTest { private final Path root = PathUtils.get(".").toAbsolutePath().getRoot(); @Override protected Settings nodeSettings(int nodeOrdinal) { String cwdToRoot = getRelativePath(PathUtils.get(".").toAbsolutePath()); Path pluginDir = PathUtils.get( cwdToRoot, relativizeToRootIfNecessary(getDataPath("/org/elasticsearch/test_plugins")).toString()); Path tempDir = createTempDir(); boolean useRelativeInMiddleOfPath = randomBoolean(); if (useRelativeInMiddleOfPath) { pluginDir = PathUtils.get(tempDir.toString(), getRelativePath(tempDir), pluginDir.toString()); } return settingsBuilder() .put(super.nodeSettings(nodeOrdinal)) .put("path.plugins", pluginDir) .put("force.http.enabled", true) .build(); } @Test public void testThatRelativePathsDontAffectPlugins() throws Exception { HttpResponse response = httpClient().method("GET").path("/_plugin/dummy/").execute(); assertThat(response, hasStatus(OK)); } private Path relativizeToRootIfNecessary(Path path) { if (WINDOWS) { return root.relativize(path); } return path; } private String getRelativePath(Path path) { StringBuilder sb = new StringBuilder(); for (int i = 0; i < path.getNameCount(); i++) { sb.append(".."); sb.append(path.getFileSystem().getSeparator()); } return sb.toString(); } public HttpRequestBuilder httpClient() { CloseableHttpClient httpClient = HttpClients.createDefault(); return new HttpRequestBuilder(httpClient) .httpTransport(internalCluster().getDataNodeInstance(HttpServerTransport.class)); } }
@Override protected Settings nodeSettings(int nodeOrdinal) { String cwdToRoot = getRelativePath(PathUtils.get(".").toAbsolutePath()); Path pluginDir = PathUtils.get( cwdToRoot, relativizeToRootIfNecessary(getDataPath("/org/elasticsearch/test_plugins")).toString()); Path tempDir = createTempDir(); boolean useRelativeInMiddleOfPath = randomBoolean(); if (useRelativeInMiddleOfPath) { pluginDir = PathUtils.get(tempDir.toString(), getRelativePath(tempDir), pluginDir.toString()); } return settingsBuilder() .put(super.nodeSettings(nodeOrdinal)) .put("path.plugins", pluginDir) .put("force.http.enabled", true) .build(); }
@BeforeClass public static void installMockUsableSpaceFS() throws Exception { // Necessary so when Environment.clinit runs, to gather all FileStores, it sees ours: origFileSystem = FileSystems.getDefault(); Field field = PathUtils.class.getDeclaredField("DEFAULT"); field.setAccessible(true); FileSystem mock = new MockUsableSpaceFileSystemProvider().getFileSystem(getBaseTempDirForTestClass().toUri()); field.set(null, mock); assertEquals(mock, PathUtils.getDefaultFileSystem()); }
/** * Reads and returns the specified {@code policyFile}. * * <p>Resources (e.g. jar files and directories) listed in {@code codebases} location will be * provided to the policy file via a system property of the short name: e.g. <code> * ${codebase.joda-convert-1.2.jar}</code> would map to full URL. */ @SuppressForbidden(reason = "accesses fully qualified URLs to configure security") static Policy readPolicy(URL policyFile, URL codebases[]) { try { try { // set codebase properties for (URL url : codebases) { String shortName = PathUtils.get(url.toURI()).getFileName().toString(); System.setProperty("codebase." + shortName, url.toString()); } return Policy.getInstance("JavaPolicy", new URIParameter(policyFile.toURI())); } finally { // clear codebase properties for (URL url : codebases) { String shortName = PathUtils.get(url.toURI()).getFileName().toString(); System.clearProperty("codebase." + shortName); } } } catch (NoSuchAlgorithmException | URISyntaxException e) { throw new IllegalArgumentException("unable to parse policy file `" + policyFile + "`", e); } }
@SuppressForbidden(reason = "discover nested jar") private void discoverJars(URI libPath, List<URL> cp, boolean optional) { try { Path[] jars = FileSystemUtils.files(PathUtils.get(libPath), "*.jar"); for (Path path : jars) { cp.add(path.toUri().toURL()); } } catch (IOException ex) { if (!optional) { throw new IllegalStateException("Cannot compute plugin classpath", ex); } } }
@SuppressForbidden(reason = "access /proc") private static double[] readProcLoadavg(String procLoadavg) { try { List<String> lines = Files.readAllLines(PathUtils.get(procLoadavg)); if (!lines.isEmpty()) { String[] fields = lines.get(0).split("\\s+"); return new double[] { Double.parseDouble(fields[0]), Double.parseDouble(fields[1]), Double.parseDouble(fields[2]) }; } } catch (IOException e) { // do not fail Elasticsearch if something unexpected // happens here } return null; }
private ShardRouting corruptRandomPrimaryFile(final boolean includePerCommitFiles) throws IOException { ClusterState state = client().admin().cluster().prepareState().get().getState(); Index test = state.metaData().index("test").getIndex(); GroupShardsIterator shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[] {"test"}, false); List<ShardIterator> iterators = iterableAsArrayList(shardIterators); ShardIterator shardIterator = RandomPicks.randomFrom(random(), iterators); ShardRouting shardRouting = shardIterator.nextOrNull(); assertNotNull(shardRouting); assertTrue(shardRouting.primary()); assertTrue(shardRouting.assignedToNode()); String nodeId = shardRouting.currentNodeId(); NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(nodeId).setFs(true).get(); Set<Path> files = new TreeSet<>(); // treeset makes sure iteration order is deterministic for (FsInfo.Path info : nodeStatses.getNodes().get(0).getFs()) { String path = info.getPath(); Path file = PathUtils.get(path) .resolve("indices") .resolve(test.getUUID()) .resolve(Integer.toString(shardRouting.getId())) .resolve("index"); if (Files.exists(file)) { // multi data path might only have one path in use try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) { for (Path item : stream) { if (Files.isRegularFile(item) && "write.lock".equals(item.getFileName().toString()) == false) { if (includePerCommitFiles || isPerSegmentFile(item.getFileName().toString())) { files.add(item); } } } } } } pruneOldDeleteGenerations(files); CorruptionUtils.corruptFile(random(), files.toArray(new Path[0])); return shardRouting; }
/** Adds access to classpath jars/classes for jar hell scan, etc */ @SuppressForbidden(reason = "accesses fully qualified URLs to configure security") static void addClasspathPermissions(Permissions policy) throws IOException { // add permissions to everything in classpath // really it should be covered by lib/, but there could be e.g. agents or similar configured) for (URL url : JarHell.parseClassPath()) { Path path; try { path = PathUtils.get(url.toURI()); } catch (URISyntaxException e) { throw new RuntimeException(e); } // resource itself policy.add(new FilePermission(path.toString(), "read,readlink")); // classes underneath if (Files.isDirectory(path)) { policy.add( new FilePermission( path.toString() + path.getFileSystem().getSeparator() + "-", "read,readlink")); } } }
public List<Path> listShardFiles(ShardRouting routing) throws IOException { NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(routing.currentNodeId()).setFs(true).get(); ClusterState state = client().admin().cluster().prepareState().get().getState(); final Index test = state.metaData().index("test").getIndex(); assertThat(routing.toString(), nodeStatses.getNodes().size(), equalTo(1)); List<Path> files = new ArrayList<>(); for (FsInfo.Path info : nodeStatses.getNodes().get(0).getFs()) { String path = info.getPath(); Path file = PathUtils.get(path) .resolve( "indices/" + test.getUUID() + "/" + Integer.toString(routing.getId()) + "/index"); if (Files.exists(file)) { // multi data path might only have one path in use try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) { for (Path item : stream) { files.add(item); } } } } return files; }
static { // just like bootstrap, initialize natives, then SM Bootstrap.initializeNatives(true, true); // initialize probes Bootstrap.initializeProbes(); // check for jar hell try { JarHell.checkJarHell(); } catch (Exception e) { if (Boolean.parseBoolean(System.getProperty("tests.maven"))) { throw new RuntimeException("found jar hell in test classpath", e); } else { Loggers.getLogger(BootstrapForTesting.class) .warn( "Your ide or custom test runner has jar hell issues, " + "you might want to look into that", e); } } // make sure java.io.tmpdir exists always (in case code uses it in a static initializer) Path javaTmpDir = PathUtils.get( Objects.requireNonNull( System.getProperty("java.io.tmpdir"), "please set ${java.io.tmpdir} in pom.xml")); try { Security.ensureDirectoryExists(javaTmpDir); } catch (Exception e) { throw new RuntimeException("unable to create test temp directory", e); } // install security manager if requested if (systemPropertyAsBoolean("tests.security.manager", true)) { try { Security.setCodebaseProperties(); // if its an insecure plugin, its not easy to simulate here, since we don't have a real // plugin install. // we just do our best so unit testing can work. integration tests for such plugins are // essential. String artifact = System.getProperty("tests.artifact"); String insecurePluginProp = Security.INSECURE_PLUGINS.get(artifact); if (insecurePluginProp != null) { System.setProperty(insecurePluginProp, "file:/-"); } // initialize paths the same exact way as bootstrap. Permissions perms = new Permissions(); // add permissions to everything in classpath for (URL url : JarHell.parseClassPath()) { Path path = PathUtils.get(url.toURI()); // resource itself perms.add(new FilePermission(path.toString(), "read,readlink")); // classes underneath perms.add( new FilePermission( path.toString() + path.getFileSystem().getSeparator() + "-", "read,readlink")); // crazy jython... String filename = path.getFileName().toString(); if (filename.contains("jython") && filename.endsWith(".jar")) { // just enough so it won't fail when it does not exist perms.add(new FilePermission(path.getParent().toString(), "read,readlink")); perms.add( new FilePermission(path.getParent().resolve("Lib").toString(), "read,readlink")); } } // java.io.tmpdir Security.addPath(perms, "java.io.tmpdir", javaTmpDir, "read,readlink,write,delete"); // custom test config file if (Strings.hasLength(System.getProperty("tests.config"))) { perms.add(new FilePermission(System.getProperty("tests.config"), "read,readlink")); } // jacoco coverage output file if (Boolean.getBoolean("tests.coverage")) { Path coverageDir = PathUtils.get(System.getProperty("tests.coverage.dir")); perms.add( new FilePermission(coverageDir.resolve("jacoco.exec").toString(), "read,write")); // in case we get fancy and use the -integration goals later: perms.add( new FilePermission(coverageDir.resolve("jacoco-it.exec").toString(), "read,write")); } Policy.setPolicy(new ESPolicy(perms)); System.setSecurityManager(new TestSecurityManager()); Security.selfTest(); if (insecurePluginProp != null) { // initialize the plugin class, in case it has one-time hacks (unit tests often won't do // this) String clazz = System.getProperty("tests.plugin.classname"); if (clazz == null) { throw new IllegalStateException( "plugin classname is needed for insecure plugin unit tests"); } Class.forName(clazz); } } catch (Exception e) { throw new RuntimeException("unable to install test security manager", e); } } }
public void testExceptionRegistration() throws ClassNotFoundException, IOException, URISyntaxException { final Set<Class> notRegistered = new HashSet<>(); final Set<Class> hasDedicatedWrite = new HashSet<>(); final Set<String> registered = new HashSet<>(); final String path = "/org/elasticsearch"; final Path startPath = PathUtils.get( ElasticsearchException.class .getProtectionDomain() .getCodeSource() .getLocation() .toURI()) .resolve("org") .resolve("elasticsearch"); final Set<? extends Class> ignore = Sets.newHashSet( org.elasticsearch.test.rest.parser.RestTestParseException.class, org.elasticsearch.index.query.TestQueryParsingException.class, org.elasticsearch.test.rest.client.RestException.class, org.elasticsearch.common.util.CancellableThreadsTest.CustomException.class, org.elasticsearch.rest.BytesRestResponseTests.WithHeadersException.class, org.elasticsearch.client.AbstractClientHeadersTests.InternalException.class); FileVisitor<Path> visitor = new FileVisitor<Path>() { private Path pkgPrefix = PathUtils.get(path).getParent(); @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { Path next = pkgPrefix.resolve(dir.getFileName()); if (ignore.contains(next)) { return FileVisitResult.SKIP_SUBTREE; } pkgPrefix = next; return FileVisitResult.CONTINUE; } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { try { String filename = file.getFileName().toString(); if (filename.endsWith(".class")) { Class<?> clazz = loadClass(filename); if (ignore.contains(clazz) == false) { if (Modifier.isAbstract(clazz.getModifiers()) == false && Modifier.isInterface(clazz.getModifiers()) == false && isEsException(clazz)) { if (ElasticsearchException.isRegistered(clazz.getName()) == false && ElasticsearchException.class.equals(clazz.getEnclosingClass()) == false) { notRegistered.add(clazz); } else if (ElasticsearchException.isRegistered(clazz.getName())) { registered.add(clazz.getName()); try { if (clazz.getDeclaredMethod("writeTo", StreamOutput.class) != null) { hasDedicatedWrite.add(clazz); } } catch (Exception e) { // fair enough } } } } } } catch (ClassNotFoundException e) { throw new RuntimeException(e); } return FileVisitResult.CONTINUE; } private boolean isEsException(Class<?> clazz) { return ElasticsearchException.class.isAssignableFrom(clazz); } private Class<?> loadClass(String filename) throws ClassNotFoundException { StringBuilder pkg = new StringBuilder(); for (Path p : pkgPrefix) { pkg.append(p.getFileName().toString()).append("."); } pkg.append(filename.substring(0, filename.length() - 6)); return Thread.currentThread().getContextClassLoader().loadClass(pkg.toString()); } @Override public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { throw exc; } @Override public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { pkgPrefix = pkgPrefix.getParent(); return FileVisitResult.CONTINUE; } }; Files.walkFileTree(startPath, visitor); final Path testStartPath = PathUtils.get(ExceptionSerializationTests.class.getResource(path).toURI()); Files.walkFileTree(testStartPath, visitor); assertTrue(notRegistered.remove(TestException.class)); assertTrue(notRegistered.remove(UnknownHeaderException.class)); assertTrue( "Classes subclassing ElasticsearchException must be registered \n" + notRegistered.toString(), notRegistered.isEmpty()); assertTrue(registered.removeAll(ElasticsearchException.getRegisteredKeys())); // check assertEquals(registered.toString(), 0, registered.size()); }
@SuppressForbidden(reason = "needs java.io.File api to start a process") synchronized void startInternal( Client client, Settings settings, String nodeName, String clusterName) throws IOException, InterruptedException { if (process != null) { throw new IllegalStateException("Already started"); } List<String> params = new ArrayList<>(); if (!Constants.WINDOWS) { params.add("bin/elasticsearch"); } else { params.add("bin/elasticsearch.bat"); } params.add("-Des.cluster.name=" + clusterName); params.add("-Des.node.name=" + nodeName); Settings.Builder externaNodeSettingsBuilder = Settings.builder(); for (Map.Entry<String, String> entry : settings.getAsMap().entrySet()) { switch (entry.getKey()) { case "cluster.name": case "node.name": case "path.home": case "node.mode": case "node.local": case NetworkModule.TRANSPORT_TYPE_KEY: case "discovery.type": case NetworkModule.TRANSPORT_SERVICE_TYPE_KEY: case "config.ignore_system_properties": continue; default: externaNodeSettingsBuilder.put(entry.getKey(), entry.getValue()); } } this.externalNodeSettings = externaNodeSettingsBuilder.put(REQUIRED_SETTINGS).build(); for (Map.Entry<String, String> entry : externalNodeSettings.getAsMap().entrySet()) { params.add("-Des." + entry.getKey() + "=" + entry.getValue()); } params.add("-Des.path.home=" + PathUtils.get(".").toAbsolutePath()); params.add("-Des.path.conf=" + path + "/config"); ProcessBuilder builder = new ProcessBuilder(params); builder.directory(path.toFile()); builder.inheritIO(); boolean success = false; try { logger.info("starting external node [{}] with: {}", nodeName, builder.command()); process = builder.start(); this.nodeInfo = null; if (waitForNode(client, nodeName)) { nodeInfo = nodeInfo(client, nodeName); assert nodeInfo != null; logger.info( "external node {} found, version [{}], build {}", nodeInfo.getNode(), nodeInfo.getVersion(), nodeInfo.getBuild()); } else { throw new IllegalStateException("Node [" + nodeName + "] didn't join the cluster"); } success = true; } finally { if (!success) { stop(); } } }