List<String> getIndexSettingsValidationErrors(Settings settings) {
String customPath = IndexMetaData.INDEX_DATA_PATH_SETTING.get(settings);
List<String> validationErrors = new ArrayList<>();
if (Strings.isEmpty(customPath) == false && env.sharedDataFile() == null) {
validationErrors.add("path.shared_data must be set in order to use custom data paths");
} else if (Strings.isEmpty(customPath) == false) {
Path resolvedPath = PathUtils.get(new Path[] {env.sharedDataFile()}, customPath);
if (resolvedPath == null) {
validationErrors.add(
"custom path ["
+ customPath
+ "] is not a sub-path of path.shared_data ["
+ env.sharedDataFile()
+ "]");
}
}
// norelease - this can be removed?
Integer number_of_primaries = settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, null);
Integer number_of_replicas = settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, null);
if (number_of_primaries != null && number_of_primaries <= 0) {
validationErrors.add("index must have 1 or more primary shards");
}
if (number_of_replicas != null && number_of_replicas < 0) {
validationErrors.add("index must have 0 or more replica shards");
}
return validationErrors;
}
@Override
protected Settings nodeSettings(int nodeOrdinal) {
Settings.Builder settings =
Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
.extendArray("plugin.types", Ec2DiscoveryPlugin.class.getName())
.put("cloud.aws.test.random", randomInt())
.put("cloud.aws.test.write_failures", 0.1)
.put("cloud.aws.test.read_failures", 0.1);
// if explicit, just load it and don't load from env
try {
if (Strings.hasText(System.getProperty("tests.config"))) {
settings.loadFromPath(PathUtils.get(System.getProperty("tests.config")));
} else {
throw new IllegalStateException(
"to run integration tests, you need to set -Dtest.thirdparty=true and -Dtests.config=/path/to/elasticsearch.yml");
}
} catch (SettingsException exception) {
throw new IllegalStateException(
"your test configuration file is incorrect: " + System.getProperty("tests.config"),
exception);
}
return settings.build();
}
public void testSelectNewPathForShard() throws Exception {
assumeFalse(
"Consistenty fails on windows ('could not remove the following files')", Constants.WINDOWS);
Path path = PathUtils.get(createTempDir().toString());
// Use 2 data paths:
String[] paths = new String[] {path.resolve("a").toString(), path.resolve("b").toString()};
Settings settings =
Settings.builder().put("path.home", path).putArray("path.data", paths).build();
NodeEnvironment nodeEnv = new NodeEnvironment(settings, new Environment(settings));
// Make sure all our mocking above actually worked:
NodePath[] nodePaths = nodeEnv.nodePaths();
assertEquals(2, nodePaths.length);
assertEquals("mocka", nodePaths[0].fileStore.name());
assertEquals("mockb", nodePaths[1].fileStore.name());
// Path a has lots of free space, but b has little, so new shard should go to a:
aFileStore.usableSpace = 100000;
bFileStore.usableSpace = 1000;
ShardId shardId = new ShardId("index", 0);
ShardPath result =
ShardPath.selectNewPathForShard(
nodeEnv, shardId, Settings.EMPTY, 100, Collections.<Path, Integer>emptyMap());
assertTrue(result.getDataPath().toString().contains(aPathPart));
// Test the reverse: b has lots of free space, but a has little, so new shard should go to b:
aFileStore.usableSpace = 1000;
bFileStore.usableSpace = 100000;
shardId = new ShardId("index", 0);
result =
ShardPath.selectNewPathForShard(
nodeEnv, shardId, Settings.EMPTY, 100, Collections.<Path, Integer>emptyMap());
assertTrue(result.getDataPath().toString().contains(bPathPart));
// Now a and be have equal usable space; we allocate two shards to the node, and each should go
// to different paths:
aFileStore.usableSpace = 100000;
bFileStore.usableSpace = 100000;
Map<Path, Integer> dataPathToShardCount = new HashMap<>();
ShardPath result1 =
ShardPath.selectNewPathForShard(
nodeEnv, shardId, Settings.EMPTY, 100, dataPathToShardCount);
dataPathToShardCount.put(NodeEnvironment.shardStatePathToDataPath(result1.getDataPath()), 1);
ShardPath result2 =
ShardPath.selectNewPathForShard(
nodeEnv, shardId, Settings.EMPTY, 100, dataPathToShardCount);
// #11122: this was the original failure: on a node with 2 disks that have nearly equal
// free space, we would always allocate all N incoming shards to the one path that
// had the most free space, never using the other drive unless new shards arrive
// after the first shards started using storage:
assertNotEquals(result1.getDataPath(), result2.getDataPath());
}
/**
* Resolve the custom path for a index's shard. Uses the {@code IndexMetaData.SETTING_DATA_PATH}
* setting to determine the root path for the index.
*
* @param indexSettings settings for the index
*/
@SuppressForbidden(
reason = "Lee is working on it: https://github.com/elastic/elasticsearch/pull/11065")
private Path resolveCustomLocation(@IndexSettings Settings indexSettings) {
assert indexSettings != Settings.EMPTY;
String customDataDir = indexSettings.get(IndexMetaData.SETTING_DATA_PATH);
if (customDataDir != null) {
// This assert is because this should be caught by MetaDataCreateIndexService
assert customPathsEnabled;
if (addNodeId) {
return PathUtils.get(customDataDir).resolve(Integer.toString(this.localNodeId));
} else {
return PathUtils.get(customDataDir);
}
} else {
throw new IllegalArgumentException(
"no custom " + IndexMetaData.SETTING_DATA_PATH + " setting available");
}
}
@ClusterScope(scope = SUITE, numDataNodes = 1)
public class SitePluginRelativePathConfigTests extends ElasticsearchIntegrationTest {
private final Path root = PathUtils.get(".").toAbsolutePath().getRoot();
@Override
protected Settings nodeSettings(int nodeOrdinal) {
String cwdToRoot = getRelativePath(PathUtils.get(".").toAbsolutePath());
Path pluginDir =
PathUtils.get(
cwdToRoot,
relativizeToRootIfNecessary(getDataPath("/org/elasticsearch/test_plugins")).toString());
Path tempDir = createTempDir();
boolean useRelativeInMiddleOfPath = randomBoolean();
if (useRelativeInMiddleOfPath) {
pluginDir = PathUtils.get(tempDir.toString(), getRelativePath(tempDir), pluginDir.toString());
}
return settingsBuilder()
.put(super.nodeSettings(nodeOrdinal))
.put("path.plugins", pluginDir)
.put("force.http.enabled", true)
.build();
}
@Test
public void testThatRelativePathsDontAffectPlugins() throws Exception {
HttpResponse response = httpClient().method("GET").path("/_plugin/dummy/").execute();
assertThat(response, hasStatus(OK));
}
private Path relativizeToRootIfNecessary(Path path) {
if (WINDOWS) {
return root.relativize(path);
}
return path;
}
private String getRelativePath(Path path) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < path.getNameCount(); i++) {
sb.append("..");
sb.append(path.getFileSystem().getSeparator());
}
return sb.toString();
}
public HttpRequestBuilder httpClient() {
CloseableHttpClient httpClient = HttpClients.createDefault();
return new HttpRequestBuilder(httpClient)
.httpTransport(internalCluster().getDataNodeInstance(HttpServerTransport.class));
}
}
@Override
protected Settings nodeSettings(int nodeOrdinal) {
String cwdToRoot = getRelativePath(PathUtils.get(".").toAbsolutePath());
Path pluginDir =
PathUtils.get(
cwdToRoot,
relativizeToRootIfNecessary(getDataPath("/org/elasticsearch/test_plugins")).toString());
Path tempDir = createTempDir();
boolean useRelativeInMiddleOfPath = randomBoolean();
if (useRelativeInMiddleOfPath) {
pluginDir = PathUtils.get(tempDir.toString(), getRelativePath(tempDir), pluginDir.toString());
}
return settingsBuilder()
.put(super.nodeSettings(nodeOrdinal))
.put("path.plugins", pluginDir)
.put("force.http.enabled", true)
.build();
}
/**
* Reads and returns the specified {@code policyFile}.
*
* <p>Resources (e.g. jar files and directories) listed in {@code codebases} location will be
* provided to the policy file via a system property of the short name: e.g. <code>
* ${codebase.joda-convert-1.2.jar}</code> would map to full URL.
*/
@SuppressForbidden(reason = "accesses fully qualified URLs to configure security")
static Policy readPolicy(URL policyFile, URL codebases[]) {
try {
try {
// set codebase properties
for (URL url : codebases) {
String shortName = PathUtils.get(url.toURI()).getFileName().toString();
System.setProperty("codebase." + shortName, url.toString());
}
return Policy.getInstance("JavaPolicy", new URIParameter(policyFile.toURI()));
} finally {
// clear codebase properties
for (URL url : codebases) {
String shortName = PathUtils.get(url.toURI()).getFileName().toString();
System.clearProperty("codebase." + shortName);
}
}
} catch (NoSuchAlgorithmException | URISyntaxException e) {
throw new IllegalArgumentException("unable to parse policy file `" + policyFile + "`", e);
}
}
@SuppressForbidden(reason = "discover nested jar")
private void discoverJars(URI libPath, List<URL> cp, boolean optional) {
try {
Path[] jars = FileSystemUtils.files(PathUtils.get(libPath), "*.jar");
for (Path path : jars) {
cp.add(path.toUri().toURL());
}
} catch (IOException ex) {
if (!optional) {
throw new IllegalStateException("Cannot compute plugin classpath", ex);
}
}
}
@SuppressForbidden(reason = "access /proc")
private static double[] readProcLoadavg(String procLoadavg) {
try {
List<String> lines = Files.readAllLines(PathUtils.get(procLoadavg));
if (!lines.isEmpty()) {
String[] fields = lines.get(0).split("\\s+");
return new double[] {
Double.parseDouble(fields[0]),
Double.parseDouble(fields[1]),
Double.parseDouble(fields[2])
};
}
} catch (IOException e) {
// do not fail Elasticsearch if something unexpected
// happens here
}
return null;
}
private ShardRouting corruptRandomPrimaryFile(final boolean includePerCommitFiles)
throws IOException {
ClusterState state = client().admin().cluster().prepareState().get().getState();
Index test = state.metaData().index("test").getIndex();
GroupShardsIterator shardIterators =
state.getRoutingTable().activePrimaryShardsGrouped(new String[] {"test"}, false);
List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
ShardIterator shardIterator = RandomPicks.randomFrom(random(), iterators);
ShardRouting shardRouting = shardIterator.nextOrNull();
assertNotNull(shardRouting);
assertTrue(shardRouting.primary());
assertTrue(shardRouting.assignedToNode());
String nodeId = shardRouting.currentNodeId();
NodesStatsResponse nodeStatses =
client().admin().cluster().prepareNodesStats(nodeId).setFs(true).get();
Set<Path> files = new TreeSet<>(); // treeset makes sure iteration order is deterministic
for (FsInfo.Path info : nodeStatses.getNodes().get(0).getFs()) {
String path = info.getPath();
Path file =
PathUtils.get(path)
.resolve("indices")
.resolve(test.getUUID())
.resolve(Integer.toString(shardRouting.getId()))
.resolve("index");
if (Files.exists(file)) { // multi data path might only have one path in use
try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) {
for (Path item : stream) {
if (Files.isRegularFile(item)
&& "write.lock".equals(item.getFileName().toString()) == false) {
if (includePerCommitFiles || isPerSegmentFile(item.getFileName().toString())) {
files.add(item);
}
}
}
}
}
}
pruneOldDeleteGenerations(files);
CorruptionUtils.corruptFile(random(), files.toArray(new Path[0]));
return shardRouting;
}
/** Adds access to classpath jars/classes for jar hell scan, etc */
@SuppressForbidden(reason = "accesses fully qualified URLs to configure security")
static void addClasspathPermissions(Permissions policy) throws IOException {
// add permissions to everything in classpath
// really it should be covered by lib/, but there could be e.g. agents or similar configured)
for (URL url : JarHell.parseClassPath()) {
Path path;
try {
path = PathUtils.get(url.toURI());
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
// resource itself
policy.add(new FilePermission(path.toString(), "read,readlink"));
// classes underneath
if (Files.isDirectory(path)) {
policy.add(
new FilePermission(
path.toString() + path.getFileSystem().getSeparator() + "-", "read,readlink"));
}
}
}
public List<Path> listShardFiles(ShardRouting routing) throws IOException {
NodesStatsResponse nodeStatses =
client().admin().cluster().prepareNodesStats(routing.currentNodeId()).setFs(true).get();
ClusterState state = client().admin().cluster().prepareState().get().getState();
final Index test = state.metaData().index("test").getIndex();
assertThat(routing.toString(), nodeStatses.getNodes().size(), equalTo(1));
List<Path> files = new ArrayList<>();
for (FsInfo.Path info : nodeStatses.getNodes().get(0).getFs()) {
String path = info.getPath();
Path file =
PathUtils.get(path)
.resolve(
"indices/" + test.getUUID() + "/" + Integer.toString(routing.getId()) + "/index");
if (Files.exists(file)) { // multi data path might only have one path in use
try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) {
for (Path item : stream) {
files.add(item);
}
}
}
}
return files;
}
@SuppressForbidden(reason = "needs java.io.File api to start a process")
synchronized void startInternal(
Client client, Settings settings, String nodeName, String clusterName)
throws IOException, InterruptedException {
if (process != null) {
throw new IllegalStateException("Already started");
}
List<String> params = new ArrayList<>();
if (!Constants.WINDOWS) {
params.add("bin/elasticsearch");
} else {
params.add("bin/elasticsearch.bat");
}
params.add("-Des.cluster.name=" + clusterName);
params.add("-Des.node.name=" + nodeName);
Settings.Builder externaNodeSettingsBuilder = Settings.builder();
for (Map.Entry<String, String> entry : settings.getAsMap().entrySet()) {
switch (entry.getKey()) {
case "cluster.name":
case "node.name":
case "path.home":
case "node.mode":
case "node.local":
case NetworkModule.TRANSPORT_TYPE_KEY:
case "discovery.type":
case NetworkModule.TRANSPORT_SERVICE_TYPE_KEY:
case "config.ignore_system_properties":
continue;
default:
externaNodeSettingsBuilder.put(entry.getKey(), entry.getValue());
}
}
this.externalNodeSettings = externaNodeSettingsBuilder.put(REQUIRED_SETTINGS).build();
for (Map.Entry<String, String> entry : externalNodeSettings.getAsMap().entrySet()) {
params.add("-Des." + entry.getKey() + "=" + entry.getValue());
}
params.add("-Des.path.home=" + PathUtils.get(".").toAbsolutePath());
params.add("-Des.path.conf=" + path + "/config");
ProcessBuilder builder = new ProcessBuilder(params);
builder.directory(path.toFile());
builder.inheritIO();
boolean success = false;
try {
logger.info("starting external node [{}] with: {}", nodeName, builder.command());
process = builder.start();
this.nodeInfo = null;
if (waitForNode(client, nodeName)) {
nodeInfo = nodeInfo(client, nodeName);
assert nodeInfo != null;
logger.info(
"external node {} found, version [{}], build {}",
nodeInfo.getNode(),
nodeInfo.getVersion(),
nodeInfo.getBuild());
} else {
throw new IllegalStateException("Node [" + nodeName + "] didn't join the cluster");
}
success = true;
} finally {
if (!success) {
stop();
}
}
}
static {
// just like bootstrap, initialize natives, then SM
Bootstrap.initializeNatives(true, true);
// initialize probes
Bootstrap.initializeProbes();
// check for jar hell
try {
JarHell.checkJarHell();
} catch (Exception e) {
if (Boolean.parseBoolean(System.getProperty("tests.maven"))) {
throw new RuntimeException("found jar hell in test classpath", e);
} else {
Loggers.getLogger(BootstrapForTesting.class)
.warn(
"Your ide or custom test runner has jar hell issues, "
+ "you might want to look into that",
e);
}
}
// make sure java.io.tmpdir exists always (in case code uses it in a static initializer)
Path javaTmpDir =
PathUtils.get(
Objects.requireNonNull(
System.getProperty("java.io.tmpdir"), "please set ${java.io.tmpdir} in pom.xml"));
try {
Security.ensureDirectoryExists(javaTmpDir);
} catch (Exception e) {
throw new RuntimeException("unable to create test temp directory", e);
}
// install security manager if requested
if (systemPropertyAsBoolean("tests.security.manager", true)) {
try {
Security.setCodebaseProperties();
// if its an insecure plugin, its not easy to simulate here, since we don't have a real
// plugin install.
// we just do our best so unit testing can work. integration tests for such plugins are
// essential.
String artifact = System.getProperty("tests.artifact");
String insecurePluginProp = Security.INSECURE_PLUGINS.get(artifact);
if (insecurePluginProp != null) {
System.setProperty(insecurePluginProp, "file:/-");
}
// initialize paths the same exact way as bootstrap.
Permissions perms = new Permissions();
// add permissions to everything in classpath
for (URL url : JarHell.parseClassPath()) {
Path path = PathUtils.get(url.toURI());
// resource itself
perms.add(new FilePermission(path.toString(), "read,readlink"));
// classes underneath
perms.add(
new FilePermission(
path.toString() + path.getFileSystem().getSeparator() + "-", "read,readlink"));
// crazy jython...
String filename = path.getFileName().toString();
if (filename.contains("jython") && filename.endsWith(".jar")) {
// just enough so it won't fail when it does not exist
perms.add(new FilePermission(path.getParent().toString(), "read,readlink"));
perms.add(
new FilePermission(path.getParent().resolve("Lib").toString(), "read,readlink"));
}
}
// java.io.tmpdir
Security.addPath(perms, "java.io.tmpdir", javaTmpDir, "read,readlink,write,delete");
// custom test config file
if (Strings.hasLength(System.getProperty("tests.config"))) {
perms.add(new FilePermission(System.getProperty("tests.config"), "read,readlink"));
}
// jacoco coverage output file
if (Boolean.getBoolean("tests.coverage")) {
Path coverageDir = PathUtils.get(System.getProperty("tests.coverage.dir"));
perms.add(
new FilePermission(coverageDir.resolve("jacoco.exec").toString(), "read,write"));
// in case we get fancy and use the -integration goals later:
perms.add(
new FilePermission(coverageDir.resolve("jacoco-it.exec").toString(), "read,write"));
}
Policy.setPolicy(new ESPolicy(perms));
System.setSecurityManager(new TestSecurityManager());
Security.selfTest();
if (insecurePluginProp != null) {
// initialize the plugin class, in case it has one-time hacks (unit tests often won't do
// this)
String clazz = System.getProperty("tests.plugin.classname");
if (clazz == null) {
throw new IllegalStateException(
"plugin classname is needed for insecure plugin unit tests");
}
Class.forName(clazz);
}
} catch (Exception e) {
throw new RuntimeException("unable to install test security manager", e);
}
}
}
public void testExceptionRegistration()
throws ClassNotFoundException, IOException, URISyntaxException {
final Set<Class> notRegistered = new HashSet<>();
final Set<Class> hasDedicatedWrite = new HashSet<>();
final Set<String> registered = new HashSet<>();
final String path = "/org/elasticsearch";
final Path startPath =
PathUtils.get(
ElasticsearchException.class
.getProtectionDomain()
.getCodeSource()
.getLocation()
.toURI())
.resolve("org")
.resolve("elasticsearch");
final Set<? extends Class> ignore =
Sets.newHashSet(
org.elasticsearch.test.rest.parser.RestTestParseException.class,
org.elasticsearch.index.query.TestQueryParsingException.class,
org.elasticsearch.test.rest.client.RestException.class,
org.elasticsearch.common.util.CancellableThreadsTest.CustomException.class,
org.elasticsearch.rest.BytesRestResponseTests.WithHeadersException.class,
org.elasticsearch.client.AbstractClientHeadersTests.InternalException.class);
FileVisitor<Path> visitor =
new FileVisitor<Path>() {
private Path pkgPrefix = PathUtils.get(path).getParent();
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
throws IOException {
Path next = pkgPrefix.resolve(dir.getFileName());
if (ignore.contains(next)) {
return FileVisitResult.SKIP_SUBTREE;
}
pkgPrefix = next;
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
try {
String filename = file.getFileName().toString();
if (filename.endsWith(".class")) {
Class<?> clazz = loadClass(filename);
if (ignore.contains(clazz) == false) {
if (Modifier.isAbstract(clazz.getModifiers()) == false
&& Modifier.isInterface(clazz.getModifiers()) == false
&& isEsException(clazz)) {
if (ElasticsearchException.isRegistered(clazz.getName()) == false
&& ElasticsearchException.class.equals(clazz.getEnclosingClass())
== false) {
notRegistered.add(clazz);
} else if (ElasticsearchException.isRegistered(clazz.getName())) {
registered.add(clazz.getName());
try {
if (clazz.getDeclaredMethod("writeTo", StreamOutput.class) != null) {
hasDedicatedWrite.add(clazz);
}
} catch (Exception e) {
// fair enough
}
}
}
}
}
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
return FileVisitResult.CONTINUE;
}
private boolean isEsException(Class<?> clazz) {
return ElasticsearchException.class.isAssignableFrom(clazz);
}
private Class<?> loadClass(String filename) throws ClassNotFoundException {
StringBuilder pkg = new StringBuilder();
for (Path p : pkgPrefix) {
pkg.append(p.getFileName().toString()).append(".");
}
pkg.append(filename.substring(0, filename.length() - 6));
return Thread.currentThread().getContextClassLoader().loadClass(pkg.toString());
}
@Override
public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
throw exc;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
pkgPrefix = pkgPrefix.getParent();
return FileVisitResult.CONTINUE;
}
};
Files.walkFileTree(startPath, visitor);
final Path testStartPath =
PathUtils.get(ExceptionSerializationTests.class.getResource(path).toURI());
Files.walkFileTree(testStartPath, visitor);
assertTrue(notRegistered.remove(TestException.class));
assertTrue(notRegistered.remove(UnknownHeaderException.class));
assertTrue(
"Classes subclassing ElasticsearchException must be registered \n"
+ notRegistered.toString(),
notRegistered.isEmpty());
assertTrue(registered.removeAll(ElasticsearchException.getRegisteredKeys())); // check
assertEquals(registered.toString(), 0, registered.size());
}