private static void replacePromptPlaceholders(Settings.Builder settings, Terminal terminal) {
List<String> secretToPrompt = new ArrayList<>();
List<String> textToPrompt = new ArrayList<>();
for (Map.Entry<String, String> entry : settings.internalMap().entrySet()) {
switch (entry.getValue()) {
case SECRET_PROMPT_VALUE:
secretToPrompt.add(entry.getKey());
break;
case TEXT_PROMPT_VALUE:
textToPrompt.add(entry.getKey());
break;
}
}
for (String setting : secretToPrompt) {
String secretValue = promptForValue(setting, terminal, true);
if (Strings.hasLength(secretValue)) {
settings.put(setting, secretValue);
} else {
// TODO: why do we remove settings if prompt returns empty??
settings.remove(setting);
}
}
for (String setting : textToPrompt) {
String textValue = promptForValue(setting, terminal, false);
if (Strings.hasLength(textValue)) {
settings.put(setting, textValue);
} else {
// TODO: why do we remove settings if prompt returns empty??
settings.remove(setting);
}
}
}
static void addBindPermissions(Permissions policy, Settings settings) throws IOException {
// http is simple
String httpRange = HttpTransportSettings.SETTING_HTTP_PORT.get(settings).getPortRangeString();
// listen is always called with 'localhost' but use wildcard to be sure, no name service is
// consulted.
// see SocketPermission implies() code
policy.add(new SocketPermission("*:" + httpRange, "listen,resolve"));
// transport is waaaay overengineered
Map<String, Settings> profiles =
TransportSettings.TRANSPORT_PROFILES_SETTING.get(settings).getAsGroups();
if (!profiles.containsKey(TransportSettings.DEFAULT_PROFILE)) {
profiles = new HashMap<>(profiles);
profiles.put(TransportSettings.DEFAULT_PROFILE, Settings.EMPTY);
}
// loop through all profiles and add permissions for each one, if its valid.
// (otherwise NettyTransport is lenient and ignores it)
for (Map.Entry<String, Settings> entry : profiles.entrySet()) {
Settings profileSettings = entry.getValue();
String name = entry.getKey();
String transportRange = profileSettings.get("port", TransportSettings.PORT.get(settings));
// a profile is only valid if its the default profile, or if it has an actual name and
// specifies a port
boolean valid =
TransportSettings.DEFAULT_PROFILE.equals(name)
|| (Strings.hasLength(name) && profileSettings.get("port") != null);
if (valid) {
// listen is always called with 'localhost' but use wildcard to be sure, no name service is
// consulted.
// see SocketPermission implies() code
policy.add(new SocketPermission("*:" + transportRange, "listen,resolve"));
}
}
}
private static long clusterSeed() {
String property = System.getProperty(TESTS_CLUSTER_SEED);
if (!Strings.hasLength(property)) {
return System.nanoTime();
}
return SeedUtils.parseSeed(property);
}
protected ReproduceErrorMessageBuilder appendProperties(String... properties) {
for (String sysPropName : properties) {
if (Strings.hasLength(System.getProperty(sysPropName))) {
appendOpt(sysPropName, System.getProperty(sysPropName));
}
}
return this;
}
private static String[] resolvePathsProperty(String propertyName, String defaultValue) {
String property = System.getProperty(propertyName);
if (!Strings.hasLength(property)) {
return defaultValue == null ? null : new String[] {defaultValue};
} else {
return property.split(PATHS_SEPARATOR);
}
}
/** Append a single VM option. */
@Override
public ReproduceErrorMessageBuilder appendOpt(String sysPropName, String value) {
if (sysPropName.equals(SYSPROP_ITERATIONS())) { // we don't want the iters to be in there!
return this;
}
if (Strings.hasLength(value)) {
return super.appendOpt(sysPropName, value);
}
return this;
}
void onStartResponse(Consumer<? super Response> onResponse, Response response) {
if (Strings.hasLength(response.getScrollId()) && response.getHits().isEmpty()) {
logger.debug(
"First response looks like a scan response. Jumping right to the second. scroll=[{}]",
response.getScrollId());
doStartNextScroll(response.getScrollId(), timeValueMillis(0), onResponse);
} else {
onResponse.accept(response);
}
}
@Inject
public PatternReplaceCharFilterFactory(
IndexSettings indexSettings, @Assisted String name, @Assisted Settings settings) {
super(indexSettings, name);
if (!Strings.hasLength(settings.get("pattern"))) {
throw new IllegalArgumentException(
"pattern is missing for [" + name + "] char filter of type 'pattern_replace'");
}
pattern = Pattern.compile(settings.get("pattern"));
replacement = settings.get("replacement", ""); // when not set or set to "", use "".
}
public Builder filter(String filter) {
if (!Strings.hasLength(filter)) {
this.filter = null;
return this;
}
try {
XContentParser parser = XContentFactory.xContent(filter).createParser(filter);
try {
filter(parser.mapOrdered());
} finally {
parser.close();
}
return this;
} catch (IOException e) {
throw new ElasticsearchGenerationException("Failed to generate [" + filter + "]", e);
}
}
/** Append a single VM option. */
@Override
public ReproduceErrorMessageBuilder appendOpt(String sysPropName, String value) {
if (sysPropName.equals(SYSPROP_ITERATIONS())) { // we don't want the iters to be in there!
return this;
}
if (sysPropName.equals(SYSPROP_TESTMETHOD())) {
// don't print out the test method, we print it ourselves in appendAllOpts
// without filtering out the parameters (needed for REST tests)
return this;
}
if (sysPropName.equals(SYSPROP_PREFIX())) {
// we always use the default prefix
return this;
}
if (Strings.hasLength(value)) {
return super.appendOpt(sysPropName, value);
}
return this;
}
/**
* Runs across all the settings set on this builder and replaces <tt>${...}</tt> elements in the
* each setting value according to the following logic:
*
* <p>
*
* <p>First, tries to resolve it against a System property ({@link System#getProperty(String)}),
* next, tries and resolve it against an environment variable ({@link System#getenv(String)}),
* and last, tries and replace it with another setting already set on this builder.
*/
public Builder replacePropertyPlaceholders() {
PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false);
PropertyPlaceholder.PlaceholderResolver placeholderResolver =
new PropertyPlaceholder.PlaceholderResolver() {
@Override
public String resolvePlaceholder(String placeholderName) {
if (placeholderName.startsWith("env.")) {
// explicit env var prefix
return System.getenv(placeholderName.substring("env.".length()));
}
String value = System.getProperty(placeholderName);
if (value != null) {
return value;
}
value = System.getenv(placeholderName);
if (value != null) {
return value;
}
return map.get(placeholderName);
}
@Override
public boolean shouldIgnoreMissing(String placeholderName) {
// if its an explicit env var, we are ok with not having a value for it and treat it
// as optional
if (placeholderName.startsWith("env.")) {
return true;
}
return false;
}
};
for (Map.Entry<String, String> entry : map.entrySet()) {
String value =
propertyPlaceholder.replacePlaceholders(entry.getValue(), placeholderResolver);
if (Strings.hasLength(value)) {
map.put(entry.getKey(), value);
}
}
return this;
}
static {
// just like bootstrap, initialize natives, then SM
Bootstrap.initializeNatives(true, true);
// initialize probes
Bootstrap.initializeProbes();
// check for jar hell
try {
JarHell.checkJarHell();
} catch (Exception e) {
if (Boolean.parseBoolean(System.getProperty("tests.maven"))) {
throw new RuntimeException("found jar hell in test classpath", e);
} else {
Loggers.getLogger(BootstrapForTesting.class)
.warn(
"Your ide or custom test runner has jar hell issues, "
+ "you might want to look into that",
e);
}
}
// make sure java.io.tmpdir exists always (in case code uses it in a static initializer)
Path javaTmpDir =
PathUtils.get(
Objects.requireNonNull(
System.getProperty("java.io.tmpdir"), "please set ${java.io.tmpdir} in pom.xml"));
try {
Security.ensureDirectoryExists(javaTmpDir);
} catch (Exception e) {
throw new RuntimeException("unable to create test temp directory", e);
}
// install security manager if requested
if (systemPropertyAsBoolean("tests.security.manager", true)) {
try {
Security.setCodebaseProperties();
// if its an insecure plugin, its not easy to simulate here, since we don't have a real
// plugin install.
// we just do our best so unit testing can work. integration tests for such plugins are
// essential.
String artifact = System.getProperty("tests.artifact");
String insecurePluginProp = Security.INSECURE_PLUGINS.get(artifact);
if (insecurePluginProp != null) {
System.setProperty(insecurePluginProp, "file:/-");
}
// initialize paths the same exact way as bootstrap.
Permissions perms = new Permissions();
// add permissions to everything in classpath
for (URL url : JarHell.parseClassPath()) {
Path path = PathUtils.get(url.toURI());
// resource itself
perms.add(new FilePermission(path.toString(), "read,readlink"));
// classes underneath
perms.add(
new FilePermission(
path.toString() + path.getFileSystem().getSeparator() + "-", "read,readlink"));
// crazy jython...
String filename = path.getFileName().toString();
if (filename.contains("jython") && filename.endsWith(".jar")) {
// just enough so it won't fail when it does not exist
perms.add(new FilePermission(path.getParent().toString(), "read,readlink"));
perms.add(
new FilePermission(path.getParent().resolve("Lib").toString(), "read,readlink"));
}
}
// java.io.tmpdir
Security.addPath(perms, "java.io.tmpdir", javaTmpDir, "read,readlink,write,delete");
// custom test config file
if (Strings.hasLength(System.getProperty("tests.config"))) {
perms.add(new FilePermission(System.getProperty("tests.config"), "read,readlink"));
}
// jacoco coverage output file
if (Boolean.getBoolean("tests.coverage")) {
Path coverageDir = PathUtils.get(System.getProperty("tests.coverage.dir"));
perms.add(
new FilePermission(coverageDir.resolve("jacoco.exec").toString(), "read,write"));
// in case we get fancy and use the -integration goals later:
perms.add(
new FilePermission(coverageDir.resolve("jacoco-it.exec").toString(), "read,write"));
}
Policy.setPolicy(new ESPolicy(perms));
System.setSecurityManager(new TestSecurityManager());
Security.selfTest();
if (insecurePluginProp != null) {
// initialize the plugin class, in case it has one-time hacks (unit tests often won't do
// this)
String clazz = System.getProperty("tests.plugin.classname");
if (clazz == null) {
throw new IllegalStateException(
"plugin classname is needed for insecure plugin unit tests");
}
Class.forName(clazz);
}
} catch (Exception e) {
throw new RuntimeException("unable to install test security manager", e);
}
}
}
/**
* Constructs new shared file system repository
*
* @param name repository name
* @param repositorySettings repository settings
* @param indexShardRepository index shard repository
* @param s3Service S3 service
* @throws IOException
*/
@Inject
public S3Repository(
RepositoryName name,
RepositorySettings repositorySettings,
IndexShardRepository indexShardRepository,
AwsS3Service s3Service)
throws IOException {
super(name.getName(), repositorySettings, indexShardRepository);
String bucket = repositorySettings.settings().get("bucket", componentSettings.get("bucket"));
if (bucket == null) {
throw new RepositoryException(name.name(), "No bucket defined for s3 gateway");
}
String region = repositorySettings.settings().get("region", componentSettings.get("region"));
if (region == null) {
// Bucket setting is not set - use global region setting
String regionSetting =
repositorySettings.settings().get("cloud.aws.region", settings.get("cloud.aws.region"));
if (regionSetting != null) {
regionSetting = regionSetting.toLowerCase(Locale.ENGLISH);
if ("us-east".equals(regionSetting)) {
// Default bucket - setting region to null
region = null;
} else if ("us-east-1".equals(regionSetting)) {
region = null;
} else if ("us-west".equals(regionSetting)) {
region = "us-west-1";
} else if ("us-west-1".equals(regionSetting)) {
region = "us-west-1";
} else if ("us-west-2".equals(regionSetting)) {
region = "us-west-2";
} else if ("ap-southeast".equals(regionSetting)) {
region = "ap-southeast-1";
} else if ("ap-southeast-1".equals(regionSetting)) {
region = "ap-southeast-1";
} else if ("ap-southeast-2".equals(regionSetting)) {
region = "ap-southeast-2";
} else if ("ap-northeast".equals(regionSetting)) {
region = "ap-northeast-1";
} else if ("ap-northeast-1".equals(regionSetting)) {
region = "ap-northeast-1";
} else if ("eu-west".equals(regionSetting)) {
region = "EU";
} else if ("eu-west-1".equals(regionSetting)) {
region = "EU";
} else if ("sa-east".equals(regionSetting)) {
region = "sa-east-1";
} else if ("sa-east-1".equals(regionSetting)) {
region = "sa-east-1";
}
}
}
int concurrentStreams =
repositorySettings
.settings()
.getAsInt("concurrent_streams", componentSettings.getAsInt("concurrent_streams", 5));
ExecutorService concurrentStreamPool =
EsExecutors.newScaling(
1,
concurrentStreams,
5,
TimeUnit.SECONDS,
EsExecutors.daemonThreadFactory(settings, "[s3_stream]"));
boolean serverSideEncryption =
repositorySettings
.settings()
.getAsBoolean(
"server_side_encryption",
componentSettings.getAsBoolean("server_side_encryption", false));
logger.debug(
"using bucket [{}], region [{}], chunk_size [{}], concurrent_streams [{}], server_side_encryption [{}]",
bucket,
region,
chunkSize,
concurrentStreams,
serverSideEncryption);
blobStore =
new S3BlobStore(
settings,
s3Service.client(
region,
repositorySettings.settings().get("access_key"),
repositorySettings.settings().get("secret_key")),
bucket,
region,
concurrentStreamPool,
serverSideEncryption);
this.chunkSize =
repositorySettings
.settings()
.getAsBytesSize(
"chunk_size",
componentSettings.getAsBytesSize(
"chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB)));
this.compress =
repositorySettings
.settings()
.getAsBoolean("compress", componentSettings.getAsBoolean("compress", false));
String basePath = repositorySettings.settings().get("base_path", null);
if (Strings.hasLength(basePath)) {
BlobPath path = new BlobPath();
for (String elem : Strings.splitStringToArray(basePath, '/')) {
path = path.add(elem);
}
this.basePath = path;
} else {
this.basePath = BlobPath.cleanPath();
}
}
