private static void replacePromptPlaceholders(Settings.Builder settings, Terminal terminal) { List<String> secretToPrompt = new ArrayList<>(); List<String> textToPrompt = new ArrayList<>(); for (Map.Entry<String, String> entry : settings.internalMap().entrySet()) { switch (entry.getValue()) { case SECRET_PROMPT_VALUE: secretToPrompt.add(entry.getKey()); break; case TEXT_PROMPT_VALUE: textToPrompt.add(entry.getKey()); break; } } for (String setting : secretToPrompt) { String secretValue = promptForValue(setting, terminal, true); if (Strings.hasLength(secretValue)) { settings.put(setting, secretValue); } else { // TODO: why do we remove settings if prompt returns empty?? settings.remove(setting); } } for (String setting : textToPrompt) { String textValue = promptForValue(setting, terminal, false); if (Strings.hasLength(textValue)) { settings.put(setting, textValue); } else { // TODO: why do we remove settings if prompt returns empty?? settings.remove(setting); } } }
static void addBindPermissions(Permissions policy, Settings settings) throws IOException { // http is simple String httpRange = HttpTransportSettings.SETTING_HTTP_PORT.get(settings).getPortRangeString(); // listen is always called with 'localhost' but use wildcard to be sure, no name service is // consulted. // see SocketPermission implies() code policy.add(new SocketPermission("*:" + httpRange, "listen,resolve")); // transport is waaaay overengineered Map<String, Settings> profiles = TransportSettings.TRANSPORT_PROFILES_SETTING.get(settings).getAsGroups(); if (!profiles.containsKey(TransportSettings.DEFAULT_PROFILE)) { profiles = new HashMap<>(profiles); profiles.put(TransportSettings.DEFAULT_PROFILE, Settings.EMPTY); } // loop through all profiles and add permissions for each one, if its valid. // (otherwise NettyTransport is lenient and ignores it) for (Map.Entry<String, Settings> entry : profiles.entrySet()) { Settings profileSettings = entry.getValue(); String name = entry.getKey(); String transportRange = profileSettings.get("port", TransportSettings.PORT.get(settings)); // a profile is only valid if its the default profile, or if it has an actual name and // specifies a port boolean valid = TransportSettings.DEFAULT_PROFILE.equals(name) || (Strings.hasLength(name) && profileSettings.get("port") != null); if (valid) { // listen is always called with 'localhost' but use wildcard to be sure, no name service is // consulted. // see SocketPermission implies() code policy.add(new SocketPermission("*:" + transportRange, "listen,resolve")); } } }
private static long clusterSeed() { String property = System.getProperty(TESTS_CLUSTER_SEED); if (!Strings.hasLength(property)) { return System.nanoTime(); } return SeedUtils.parseSeed(property); }
protected ReproduceErrorMessageBuilder appendProperties(String... properties) { for (String sysPropName : properties) { if (Strings.hasLength(System.getProperty(sysPropName))) { appendOpt(sysPropName, System.getProperty(sysPropName)); } } return this; }
private static String[] resolvePathsProperty(String propertyName, String defaultValue) { String property = System.getProperty(propertyName); if (!Strings.hasLength(property)) { return defaultValue == null ? null : new String[] {defaultValue}; } else { return property.split(PATHS_SEPARATOR); } }
/** Append a single VM option. */ @Override public ReproduceErrorMessageBuilder appendOpt(String sysPropName, String value) { if (sysPropName.equals(SYSPROP_ITERATIONS())) { // we don't want the iters to be in there! return this; } if (Strings.hasLength(value)) { return super.appendOpt(sysPropName, value); } return this; }
void onStartResponse(Consumer<? super Response> onResponse, Response response) { if (Strings.hasLength(response.getScrollId()) && response.getHits().isEmpty()) { logger.debug( "First response looks like a scan response. Jumping right to the second. scroll=[{}]", response.getScrollId()); doStartNextScroll(response.getScrollId(), timeValueMillis(0), onResponse); } else { onResponse.accept(response); } }
@Inject public PatternReplaceCharFilterFactory( IndexSettings indexSettings, @Assisted String name, @Assisted Settings settings) { super(indexSettings, name); if (!Strings.hasLength(settings.get("pattern"))) { throw new IllegalArgumentException( "pattern is missing for [" + name + "] char filter of type 'pattern_replace'"); } pattern = Pattern.compile(settings.get("pattern")); replacement = settings.get("replacement", ""); // when not set or set to "", use "". }
public Builder filter(String filter) { if (!Strings.hasLength(filter)) { this.filter = null; return this; } try { XContentParser parser = XContentFactory.xContent(filter).createParser(filter); try { filter(parser.mapOrdered()); } finally { parser.close(); } return this; } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate [" + filter + "]", e); } }
/** Append a single VM option. */ @Override public ReproduceErrorMessageBuilder appendOpt(String sysPropName, String value) { if (sysPropName.equals(SYSPROP_ITERATIONS())) { // we don't want the iters to be in there! return this; } if (sysPropName.equals(SYSPROP_TESTMETHOD())) { // don't print out the test method, we print it ourselves in appendAllOpts // without filtering out the parameters (needed for REST tests) return this; } if (sysPropName.equals(SYSPROP_PREFIX())) { // we always use the default prefix return this; } if (Strings.hasLength(value)) { return super.appendOpt(sysPropName, value); } return this; }
/** * Runs across all the settings set on this builder and replaces <tt>${...}</tt> elements in the * each setting value according to the following logic: * * <p> * * <p>First, tries to resolve it against a System property ({@link System#getProperty(String)}), * next, tries and resolve it against an environment variable ({@link System#getenv(String)}), * and last, tries and replace it with another setting already set on this builder. */ public Builder replacePropertyPlaceholders() { PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false); PropertyPlaceholder.PlaceholderResolver placeholderResolver = new PropertyPlaceholder.PlaceholderResolver() { @Override public String resolvePlaceholder(String placeholderName) { if (placeholderName.startsWith("env.")) { // explicit env var prefix return System.getenv(placeholderName.substring("env.".length())); } String value = System.getProperty(placeholderName); if (value != null) { return value; } value = System.getenv(placeholderName); if (value != null) { return value; } return map.get(placeholderName); } @Override public boolean shouldIgnoreMissing(String placeholderName) { // if its an explicit env var, we are ok with not having a value for it and treat it // as optional if (placeholderName.startsWith("env.")) { return true; } return false; } }; for (Map.Entry<String, String> entry : map.entrySet()) { String value = propertyPlaceholder.replacePlaceholders(entry.getValue(), placeholderResolver); if (Strings.hasLength(value)) { map.put(entry.getKey(), value); } } return this; }
static { // just like bootstrap, initialize natives, then SM Bootstrap.initializeNatives(true, true); // initialize probes Bootstrap.initializeProbes(); // check for jar hell try { JarHell.checkJarHell(); } catch (Exception e) { if (Boolean.parseBoolean(System.getProperty("tests.maven"))) { throw new RuntimeException("found jar hell in test classpath", e); } else { Loggers.getLogger(BootstrapForTesting.class) .warn( "Your ide or custom test runner has jar hell issues, " + "you might want to look into that", e); } } // make sure java.io.tmpdir exists always (in case code uses it in a static initializer) Path javaTmpDir = PathUtils.get( Objects.requireNonNull( System.getProperty("java.io.tmpdir"), "please set ${java.io.tmpdir} in pom.xml")); try { Security.ensureDirectoryExists(javaTmpDir); } catch (Exception e) { throw new RuntimeException("unable to create test temp directory", e); } // install security manager if requested if (systemPropertyAsBoolean("tests.security.manager", true)) { try { Security.setCodebaseProperties(); // if its an insecure plugin, its not easy to simulate here, since we don't have a real // plugin install. // we just do our best so unit testing can work. integration tests for such plugins are // essential. String artifact = System.getProperty("tests.artifact"); String insecurePluginProp = Security.INSECURE_PLUGINS.get(artifact); if (insecurePluginProp != null) { System.setProperty(insecurePluginProp, "file:/-"); } // initialize paths the same exact way as bootstrap. Permissions perms = new Permissions(); // add permissions to everything in classpath for (URL url : JarHell.parseClassPath()) { Path path = PathUtils.get(url.toURI()); // resource itself perms.add(new FilePermission(path.toString(), "read,readlink")); // classes underneath perms.add( new FilePermission( path.toString() + path.getFileSystem().getSeparator() + "-", "read,readlink")); // crazy jython... String filename = path.getFileName().toString(); if (filename.contains("jython") && filename.endsWith(".jar")) { // just enough so it won't fail when it does not exist perms.add(new FilePermission(path.getParent().toString(), "read,readlink")); perms.add( new FilePermission(path.getParent().resolve("Lib").toString(), "read,readlink")); } } // java.io.tmpdir Security.addPath(perms, "java.io.tmpdir", javaTmpDir, "read,readlink,write,delete"); // custom test config file if (Strings.hasLength(System.getProperty("tests.config"))) { perms.add(new FilePermission(System.getProperty("tests.config"), "read,readlink")); } // jacoco coverage output file if (Boolean.getBoolean("tests.coverage")) { Path coverageDir = PathUtils.get(System.getProperty("tests.coverage.dir")); perms.add( new FilePermission(coverageDir.resolve("jacoco.exec").toString(), "read,write")); // in case we get fancy and use the -integration goals later: perms.add( new FilePermission(coverageDir.resolve("jacoco-it.exec").toString(), "read,write")); } Policy.setPolicy(new ESPolicy(perms)); System.setSecurityManager(new TestSecurityManager()); Security.selfTest(); if (insecurePluginProp != null) { // initialize the plugin class, in case it has one-time hacks (unit tests often won't do // this) String clazz = System.getProperty("tests.plugin.classname"); if (clazz == null) { throw new IllegalStateException( "plugin classname is needed for insecure plugin unit tests"); } Class.forName(clazz); } } catch (Exception e) { throw new RuntimeException("unable to install test security manager", e); } } }
/** * Constructs new shared file system repository * * @param name repository name * @param repositorySettings repository settings * @param indexShardRepository index shard repository * @param s3Service S3 service * @throws IOException */ @Inject public S3Repository( RepositoryName name, RepositorySettings repositorySettings, IndexShardRepository indexShardRepository, AwsS3Service s3Service) throws IOException { super(name.getName(), repositorySettings, indexShardRepository); String bucket = repositorySettings.settings().get("bucket", componentSettings.get("bucket")); if (bucket == null) { throw new RepositoryException(name.name(), "No bucket defined for s3 gateway"); } String region = repositorySettings.settings().get("region", componentSettings.get("region")); if (region == null) { // Bucket setting is not set - use global region setting String regionSetting = repositorySettings.settings().get("cloud.aws.region", settings.get("cloud.aws.region")); if (regionSetting != null) { regionSetting = regionSetting.toLowerCase(Locale.ENGLISH); if ("us-east".equals(regionSetting)) { // Default bucket - setting region to null region = null; } else if ("us-east-1".equals(regionSetting)) { region = null; } else if ("us-west".equals(regionSetting)) { region = "us-west-1"; } else if ("us-west-1".equals(regionSetting)) { region = "us-west-1"; } else if ("us-west-2".equals(regionSetting)) { region = "us-west-2"; } else if ("ap-southeast".equals(regionSetting)) { region = "ap-southeast-1"; } else if ("ap-southeast-1".equals(regionSetting)) { region = "ap-southeast-1"; } else if ("ap-southeast-2".equals(regionSetting)) { region = "ap-southeast-2"; } else if ("ap-northeast".equals(regionSetting)) { region = "ap-northeast-1"; } else if ("ap-northeast-1".equals(regionSetting)) { region = "ap-northeast-1"; } else if ("eu-west".equals(regionSetting)) { region = "EU"; } else if ("eu-west-1".equals(regionSetting)) { region = "EU"; } else if ("sa-east".equals(regionSetting)) { region = "sa-east-1"; } else if ("sa-east-1".equals(regionSetting)) { region = "sa-east-1"; } } } int concurrentStreams = repositorySettings .settings() .getAsInt("concurrent_streams", componentSettings.getAsInt("concurrent_streams", 5)); ExecutorService concurrentStreamPool = EsExecutors.newScaling( 1, concurrentStreams, 5, TimeUnit.SECONDS, EsExecutors.daemonThreadFactory(settings, "[s3_stream]")); boolean serverSideEncryption = repositorySettings .settings() .getAsBoolean( "server_side_encryption", componentSettings.getAsBoolean("server_side_encryption", false)); logger.debug( "using bucket [{}], region [{}], chunk_size [{}], concurrent_streams [{}], server_side_encryption [{}]", bucket, region, chunkSize, concurrentStreams, serverSideEncryption); blobStore = new S3BlobStore( settings, s3Service.client( region, repositorySettings.settings().get("access_key"), repositorySettings.settings().get("secret_key")), bucket, region, concurrentStreamPool, serverSideEncryption); this.chunkSize = repositorySettings .settings() .getAsBytesSize( "chunk_size", componentSettings.getAsBytesSize( "chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB))); this.compress = repositorySettings .settings() .getAsBoolean("compress", componentSettings.getAsBoolean("compress", false)); String basePath = repositorySettings.settings().get("base_path", null); if (Strings.hasLength(basePath)) { BlobPath path = new BlobPath(); for (String elem : Strings.splitStringToArray(basePath, '/')) { path = path.add(elem); } this.basePath = path; } else { this.basePath = BlobPath.cleanPath(); } }