protected Config() { try { config = new PropertiesConfiguration(CONFIG_FILENAME); } catch (ConfigurationException e) { log.warn(String.format("error reading %s: %s", CONFIG_FILENAME, e)); config = new PropertiesConfiguration(); } lsrnIdTypeToUriPattern = new Hashtable<Resource, List<Pattern>>(); Configuration subconfig = config.subset(ROOT_CONFIG_KEY); for (Iterator<?> i = subconfig.getKeys(); i.hasNext(); ) { String key = (String) i.next(); String parts[] = key.split("\\.", 2); if (parts.length == 2) { String lsrnNamespace = parts[0]; Resource lsrnIdType = ResourceFactory.createResource(getIdentifierClassURI(lsrnNamespace)); String subkey = parts[1]; for (Object patternStr : subconfig.getList(key)) { Pattern pattern = Pattern.compile((String) patternStr); if (subkey.equals(URI_PATTERN_CONFIG_KEY)) addURIPattern(lsrnIdType, pattern); else if (subkey.equals(FAILSAFE_URI_PATTERN_CONFIG_KEY)) setFailsafeURIPattern(lsrnIdType, pattern); } } } }
@Test public void testHdfs() throws Exception { Configuration conf = HadoopConfigurationBuilder.buildHdfsConfiguration(clusterSpec, cluster, defaults); assertThat(Iterators.size(conf.getKeys()), is(1)); assertThat(conf.getString("p1"), is("hdfs1")); }
private static List<String> asYamlLines(Configuration config, int depth) { List<String> lines = Lists.newArrayList(); Set<String> prefixes = Sets.newHashSet(); Iterator<String> keys = config.getKeys(); while (keys.hasNext()) { String key = keys.next(); String[] parts = key.split("\\."); String prefix = parts[0]; if (prefixes.contains(prefix)) { continue; // skip parsed set of keys } if (parts.length == 1) { lines.add(spaces(depth * 2) + key + ": " + config.getProperty(key)); } else if (parts.length > 1) { lines.add(spaces(depth * 2) + prefix + ":"); lines.addAll(asYamlLines(config.subset(prefix), depth + 1)); } prefixes.add(prefix); } return lines; }
public static void create(final Configuration configuration) { final SparkConf sparkConf = new SparkConf(); configuration .getKeys() .forEachRemaining(key -> sparkConf.set(key, configuration.getProperty(key).toString())); sparkConf.setAppName("Apache TinkerPop's Spark-Gremlin"); CONTEXT = SparkContext.getOrCreate(sparkConf); }
/** Asserts that the actual configuration contains the expected ones. */ private static void assertContains(Configuration expected, Configuration actual) { @SuppressWarnings("rawtypes") Iterator keys = expected.getKeys(); while (keys.hasNext()) { String key = (String) keys.next(); assertPropertyEquals(key, expected.getString(key), actual.getString(key)); } }
public static void main(String argsp[]) { Iterator i = config.getKeys("whitetext"); while (i.hasNext()) { String key = (String) i.next(); System.out.println(key + ":" + config.getString(key)); } }
@Test public void testCommon() throws Exception { Configuration conf = HadoopConfigurationBuilder.buildCommonConfiguration(clusterSpec, cluster, defaults); assertThat(Iterators.size(conf.getKeys()), is(3)); assertThat(conf.getString("p1"), is("common1")); assertThat(conf.getString("p2"), is("common2")); assertThat(conf.getString("fs.default.name"), matches("hdfs://.+:8020/")); }
private static void logConfiguration(Configuration configuration) { if (LOG.isDebugEnabled()) { Iterator<String> keys = configuration.getKeys(); LOG.debug("Configuration loaded:"); while (keys.hasNext()) { String key = keys.next(); LOG.debug("{} = {}", key, configuration.getProperty(key)); } } }
public static synchronized SenseiPluginRegistry build(Configuration conf) { if (cachedRegistries.containsKey(conf)) { return cachedRegistries.get(conf); } SenseiPluginRegistry ret = new SenseiPluginRegistry(); ret.configuration = conf; Iterator<?> keysIterator = conf.getKeys(); while (keysIterator.hasNext()) { String key = (String) keysIterator.next(); if (key.endsWith(".class")) { String prefix = key.substring(0, key.indexOf(".class")); String pluginName = getNameByPrefix(prefix); String pluginClass = conf.getString(key); ret.plugins.add(new PluginHolder(ret, pluginClass, pluginName, prefix)); } if (key.endsWith(".instance")) { String prefix = key.substring(0, key.indexOf(".instance")); String pluginName = getNameByPrefix(prefix); Object pluginInstance = conf.getProperty(key); ret.plugins.add(new PluginHolder(ret, pluginInstance, pluginName, prefix)); } } for (PluginHolder pluginHolder : ret.plugins) { ret.pluginsByPrefix.put(pluginHolder.fullPrefix, pluginHolder); ret.pluginsByNames.put(pluginHolder.pluginName, pluginHolder); Iterator<?> propertyIterator = conf.getKeys(pluginHolder.fullPrefix); while (propertyIterator.hasNext()) { String propertyName = (String) propertyIterator.next(); if (propertyName.endsWith(".class")) { continue; } String property = propertyName; if (propertyName.contains(pluginHolder.fullPrefix)) { property = propertyName.substring(pluginHolder.fullPrefix.length() + 1); } pluginHolder.properties.put(property, conf.getProperty(propertyName).toString()); } } cachedRegistries.put(conf, ret); return ret; }
public ZooKeeperBenchmark(Configuration conf) throws IOException { LinkedList<String> serverList = new LinkedList<String>(); Iterator<String> serverNames = conf.getKeys("server"); while (serverNames.hasNext()) { String serverName = serverNames.next(); String address = conf.getString(serverName); serverList.add(address); } if (serverList.size() == 0) { throw new IllegalArgumentException("ZooKeeper server addresses required"); } _interval = conf.getInt("interval"); _totalOps = conf.getInt("totalOperations"); _lowerbound = conf.getInt("lowerbound"); int totaltime = conf.getInt("totalTime"); _totalTimeSeconds = Math.round((double) totaltime / 1000.0); boolean sync = conf.getBoolean("sync"); _running = new HashMap<Integer, Thread>(); _clients = new BenchmarkClient[serverList.size()]; _barrier = new CyclicBarrier(_clients.length + 1); _deadline = totaltime / _interval; LOG.info( "benchmark set with: interval: " + _interval + " total number: " + _totalOps + " threshold: " + _lowerbound + " time: " + totaltime + " sync: " + (sync ? "SYNC" : "ASYNC")); _data = ""; for (int i = 0; i < 20; i++) { // 100 bytes of important data _data += "!!!!!"; } int avgOps = _totalOps / serverList.size(); for (int i = 0; i < serverList.size(); i++) { if (sync) { _clients[i] = new SyncBenchmarkClient(this, serverList.get(i), "/zkTest", avgOps, i); } else { _clients[i] = new AsyncBenchmarkClient(this, serverList.get(i), "/zkTest", avgOps, i); } } }
/** * Asserts that the configuration matches the properties and that they have the same number of * entries. If the value for a key contains the list delimiter then this method will only confirm * that the <em>first</em> value in the delimited list matches the key's value in <code>properties * </code>. */ private static void assertPropertiesEquals( Map<String, String> properties, Configuration configuration) { int count = 0; @SuppressWarnings("rawtypes") Iterator keys = configuration.getKeys(); while (keys.hasNext()) { String key = (String) keys.next(); assertPropertyEquals(key, properties.get(key), configuration.getString(key)); count++; } assertEquals( "Configuration does not have the same number of properties", properties.size(), count); }
/** * Gets all properties in the engine configuration that are prefixed with the given prefix. * * @param prefix the prefix * @return all prefixed properties */ private Map<String, String> getConfigMap(String prefix) { Map<String, String> rval = new HashMap<>(); Iterator<?> keys = config.getKeys(prefix); while (keys.hasNext()) { String key = String.valueOf(keys.next()); if (key.equals(prefix)) { continue; } String shortKey = key.substring(prefix.length() + 1); rval.put(shortKey, config.getString(key)); } return rval; }
private Options getAsciiDocOptionsAndAttributes(ParserContext context) { Configuration config = context.getConfig(); final AttributesBuilder attributes = attributes(config.getStringArray(Keys.ASCIIDOCTOR_ATTRIBUTES)); if (config.getBoolean(Keys.ASCIIDOCTOR_ATTRIBUTES_EXPORT, false)) { final String prefix = config.getString(Keys.ASCIIDOCTOR_ATTRIBUTES_EXPORT_PREFIX, ""); for (final Iterator<String> it = config.getKeys(); it.hasNext(); ) { final String key = it.next(); if (!key.startsWith("asciidoctor")) { attributes.attribute(prefix + key.replace(".", "_"), config.getProperty(key)); } } } final Configuration optionsSubset = config.subset(Keys.ASCIIDOCTOR_OPTION); final Options options = options().attributes(attributes.get()).get(); for (final Iterator<String> iterator = optionsSubset.getKeys(); iterator.hasNext(); ) { final String name = iterator.next(); options.setOption(name, guessTypeByContent(optionsSubset.getString(name))); } options.setBaseDir(context.getFile().getParentFile().getAbsolutePath()); options.setSafe(UNSAFE); return options; }
private void append(Map<String, Object> map, Configuration config) { if (config instanceof ConfigurableEnvironmentConfiguration) { return; } if (config instanceof SystemConfiguration) { return; } if (config instanceof EnvironmentConfiguration) { return; } for (Iterator<String> iter = config.getKeys(); iter.hasNext(); ) { String key = iter.next(); map.put(key, config.getProperty(key)); } }
public HBaseStoreManager(org.apache.commons.configuration.Configuration config) throws StorageException { super(config, PORT_DEFAULT); this.tableName = config.getString(TABLE_NAME_KEY, TABLE_NAME_DEFAULT); this.hconf = HBaseConfiguration.create(); for (Map.Entry<String, String> confEntry : HBASE_CONFIGURATION.entrySet()) { if (config.containsKey(confEntry.getKey())) { hconf.set(confEntry.getValue(), config.getString(confEntry.getKey())); } } // Copy a subset of our commons config into a Hadoop config org.apache.commons.configuration.Configuration hbCommons = config.subset(HBASE_CONFIGURATION_NAMESPACE); @SuppressWarnings("unchecked") // I hope commons-config eventually fixes this Iterator<String> keys = hbCommons.getKeys(); int keysLoaded = 0; while (keys.hasNext()) { String key = keys.next(); String value = hbCommons.getString(key); logger.debug("HBase configuration: setting {}={}", key, value); hconf.set(key, value); keysLoaded++; } logger.debug("HBase configuration: set a total of {} configuration values", keysLoaded); connectionPool = new HTablePool(hconf, connectionPoolSize); openStores = new ConcurrentHashMap<String, HBaseKeyColumnValueStore>(); // TODO: allowing publicly mutate fields is bad, should be fixed features = new StoreFeatures(); features.supportsScan = true; features.supportsBatchMutation = true; features.supportsTransactions = false; features.supportsConsistentKeyOperations = true; features.supportsLocking = false; features.isKeyOrdered = false; features.isDistributed = true; features.hasLocalKeyPartition = false; }
@Test public void testOverrides() throws Exception { Configuration overrides = new PropertiesConfiguration(); overrides.addProperty("hadoop-common.p1", "overridden1"); overrides.addProperty("hadoop-common.p2", "overridden2"); overrides.addProperty("hadoop-common.fs.default.name", "not-overridden"); clusterSpec = ClusterSpec.withNoDefaults(overrides); Configuration conf = HadoopConfigurationBuilder.buildCommonConfiguration(clusterSpec, cluster, defaults); assertThat(Iterators.size(conf.getKeys()), is(3)); assertThat(conf.getString("p1"), is("overridden1")); assertThat(conf.getString("p2"), is("overridden2")); assertThat( "Can't override dynamically set properties", conf.getString("fs.default.name"), matches("hdfs://.+:8020/")); }
/** * Load properties for a given client. It first loads the default values for all properties, and * any properties already defined with Archaius ConfigurationManager. */ @Override public void loadProperties(String restClientName) { enableDynamicProperties = true; setClientName(restClientName); loadDefaultValues(); Configuration props = ConfigurationManager.getConfigInstance().subset(restClientName); for (Iterator<String> keys = props.getKeys(); keys.hasNext(); ) { String key = keys.next(); String prop = key; try { if (prop.startsWith(getNameSpace())) { prop = prop.substring(getNameSpace().length() + 1); } setPropertyInternal(prop, getStringValue(props, key)); } catch (Exception ex) { throw new RuntimeException(String.format("Property %s is invalid", prop)); } } }
public CommonsConfigurationAccessor(Configuration configuration) { accessors = new ArrayList<CommonsConfigurationEntryAccessor>(); int i = 0; Iterator keys = configuration.getKeys(); while (keys.hasNext()) { String name = (String) keys.next(); accessors.add(new CommonsConfigurationEntryAccessor(name)); i++; } // sort alphabetically Collections.sort( accessors, new Comparator<CommonsConfigurationEntryAccessor>() { public int compare( CommonsConfigurationEntryAccessor o1, CommonsConfigurationEntryAccessor o2) { return o1.getName().compareTo(o2.getName()); } }); }