protected Config() {
try {
config = new PropertiesConfiguration(CONFIG_FILENAME);
} catch (ConfigurationException e) {
log.warn(String.format("error reading %s: %s", CONFIG_FILENAME, e));
config = new PropertiesConfiguration();
}
lsrnIdTypeToUriPattern = new Hashtable<Resource, List<Pattern>>();
Configuration subconfig = config.subset(ROOT_CONFIG_KEY);
for (Iterator<?> i = subconfig.getKeys(); i.hasNext(); ) {
String key = (String) i.next();
String parts[] = key.split("\\.", 2);
if (parts.length == 2) {
String lsrnNamespace = parts[0];
Resource lsrnIdType =
ResourceFactory.createResource(getIdentifierClassURI(lsrnNamespace));
String subkey = parts[1];
for (Object patternStr : subconfig.getList(key)) {
Pattern pattern = Pattern.compile((String) patternStr);
if (subkey.equals(URI_PATTERN_CONFIG_KEY)) addURIPattern(lsrnIdType, pattern);
else if (subkey.equals(FAILSAFE_URI_PATTERN_CONFIG_KEY))
setFailsafeURIPattern(lsrnIdType, pattern);
}
}
}
}
@Test
public void testHdfs() throws Exception {
Configuration conf =
HadoopConfigurationBuilder.buildHdfsConfiguration(clusterSpec, cluster, defaults);
assertThat(Iterators.size(conf.getKeys()), is(1));
assertThat(conf.getString("p1"), is("hdfs1"));
}
private static List<String> asYamlLines(Configuration config, int depth) {
List<String> lines = Lists.newArrayList();
Set<String> prefixes = Sets.newHashSet();
Iterator<String> keys = config.getKeys();
while (keys.hasNext()) {
String key = keys.next();
String[] parts = key.split("\\.");
String prefix = parts[0];
if (prefixes.contains(prefix)) {
continue; // skip parsed set of keys
}
if (parts.length == 1) {
lines.add(spaces(depth * 2) + key + ": " + config.getProperty(key));
} else if (parts.length > 1) {
lines.add(spaces(depth * 2) + prefix + ":");
lines.addAll(asYamlLines(config.subset(prefix), depth + 1));
}
prefixes.add(prefix);
}
return lines;
}
public static void create(final Configuration configuration) {
final SparkConf sparkConf = new SparkConf();
configuration
.getKeys()
.forEachRemaining(key -> sparkConf.set(key, configuration.getProperty(key).toString()));
sparkConf.setAppName("Apache TinkerPop's Spark-Gremlin");
CONTEXT = SparkContext.getOrCreate(sparkConf);
}
/** Asserts that the actual configuration contains the expected ones. */
private static void assertContains(Configuration expected, Configuration actual) {
@SuppressWarnings("rawtypes")
Iterator keys = expected.getKeys();
while (keys.hasNext()) {
String key = (String) keys.next();
assertPropertyEquals(key, expected.getString(key), actual.getString(key));
}
}
public static void main(String argsp[]) {
Iterator i = config.getKeys("whitetext");
while (i.hasNext()) {
String key = (String) i.next();
System.out.println(key + ":" + config.getString(key));
}
}
@Test
public void testCommon() throws Exception {
Configuration conf =
HadoopConfigurationBuilder.buildCommonConfiguration(clusterSpec, cluster, defaults);
assertThat(Iterators.size(conf.getKeys()), is(3));
assertThat(conf.getString("p1"), is("common1"));
assertThat(conf.getString("p2"), is("common2"));
assertThat(conf.getString("fs.default.name"), matches("hdfs://.+:8020/"));
}
private static void logConfiguration(Configuration configuration) {
if (LOG.isDebugEnabled()) {
Iterator<String> keys = configuration.getKeys();
LOG.debug("Configuration loaded:");
while (keys.hasNext()) {
String key = keys.next();
LOG.debug("{} = {}", key, configuration.getProperty(key));
}
}
}
public static synchronized SenseiPluginRegistry build(Configuration conf) {
if (cachedRegistries.containsKey(conf)) {
return cachedRegistries.get(conf);
}
SenseiPluginRegistry ret = new SenseiPluginRegistry();
ret.configuration = conf;
Iterator<?> keysIterator = conf.getKeys();
while (keysIterator.hasNext()) {
String key = (String) keysIterator.next();
if (key.endsWith(".class")) {
String prefix = key.substring(0, key.indexOf(".class"));
String pluginName = getNameByPrefix(prefix);
String pluginClass = conf.getString(key);
ret.plugins.add(new PluginHolder(ret, pluginClass, pluginName, prefix));
}
if (key.endsWith(".instance")) {
String prefix = key.substring(0, key.indexOf(".instance"));
String pluginName = getNameByPrefix(prefix);
Object pluginInstance = conf.getProperty(key);
ret.plugins.add(new PluginHolder(ret, pluginInstance, pluginName, prefix));
}
}
for (PluginHolder pluginHolder : ret.plugins) {
ret.pluginsByPrefix.put(pluginHolder.fullPrefix, pluginHolder);
ret.pluginsByNames.put(pluginHolder.pluginName, pluginHolder);
Iterator<?> propertyIterator = conf.getKeys(pluginHolder.fullPrefix);
while (propertyIterator.hasNext()) {
String propertyName = (String) propertyIterator.next();
if (propertyName.endsWith(".class")) {
continue;
}
String property = propertyName;
if (propertyName.contains(pluginHolder.fullPrefix)) {
property = propertyName.substring(pluginHolder.fullPrefix.length() + 1);
}
pluginHolder.properties.put(property, conf.getProperty(propertyName).toString());
}
}
cachedRegistries.put(conf, ret);
return ret;
}
public ZooKeeperBenchmark(Configuration conf) throws IOException {
LinkedList<String> serverList = new LinkedList<String>();
Iterator<String> serverNames = conf.getKeys("server");
while (serverNames.hasNext()) {
String serverName = serverNames.next();
String address = conf.getString(serverName);
serverList.add(address);
}
if (serverList.size() == 0) {
throw new IllegalArgumentException("ZooKeeper server addresses required");
}
_interval = conf.getInt("interval");
_totalOps = conf.getInt("totalOperations");
_lowerbound = conf.getInt("lowerbound");
int totaltime = conf.getInt("totalTime");
_totalTimeSeconds = Math.round((double) totaltime / 1000.0);
boolean sync = conf.getBoolean("sync");
_running = new HashMap<Integer, Thread>();
_clients = new BenchmarkClient[serverList.size()];
_barrier = new CyclicBarrier(_clients.length + 1);
_deadline = totaltime / _interval;
LOG.info(
"benchmark set with: interval: "
+ _interval
+ " total number: "
+ _totalOps
+ " threshold: "
+ _lowerbound
+ " time: "
+ totaltime
+ " sync: "
+ (sync ? "SYNC" : "ASYNC"));
_data = "";
for (int i = 0; i < 20; i++) { // 100 bytes of important data
_data += "!!!!!";
}
int avgOps = _totalOps / serverList.size();
for (int i = 0; i < serverList.size(); i++) {
if (sync) {
_clients[i] = new SyncBenchmarkClient(this, serverList.get(i), "/zkTest", avgOps, i);
} else {
_clients[i] = new AsyncBenchmarkClient(this, serverList.get(i), "/zkTest", avgOps, i);
}
}
}
/**
* Asserts that the configuration matches the properties and that they have the same number of
* entries. If the value for a key contains the list delimiter then this method will only confirm
* that the <em>first</em> value in the delimited list matches the key's value in <code>properties
* </code>.
*/
private static void assertPropertiesEquals(
Map<String, String> properties, Configuration configuration) {
int count = 0;
@SuppressWarnings("rawtypes")
Iterator keys = configuration.getKeys();
while (keys.hasNext()) {
String key = (String) keys.next();
assertPropertyEquals(key, properties.get(key), configuration.getString(key));
count++;
}
assertEquals(
"Configuration does not have the same number of properties", properties.size(), count);
}
/**
* Gets all properties in the engine configuration that are prefixed with the given prefix.
*
* @param prefix the prefix
* @return all prefixed properties
*/
private Map<String, String> getConfigMap(String prefix) {
Map<String, String> rval = new HashMap<>();
Iterator<?> keys = config.getKeys(prefix);
while (keys.hasNext()) {
String key = String.valueOf(keys.next());
if (key.equals(prefix)) {
continue;
}
String shortKey = key.substring(prefix.length() + 1);
rval.put(shortKey, config.getString(key));
}
return rval;
}
private Options getAsciiDocOptionsAndAttributes(ParserContext context) {
Configuration config = context.getConfig();
final AttributesBuilder attributes =
attributes(config.getStringArray(Keys.ASCIIDOCTOR_ATTRIBUTES));
if (config.getBoolean(Keys.ASCIIDOCTOR_ATTRIBUTES_EXPORT, false)) {
final String prefix = config.getString(Keys.ASCIIDOCTOR_ATTRIBUTES_EXPORT_PREFIX, "");
for (final Iterator<String> it = config.getKeys(); it.hasNext(); ) {
final String key = it.next();
if (!key.startsWith("asciidoctor")) {
attributes.attribute(prefix + key.replace(".", "_"), config.getProperty(key));
}
}
}
final Configuration optionsSubset = config.subset(Keys.ASCIIDOCTOR_OPTION);
final Options options = options().attributes(attributes.get()).get();
for (final Iterator<String> iterator = optionsSubset.getKeys(); iterator.hasNext(); ) {
final String name = iterator.next();
options.setOption(name, guessTypeByContent(optionsSubset.getString(name)));
}
options.setBaseDir(context.getFile().getParentFile().getAbsolutePath());
options.setSafe(UNSAFE);
return options;
}
private void append(Map<String, Object> map, Configuration config) {
if (config instanceof ConfigurableEnvironmentConfiguration) {
return;
}
if (config instanceof SystemConfiguration) {
return;
}
if (config instanceof EnvironmentConfiguration) {
return;
}
for (Iterator<String> iter = config.getKeys(); iter.hasNext(); ) {
String key = iter.next();
map.put(key, config.getProperty(key));
}
}
public HBaseStoreManager(org.apache.commons.configuration.Configuration config)
throws StorageException {
super(config, PORT_DEFAULT);
this.tableName = config.getString(TABLE_NAME_KEY, TABLE_NAME_DEFAULT);
this.hconf = HBaseConfiguration.create();
for (Map.Entry<String, String> confEntry : HBASE_CONFIGURATION.entrySet()) {
if (config.containsKey(confEntry.getKey())) {
hconf.set(confEntry.getValue(), config.getString(confEntry.getKey()));
}
}
// Copy a subset of our commons config into a Hadoop config
org.apache.commons.configuration.Configuration hbCommons =
config.subset(HBASE_CONFIGURATION_NAMESPACE);
@SuppressWarnings("unchecked") // I hope commons-config eventually fixes this
Iterator<String> keys = hbCommons.getKeys();
int keysLoaded = 0;
while (keys.hasNext()) {
String key = keys.next();
String value = hbCommons.getString(key);
logger.debug("HBase configuration: setting {}={}", key, value);
hconf.set(key, value);
keysLoaded++;
}
logger.debug("HBase configuration: set a total of {} configuration values", keysLoaded);
connectionPool = new HTablePool(hconf, connectionPoolSize);
openStores = new ConcurrentHashMap<String, HBaseKeyColumnValueStore>();
// TODO: allowing publicly mutate fields is bad, should be fixed
features = new StoreFeatures();
features.supportsScan = true;
features.supportsBatchMutation = true;
features.supportsTransactions = false;
features.supportsConsistentKeyOperations = true;
features.supportsLocking = false;
features.isKeyOrdered = false;
features.isDistributed = true;
features.hasLocalKeyPartition = false;
}
@Test
public void testOverrides() throws Exception {
Configuration overrides = new PropertiesConfiguration();
overrides.addProperty("hadoop-common.p1", "overridden1");
overrides.addProperty("hadoop-common.p2", "overridden2");
overrides.addProperty("hadoop-common.fs.default.name", "not-overridden");
clusterSpec = ClusterSpec.withNoDefaults(overrides);
Configuration conf =
HadoopConfigurationBuilder.buildCommonConfiguration(clusterSpec, cluster, defaults);
assertThat(Iterators.size(conf.getKeys()), is(3));
assertThat(conf.getString("p1"), is("overridden1"));
assertThat(conf.getString("p2"), is("overridden2"));
assertThat(
"Can't override dynamically set properties",
conf.getString("fs.default.name"),
matches("hdfs://.+:8020/"));
}
/**
* Load properties for a given client. It first loads the default values for all properties, and
* any properties already defined with Archaius ConfigurationManager.
*/
@Override
public void loadProperties(String restClientName) {
enableDynamicProperties = true;
setClientName(restClientName);
loadDefaultValues();
Configuration props = ConfigurationManager.getConfigInstance().subset(restClientName);
for (Iterator<String> keys = props.getKeys(); keys.hasNext(); ) {
String key = keys.next();
String prop = key;
try {
if (prop.startsWith(getNameSpace())) {
prop = prop.substring(getNameSpace().length() + 1);
}
setPropertyInternal(prop, getStringValue(props, key));
} catch (Exception ex) {
throw new RuntimeException(String.format("Property %s is invalid", prop));
}
}
}
public CommonsConfigurationAccessor(Configuration configuration) {
accessors = new ArrayList<CommonsConfigurationEntryAccessor>();
int i = 0;
Iterator keys = configuration.getKeys();
while (keys.hasNext()) {
String name = (String) keys.next();
accessors.add(new CommonsConfigurationEntryAccessor(name));
i++;
}
// sort alphabetically
Collections.sort(
accessors,
new Comparator<CommonsConfigurationEntryAccessor>() {
public int compare(
CommonsConfigurationEntryAccessor o1, CommonsConfigurationEntryAccessor o2) {
return o1.getName().compareTo(o2.getName());
}
});
}
