private static void checkMongoClient(
Configuration configuration, Class<?> mappedClass, String clientName, String dbName) {
Configuration mongodbClientConfiguration =
getMongoClientConfiguration(configuration, clientName);
if (mongodbClientConfiguration.isEmpty()) {
throw SeedException.createNew(MongoDbErrorCodes.UNKNOWN_CLIENT_SPECIFIED)
.put("aggregate", mappedClass.getName())
.put("clientName", clientName)
.put("dbName", dbName);
}
boolean async = mongodbClientConfiguration.getBoolean("async", false);
if (async) {
throw SeedException.createNew(MorphiaErrorCodes.ERROR_ASYNC_CLIENT)
.put("aggregate", mappedClass.getName())
.put("clientName", clientName)
.put("dbName", dbName);
}
String[] dbNames = mongodbClientConfiguration.getStringArray("databases");
boolean found = false;
for (String nameToCheck : dbNames) {
if (nameToCheck.equals(resolveDatabaseAlias(mongodbClientConfiguration, dbName))) {
found = true;
break;
}
}
if (!found) {
throw SeedException.createNew(MorphiaErrorCodes.UNKNOW_DATABASE_NAME)
.put("aggregate", mappedClass.getName())
.put("clientName", clientName)
.put("dbName", dbName);
}
}
/**
* This is to workaround the issue that {@link AbstractConfiguration} by default automatically
* convert comma delimited string to array
*/
protected static String getStringValue(Configuration config, String key) {
try {
String values[] = config.getStringArray(key);
if (values == null) {
return null;
}
if (values.length == 0) {
return config.getString(key);
} else if (values.length == 1) {
return values[0];
}
StringBuilder sb = new StringBuilder();
for (int i = 0; i < values.length; i++) {
sb.append(values[i]);
if (i != values.length - 1) {
sb.append(",");
}
}
return sb.toString();
} catch (Exception e) {
Object v = config.getProperty(key);
if (v != null) {
return String.valueOf(v);
} else {
return null;
}
}
}
@Test
public void testFromSystem_containsListValues() throws Exception {
AbstractConfiguration.setDefaultListDelimiter('|');
Map<String, String> properties = Maps.newHashMap();
properties.put("testProperty", "b,bee");
for (Entry<String, String> entry : properties.entrySet()) {
System.setProperty(entry.getKey(), entry.getValue());
}
Splitter splitter = Splitter.on(',');
Configuration systemConfiguration = configurationHelper.fromSystem();
for (Entry<String, String> entry : properties.entrySet()) {
String[] actualValues = systemConfiguration.getStringArray(entry.getKey());
String[] expectedValues;
if ("line.separator".equals(entry.getKey())) {
expectedValues = new String[] {SystemUtils.LINE_SEPARATOR};
} else {
expectedValues = splitter.splitToList(entry.getValue()).toArray(new String[0]);
}
assertArrayEquals(
String.format("Values for key %s do not match", entry.getKey()),
expectedValues,
actualValues);
}
}
public void init() {
// read network configuration
Configuration config = null;
try {
config = new PropertiesConfiguration("network.cfg");
} catch (ConfigurationException e) {
e.printStackTrace();
}
urls = config.getStringArray("node.url");
processes = new ArrayList<DA_Schiper_Eggli_Sandoz_RMI>();
// locate processes
for (String url : urls) {
try {
DA_Schiper_Eggli_Sandoz_RMI process = (DA_Schiper_Eggli_Sandoz_RMI) Naming.lookup(url);
process.reset();
processes.add(process);
} catch (RemoteException e1) {
e1.printStackTrace();
} catch (NotBoundException e2) {
e2.printStackTrace();
} catch (MalformedURLException e3) {
e3.printStackTrace();
}
}
}
/**
* Asserts that reading list values from a properties file works properly when the default list
* delimiter is not modified.
*/
@Test
public void testFromFile_listValuesWithDefaultDelimiterUnchanged() throws Exception {
Configuration configuration =
configurationHelper.fromFile(
ConfigurationHelperTest.class.getResource("props/test3.properties"));
assertPropertiesEquals(test3Properties, configuration);
String[] stringArray = configuration.getStringArray("i.j.k");
assertArrayEquals(new String[] {"foo", "bar"}, stringArray);
}
/**
* Resolve the real database name given an alias.
*
* @param clientConfiguration the configuration of the client.
* @param dbName the name of the alias or the database.
* @return the resolved database name (may be the provided database name if no alias is defined).
*/
public static String resolveDatabaseAlias(Configuration clientConfiguration, String dbName) {
String[] databases = clientConfiguration.getStringArray("databases");
if (databases != null) {
for (String database : databases) {
if (dbName.equals(
clientConfiguration.getString(String.format("alias.%s", database), dbName))) {
return database;
}
}
}
return dbName;
}
public void configure(final Configuration config) throws ConfigurationException {
this.timestampField = config.getString("timestampField");
String[] startTimeStrings = config.getStringArray("startTimes");
String[] endTimeStrings = config.getStringArray("endTimes");
Preconditions.checkNotNull(this.timestampField, "Missing property: timestampField");
Preconditions.checkArgument(
startTimeStrings.length > 0 && startTimeStrings.length == endTimeStrings.length,
"Invalid start/end time property values!");
int count = startTimeStrings.length;
DateFormat format = formatProvider.get();
this.startTimes = new long[count];
this.endTimes = new long[count];
try {
for (int ii = 0; ii < startTimeStrings.length; ii++) {
this.startTimes[ii] = format.parse(startTimeStrings[ii]).getTime();
this.endTimes[ii] = format.parse(endTimeStrings[ii]).getTime();
}
} catch (ParseException pe) {
throw new ConfigurationException("Failed to parse start/end time string", pe);
}
}
public void start(File startDir, Collection results, String xmlFileName) {
logger.debug("File: " + xmlFileName);
File fXml = new File(xmlFileName);
if (fXml.exists() && fXml.isFile()) {
config = ConfigLoader.load(xmlFileName);
anzElements = config.getStringArray("substitute/from").length;
logger.info("HtmlTagSubstitute elements=" + anzElements);
try {
walk(startDir, results);
} catch (IOException e) {
logger.error("", e);
}
logger.debug("Processed files: " + results.size());
}
}
static {
if (config == null) {
synchronized (ApiConf.class) {
if (config == null)
try {
entity = new HashMap();
config = new XMLConfiguration("entity.xml");
String[] nrs = config.getStringArray("entities.entity.nr");
String[] ids = config.getStringArray("entities.entity.id");
for (int i = 0; i < nrs.length; i++) {
entity.put(nrs[i], ids[i]);
}
} catch (ConfigurationException e) {
e.printStackTrace();
}
}
}
}
private Map<String, RoutingTableBuilder> getTableToRoutingTableBuilderMap(
Configuration routingTableBuilderConfig) {
if (routingTableBuilderConfig.containsKey(TABLES_KEY)) {
String[] tables = routingTableBuilderConfig.getStringArray(TABLES_KEY);
if ((tables != null) && (tables.length > 0)) {
Map<String, RoutingTableBuilder> routingTableBuilderMap =
new HashMap<String, RoutingTableBuilder>();
for (String table : tables) {
RoutingTableBuilder routingTableBuilder =
getRoutingTableBuilder(routingTableBuilderConfig.subset(table));
if (routingTableBuilder == null) {
LOGGER.error("RoutingTableBuilder is null for table : " + table);
} else {
routingTableBuilderMap.put(table, routingTableBuilder);
}
}
return routingTableBuilderMap;
}
}
return null;
}
/** The constructor. */
private ConfigLoader() {
Configuration conf;
try {
conf = new PropertiesConfiguration(RESOURCES_TEXGEN_PROPERTIES);
} catch (ConfigurationException e) {
throw new IllegalArgumentException(e);
}
String[] groups = conf.getStringArray(TEX_GEN_GROUPS);
for (int i = 0; i < groups.length; i++) {
String group = groups[i];
m_groups.add(new ConfigGroup(group, conf));
}
m_toolkitConfig =
new ToolkitConfig(
conf.getString(BASEDIR),
conf.getString(XML_PATH),
conf.getString(RESOURCE_BUNDLE_PATH),
conf.getString(RESOURCE_BUNDLE_FQN),
conf.getString(OUTPUTDIR),
conf.getList(TOOLKIT_NAMES));
}
private Options getAsciiDocOptionsAndAttributes(ParserContext context) {
Configuration config = context.getConfig();
final AttributesBuilder attributes =
attributes(config.getStringArray(Keys.ASCIIDOCTOR_ATTRIBUTES));
if (config.getBoolean(Keys.ASCIIDOCTOR_ATTRIBUTES_EXPORT, false)) {
final String prefix = config.getString(Keys.ASCIIDOCTOR_ATTRIBUTES_EXPORT_PREFIX, "");
for (final Iterator<String> it = config.getKeys(); it.hasNext(); ) {
final String key = it.next();
if (!key.startsWith("asciidoctor")) {
attributes.attribute(prefix + key.replace(".", "_"), config.getProperty(key));
}
}
}
final Configuration optionsSubset = config.subset(Keys.ASCIIDOCTOR_OPTION);
final Options options = options().attributes(attributes.get()).get();
for (final Iterator<String> iterator = optionsSubset.getKeys(); iterator.hasNext(); ) {
final String name = iterator.next();
options.setOption(name, guessTypeByContent(optionsSubset.getString(name)));
}
options.setBaseDir(context.getFile().getParentFile().getAbsolutePath());
options.setSafe(UNSAFE);
return options;
}
public void run(String[] args) {
try {
OptionParser parser = new OptionParser();
parser.accepts("c", "configuration file").withRequiredArg().ofType(String.class);
parser.accepts("help", "print help statement");
OptionSet options = parser.parse(args);
if (options.has("help")) {
parser.printHelpOn(System.out);
System.exit(-1);
}
// Logger configuration: log to the console
BasicConfigurator.configure();
LOG.setLevel(Level.DEBUG);
Configuration conf = new PropertiesConfiguration();
if (options.has("c")) {
String configFile = (String) options.valueOf("c");
conf = new PropertiesConfiguration(configFile);
}
double warmup_lambda =
conf.getDouble("warmup_job_arrival_rate_s", DEFAULT_WARMUP_JOB_ARRIVAL_RATE_S);
int warmup_duration_s = conf.getInt("warmup_s", DEFAULT_WARMUP_S);
int post_warmup_s = conf.getInt("post_warmup_s", DEFAULT_POST_WARMUP_S);
// We use this to represent the the rate to fully load the cluster. This is a hack.
double lambda = conf.getDouble("job_arrival_rate_s", DEFAULT_JOB_ARRIVAL_RATE_S);
int experiment_duration_s = conf.getInt("experiment_s", DEFAULT_EXPERIMENT_S);
LOG.debug(
"Using arrival rate of "
+ lambda
+ " tasks per second and running experiment for "
+ experiment_duration_s
+ " seconds.");
int tasksPerJob = conf.getInt("tasks_per_job", DEFAULT_TASKS_PER_JOB);
int numPreferredNodes = conf.getInt("num_preferred_nodes", DEFAULT_NUM_PREFERRED_NODES);
LOG.debug("Using " + numPreferredNodes + " preferred nodes for each task.");
int benchmarkIterations = conf.getInt("benchmark.iterations", DEFAULT_BENCHMARK_ITERATIONS);
int benchmarkId = conf.getInt("benchmark.id", DEFAULT_TASK_BENCHMARK);
List<String> backends = new ArrayList<String>();
if (numPreferredNodes > 0) {
/* Attempt to parse the list of slaves, which we'll need to (randomly) select preferred
* nodes. */
if (!conf.containsKey(BACKENDS)) {
LOG.fatal(
"Missing configuration backend list, which is needed to randomly select "
+ "preferred nodes (num_preferred_nodes set to "
+ numPreferredNodes
+ ")");
}
for (String node : conf.getStringArray(BACKENDS)) {
backends.add(node);
}
if (backends.size() < numPreferredNodes) {
LOG.fatal("Number of backends smaller than number of preferred nodes!");
}
}
List<SubExperiment> experiments = new ArrayList<SubExperiment>();
double fullyUtilizedArrivalRate = lambda;
// For the first twenty seconds, the first user submits at a rate to fully utilize the
// cluster.
List<UserInfo> onlyUser0 = new ArrayList<UserInfo>();
onlyUser0.add(new UserInfo("user0", 1, 0));
experiments.add(new SubExperiment(onlyUser0, 20, fullyUtilizedArrivalRate));
// For the next 10 seconds, user1 increases her rate to 25% of the cluster.
List<UserInfo> user1QuarterDemand = new ArrayList<UserInfo>();
user1QuarterDemand.add(new UserInfo("user0", 4, 0));
user1QuarterDemand.add(new UserInfo("user1", 5, 0));
experiments.add(new SubExperiment(user1QuarterDemand, 10, 1.25 * fullyUtilizedArrivalRate));
// For the next 10 seconds, user 1 increases her rate to 50% of the cluster (using exactly
// her share, but no more).
List<UserInfo> user1HalfDemand = new ArrayList<UserInfo>();
user1HalfDemand.add(new UserInfo("user0", 2, 0));
user1HalfDemand.add(new UserInfo("user1", 3, 0));
experiments.add(new SubExperiment(user1HalfDemand, 10, 1.5 * fullyUtilizedArrivalRate));
// Next user 1 goes back down to 25%.
experiments.add(new SubExperiment(user1QuarterDemand, 10, 1.25 * fullyUtilizedArrivalRate));
// Finally user 1 goes back to 0.
experiments.add(new SubExperiment(onlyUser0, 20, fullyUtilizedArrivalRate));
SparrowFrontendClient client = new SparrowFrontendClient();
int schedulerPort =
conf.getInt("scheduler_port", SchedulerThrift.DEFAULT_SCHEDULER_THRIFT_PORT);
client.initialize(new InetSocketAddress("localhost", schedulerPort), APPLICATION_ID, this);
if (warmup_duration_s > 0) {
List<SubExperiment> warmupExperiment = new ArrayList<SubExperiment>();
List<UserInfo> warmupUsers = new ArrayList<UserInfo>();
warmupUsers.add(new UserInfo("warmupUser", 1, 0));
warmupExperiment.add(new SubExperiment(warmupUsers, warmup_duration_s, warmup_lambda));
LOG.debug(
"Warming up for "
+ warmup_duration_s
+ " seconds at arrival rate of "
+ warmup_lambda
+ " jobs per second");
launchTasks(
warmupExperiment,
tasksPerJob,
numPreferredNodes,
benchmarkIterations,
benchmarkId,
backends,
client);
LOG.debug(
"Waiting for queues to drain after warmup (waiting " + post_warmup_s + " seconds)");
Thread.sleep(post_warmup_s * 1000);
}
LOG.debug("Launching experiment for " + experiment_duration_s + " seconds");
launchTasks(
experiments,
tasksPerJob,
numPreferredNodes,
benchmarkIterations,
benchmarkId,
backends,
client);
} catch (Exception e) {
LOG.error("Fatal exception", e);
}
}
