/** Use the native EC2 discovery module on AWS */
private static void addDefaultsForEC2(ClusterSpec spec, CompositeConfiguration config) {
config.addProperty("es.discovery.type", "ec2");
if (!config.containsKey("es.cloud.aws.access_key")) {
config.addProperty("es.cloud.aws.access_key", spec.getIdentity());
}
if (!config.containsKey("es.cloud.aws.secret_key")) {
config.addProperty("es.cloud.aws.secret_key", spec.getCredential());
}
if (!config.getList("es.plugins", Lists.newLinkedList()).contains("cloud-aws")) {
config.addProperty("es.plugins", "cloud-aws");
}
}
@Test
public void testOverridesNumberOfReducers() throws Exception {
Configuration overrides = new PropertiesConfiguration();
overrides.addProperty("hadoop-mapreduce.mapred.reduce.tasks", "7");
clusterSpec = ClusterSpec.withNoDefaults(overrides);
Configuration conf =
HadoopConfigurationBuilder.buildMapReduceConfiguration(clusterSpec, cluster, defaults);
assertThat(conf.getString("mapred.reduce.tasks"), is("7"));
}
@Before
public void setUp() throws Exception {
defaults = new PropertiesConfiguration();
defaults.addProperty("hadoop-common.p1", "common1");
defaults.addProperty("hadoop-common.p2", "common2");
defaults.addProperty("hadoop-hdfs.p1", "hdfs1");
defaults.addProperty("hadoop-mapreduce.p1", "mapred1");
clusterSpec = ClusterSpec.withTemporaryKeys();
cluster = newCluster(1);
}
@Before
public void setUp() throws Exception {
CompositeConfiguration config = new CompositeConfiguration();
config.addConfiguration(new PropertiesConfiguration("whirr-elasticsearch-test.properties"));
if (System.getProperty("config") != null) {
config.addConfiguration(new PropertiesConfiguration(System.getProperty("config")));
}
clusterSpec = ClusterSpec.withTemporaryKeys(config);
controller = new ClusterController();
cluster = controller.launchCluster(clusterSpec);
}
/** Build a configuration by adding the expected defaults */
public static Configuration buildConfig(ClusterSpec spec, Cluster cluster) {
CompositeConfiguration config = new CompositeConfiguration();
config.addConfiguration(spec.getConfiguration());
try {
config.addConfiguration(
new PropertiesConfiguration("whirr-elasticsearch-default.properties"));
} catch (ConfigurationException e) {
LOG.error("Configuration error", e); // this should never happen
}
if ("aws-ec2".equals(spec.getProvider()) || "ec2".equals(spec.getProvider())) {
addDefaultsForEC2(spec, config);
} else {
addDefaultsForUnicast(cluster, config);
}
if (!config.containsKey("es.cluster.name")) {
config.addProperty("es.cluster.name", spec.getClusterName());
}
return config;
}
@Test
public void testOverrides() throws Exception {
Configuration overrides = new PropertiesConfiguration();
overrides.addProperty("hadoop-common.p1", "overridden1");
overrides.addProperty("hadoop-common.p2", "overridden2");
overrides.addProperty("hadoop-common.fs.default.name", "not-overridden");
clusterSpec = ClusterSpec.withNoDefaults(overrides);
Configuration conf =
HadoopConfigurationBuilder.buildCommonConfiguration(clusterSpec, cluster, defaults);
assertThat(Iterators.size(conf.getKeys()), is(3));
assertThat(conf.getString("p1"), is("overridden1"));
assertThat(conf.getString("p2"), is("overridden2"));
assertThat(
"Can't override dynamically set properties",
conf.getString("fs.default.name"),
matches("hdfs://.+:8020/"));
}