@Test
public void testHdfs() throws Exception {
Configuration conf =
HadoopConfigurationBuilder.buildHdfsConfiguration(clusterSpec, cluster, defaults);
assertThat(Iterators.size(conf.getKeys()), is(1));
assertThat(conf.getString("p1"), is("hdfs1"));
}
@Test
public void testOverridesNumberOfReducers() throws Exception {
Configuration overrides = new PropertiesConfiguration();
overrides.addProperty("hadoop-mapreduce.mapred.reduce.tasks", "7");
clusterSpec = ClusterSpec.withNoDefaults(overrides);
Configuration conf =
HadoopConfigurationBuilder.buildMapReduceConfiguration(clusterSpec, cluster, defaults);
assertThat(conf.getString("mapred.reduce.tasks"), is("7"));
}
@Test
public void testCommon() throws Exception {
Configuration conf =
HadoopConfigurationBuilder.buildCommonConfiguration(clusterSpec, cluster, defaults);
assertThat(Iterators.size(conf.getKeys()), is(3));
assertThat(conf.getString("p1"), is("common1"));
assertThat(conf.getString("p2"), is("common2"));
assertThat(conf.getString("fs.default.name"), matches("hdfs://.+:8020/"));
}
@Test
public void testMapReduce() throws Exception {
Cluster cluster = newCluster(5);
Configuration conf =
HadoopConfigurationBuilder.buildMapReduceConfiguration(clusterSpec, cluster, defaults);
assertThat(conf.getString("p1"), is("mapred1"));
assertThat(conf.getString("mapred.job.tracker"), matches(".+:8021"));
assertThat(conf.getString("mapred.tasktracker.map.tasks.maximum"), is("4"));
assertThat(conf.getString("mapred.tasktracker.reduce.tasks.maximum"), is("3"));
assertThat(conf.getString("mapred.reduce.tasks"), is("15"));
}
@Test
public void testOverrides() throws Exception {
Configuration overrides = new PropertiesConfiguration();
overrides.addProperty("hadoop-common.p1", "overridden1");
overrides.addProperty("hadoop-common.p2", "overridden2");
overrides.addProperty("hadoop-common.fs.default.name", "not-overridden");
clusterSpec = ClusterSpec.withNoDefaults(overrides);
Configuration conf =
HadoopConfigurationBuilder.buildCommonConfiguration(clusterSpec, cluster, defaults);
assertThat(Iterators.size(conf.getKeys()), is(3));
assertThat(conf.getString("p1"), is("overridden1"));
assertThat(conf.getString("p2"), is("overridden2"));
assertThat(
"Can't override dynamically set properties",
conf.getString("fs.default.name"),
matches("hdfs://.+:8020/"));
}