/** * Test for {@link DFSUtil#getNNServiceRpcAddresses(Configuration)} {@link * DFSUtil#getNameServiceIdFromAddress(Configuration, InetSocketAddress, String...) * (Configuration)} */ @Test public void testMultipleNamenodes() throws IOException { HdfsConfiguration conf = new HdfsConfiguration(); conf.set(DFS_NAMESERVICES, "nn1,nn2"); // Test - configured list of namenodes are returned final String NN1_ADDRESS = "localhost:9000"; final String NN2_ADDRESS = "localhost:9001"; final String NN3_ADDRESS = "localhost:9002"; conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "nn1"), NN1_ADDRESS); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "nn2"), NN2_ADDRESS); Map<String, Map<String, InetSocketAddress>> nnMap = DFSUtil.getNNServiceRpcAddresses(conf); assertEquals(2, nnMap.size()); Map<String, InetSocketAddress> nn1Map = nnMap.get("nn1"); assertEquals(1, nn1Map.size()); InetSocketAddress addr = nn1Map.get(null); assertEquals("localhost", addr.getHostName()); assertEquals(9000, addr.getPort()); Map<String, InetSocketAddress> nn2Map = nnMap.get("nn2"); assertEquals(1, nn2Map.size()); addr = nn2Map.get(null); assertEquals("localhost", addr.getHostName()); assertEquals(9001, addr.getPort()); // Test - can look up nameservice ID from service address checkNameServiceId(conf, NN1_ADDRESS, "nn1"); checkNameServiceId(conf, NN2_ADDRESS, "nn2"); checkNameServiceId(conf, NN3_ADDRESS, null); // HA is not enabled in a purely federated config assertFalse(HAUtil.isHAEnabled(conf, "nn1")); assertFalse(HAUtil.isHAEnabled(conf, "nn2")); }
/** * Test {@link DFSUtil#getNamenodeNameServiceId(Configuration)} to ensure exception is thrown when * multiple rpc addresses match the local node's address */ @Test(expected = HadoopIllegalArgumentException.class) public void testGetNameServiceIdException() { HdfsConfiguration conf = new HdfsConfiguration(); conf.set(DFS_NAMESERVICES, "nn1,nn2"); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "nn1"), "localhost:9000"); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "nn2"), "localhost:9001"); DFSUtil.getNamenodeNameServiceId(conf); fail("Expected exception is not thrown"); }
/** * Test to ensure nameservice specific keys in the configuration are copied to generic keys when * the namenode starts. */ @Test public void testConfModificationFederationAndHa() { final HdfsConfiguration conf = new HdfsConfiguration(); String nsId = "ns1"; String nnId = "nn1"; conf.set(DFS_NAMESERVICES, nsId); conf.set(DFS_NAMESERVICE_ID, nsId); conf.set(DFS_HA_NAMENODES_KEY_PREFIX + "." + nsId, nnId); // Set the nameservice specific keys with nameserviceId in the config key for (String key : NameNode.NAMENODE_SPECIFIC_KEYS) { // Note: value is same as the key conf.set(DFSUtil.addKeySuffixes(key, nsId, nnId), key); } // Initialize generic keys from specific keys NameNode.initializeGenericKeys(conf, nsId, nnId); // Retrieve the keys without nameserviceId and Ensure generic keys are set // to the correct value for (String key : NameNode.NAMENODE_SPECIFIC_KEYS) { assertEquals(key, conf.get(key)); } }
@Test public void getNameNodeServiceAddr() throws IOException { HdfsConfiguration conf = new HdfsConfiguration(); // One nameservice with two NNs final String NS1_NN1_HOST = "ns1-nn1.example.com:8020"; final String NS1_NN1_HOST_SVC = "ns1-nn2.example.com:8021"; final String NS1_NN2_HOST = "ns1-nn1.example.com:8020"; final String NS1_NN2_HOST_SVC = "ns1-nn2.example.com:8021"; conf.set(DFS_NAMESERVICES, "ns1"); conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"), "nn1,nn2"); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn1"), NS1_NN1_HOST); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn2"), NS1_NN2_HOST); // The rpc address is used if no service address is defined assertEquals(NS1_NN1_HOST, DFSUtil.getNamenodeServiceAddr(conf, null, "nn1")); assertEquals(NS1_NN2_HOST, DFSUtil.getNamenodeServiceAddr(conf, null, "nn2")); // A nameservice is specified explicitly assertEquals(NS1_NN1_HOST, DFSUtil.getNamenodeServiceAddr(conf, "ns1", "nn1")); assertEquals(null, DFSUtil.getNamenodeServiceAddr(conf, "invalid", "nn1")); // The service addrs are used when they are defined conf.set( DFSUtil.addKeySuffixes(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns1", "nn1"), NS1_NN1_HOST_SVC); conf.set( DFSUtil.addKeySuffixes(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns1", "nn2"), NS1_NN2_HOST_SVC); assertEquals(NS1_NN1_HOST_SVC, DFSUtil.getNamenodeServiceAddr(conf, null, "nn1")); assertEquals(NS1_NN2_HOST_SVC, DFSUtil.getNamenodeServiceAddr(conf, null, "nn2")); // We can determine the nameservice ID, there's only one listed assertEquals("ns1", DFSUtil.getNamenodeNameServiceId(conf)); assertEquals("ns1", DFSUtil.getSecondaryNameServiceId(conf)); }
@Test public void testHANameNodesWithFederation() throws URISyntaxException { HdfsConfiguration conf = new HdfsConfiguration(); final String NS1_NN1_HOST = "ns1-nn1.example.com:8020"; final String NS1_NN2_HOST = "ns1-nn2.example.com:8020"; final String NS2_NN1_HOST = "ns2-nn1.example.com:8020"; final String NS2_NN2_HOST = "ns2-nn2.example.com:8020"; conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://ns1"); // Two nameservices, each with two NNs. conf.set(DFS_NAMESERVICES, "ns1,ns2"); conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"), "ns1-nn1,ns1-nn2"); conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns2"), "ns2-nn1,ns2-nn2"); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "ns1-nn1"), NS1_NN1_HOST); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "ns1-nn2"), NS1_NN2_HOST); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns2", "ns2-nn1"), NS2_NN1_HOST); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns2", "ns2-nn2"), NS2_NN2_HOST); Map<String, Map<String, InetSocketAddress>> map = DFSUtil.getHaNnRpcAddresses(conf); assertTrue(HAUtil.isHAEnabled(conf, "ns1")); assertTrue(HAUtil.isHAEnabled(conf, "ns2")); assertFalse(HAUtil.isHAEnabled(conf, "ns3")); assertEquals(NS1_NN1_HOST, map.get("ns1").get("ns1-nn1").toString()); assertEquals(NS1_NN2_HOST, map.get("ns1").get("ns1-nn2").toString()); assertEquals(NS2_NN1_HOST, map.get("ns2").get("ns2-nn1").toString()); assertEquals(NS2_NN2_HOST, map.get("ns2").get("ns2-nn2").toString()); assertEquals(NS1_NN1_HOST, DFSUtil.getNamenodeServiceAddr(conf, "ns1", "ns1-nn1")); assertEquals(NS1_NN2_HOST, DFSUtil.getNamenodeServiceAddr(conf, "ns1", "ns1-nn2")); assertEquals(NS2_NN1_HOST, DFSUtil.getNamenodeServiceAddr(conf, "ns2", "ns2-nn1")); // No nameservice was given and we can't determine which service addr // to use as two nameservices could share a namenode ID. assertEquals(null, DFSUtil.getNamenodeServiceAddr(conf, null, "ns1-nn1")); // Ditto for nameservice IDs, if multiple are defined assertEquals(null, DFSUtil.getNamenodeNameServiceId(conf)); assertEquals(null, DFSUtil.getSecondaryNameServiceId(conf)); Collection<URI> uris = DFSUtil.getNameServiceUris(conf, DFS_NAMENODE_RPC_ADDRESS_KEY); assertEquals(2, uris.size()); assertTrue(uris.contains(new URI("hdfs://ns1"))); assertTrue(uris.contains(new URI("hdfs://ns2"))); }
private Configuration setupAddress(String key) { HdfsConfiguration conf = new HdfsConfiguration(); conf.set(DFS_NAMESERVICES, "nn1"); conf.set(DFSUtil.addKeySuffixes(key, "nn1"), "localhost:9000"); return conf; }
@Test public void testGetNNUris() throws Exception { HdfsConfiguration conf = new HdfsConfiguration(); final String NS1_NN1_ADDR = "ns1-nn1.example.com:8020"; final String NS1_NN2_ADDR = "ns1-nn2.example.com:8020"; final String NS2_NN_ADDR = "ns2-nn.example.com:8020"; final String NN1_ADDR = "nn.example.com:8020"; final String NN1_SRVC_ADDR = "nn.example.com:8021"; final String NN2_ADDR = "nn2.example.com:8020"; conf.set(DFS_NAMESERVICES, "ns1,ns2"); conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"), "nn1,nn2"); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn1"), NS1_NN1_ADDR); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn2"), NS1_NN2_ADDR); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns2"), NS2_NN_ADDR); conf.set(DFS_NAMENODE_RPC_ADDRESS_KEY, "hdfs://" + NN1_ADDR); conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + NN2_ADDR); Collection<URI> uris = DFSUtil.getNameServiceUris( conf, DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_ADDRESS_KEY); assertEquals(4, uris.size()); assertTrue(uris.contains(new URI("hdfs://ns1"))); assertTrue(uris.contains(new URI("hdfs://" + NS2_NN_ADDR))); assertTrue(uris.contains(new URI("hdfs://" + NN1_ADDR))); assertTrue(uris.contains(new URI("hdfs://" + NN2_ADDR))); // Make sure that non-HDFS URIs in fs.defaultFS don't get included. conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "viewfs://vfs-name.example.com"); uris = DFSUtil.getNameServiceUris( conf, DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_ADDRESS_KEY); assertEquals(3, uris.size()); assertTrue(uris.contains(new URI("hdfs://ns1"))); assertTrue(uris.contains(new URI("hdfs://" + NS2_NN_ADDR))); assertTrue(uris.contains(new URI("hdfs://" + NN1_ADDR))); // Make sure that an HA URI being the default URI doesn't result in multiple // entries being returned. conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://ns1"); uris = DFSUtil.getNameServiceUris( conf, DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_ADDRESS_KEY); assertEquals(3, uris.size()); assertTrue(uris.contains(new URI("hdfs://ns1"))); assertTrue(uris.contains(new URI("hdfs://" + NS2_NN_ADDR))); assertTrue(uris.contains(new URI("hdfs://" + NN1_ADDR))); // Make sure that when a service RPC address is used that is distinct from // the client RPC address, and that client RPC address is also used as the // default URI, that the client URI does not end up in the set of URIs // returned. conf = new HdfsConfiguration(); conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + NN1_ADDR); conf.set(DFS_NAMENODE_RPC_ADDRESS_KEY, NN1_ADDR); conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, NN1_SRVC_ADDR); uris = DFSUtil.getNameServiceUris( conf, DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_ADDRESS_KEY); assertEquals(1, uris.size()); assertTrue(uris.contains(new URI("hdfs://" + NN1_SRVC_ADDR))); }