コード例 #1
0
  /**
   * Test for {@link DFSUtil#getNNServiceRpcAddresses(Configuration)} {@link
   * DFSUtil#getNameServiceIdFromAddress(Configuration, InetSocketAddress, String...)
   * (Configuration)}
   */
  @Test
  public void testMultipleNamenodes() throws IOException {
    HdfsConfiguration conf = new HdfsConfiguration();
    conf.set(DFS_NAMESERVICES, "nn1,nn2");
    // Test - configured list of namenodes are returned
    final String NN1_ADDRESS = "localhost:9000";
    final String NN2_ADDRESS = "localhost:9001";
    final String NN3_ADDRESS = "localhost:9002";
    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "nn1"), NN1_ADDRESS);
    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "nn2"), NN2_ADDRESS);

    Map<String, Map<String, InetSocketAddress>> nnMap = DFSUtil.getNNServiceRpcAddresses(conf);
    assertEquals(2, nnMap.size());

    Map<String, InetSocketAddress> nn1Map = nnMap.get("nn1");
    assertEquals(1, nn1Map.size());
    InetSocketAddress addr = nn1Map.get(null);
    assertEquals("localhost", addr.getHostName());
    assertEquals(9000, addr.getPort());

    Map<String, InetSocketAddress> nn2Map = nnMap.get("nn2");
    assertEquals(1, nn2Map.size());
    addr = nn2Map.get(null);
    assertEquals("localhost", addr.getHostName());
    assertEquals(9001, addr.getPort());

    // Test - can look up nameservice ID from service address
    checkNameServiceId(conf, NN1_ADDRESS, "nn1");
    checkNameServiceId(conf, NN2_ADDRESS, "nn2");
    checkNameServiceId(conf, NN3_ADDRESS, null);

    // HA is not enabled in a purely federated config
    assertFalse(HAUtil.isHAEnabled(conf, "nn1"));
    assertFalse(HAUtil.isHAEnabled(conf, "nn2"));
  }
コード例 #2
0
 /**
  * Test {@link DFSUtil#getNamenodeNameServiceId(Configuration)} to ensure exception is thrown when
  * multiple rpc addresses match the local node's address
  */
 @Test(expected = HadoopIllegalArgumentException.class)
 public void testGetNameServiceIdException() {
   HdfsConfiguration conf = new HdfsConfiguration();
   conf.set(DFS_NAMESERVICES, "nn1,nn2");
   conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "nn1"), "localhost:9000");
   conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "nn2"), "localhost:9001");
   DFSUtil.getNamenodeNameServiceId(conf);
   fail("Expected exception is not thrown");
 }
コード例 #3
0
  /**
   * Test to ensure nameservice specific keys in the configuration are copied to generic keys when
   * the namenode starts.
   */
  @Test
  public void testConfModificationFederationAndHa() {
    final HdfsConfiguration conf = new HdfsConfiguration();
    String nsId = "ns1";
    String nnId = "nn1";

    conf.set(DFS_NAMESERVICES, nsId);
    conf.set(DFS_NAMESERVICE_ID, nsId);
    conf.set(DFS_HA_NAMENODES_KEY_PREFIX + "." + nsId, nnId);

    // Set the nameservice specific keys with nameserviceId in the config key
    for (String key : NameNode.NAMENODE_SPECIFIC_KEYS) {
      // Note: value is same as the key
      conf.set(DFSUtil.addKeySuffixes(key, nsId, nnId), key);
    }

    // Initialize generic keys from specific keys
    NameNode.initializeGenericKeys(conf, nsId, nnId);

    // Retrieve the keys without nameserviceId and Ensure generic keys are set
    // to the correct value
    for (String key : NameNode.NAMENODE_SPECIFIC_KEYS) {
      assertEquals(key, conf.get(key));
    }
  }
コード例 #4
0
  @Test
  public void getNameNodeServiceAddr() throws IOException {
    HdfsConfiguration conf = new HdfsConfiguration();

    // One nameservice with two NNs
    final String NS1_NN1_HOST = "ns1-nn1.example.com:8020";
    final String NS1_NN1_HOST_SVC = "ns1-nn2.example.com:8021";
    final String NS1_NN2_HOST = "ns1-nn1.example.com:8020";
    final String NS1_NN2_HOST_SVC = "ns1-nn2.example.com:8021";

    conf.set(DFS_NAMESERVICES, "ns1");
    conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"), "nn1,nn2");

    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn1"), NS1_NN1_HOST);
    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn2"), NS1_NN2_HOST);

    // The rpc address is used if no service address is defined
    assertEquals(NS1_NN1_HOST, DFSUtil.getNamenodeServiceAddr(conf, null, "nn1"));
    assertEquals(NS1_NN2_HOST, DFSUtil.getNamenodeServiceAddr(conf, null, "nn2"));

    // A nameservice is specified explicitly
    assertEquals(NS1_NN1_HOST, DFSUtil.getNamenodeServiceAddr(conf, "ns1", "nn1"));
    assertEquals(null, DFSUtil.getNamenodeServiceAddr(conf, "invalid", "nn1"));

    // The service addrs are used when they are defined
    conf.set(
        DFSUtil.addKeySuffixes(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns1", "nn1"),
        NS1_NN1_HOST_SVC);
    conf.set(
        DFSUtil.addKeySuffixes(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns1", "nn2"),
        NS1_NN2_HOST_SVC);

    assertEquals(NS1_NN1_HOST_SVC, DFSUtil.getNamenodeServiceAddr(conf, null, "nn1"));
    assertEquals(NS1_NN2_HOST_SVC, DFSUtil.getNamenodeServiceAddr(conf, null, "nn2"));

    // We can determine the nameservice ID, there's only one listed
    assertEquals("ns1", DFSUtil.getNamenodeNameServiceId(conf));
    assertEquals("ns1", DFSUtil.getSecondaryNameServiceId(conf));
  }
コード例 #5
0
  @Test
  public void testHANameNodesWithFederation() throws URISyntaxException {
    HdfsConfiguration conf = new HdfsConfiguration();

    final String NS1_NN1_HOST = "ns1-nn1.example.com:8020";
    final String NS1_NN2_HOST = "ns1-nn2.example.com:8020";
    final String NS2_NN1_HOST = "ns2-nn1.example.com:8020";
    final String NS2_NN2_HOST = "ns2-nn2.example.com:8020";
    conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://ns1");

    // Two nameservices, each with two NNs.
    conf.set(DFS_NAMESERVICES, "ns1,ns2");
    conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"), "ns1-nn1,ns1-nn2");
    conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns2"), "ns2-nn1,ns2-nn2");
    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "ns1-nn1"), NS1_NN1_HOST);
    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "ns1-nn2"), NS1_NN2_HOST);
    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns2", "ns2-nn1"), NS2_NN1_HOST);
    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns2", "ns2-nn2"), NS2_NN2_HOST);

    Map<String, Map<String, InetSocketAddress>> map = DFSUtil.getHaNnRpcAddresses(conf);

    assertTrue(HAUtil.isHAEnabled(conf, "ns1"));
    assertTrue(HAUtil.isHAEnabled(conf, "ns2"));
    assertFalse(HAUtil.isHAEnabled(conf, "ns3"));

    assertEquals(NS1_NN1_HOST, map.get("ns1").get("ns1-nn1").toString());
    assertEquals(NS1_NN2_HOST, map.get("ns1").get("ns1-nn2").toString());
    assertEquals(NS2_NN1_HOST, map.get("ns2").get("ns2-nn1").toString());
    assertEquals(NS2_NN2_HOST, map.get("ns2").get("ns2-nn2").toString());

    assertEquals(NS1_NN1_HOST, DFSUtil.getNamenodeServiceAddr(conf, "ns1", "ns1-nn1"));
    assertEquals(NS1_NN2_HOST, DFSUtil.getNamenodeServiceAddr(conf, "ns1", "ns1-nn2"));
    assertEquals(NS2_NN1_HOST, DFSUtil.getNamenodeServiceAddr(conf, "ns2", "ns2-nn1"));

    // No nameservice was given and we can't determine which service addr
    // to use as two nameservices could share a namenode ID.
    assertEquals(null, DFSUtil.getNamenodeServiceAddr(conf, null, "ns1-nn1"));

    // Ditto for nameservice IDs, if multiple are defined
    assertEquals(null, DFSUtil.getNamenodeNameServiceId(conf));
    assertEquals(null, DFSUtil.getSecondaryNameServiceId(conf));

    Collection<URI> uris = DFSUtil.getNameServiceUris(conf, DFS_NAMENODE_RPC_ADDRESS_KEY);
    assertEquals(2, uris.size());
    assertTrue(uris.contains(new URI("hdfs://ns1")));
    assertTrue(uris.contains(new URI("hdfs://ns2")));
  }
コード例 #6
0
 private Configuration setupAddress(String key) {
   HdfsConfiguration conf = new HdfsConfiguration();
   conf.set(DFS_NAMESERVICES, "nn1");
   conf.set(DFSUtil.addKeySuffixes(key, "nn1"), "localhost:9000");
   return conf;
 }
コード例 #7
0
  @Test
  public void testGetNNUris() throws Exception {
    HdfsConfiguration conf = new HdfsConfiguration();

    final String NS1_NN1_ADDR = "ns1-nn1.example.com:8020";
    final String NS1_NN2_ADDR = "ns1-nn2.example.com:8020";
    final String NS2_NN_ADDR = "ns2-nn.example.com:8020";
    final String NN1_ADDR = "nn.example.com:8020";
    final String NN1_SRVC_ADDR = "nn.example.com:8021";
    final String NN2_ADDR = "nn2.example.com:8020";

    conf.set(DFS_NAMESERVICES, "ns1,ns2");
    conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"), "nn1,nn2");
    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn1"), NS1_NN1_ADDR);
    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn2"), NS1_NN2_ADDR);

    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns2"), NS2_NN_ADDR);

    conf.set(DFS_NAMENODE_RPC_ADDRESS_KEY, "hdfs://" + NN1_ADDR);

    conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + NN2_ADDR);

    Collection<URI> uris =
        DFSUtil.getNameServiceUris(
            conf, DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_ADDRESS_KEY);

    assertEquals(4, uris.size());
    assertTrue(uris.contains(new URI("hdfs://ns1")));
    assertTrue(uris.contains(new URI("hdfs://" + NS2_NN_ADDR)));
    assertTrue(uris.contains(new URI("hdfs://" + NN1_ADDR)));
    assertTrue(uris.contains(new URI("hdfs://" + NN2_ADDR)));

    // Make sure that non-HDFS URIs in fs.defaultFS don't get included.
    conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "viewfs://vfs-name.example.com");

    uris =
        DFSUtil.getNameServiceUris(
            conf, DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_ADDRESS_KEY);

    assertEquals(3, uris.size());
    assertTrue(uris.contains(new URI("hdfs://ns1")));
    assertTrue(uris.contains(new URI("hdfs://" + NS2_NN_ADDR)));
    assertTrue(uris.contains(new URI("hdfs://" + NN1_ADDR)));

    // Make sure that an HA URI being the default URI doesn't result in multiple
    // entries being returned.
    conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://ns1");

    uris =
        DFSUtil.getNameServiceUris(
            conf, DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_ADDRESS_KEY);

    assertEquals(3, uris.size());
    assertTrue(uris.contains(new URI("hdfs://ns1")));
    assertTrue(uris.contains(new URI("hdfs://" + NS2_NN_ADDR)));
    assertTrue(uris.contains(new URI("hdfs://" + NN1_ADDR)));

    // Make sure that when a service RPC address is used that is distinct from
    // the client RPC address, and that client RPC address is also used as the
    // default URI, that the client URI does not end up in the set of URIs
    // returned.
    conf = new HdfsConfiguration();
    conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + NN1_ADDR);
    conf.set(DFS_NAMENODE_RPC_ADDRESS_KEY, NN1_ADDR);
    conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, NN1_SRVC_ADDR);

    uris =
        DFSUtil.getNameServiceUris(
            conf, DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_ADDRESS_KEY);

    assertEquals(1, uris.size());
    assertTrue(uris.contains(new URI("hdfs://" + NN1_SRVC_ADDR)));
  }