Ejemplo n.º 1
0
  /** Test conversion of LocatedBlock to BlockLocation */
  @Test
  public void testLocatedBlocks2Locations() {
    DatanodeInfo d = DFSTestUtil.getLocalDatanodeInfo();
    DatanodeInfo[] ds = new DatanodeInfo[1];
    ds[0] = d;

    // ok
    ExtendedBlock b1 = new ExtendedBlock("bpid", 1, 1, 1);
    LocatedBlock l1 = new LocatedBlock(b1, ds, 0, false);

    // corrupt
    ExtendedBlock b2 = new ExtendedBlock("bpid", 2, 1, 1);
    LocatedBlock l2 = new LocatedBlock(b2, ds, 0, true);

    List<LocatedBlock> ls = Arrays.asList(l1, l2);
    LocatedBlocks lbs = new LocatedBlocks(10, false, ls, l2, true, null);

    BlockLocation[] bs = DFSUtil.locatedBlocks2Locations(lbs);

    assertTrue("expected 2 blocks but got " + bs.length, bs.length == 2);

    int corruptCount = 0;
    for (BlockLocation b : bs) {
      if (b.isCorrupt()) {
        corruptCount++;
      }
    }

    assertTrue("expected 1 corrupt files but got " + corruptCount, corruptCount == 1);

    // test an empty location
    bs = DFSUtil.locatedBlocks2Locations(new LocatedBlocks());
    assertEquals(0, bs.length);
  }
Ejemplo n.º 2
0
  /**
   * Test for {@link DFSUtil#getNNServiceRpcAddresses(Configuration)} {@link
   * DFSUtil#getNameServiceIdFromAddress(Configuration, InetSocketAddress, String...)
   * (Configuration)}
   */
  @Test
  public void testMultipleNamenodes() throws IOException {
    HdfsConfiguration conf = new HdfsConfiguration();
    conf.set(DFS_NAMESERVICES, "nn1,nn2");
    // Test - configured list of namenodes are returned
    final String NN1_ADDRESS = "localhost:9000";
    final String NN2_ADDRESS = "localhost:9001";
    final String NN3_ADDRESS = "localhost:9002";
    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "nn1"), NN1_ADDRESS);
    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "nn2"), NN2_ADDRESS);

    Map<String, Map<String, InetSocketAddress>> nnMap = DFSUtil.getNNServiceRpcAddresses(conf);
    assertEquals(2, nnMap.size());

    Map<String, InetSocketAddress> nn1Map = nnMap.get("nn1");
    assertEquals(1, nn1Map.size());
    InetSocketAddress addr = nn1Map.get(null);
    assertEquals("localhost", addr.getHostName());
    assertEquals(9000, addr.getPort());

    Map<String, InetSocketAddress> nn2Map = nnMap.get("nn2");
    assertEquals(1, nn2Map.size());
    addr = nn2Map.get(null);
    assertEquals("localhost", addr.getHostName());
    assertEquals(9001, addr.getPort());

    // Test - can look up nameservice ID from service address
    checkNameServiceId(conf, NN1_ADDRESS, "nn1");
    checkNameServiceId(conf, NN2_ADDRESS, "nn2");
    checkNameServiceId(conf, NN3_ADDRESS, null);

    // HA is not enabled in a purely federated config
    assertFalse(HAUtil.isHAEnabled(conf, "nn1"));
    assertFalse(HAUtil.isHAEnabled(conf, "nn2"));
  }
Ejemplo n.º 3
0
 /**
  * Test {@link DFSUtil#getNamenodeNameServiceId(Configuration)} to ensure exception is thrown when
  * multiple rpc addresses match the local node's address
  */
 @Test(expected = HadoopIllegalArgumentException.class)
 public void testGetNameServiceIdException() {
   HdfsConfiguration conf = new HdfsConfiguration();
   conf.set(DFS_NAMESERVICES, "nn1,nn2");
   conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "nn1"), "localhost:9000");
   conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "nn2"), "localhost:9001");
   DFSUtil.getNamenodeNameServiceId(conf);
   fail("Expected exception is not thrown");
 }
Ejemplo n.º 4
0
 @Test
 public void testGetOnlyNameServiceIdOrNull() {
   HdfsConfiguration conf = new HdfsConfiguration();
   conf.set(DFS_NAMESERVICES, "ns1,ns2");
   assertNull(DFSUtil.getOnlyNameServiceIdOrNull(conf));
   conf.set(DFS_NAMESERVICES, "");
   assertNull(DFSUtil.getOnlyNameServiceIdOrNull(conf));
   conf.set(DFS_NAMESERVICES, "ns1");
   assertEquals("ns1", DFSUtil.getOnlyNameServiceIdOrNull(conf));
 }
Ejemplo n.º 5
0
  @Test
  public void testGetPassword() throws Exception {
    File testDir = new File(System.getProperty("test.build.data", "target/test-dir"));

    Configuration conf = new Configuration();
    final String ourUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks";

    File file = new File(testDir, "test.jks");
    file.delete();
    conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl);

    CredentialProvider provider = CredentialProviderFactory.getProviders(conf).get(0);
    char[] keypass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
    char[] storepass = {'s', 't', 'o', 'r', 'e', 'p', 'a', 's', 's'};
    char[] trustpass = {'t', 'r', 'u', 's', 't', 'p', 'a', 's', 's'};

    // ensure that we get nulls when the key isn't there
    assertEquals(null, provider.getCredentialEntry(DFS_SERVER_HTTPS_KEYPASSWORD_KEY));
    assertEquals(null, provider.getCredentialEntry(DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY));
    assertEquals(null, provider.getCredentialEntry(DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY));

    // create new aliases
    try {
      provider.createCredentialEntry(DFS_SERVER_HTTPS_KEYPASSWORD_KEY, keypass);

      provider.createCredentialEntry(DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY, storepass);

      provider.createCredentialEntry(DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY, trustpass);

      // write out so that it can be found in checks
      provider.flush();
    } catch (Exception e) {
      e.printStackTrace();
      throw e;
    }
    // make sure we get back the right key directly from api
    assertArrayEquals(
        keypass, provider.getCredentialEntry(DFS_SERVER_HTTPS_KEYPASSWORD_KEY).getCredential());
    assertArrayEquals(
        storepass,
        provider.getCredentialEntry(DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY).getCredential());
    assertArrayEquals(
        trustpass,
        provider.getCredentialEntry(DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY).getCredential());

    // use WebAppUtils as would be used by loadSslConfiguration
    Assert.assertEquals("keypass", DFSUtil.getPassword(conf, DFS_SERVER_HTTPS_KEYPASSWORD_KEY));
    Assert.assertEquals(
        "storepass", DFSUtil.getPassword(conf, DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY));
    Assert.assertEquals(
        "trustpass", DFSUtil.getPassword(conf, DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY));

    // let's make sure that a password that doesn't exist returns null
    Assert.assertEquals(null, DFSUtil.getPassword(conf, "invalid-alias"));
  }
Ejemplo n.º 6
0
 private static void checkAllResults(Long[] toCheck, boolean shouldSucceed) {
   if (shouldSucceed) {
     DFSUtil.assertAllResultsEqual(Arrays.asList(toCheck));
   } else {
     try {
       DFSUtil.assertAllResultsEqual(Arrays.asList(toCheck));
       fail("Should not have succeeded with input: " + Arrays.toString(toCheck));
     } catch (AssertionError ae) {
       GenericTestUtils.assertExceptionContains("Not all elements match", ae);
     }
   }
 }
Ejemplo n.º 7
0
  private static Configuration createWebHDFSHAConfiguration(
      String logicalHostName, String nnaddr1, String nnaddr2) {
    HdfsConfiguration conf = new HdfsConfiguration();

    conf.set(DFS_NAMESERVICES, "ns1");
    conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"), "nn1,nn2");
    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_HTTP_ADDRESS_KEY, "ns1", "nn1"), nnaddr1);
    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_HTTP_ADDRESS_KEY, "ns1", "nn2"), nnaddr2);

    conf.set(
        DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "." + logicalHostName,
        ConfiguredFailoverProxyProvider.class.getName());
    return conf;
  }
Ejemplo n.º 8
0
 public void checkNameServiceId(Configuration conf, String addr, String expectedNameServiceId) {
   InetSocketAddress s = NetUtils.createSocketAddr(addr);
   String nameserviceId =
       DFSUtil.getNameServiceIdFromAddress(
           conf, s, DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_ADDRESS_KEY);
   assertEquals(expectedNameServiceId, nameserviceId);
 }
 public static void setFederatedConfiguration(MiniDFSCluster cluster, Configuration conf) {
   Set<String> nameservices = new HashSet<String>();
   for (NameNodeInfo info : cluster.getNameNodeInfos()) {
     assert info.nameserviceId != null;
     nameservices.add(info.nameserviceId);
     conf.set(
         DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, info.nameserviceId),
         DFSUtil.createUri(HdfsConstants.HDFS_URI_SCHEME, info.nameNode.getNameNodeAddress())
             .toString());
     conf.set(
         DFSUtil.addKeySuffixes(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, info.nameserviceId),
         DFSUtil.createUri(HdfsConstants.HDFS_URI_SCHEME, info.nameNode.getNameNodeAddress())
             .toString());
   }
   conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, Joiner.on(",").join(nameservices));
 }
Ejemplo n.º 10
0
 @Test(timeout = 1000)
 public void testDurationToString() throws Exception {
   assertEquals("000:00:00:00.000", DFSUtil.durationToString(0));
   assertEquals(
       "001:01:01:01.000",
       DFSUtil.durationToString(((24 * 60 * 60) + (60 * 60) + (60) + 1) * 1000));
   assertEquals(
       "000:23:59:59.999",
       DFSUtil.durationToString(((23 * 60 * 60) + (59 * 60) + (59)) * 1000 + 999));
   assertEquals(
       "-001:01:01:01.000",
       DFSUtil.durationToString(-((24 * 60 * 60) + (60 * 60) + (60) + 1) * 1000));
   assertEquals(
       "-000:23:59:59.574",
       DFSUtil.durationToString(-(((23 * 60 * 60) + (59 * 60) + (59)) * 1000 + 574)));
 }
Ejemplo n.º 11
0
  @Test
  public void testGetInfoServer() throws IOException {
    HdfsConfiguration conf = new HdfsConfiguration();
    conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    UserGroupInformation.setConfiguration(conf);

    String httpsport = DFSUtil.getInfoServer(null, conf, true);
    assertEquals("0.0.0.0:" + DFS_NAMENODE_HTTPS_PORT_DEFAULT, httpsport);

    String httpport = DFSUtil.getInfoServer(null, conf, false);
    assertEquals("0.0.0.0:" + DFS_NAMENODE_HTTP_PORT_DEFAULT, httpport);

    String httpAddress =
        DFSUtil.getInfoServer(new InetSocketAddress("localhost", 8020), conf, false);
    assertEquals("localhost:" + DFS_NAMENODE_HTTP_PORT_DEFAULT, httpAddress);
  }
Ejemplo n.º 12
0
  /**
   * Test to ensure nameservice specific keys in the configuration are copied to generic keys when
   * the namenode starts.
   */
  @Test
  public void testConfModificationFederationAndHa() {
    final HdfsConfiguration conf = new HdfsConfiguration();
    String nsId = "ns1";
    String nnId = "nn1";

    conf.set(DFS_NAMESERVICES, nsId);
    conf.set(DFS_NAMESERVICE_ID, nsId);
    conf.set(DFS_HA_NAMENODES_KEY_PREFIX + "." + nsId, nnId);

    // Set the nameservice specific keys with nameserviceId in the config key
    for (String key : NameNode.NAMENODE_SPECIFIC_KEYS) {
      // Note: value is same as the key
      conf.set(DFSUtil.addKeySuffixes(key, nsId, nnId), key);
    }

    // Initialize generic keys from specific keys
    NameNode.initializeGenericKeys(conf, nsId, nnId);

    // Retrieve the keys without nameserviceId and Ensure generic keys are set
    // to the correct value
    for (String key : NameNode.NAMENODE_SPECIFIC_KEYS) {
      assertEquals(key, conf.get(key));
    }
  }
Ejemplo n.º 13
0
  @Test
  public void testGetInfoServer() throws IOException, URISyntaxException {
    HdfsConfiguration conf = new HdfsConfiguration();

    URI httpsport = DFSUtil.getInfoServer(null, conf, "https");
    assertEquals(
        new URI("https", null, "0.0.0.0", DFS_NAMENODE_HTTPS_PORT_DEFAULT, null, null, null),
        httpsport);

    URI httpport = DFSUtil.getInfoServer(null, conf, "http");
    assertEquals(
        new URI("http", null, "0.0.0.0", DFS_NAMENODE_HTTP_PORT_DEFAULT, null, null, null),
        httpport);

    URI httpAddress = DFSUtil.getInfoServer(new InetSocketAddress("localhost", 8020), conf, "http");
    assertEquals(URI.create("http://localhost:" + DFS_NAMENODE_HTTP_PORT_DEFAULT), httpAddress);
  }
  private DNAddrPair chooseDataNode(LocatedBlock block) throws IOException {
    while (true) {
      DatanodeInfo[] nodes = block.getLocations();
      try {
        DatanodeInfo chosenNode = bestNode(nodes, deadNodes);
        InetSocketAddress targetAddr = NetUtils.createSocketAddr(chosenNode.getXferAddr());
        return new DNAddrPair(chosenNode, targetAddr);
      } catch (IOException ie) {
        String blockInfo = block.getBlock() + " file=" + src;
        if (failures >= dfsClient.getMaxBlockAcquireFailures()) {
          throw new BlockMissingException(
              src, "Could not obtain block: " + blockInfo, block.getStartOffset());
        }

        if (nodes == null || nodes.length == 0) {
          DFSClient.LOG.info("No node available for block: " + blockInfo);
        }
        DFSClient.LOG.info(
            "Could not obtain block "
                + block.getBlock()
                + " from any node: "
                + ie
                + ". Will get new block locations from namenode and retry...");
        try {
          // Introducing a random factor to the wait time before another retry.
          // The wait time is dependent on # of failures and a random factor.
          // At the first time of getting a BlockMissingException, the wait time
          // is a random number between 0..3000 ms. If the first retry
          // still fails, we will wait 3000 ms grace period before the 2nd retry.
          // Also at the second retry, the waiting window is expanded to 6000 ms
          // alleviating the request rate from the server. Similarly the 3rd retry
          // will wait 6000ms grace period before retry and the waiting window is
          // expanded to 9000ms.
          double waitTime =
              timeWindow * failures
                  + // grace period for the last round of attempt
                  timeWindow
                      * (failures + 1)
                      * DFSUtil.getRandom().nextDouble(); // expanding time window for each failure
          DFSClient.LOG.warn(
              "DFS chooseDataNode: got # "
                  + (failures + 1)
                  + " IOException, will wait for "
                  + waitTime
                  + " msec.");
          Thread.sleep((long) waitTime);
        } catch (InterruptedException iex) {
        }
        deadNodes.clear(); // 2nd option is to remove only nodes[blockId]
        openInfo();
        block = getBlockAt(block.getStartOffset(), false);
        failures++;
        continue;
      }
    }
  }
Ejemplo n.º 15
0
 /** Test {@link DFSUtil#getNameServiceIds(Configuration)} */
 @Test
 public void testGetNameServiceIds() {
   HdfsConfiguration conf = new HdfsConfiguration();
   conf.set(DFS_NAMESERVICES, "nn1,nn2");
   Collection<String> nameserviceIds = DFSUtil.getNameServiceIds(conf);
   Iterator<String> it = nameserviceIds.iterator();
   assertEquals(2, nameserviceIds.size());
   assertEquals("nn1", it.next().toString());
   assertEquals("nn2", it.next().toString());
 }
Ejemplo n.º 16
0
  @Test(timeout = 5000)
  public void testGetSpnegoKeytabKey() {
    HdfsConfiguration conf = new HdfsConfiguration();
    String defaultKey = "default.spengo.key";
    conf.unset(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY);
    assertEquals(
        "Test spnego key in config is null",
        defaultKey,
        DFSUtil.getSpnegoKeytabKey(conf, defaultKey));

    conf.set(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY, "");
    assertEquals(
        "Test spnego key is empty", defaultKey, DFSUtil.getSpnegoKeytabKey(conf, defaultKey));

    String spengoKey = "spengo.key";
    conf.set(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY, spengoKey);
    assertEquals(
        "Test spnego key is NOT null",
        DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY,
        DFSUtil.getSpnegoKeytabKey(conf, defaultKey));
  }
Ejemplo n.º 17
0
 @Test(timeout = 15000)
 public void testIsValidName() {
   assertFalse(DFSUtil.isValidName("/foo/../bar"));
   assertFalse(DFSUtil.isValidName("/foo/./bar"));
   assertFalse(DFSUtil.isValidName("/foo//bar"));
   assertTrue(DFSUtil.isValidName("/"));
   assertTrue(DFSUtil.isValidName("/bar/"));
   assertFalse(DFSUtil.isValidName("/foo/:/bar"));
   assertFalse(DFSUtil.isValidName("/foo:bar"));
 }
  /** Read the block length from one of the datanodes. */
  private long readBlockLength(LocatedBlock locatedblock) throws IOException {
    assert locatedblock != null : "LocatedBlock cannot be null";
    int replicaNotFoundCount = locatedblock.getLocations().length;

    for (DatanodeInfo datanode : locatedblock.getLocations()) {
      ClientDatanodeProtocol cdp = null;

      try {
        cdp =
            DFSUtil.createClientDatanodeProtocolProxy(
                datanode, dfsClient.conf, dfsClient.getConf().socketTimeout, locatedblock);

        final long n = cdp.getReplicaVisibleLength(locatedblock.getBlock());

        if (n >= 0) {
          return n;
        }
      } catch (IOException ioe) {
        if (ioe instanceof RemoteException
            && (((RemoteException) ioe).unwrapRemoteException()
                instanceof ReplicaNotFoundException)) {
          // special case : replica might not be on the DN, treat as 0 length
          replicaNotFoundCount--;
        }

        if (DFSClient.LOG.isDebugEnabled()) {
          DFSClient.LOG.debug(
              "Failed to getReplicaVisibleLength from datanode "
                  + datanode
                  + " for block "
                  + locatedblock.getBlock(),
              ioe);
        }
      } finally {
        if (cdp != null) {
          RPC.stopProxy(cdp);
        }
      }
    }

    // Namenode told us about these locations, but none know about the replica
    // means that we hit the race between pipeline creation start and end.
    // we require all 3 because some other exception could have happened
    // on a DN that has it.  we want to report that error
    if (replicaNotFoundCount == 0) {
      return 0;
    }

    throw new IOException("Cannot obtain block length for " + locatedblock);
  }
Ejemplo n.º 19
0
 public void initGeneralConf(Configuration conf, String nameserviceId) {
   // overwrite relevant settings
   this.nameserviceId = nameserviceId;
   // avatar nodes
   conf.set("fs.default.name", "hdfs://localhost:" + nnPort);
   conf.set("fs.default.name0", "hdfs://localhost:" + nn0Port);
   conf.set("fs.default.name1", "hdfs://localhost:" + nn1Port);
   if (federation) {
     conf.set("dfs.namenode.rpc-address0", "localhost:" + rpc0Port);
     conf.set("dfs.namenode.rpc-address1", "localhost:" + rpc1Port);
   }
   conf.set("dfs.namenode.dn-address", "localhost:" + nnDnPort);
   conf.set("dfs.namenode.dn-address0", "localhost:" + nnDn0Port);
   conf.set("dfs.namenode.dn-address1", "localhost:" + nnDn1Port);
   conf.set("dfs.http.address", "localhost:" + httpPort);
   conf.set("dfs.http.address0", "localhost:" + http0Port);
   conf.set("dfs.http.address1", "localhost:" + http1Port);
   conf.set("dfs.name.dir.shared0", fsimage0Dir);
   conf.set("dfs.name.dir.shared1", fsimage1Dir);
   conf.set("dfs.name.edits.dir.shared0", fsedits0Dir);
   conf.set("dfs.name.edits.dir.shared1", fsedits1Dir);
   if (federation) {
     for (String key : AvatarNode.AVATARSERVICE_SPECIFIC_KEYS) {
       String value = conf.get(key);
       if (value != null) {
         String newKey = DFSUtil.getNameServiceIdKey(key, nameserviceId);
         conf.set(newKey, value);
       }
     }
     String rpcKey =
         DFSUtil.getNameServiceIdKey(AvatarNode.DFS_NAMENODE_RPC_ADDRESS_KEY, nameserviceId);
     conf.set(rpcKey, "localhost:" + rpcPort);
     String dnKey =
         DFSUtil.getNameServiceIdKey(NameNode.DATANODE_PROTOCOL_ADDRESS, nameserviceId);
     conf.set(dnKey, "localhost:" + nnDnPort);
   }
 }
Ejemplo n.º 20
0
  @Test
  public void testGetHaNnHttpAddresses() throws IOException {
    final String LOGICAL_HOST_NAME = "ns1";
    final String NS1_NN1_ADDR = "ns1-nn1.example.com:8020";
    final String NS1_NN2_ADDR = "ns1-nn2.example.com:8020";

    Configuration conf =
        createWebHDFSHAConfiguration(LOGICAL_HOST_NAME, NS1_NN1_ADDR, NS1_NN2_ADDR);

    Map<String, Map<String, InetSocketAddress>> map =
        DFSUtil.getHaNnWebHdfsAddresses(conf, "webhdfs");

    assertEquals(NS1_NN1_ADDR, map.get("ns1").get("nn1").toString());
    assertEquals(NS1_NN2_ADDR, map.get("ns1").get("nn2").toString());
  }
Ejemplo n.º 21
0
  /** Tests to ensure default namenode is used as fallback */
  @Test
  public void testDefaultNamenode() throws IOException {
    HdfsConfiguration conf = new HdfsConfiguration();
    final String hdfs_default = "hdfs://localhost:9999/";
    conf.set(FS_DEFAULT_NAME_KEY, hdfs_default);
    // If DFS_FEDERATION_NAMESERVICES is not set, verify that
    // default namenode address is returned.
    Map<String, Map<String, InetSocketAddress>> addrMap = DFSUtil.getNNServiceRpcAddresses(conf);
    assertEquals(1, addrMap.size());

    Map<String, InetSocketAddress> defaultNsMap = addrMap.get(null);
    assertEquals(1, defaultNsMap.size());

    assertEquals(9999, defaultNsMap.get(null).getPort());
  }
Ejemplo n.º 22
0
  @Test(timeout = 15000)
  public void testLocalhostReverseLookup() {
    // 127.0.0.1 -> localhost reverse resolution does not happen on Windows.
    Assume.assumeTrue(!Shell.WINDOWS);

    // Make sure when config FS_DEFAULT_NAME_KEY using IP address,
    // it will automatically convert it to hostname
    HdfsConfiguration conf = new HdfsConfiguration();
    conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://127.0.0.1:8020");
    Collection<URI> uris = DFSUtil.getNameServiceUris(conf);
    assertEquals(1, uris.size());
    for (URI uri : uris) {
      assertThat(uri.getHost(), not("127.0.0.1"));
    }
  }
Ejemplo n.º 23
0
 @Test(timeout = 5000)
 public void testRelativeTimeConversion() throws Exception {
   try {
     DFSUtil.parseRelativeTime("1");
   } catch (IOException e) {
     assertExceptionContains("too short", e);
   }
   try {
     DFSUtil.parseRelativeTime("1z");
   } catch (IOException e) {
     assertExceptionContains("unknown time unit", e);
   }
   try {
     DFSUtil.parseRelativeTime("yyz");
   } catch (IOException e) {
     assertExceptionContains("is not a number", e);
   }
   assertEquals(61 * 1000, DFSUtil.parseRelativeTime("61s"));
   assertEquals(61 * 60 * 1000, DFSUtil.parseRelativeTime("61m"));
   assertEquals(0, DFSUtil.parseRelativeTime("0s"));
   assertEquals(25 * 60 * 60 * 1000, DFSUtil.parseRelativeTime("25h"));
   assertEquals(4 * 24 * 60 * 60 * 1000l, DFSUtil.parseRelativeTime("4d"));
   assertEquals(999 * 24 * 60 * 60 * 1000l, DFSUtil.parseRelativeTime("999d"));
 }
Ejemplo n.º 24
0
  /**
   * Tests for empty configuration, an exception is thrown from {@link
   * DFSUtil#getNNServiceRpcAddresses(Configuration)} {@link
   * DFSUtil#getBackupNodeAddresses(Configuration)} {@link
   * DFSUtil#getSecondaryNameNodeAddresses(Configuration)}
   */
  @Test
  public void testEmptyConf() {
    HdfsConfiguration conf = new HdfsConfiguration(false);
    try {
      Map<String, Map<String, InetSocketAddress>> map = DFSUtil.getNNServiceRpcAddresses(conf);
      fail("Expected IOException is not thrown, result was: " + DFSUtil.addressMapToString(map));
    } catch (IOException expected) {
      /** Expected */
    }

    try {
      Map<String, Map<String, InetSocketAddress>> map = DFSUtil.getBackupNodeAddresses(conf);
      fail("Expected IOException is not thrown, result was: " + DFSUtil.addressMapToString(map));
    } catch (IOException expected) {
      /** Expected */
    }

    try {
      Map<String, Map<String, InetSocketAddress>> map = DFSUtil.getSecondaryNameNodeAddresses(conf);
      fail("Expected IOException is not thrown, result was: " + DFSUtil.addressMapToString(map));
    } catch (IOException expected) {
      /** Expected */
    }
  }
Ejemplo n.º 25
0
  @Test
  public void testGetNNUris() throws Exception {
    HdfsConfiguration conf = new HdfsConfiguration();

    final String NS1_NN1_ADDR = "ns1-nn1.example.com:8020";
    final String NS1_NN2_ADDR = "ns1-nn2.example.com:8020";
    final String NS2_NN_ADDR = "ns2-nn.example.com:8020";
    final String NN1_ADDR = "nn.example.com:8020";
    final String NN1_SRVC_ADDR = "nn.example.com:8021";
    final String NN2_ADDR = "nn2.example.com:8020";

    conf.set(DFS_NAMESERVICES, "ns1,ns2");
    conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"), "nn1,nn2");
    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn1"), NS1_NN1_ADDR);
    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn2"), NS1_NN2_ADDR);

    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns2"), NS2_NN_ADDR);

    conf.set(DFS_NAMENODE_RPC_ADDRESS_KEY, "hdfs://" + NN1_ADDR);

    conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + NN2_ADDR);

    Collection<URI> uris =
        DFSUtil.getNameServiceUris(
            conf, DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_ADDRESS_KEY);

    assertEquals(4, uris.size());
    assertTrue(uris.contains(new URI("hdfs://ns1")));
    assertTrue(uris.contains(new URI("hdfs://" + NS2_NN_ADDR)));
    assertTrue(uris.contains(new URI("hdfs://" + NN1_ADDR)));
    assertTrue(uris.contains(new URI("hdfs://" + NN2_ADDR)));

    // Make sure that non-HDFS URIs in fs.defaultFS don't get included.
    conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "viewfs://vfs-name.example.com");

    uris =
        DFSUtil.getNameServiceUris(
            conf, DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_ADDRESS_KEY);

    assertEquals(3, uris.size());
    assertTrue(uris.contains(new URI("hdfs://ns1")));
    assertTrue(uris.contains(new URI("hdfs://" + NS2_NN_ADDR)));
    assertTrue(uris.contains(new URI("hdfs://" + NN1_ADDR)));

    // Make sure that an HA URI being the default URI doesn't result in multiple
    // entries being returned.
    conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://ns1");

    uris =
        DFSUtil.getNameServiceUris(
            conf, DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_ADDRESS_KEY);

    assertEquals(3, uris.size());
    assertTrue(uris.contains(new URI("hdfs://ns1")));
    assertTrue(uris.contains(new URI("hdfs://" + NS2_NN_ADDR)));
    assertTrue(uris.contains(new URI("hdfs://" + NN1_ADDR)));

    // Make sure that when a service RPC address is used that is distinct from
    // the client RPC address, and that client RPC address is also used as the
    // default URI, that the client URI does not end up in the set of URIs
    // returned.
    conf = new HdfsConfiguration();
    conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + NN1_ADDR);
    conf.set(DFS_NAMENODE_RPC_ADDRESS_KEY, NN1_ADDR);
    conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, NN1_SRVC_ADDR);

    uris =
        DFSUtil.getNameServiceUris(
            conf, DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_ADDRESS_KEY);

    assertEquals(1, uris.size());
    assertTrue(uris.contains(new URI("hdfs://" + NN1_SRVC_ADDR)));
  }
Ejemplo n.º 26
0
  @Test
  public void testHANameNodesWithFederation() throws URISyntaxException {
    HdfsConfiguration conf = new HdfsConfiguration();

    final String NS1_NN1_HOST = "ns1-nn1.example.com:8020";
    final String NS1_NN2_HOST = "ns1-nn2.example.com:8020";
    final String NS2_NN1_HOST = "ns2-nn1.example.com:8020";
    final String NS2_NN2_HOST = "ns2-nn2.example.com:8020";
    conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://ns1");

    // Two nameservices, each with two NNs.
    conf.set(DFS_NAMESERVICES, "ns1,ns2");
    conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"), "ns1-nn1,ns1-nn2");
    conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns2"), "ns2-nn1,ns2-nn2");
    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "ns1-nn1"), NS1_NN1_HOST);
    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "ns1-nn2"), NS1_NN2_HOST);
    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns2", "ns2-nn1"), NS2_NN1_HOST);
    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns2", "ns2-nn2"), NS2_NN2_HOST);

    Map<String, Map<String, InetSocketAddress>> map = DFSUtil.getHaNnRpcAddresses(conf);

    assertTrue(HAUtil.isHAEnabled(conf, "ns1"));
    assertTrue(HAUtil.isHAEnabled(conf, "ns2"));
    assertFalse(HAUtil.isHAEnabled(conf, "ns3"));

    assertEquals(NS1_NN1_HOST, map.get("ns1").get("ns1-nn1").toString());
    assertEquals(NS1_NN2_HOST, map.get("ns1").get("ns1-nn2").toString());
    assertEquals(NS2_NN1_HOST, map.get("ns2").get("ns2-nn1").toString());
    assertEquals(NS2_NN2_HOST, map.get("ns2").get("ns2-nn2").toString());

    assertEquals(NS1_NN1_HOST, DFSUtil.getNamenodeServiceAddr(conf, "ns1", "ns1-nn1"));
    assertEquals(NS1_NN2_HOST, DFSUtil.getNamenodeServiceAddr(conf, "ns1", "ns1-nn2"));
    assertEquals(NS2_NN1_HOST, DFSUtil.getNamenodeServiceAddr(conf, "ns2", "ns2-nn1"));

    // No nameservice was given and we can't determine which service addr
    // to use as two nameservices could share a namenode ID.
    assertEquals(null, DFSUtil.getNamenodeServiceAddr(conf, null, "ns1-nn1"));

    // Ditto for nameservice IDs, if multiple are defined
    assertEquals(null, DFSUtil.getNamenodeNameServiceId(conf));
    assertEquals(null, DFSUtil.getSecondaryNameServiceId(conf));

    Collection<URI> uris = DFSUtil.getNameServiceUris(conf, DFS_NAMENODE_RPC_ADDRESS_KEY);
    assertEquals(2, uris.size());
    assertTrue(uris.contains(new URI("hdfs://ns1")));
    assertTrue(uris.contains(new URI("hdfs://ns2")));
  }
Ejemplo n.º 27
0
  @Test
  public void getNameNodeServiceAddr() throws IOException {
    HdfsConfiguration conf = new HdfsConfiguration();

    // One nameservice with two NNs
    final String NS1_NN1_HOST = "ns1-nn1.example.com:8020";
    final String NS1_NN1_HOST_SVC = "ns1-nn2.example.com:8021";
    final String NS1_NN2_HOST = "ns1-nn1.example.com:8020";
    final String NS1_NN2_HOST_SVC = "ns1-nn2.example.com:8021";

    conf.set(DFS_NAMESERVICES, "ns1");
    conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"), "nn1,nn2");

    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn1"), NS1_NN1_HOST);
    conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn2"), NS1_NN2_HOST);

    // The rpc address is used if no service address is defined
    assertEquals(NS1_NN1_HOST, DFSUtil.getNamenodeServiceAddr(conf, null, "nn1"));
    assertEquals(NS1_NN2_HOST, DFSUtil.getNamenodeServiceAddr(conf, null, "nn2"));

    // A nameservice is specified explicitly
    assertEquals(NS1_NN1_HOST, DFSUtil.getNamenodeServiceAddr(conf, "ns1", "nn1"));
    assertEquals(null, DFSUtil.getNamenodeServiceAddr(conf, "invalid", "nn1"));

    // The service addrs are used when they are defined
    conf.set(
        DFSUtil.addKeySuffixes(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns1", "nn1"),
        NS1_NN1_HOST_SVC);
    conf.set(
        DFSUtil.addKeySuffixes(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns1", "nn2"),
        NS1_NN2_HOST_SVC);

    assertEquals(NS1_NN1_HOST_SVC, DFSUtil.getNamenodeServiceAddr(conf, null, "nn1"));
    assertEquals(NS1_NN2_HOST_SVC, DFSUtil.getNamenodeServiceAddr(conf, null, "nn2"));

    // We can determine the nameservice ID, there's only one listed
    assertEquals("ns1", DFSUtil.getNamenodeNameServiceId(conf));
    assertEquals("ns1", DFSUtil.getSecondaryNameServiceId(conf));
  }
Ejemplo n.º 28
0
 /**
  * Test {@link DFSUtil#getBackupNameServiceId(Configuration)} to ensure nameserviceId for backup
  * node is determined based on matching the address with local node's address
  */
 @Test
 public void getBackupNameServiceId() {
   Configuration conf = setupAddress(DFS_NAMENODE_BACKUP_ADDRESS_KEY);
   assertEquals("nn1", DFSUtil.getBackupNameServiceId(conf));
 }
Ejemplo n.º 29
0
 /**
  * Test {@link DFSUtil#getSecondaryNameServiceId(Configuration)} to ensure nameserviceId for
  * backup node is determined based on matching the address with local node's address
  */
 @Test
 public void getSecondaryNameServiceId() {
   Configuration conf = setupAddress(DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY);
   assertEquals("nn1", DFSUtil.getSecondaryNameServiceId(conf));
 }
Ejemplo n.º 30
0
 @Test
 public void testSubstituteForWildcardAddress() throws IOException {
   assertEquals("foo:12345", DFSUtil.substituteForWildcardAddress("0.0.0.0:12345", "foo"));
   assertEquals("127.0.0.1:12345", DFSUtil.substituteForWildcardAddress("127.0.0.1:12345", "foo"));
 }