Пример #1
1
  @Test(expected = FileNotFoundException.class)
  public final void testNoSuchFile() throws Exception {
    QueryIdFactory.reset();
    SubQueryId schid = QueryIdFactory.newSubQueryId(QueryIdFactory.newQueryId());
    QueryUnitId qid1 = QueryIdFactory.newQueryUnitId(schid);
    QueryUnitId qid2 = QueryIdFactory.newQueryUnitId(schid);

    File qid1Dir = new File(TEST_DATA + "/" + qid1.toString() + "/out");
    qid1Dir.mkdirs();
    File qid2Dir = new File(TEST_DATA + "/" + qid2.toString() + "/out");
    qid2Dir.mkdirs();

    Random rnd = new Random();
    FileWriter writer = new FileWriter(qid1Dir + "/" + "testHttp");
    String watermark1 = "test_" + rnd.nextInt();
    writer.write(watermark1);
    writer.flush();
    writer.close();

    writer = new FileWriter(qid2Dir + "/" + "testHttp");
    String watermark2 = "test_" + rnd.nextInt();
    writer.write(watermark2);
    writer.flush();
    writer.close();

    InterDataRetriever ret = new InterDataRetriever();
    HttpDataServer server = new HttpDataServer(NetUtils.createSocketAddr("127.0.0.1:0"), ret);
    server.start();

    ret.register(qid1, qid1Dir.getPath());
    InetSocketAddress addr = server.getBindAddress();
    assertDataRetrival(qid1, addr.getPort(), watermark1);
    ret.unregister(qid1);
    assertDataRetrival(qid1, addr.getPort(), watermark1);
  }
Пример #2
0
  /** Initialize SecondaryNameNode. */
  private void initialize(Configuration conf) throws IOException {
    // initiate Java VM metrics
    JvmMetrics.init("SecondaryNameNode", conf.get("session.id"));

    // Create connection to the namenode.
    shouldRun = true;
    nameNodeAddr = NameNode.getAddress(conf);

    this.conf = conf;
    this.namenode =
        (NamenodeProtocol)
            RPC.waitForProxy(
                NamenodeProtocol.class, NamenodeProtocol.versionID, nameNodeAddr, conf);

    // initialize checkpoint directories
    fsName = getInfoServer();
    checkpointDirs = FSImage.getCheckpointDirs(conf, "/tmp/hadoop/dfs/namesecondary");
    checkpointEditsDirs = FSImage.getCheckpointEditsDirs(conf, "/tmp/hadoop/dfs/namesecondary");
    checkpointImage = new CheckpointStorage(conf);
    checkpointImage.recoverCreate(checkpointDirs, checkpointEditsDirs);

    // Initialize other scheduling parameters from the configuration
    checkpointPeriod = conf.getLong("fs.checkpoint.period", 3600);
    checkpointSize = conf.getLong("fs.checkpoint.size", 4194304);

    // initialize the webserver for uploading files.
    String infoAddr =
        NetUtils.getServerAddress(
            conf,
            "dfs.secondary.info.bindAddress",
            "dfs.secondary.info.port",
            "dfs.secondary.http.address");
    InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr);
    infoBindAddress = infoSocAddr.getHostName();
    int tmpInfoPort = infoSocAddr.getPort();
    infoServer = new HttpServer("secondary", infoBindAddress, tmpInfoPort, tmpInfoPort == 0, conf);
    infoServer.setAttribute("name.system.image", checkpointImage);
    this.infoServer.setAttribute("name.conf", conf);
    infoServer.addInternalServlet("getimage", "/getimage", GetImageServlet.class);
    infoServer.start();

    // The web-server port can be ephemeral... ensure we have the correct info
    infoPort = infoServer.getPort();
    conf.set("dfs.secondary.http.address", infoBindAddress + ":" + infoPort);
    LOG.info("Secondary Web-server up at: " + infoBindAddress + ":" + infoPort);
    LOG.warn(
        "Checkpoint Period   :"
            + checkpointPeriod
            + " secs "
            + "("
            + checkpointPeriod / 60
            + " min)");
    LOG.warn(
        "Log Size Trigger    :"
            + checkpointSize
            + " bytes "
            + "("
            + checkpointSize / 1024
            + " KB)");
  }
  static ClientDatanodeProtocolPB createClientDatanodeProtocolProxy(
      DatanodeID datanodeid,
      Configuration conf,
      int socketTimeout,
      boolean connectToDnViaHostname,
      LocatedBlock locatedBlock)
      throws IOException {
    final String dnAddr = datanodeid.getIpcAddr(connectToDnViaHostname);
    InetSocketAddress addr = NetUtils.createSocketAddr(dnAddr);
    if (LOG.isDebugEnabled()) {
      LOG.debug("Connecting to datanode " + dnAddr + " addr=" + addr);
    }

    // Since we're creating a new UserGroupInformation here, we know that no
    // future RPC proxies will be able to re-use the same connection. And
    // usages of this proxy tend to be one-off calls.
    //
    // This is a temporary fix: callers should really achieve this by using
    // RPC.stopProxy() on the resulting object, but this is currently not
    // working in trunk. See the discussion on HDFS-1965.
    Configuration confWithNoIpcIdle = new Configuration(conf);
    confWithNoIpcIdle.setInt(
        CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0);

    UserGroupInformation ticket =
        UserGroupInformation.createRemoteUser(locatedBlock.getBlock().getLocalBlock().toString());
    ticket.addToken(locatedBlock.getBlockToken());
    return createClientDatanodeProtocolProxy(
        addr, ticket, confWithNoIpcIdle, NetUtils.getDefaultSocketFactory(conf), socketTimeout);
  }
Пример #4
0
  public void testSocketAddress() throws IOException {
    Configuration conf = new Configuration();
    final String defaultAddr = "host:1";
    final int defaultPort = 2;
    InetSocketAddress addr = null;

    addr = conf.getSocketAddr("myAddress", defaultAddr, defaultPort);
    assertEquals(defaultAddr, NetUtils.getHostPortString(addr));

    conf.set("myAddress", "host2");
    addr = conf.getSocketAddr("myAddress", defaultAddr, defaultPort);
    assertEquals("host2:" + defaultPort, NetUtils.getHostPortString(addr));

    conf.set("myAddress", "host2:3");
    addr = conf.getSocketAddr("myAddress", defaultAddr, defaultPort);
    assertEquals("host2:3", NetUtils.getHostPortString(addr));

    boolean threwException = false;
    conf.set("myAddress", "bad:-port");
    try {
      addr = conf.getSocketAddr("myAddress", defaultAddr, defaultPort);
    } catch (IllegalArgumentException iae) {
      threwException = true;
      assertEquals(
          "Does not contain a valid host:port authority: "
              + "bad:-port (configuration property 'myAddress')",
          iae.getMessage());

    } finally {
      assertTrue(threwException);
    }
  }
Пример #5
0
  public void testSetSocketAddress() throws IOException {
    Configuration conf = new Configuration();
    NetUtils.addStaticResolution("host", "127.0.0.1");
    final String defaultAddr = "host:1";

    InetSocketAddress addr = NetUtils.createSocketAddr(defaultAddr);
    conf.setSocketAddr("myAddress", addr);
    assertEquals(defaultAddr, NetUtils.getHostPortString(addr));
  }
Пример #6
0
 NamenodeRegistration setRegistration() throws IOException {
   nodeRegistration =
       new NamenodeRegistration(
           NetUtils.getHostPortString(rpcServer.getRpcAddress()),
           NetUtils.getHostPortString(getHttpAddress()),
           StorageInfo.getStorageInfoFromDB(),
           getRole()); // HOP change. previous code was getFSImage().getStorage()
   return nodeRegistration;
 }
  @Override
  public void start() {

    // NodeManager is the last service to start, so NodeId is available.
    this.nodeId = this.context.getNodeId();

    String httpBindAddressStr =
        getConfig()
            .get(YarnConfiguration.NM_WEBAPP_ADDRESS, YarnConfiguration.DEFAULT_NM_WEBAPP_ADDRESS);
    InetSocketAddress httpBindAddress =
        NetUtils.createSocketAddr(
            httpBindAddressStr,
            YarnConfiguration.DEFAULT_NM_WEBAPP_PORT,
            YarnConfiguration.NM_WEBAPP_ADDRESS);
    try {
      //      this.hostName = InetAddress.getLocalHost().getCanonicalHostName();
      this.httpPort = httpBindAddress.getPort();
      // Registration has to be in start so that ContainerManager can get the
      // perNM tokens needed to authenticate ContainerTokens.
      registerWithRM();
      super.start();
      startStatusUpdater();
    } catch (Exception e) {
      throw new AvroRuntimeException(e);
    }
  }
Пример #8
0
  /**
   * Get a protocol proxy that contains a proxy connection to a remote server and a set of methods
   * that are supported by the server
   *
   * @param protocol
   * @param clientVersion
   * @param addr
   * @param conf
   * @return a protocol proxy
   * @throws IOException
   */
  public static <T> ProtocolProxy<T> getProtocolProxy(
      Class<T> protocol, long clientVersion, InetSocketAddress addr, Configuration conf)
      throws IOException {

    return getProtocolProxy(
        protocol, clientVersion, addr, conf, NetUtils.getDefaultSocketFactory(conf));
  }
Пример #9
0
 /*
  * Convert the map returned from DFSUtil functions to an array of
  * addresses represented as "host:port"
  */
 private String[] toStringArray(List<ConfiguredNNAddress> list) {
   String[] ret = new String[list.size()];
   for (int i = 0; i < list.size(); i++) {
     ret[i] = NetUtils.getHostPortString(list.get(i).getAddress());
   }
   return ret;
 }
Пример #10
0
 protected synchronized void setupConnection() throws IOException {
   short ioFailures = 0;
   short timeoutFailures = 0;
   while (true) {
     try {
       this.socket = socketFactory.createSocket();
       this.socket.setTcpNoDelay(tcpNoDelay);
       this.socket.setKeepAlive(tcpKeepAlive);
       // connection time out is 20s
       NetUtils.connect(this.socket, remoteId.getAddress(), getSocketTimeout(conf));
       if (remoteId.rpcTimeout > 0) {
         pingInterval = remoteId.rpcTimeout; // overwrite pingInterval
       }
       this.socket.setSoTimeout(pingInterval);
       return;
     } catch (SocketTimeoutException toe) {
       /* The max number of retries is 45,
        * which amounts to 20s*45 = 15 minutes retries.
        */
       handleConnectionFailure(timeoutFailures++, maxRetries, toe);
     } catch (IOException ie) {
       handleConnectionFailure(ioFailures++, maxRetries, ie);
     }
   }
 }
Пример #11
0
  @Test(timeout = 90000)
  public void testRPCInterruptedSimple() throws IOException {
    final Configuration conf = new Configuration();
    Server server =
        new RPC.Builder(conf)
            .setProtocol(TestProtocol.class)
            .setInstance(new TestImpl())
            .setBindAddress(ADDRESS)
            .setPort(0)
            .setNumHandlers(5)
            .setVerbose(true)
            .setSecretManager(null)
            .build();

    server.start();
    InetSocketAddress addr = NetUtils.getConnectAddress(server);

    final TestProtocol proxy = RPC.getProxy(TestProtocol.class, TestProtocol.versionID, addr, conf);
    // Connect to the server
    proxy.ping();
    // Interrupt self, try another call
    Thread.currentThread().interrupt();
    try {
      proxy.ping();
      fail("Interruption did not cause IPC to fail");
    } catch (IOException ioe) {
      if (!ioe.toString().contains("InterruptedException")) {
        throw ioe;
      }
      // clear interrupt status for future tests
      Thread.interrupted();
    }
  }
Пример #12
0
 public void checkNameServiceId(Configuration conf, String addr, String expectedNameServiceId) {
   InetSocketAddress s = NetUtils.createSocketAddr(addr);
   String nameserviceId =
       DFSUtil.getNameServiceIdFromAddress(
           conf, s, DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_ADDRESS_KEY);
   assertEquals(expectedNameServiceId, nameserviceId);
 }
Пример #13
0
  @Test
  public void testProxyAddress() throws IOException {
    Server server =
        new RPC.Builder(conf)
            .setProtocol(TestProtocol.class)
            .setInstance(new TestImpl())
            .setBindAddress(ADDRESS)
            .setPort(0)
            .build();
    TestProtocol proxy = null;

    try {
      server.start();
      InetSocketAddress addr = NetUtils.getConnectAddress(server);

      // create a client
      proxy = RPC.getProxy(TestProtocol.class, TestProtocol.versionID, addr, conf);

      assertEquals(addr, RPC.getServerAddress(proxy));
    } finally {
      server.stop();
      if (proxy != null) {
        RPC.stopProxy(proxy);
      }
    }
  }
  void launchWorkerNode(Container container) throws IOException, YarnException {
    Map<String, String> env = System.getenv();
    ContainerLaunchContext workerContext = Records.newRecord(ContainerLaunchContext.class);

    LocalResource workerJar = Records.newRecord(LocalResource.class);
    setupWorkerJar(workerJar);
    workerContext.setLocalResources(Collections.singletonMap("socialite.jar", workerJar));
    System.out.println("[Master] workerJar:" + workerJar);

    Map<String, String> workerEnv = new HashMap<String, String>(env);
    setupWorkerEnv(workerEnv);
    workerContext.setEnvironment(workerEnv);

    String opts = "-Dsocialite.master=" + NetUtils.getHostname().split("/")[1] + " ";
    opts += "-Dsocialite.worker.num_threads=" + ClusterConf.get().getNumWorkerThreads() + " ";
    workerContext.setCommands(
        Collections.singletonList(
            "$JAVA_HOME/bin/java -ea "
                + env.get("JVM_OPTS")
                + " "
                + env.get("SOCIALITE_OPTS")
                + " "
                + opts
                + " "
                + "socialite.dist.worker.WorkerNode"
                + " 1>"
                + ApplicationConstants.LOG_DIR_EXPANSION_VAR
                + "/stdout"
                + " 2>"
                + ApplicationConstants.LOG_DIR_EXPANSION_VAR
                + "/stderr"));
    nmClient.startContainer(container, workerContext);
    L.info("Launched worker node (container:" + container.getId() + ")");
  }
Пример #15
0
  @Test(timeout = 90000)
  public void testRPCInterruptedSimple() throws Exception {
    final Configuration conf = new Configuration();
    Server server =
        RPC.getServer(TestProtocol.class, new TestImpl(), ADDRESS, 0, 5, true, conf, null);
    server.start();
    InetSocketAddress addr = NetUtils.getConnectAddress(server);

    final TestProtocol proxy = RPC.getProxy(TestProtocol.class, TestProtocol.versionID, addr, conf);
    // Connect to the server
    proxy.ping();
    // Interrupt self, try another call
    Thread.currentThread().interrupt();
    try {
      proxy.ping();
      fail("Interruption did not cause IPC to fail");
    } catch (IOException ioe) {
      if (!ioe.toString().contains("InterruptedException")) {
        throw ioe;
      }
      // clear interrupt status for future tests
      Thread.interrupted();
    } finally {
      server.stop();
    }
  }
Пример #16
0
  private void doRPCs(Configuration conf, boolean expectFailure) throws Exception {
    SecurityUtil.setPolicy(new ConfiguredPolicy(conf, new TestPolicyProvider()));

    Server server = RPC.getServer(new TestImpl(), ADDRESS, 0, 5, true, conf);

    TestProtocol proxy = null;

    server.start();

    InetSocketAddress addr = NetUtils.getConnectAddress(server);

    try {
      proxy = (TestProtocol) RPC.getProxy(TestProtocol.class, TestProtocol.versionID, addr, conf);
      proxy.ping();

      if (expectFailure) {
        fail("Expect RPC.getProxy to fail with AuthorizationException!");
      }
    } catch (RemoteException e) {
      if (expectFailure) {
        assertTrue(e.unwrapRemoteException() instanceof AuthorizationException);
      } else {
        throw e;
      }
    } finally {
      server.stop();
      if (proxy != null) {
        RPC.stopProxy(proxy);
      }
    }
  }
Пример #17
0
  public void testSerial(
      int handlerCount, boolean handlerSleep, int clientCount, int callerCount, int callCount)
      throws Exception {
    Server server = new TestServer(handlerCount, handlerSleep);
    InetSocketAddress addr = NetUtils.getConnectAddress(server);
    server.start();

    Client[] clients = new Client[clientCount];
    for (int i = 0; i < clientCount; i++) {
      clients[i] = new Client(LongWritable.class, conf);
    }

    SerialCaller[] callers = new SerialCaller[callerCount];
    for (int i = 0; i < callerCount; i++) {
      callers[i] = new SerialCaller(clients[i % clientCount], addr, callCount);
      callers[i].start();
    }
    for (int i = 0; i < callerCount; i++) {
      callers[i].join();
      assertFalse(callers[i].failed);
    }
    for (int i = 0; i < clientCount; i++) {
      clients[i].stop();
    }
    server.stop();
  }
 /**
  * Constructor.
  *
  * @param datanodeid Datanode to connect to.
  * @param conf Configuration.
  * @param socketTimeout Socket timeout to use.
  * @param connectToDnViaHostname connect to the Datanode using its hostname
  * @throws IOException
  */
 public ClientDatanodeProtocolTranslatorPB(
     DatanodeID datanodeid, Configuration conf, int socketTimeout, boolean connectToDnViaHostname)
     throws IOException {
   final String dnAddr = datanodeid.getIpcAddr(connectToDnViaHostname);
   InetSocketAddress addr = NetUtils.createSocketAddr(dnAddr);
   if (LOG.isDebugEnabled()) {
     LOG.debug("Connecting to datanode " + dnAddr + " addr=" + addr);
   }
   rpcProxy =
       createClientDatanodeProtocolProxy(
           addr,
           UserGroupInformation.getCurrentUser(),
           conf,
           NetUtils.getDefaultSocketFactory(conf),
           socketTimeout);
 }
Пример #19
0
  @Test
  public final void testHttpDataServer() throws Exception {
    Random rnd = new Random();
    FileWriter writer = new FileWriter(TEST_DATA + "/" + "testHttp");
    String watermark = "test_" + rnd.nextInt();
    writer.write(watermark + "\n");
    writer.flush();
    writer.close();

    DataRetriever ret = new DirectoryRetriever(TEST_DATA);
    HttpDataServer server = new HttpDataServer(NetUtils.createSocketAddr("127.0.0.1:0"), ret);
    server.start();

    InetSocketAddress addr = server.getBindAddress();
    URL url = new URL("http://127.0.0.1:" + addr.getPort() + "/testHttp");
    BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream()));
    String line = null;
    boolean found = false;
    while ((line = in.readLine()) != null) {
      System.out.println(line);
      if (line.equals(watermark)) found = true;
    }
    assertTrue(found);
    in.close();
    server.stop();
  }
  public void initializeServer() throws IOException {

    String serverAddr = conf.get(CLUSTER_BALANCER_ADDR, "localhost:9143");
    InetSocketAddress addr = NetUtils.createSocketAddr(serverAddr);
    clusterDaemonServer = RPC.getServer(this, addr.getHostName(), addr.getPort(), conf);
    clusterDaemonServer.start();

    // Http server
    String infoServerAddr = conf.get(CLUSTER_HTTP_BALANCER_ADDR, "localhost:50143");
    InetSocketAddress infoAddr = NetUtils.createSocketAddr(infoServerAddr);
    infoServer =
        new HttpServer(
            "cb", infoAddr.getHostName(), infoAddr.getPort(), infoAddr.getPort() == 0, conf);
    infoServer.setAttribute("cluster.balancer", this);
    infoServer.start();
  }
Пример #21
0
  public void launchSupervisorOnContainer(Container container) throws IOException {
    LOG.info("Connecting to ContainerManager for containerid=" + container.getId());
    String cmIpPortStr = container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
    InetSocketAddress cmAddress = NetUtils.createSocketAddr(cmIpPortStr);
    LOG.info("Connecting to ContainerManager at " + cmIpPortStr);
    ContainerManager proxy =
        ((ContainerManager) rpc.getProxy(ContainerManager.class, cmAddress, hadoopConf));

    LOG.info("launchSupervisorOnContainer( id:" + container.getId() + " )");
    ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class);

    launchContext.setContainerId(container.getId());
    launchContext.setResource(container.getResource());

    try {
      launchContext.setUser(UserGroupInformation.getCurrentUser().getShortUserName());
    } catch (IOException e) {
      LOG.info(
          "Getting current user info failed when trying to launch the container" + e.getMessage());
    }

    Map<String, String> env = new HashMap<String, String>();
    launchContext.setEnvironment(env);

    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    String stormVersion = Util.getStormVersion(this.storm_conf);
    Path zip = new Path("/lib/storm/" + stormVersion + "/storm.zip");
    FileSystem fs = FileSystem.get(this.hadoopConf);
    localResources.put(
        "storm",
        Util.newYarnAppResource(
            fs, zip, LocalResourceType.ARCHIVE, LocalResourceVisibility.PUBLIC));

    String appHome = Util.getApplicationHomeForId(this.appAttemptId.toString());

    Path dirDst = Util.createConfigurationFileInFs(fs, appHome, this.storm_conf, this.hadoopConf);

    localResources.put("conf", Util.newYarnAppResource(fs, dirDst));

    launchContext.setLocalResources(localResources);

    List<String> supervisorArgs = Util.buildSupervisorCommands(this.storm_conf);

    launchContext.setCommands(supervisorArgs);

    StartContainerRequest startRequest = Records.newRecord(StartContainerRequest.class);
    startRequest.setContainerLaunchContext(launchContext);

    LOG.info(
        "launchSupervisorOnContainer: startRequest prepared, calling startContainer. "
            + startRequest);
    try {
      StartContainerResponse response = proxy.startContainer(startRequest);
      LOG.info("Got a start container response " + response);
    } catch (Exception e) {
      LOG.error("Caught an exception while trying to start a container", e);
      System.exit(-1);
    }
  }
Пример #22
0
 /** Returns the Jetty server that the Namenode is listening on. */
 private String getInfoServer() throws IOException {
   URI fsName = FileSystem.getDefaultUri(conf);
   if (!"hdfs".equals(fsName.getScheme())) {
     throw new IOException("This is not a DFS");
   }
   return NetUtils.getServerAddress(
       conf, "dfs.info.bindAddress", "dfs.info.port", "dfs.http.address");
 }
Пример #23
0
 public static String jobTrackUrl(String requestUrl, Configuration conf, RunningJob job) {
   // Create a link to the status of the running job
   String trackerAddress = conf.get("mapred.job.tracker.http.address");
   InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(trackerAddress);
   int cutoff = requestUrl.indexOf('/', requestUrl.lastIndexOf(':'));
   requestUrl = requestUrl.substring(0, cutoff);
   InetSocketAddress requestSocAddr = NetUtils.createSocketAddr(requestUrl);
   String address =
       "http://"
           + requestSocAddr.getHostName()
           + ":"
           + infoSocAddr.getPort()
           + "/jobdetails.jsp?jobid="
           + job.getID()
           + "&amp;refresh=30";
   return address;
 }
Пример #24
0
    /**
     * Connect to the server and set up the I/O streams. It then sends a header to the server and
     * starts the connection thread that waits for responses.
     *
     * @throws java.io.IOException e
     */
    protected synchronized void setupIOstreams() throws IOException, InterruptedException {

      if (socket != null || shouldCloseConnection.get()) {
        return;
      }

      if (failedServers.isFailedServer(remoteId.getAddress())) {
        if (LOG.isDebugEnabled()) {
          LOG.debug(
              "Not trying to connect to "
                  + remoteId.getAddress()
                  + " this server is in the failed servers list");
        }
        IOException e =
            new FailedServerException(
                "This server is in the failed servers list: " + remoteId.getAddress());
        markClosed(e);
        close();
        throw e;
      }

      try {
        if (LOG.isDebugEnabled()) {
          LOG.debug("Connecting to " + remoteId);
        }
        setupConnection();
        this.in =
            new DataInputStream(
                new BufferedInputStream(new PingInputStream(NetUtils.getInputStream(socket))));
        this.out = new DataOutputStream(new BufferedOutputStream(NetUtils.getOutputStream(socket)));
        writeHeader();

        // update last activity time
        touch();

        // start the receiver thread after the socket connection has been set up
        start();
      } catch (IOException e) {
        failedServers.addToFailedServers(remoteId.address);
        markClosed(e);
        close();

        throw e;
      }
    }
    public void start(Configuration conf) {
      YarnRPC rpc = YarnRPC.create(conf);
      // TODO : use fixed port ??
      InetSocketAddress address = NetUtils.createSocketAddr(hostAddress);
      InetAddress hostNameResolved = null;
      try {
        address.getAddress();
        hostNameResolved = InetAddress.getLocalHost();
      } catch (UnknownHostException e) {
        throw new YarnRuntimeException(e);
      }

      server = rpc.getServer(protocol, this, address, conf, null, 1);
      server.start();
      this.bindAddress = NetUtils.getConnectAddress(server);
      super.start();
      amRunning = true;
    }
Пример #26
0
  public void testUpdateSocketAddress() throws IOException {
    InetSocketAddress addr = NetUtils.createSocketAddrForHost("host", 1);
    InetSocketAddress connectAddr = conf.updateConnectAddr("myAddress", addr);
    assertEquals(connectAddr.getHostName(), addr.getHostName());

    addr = new InetSocketAddress(1);
    connectAddr = conf.updateConnectAddr("myAddress", addr);
    assertEquals(connectAddr.getHostName(), InetAddress.getLocalHost().getHostName());
  }
  private DNAddrPair chooseDataNode(LocatedBlock block) throws IOException {
    while (true) {
      DatanodeInfo[] nodes = block.getLocations();
      try {
        DatanodeInfo chosenNode = bestNode(nodes, deadNodes);
        InetSocketAddress targetAddr = NetUtils.createSocketAddr(chosenNode.getXferAddr());
        return new DNAddrPair(chosenNode, targetAddr);
      } catch (IOException ie) {
        String blockInfo = block.getBlock() + " file=" + src;
        if (failures >= dfsClient.getMaxBlockAcquireFailures()) {
          throw new BlockMissingException(
              src, "Could not obtain block: " + blockInfo, block.getStartOffset());
        }

        if (nodes == null || nodes.length == 0) {
          DFSClient.LOG.info("No node available for block: " + blockInfo);
        }
        DFSClient.LOG.info(
            "Could not obtain block "
                + block.getBlock()
                + " from any node: "
                + ie
                + ". Will get new block locations from namenode and retry...");
        try {
          // Introducing a random factor to the wait time before another retry.
          // The wait time is dependent on # of failures and a random factor.
          // At the first time of getting a BlockMissingException, the wait time
          // is a random number between 0..3000 ms. If the first retry
          // still fails, we will wait 3000 ms grace period before the 2nd retry.
          // Also at the second retry, the waiting window is expanded to 6000 ms
          // alleviating the request rate from the server. Similarly the 3rd retry
          // will wait 6000ms grace period before retry and the waiting window is
          // expanded to 9000ms.
          double waitTime =
              timeWindow * failures
                  + // grace period for the last round of attempt
                  timeWindow
                      * (failures + 1)
                      * DFSUtil.getRandom().nextDouble(); // expanding time window for each failure
          DFSClient.LOG.warn(
              "DFS chooseDataNode: got # "
                  + (failures + 1)
                  + " IOException, will wait for "
                  + waitTime
                  + " msec.");
          Thread.sleep((long) waitTime);
        } catch (InterruptedException iex) {
        }
        deadNodes.clear(); // 2nd option is to remove only nodes[blockId]
        openInfo();
        block = getBlockAt(block.getStartOffset(), false);
        failures++;
        continue;
      }
    }
  }
 protected ResourceTracker getRMClient() {
   Configuration conf = getConfig();
   YarnRPC rpc = YarnRPC.create(conf);
   InetSocketAddress rmAddress =
       NetUtils.createSocketAddr(
           this.rmAddress,
           YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT,
           YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS);
   return (ResourceTracker) rpc.getProxy(ResourceTracker.class, rmAddress, conf);
 }
 @Override
 protected void serviceInit(Configuration conf) throws Exception {
   clientServiceBindAddress = RMADDRESS;
   /*
   clientServiceBindAddress = conf.get(
       YarnConfiguration.APPSMANAGER_ADDRESS,
       YarnConfiguration.DEFAULT_APPSMANAGER_BIND_ADDRESS);
       */
   clientBindAddress = NetUtils.createSocketAddr(clientServiceBindAddress);
   super.serviceInit(conf);
 }
Пример #30
0
 /**
  * Get a delegation token for the user from the JobTracker.
  *
  * @param renewer the user who can renew the token
  * @return the new token
  * @throws IOException
  */
 public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
     throws IOException, InterruptedException {
   Token<DelegationTokenIdentifier> result = client.getDelegationToken(renewer);
   InetSocketAddress addr = Master.getMasterAddress(conf);
   StringBuilder service = new StringBuilder();
   service.append(NetUtils.normalizeHostName(addr.getAddress().getHostAddress()));
   service.append(':');
   service.append(addr.getPort());
   result.setService(new Text(service.toString()));
   return result;
 }