Ejemplo n.º 1
0
  @Test(timeout = 90000)
  public void testRPCInterruptedSimple() throws Exception {
    final Configuration conf = new Configuration();
    Server server =
        RPC.getServer(TestProtocol.class, new TestImpl(), ADDRESS, 0, 5, true, conf, null);
    server.start();
    InetSocketAddress addr = NetUtils.getConnectAddress(server);

    final TestProtocol proxy = RPC.getProxy(TestProtocol.class, TestProtocol.versionID, addr, conf);
    // Connect to the server
    proxy.ping();
    // Interrupt self, try another call
    Thread.currentThread().interrupt();
    try {
      proxy.ping();
      fail("Interruption did not cause IPC to fail");
    } catch (IOException ioe) {
      if (!ioe.toString().contains("InterruptedException")) {
        throw ioe;
      }
      // clear interrupt status for future tests
      Thread.interrupted();
    } finally {
      server.stop();
    }
  }
Ejemplo n.º 2
0
  @Test
  public void testProxyAddress() throws IOException {
    Server server =
        new RPC.Builder(conf)
            .setProtocol(TestProtocol.class)
            .setInstance(new TestImpl())
            .setBindAddress(ADDRESS)
            .setPort(0)
            .build();
    TestProtocol proxy = null;

    try {
      server.start();
      InetSocketAddress addr = NetUtils.getConnectAddress(server);

      // create a client
      proxy = RPC.getProxy(TestProtocol.class, TestProtocol.versionID, addr, conf);

      assertEquals(addr, RPC.getServerAddress(proxy));
    } finally {
      server.stop();
      if (proxy != null) {
        RPC.stopProxy(proxy);
      }
    }
  }
Ejemplo n.º 3
0
  @Test
  public void testConnectionPing() throws Exception {
    Configuration conf = new Configuration();
    int pingInterval = 50;
    conf.setBoolean(CommonConfigurationKeys.IPC_CLIENT_PING_KEY, true);
    conf.setInt(CommonConfigurationKeys.IPC_PING_INTERVAL_KEY, pingInterval);
    final Server server =
        new RPC.Builder(conf)
            .setProtocol(TestProtocol.class)
            .setInstance(new TestImpl())
            .setBindAddress(ADDRESS)
            .setPort(0)
            .setNumHandlers(5)
            .setVerbose(true)
            .build();
    server.start();

    final TestProtocol proxy =
        RPC.getProxy(TestProtocol.class, TestProtocol.versionID, server.getListenerAddress(), conf);
    try {
      // this call will throw exception if server couldn't decode the ping
      proxy.sleep(pingInterval * 4);
    } finally {
      if (proxy != null) RPC.stopProxy(proxy);
    }
  }
Ejemplo n.º 4
0
    // Register  protocol and its impl for rpc calls
    void registerProtocolAndImpl(RpcKind rpcKind, Class<?> protocolClass, Object protocolImpl) {
      String protocolName = RPC.getProtocolName(protocolClass);
      long version;

      try {
        version = RPC.getProtocolVersion(protocolClass);
      } catch (Exception ex) {
        LOG.warn("Protocol " + protocolClass + " NOT registered as cannot get protocol version ");
        return;
      }

      getProtocolImplMap(rpcKind)
          .put(
              new ProtoNameVer(protocolName, version),
              new ProtoClassProtoImpl(protocolClass, protocolImpl));
      LOG.debug(
          "RpcKind = "
              + rpcKind
              + " Protocol Name = "
              + protocolName
              + " version="
              + version
              + " ProtocolImpl="
              + protocolImpl.getClass().getName()
              + " protocolClass="
              + protocolClass.getName());
    }
Ejemplo n.º 5
0
  private void doRPCs(Configuration conf, boolean expectFailure) throws Exception {
    SecurityUtil.setPolicy(new ConfiguredPolicy(conf, new TestPolicyProvider()));

    Server server = RPC.getServer(new TestImpl(), ADDRESS, 0, 5, true, conf);

    TestProtocol proxy = null;

    server.start();

    InetSocketAddress addr = NetUtils.getConnectAddress(server);

    try {
      proxy = (TestProtocol) RPC.getProxy(TestProtocol.class, TestProtocol.versionID, addr, conf);
      proxy.ping();

      if (expectFailure) {
        fail("Expect RPC.getProxy to fail with AuthorizationException!");
      }
    } catch (RemoteException e) {
      if (expectFailure) {
        assertTrue(e.unwrapRemoteException() instanceof AuthorizationException);
      } else {
        throw e;
      }
    } finally {
      server.stop();
      if (proxy != null) {
        RPC.stopProxy(proxy);
      }
    }
  }
Ejemplo n.º 6
0
 public static void main(String[] args) throws Exception {
   Configuration conf = new Configuration();
   BizProtocol proxy =
       RPC.getProxy(BizProtocol.class, 10010, new InetSocketAddress("localhost", 8888), conf);
   String result = proxy.hello("world");
   System.out.println(result);
   RPC.stopProxy(proxy);
 }
Ejemplo n.º 7
0
 @Test
 public void testStopProxy() throws IOException {
   StoppedProtocol proxy =
       RPC.getProxy(StoppedProtocol.class, StoppedProtocol.versionID, null, conf);
   StoppedInvocationHandler invocationHandler =
       (StoppedInvocationHandler) Proxy.getInvocationHandler(proxy);
   assertEquals(invocationHandler.getCloseCalled(), 0);
   RPC.stopProxy(proxy);
   assertEquals(invocationHandler.getCloseCalled(), 1);
 }
Ejemplo n.º 8
0
    /**
     * Construct an RPC server.
     *
     * @param protocolClass - the protocol being registered can be null for compatibility with old
     *     usage (see below for details)
     * @param protocolImpl the protocol impl that will be called
     * @param conf the configuration to use
     * @param bindAddress the address to bind on to listen for connection
     * @param port the port to listen for connections on
     * @param numHandlers the number of method handler threads to run
     * @param verbose whether each call should be logged
     */
    public Server(
        Class<?> protocolClass,
        Object protocolImpl,
        Configuration conf,
        String bindAddress,
        int port,
        int numHandlers,
        int numReaders,
        int queueSizePerHandler,
        boolean verbose,
        SecretManager<? extends TokenIdentifier> secretManager,
        String portRangeConfig)
        throws IOException {
      super(
          bindAddress,
          port,
          null,
          numHandlers,
          numReaders,
          queueSizePerHandler,
          conf,
          classNameBase(protocolImpl.getClass().getName()),
          secretManager,
          portRangeConfig);

      this.verbose = verbose;

      Class<?>[] protocols;
      if (protocolClass == null) { // derive protocol from impl
        /*
         * In order to remain compatible with the old usage where a single
         * target protocolImpl is suppled for all protocol interfaces, and
         * the protocolImpl is derived from the protocolClass(es)
         * we register all interfaces extended by the protocolImpl
         */
        protocols = RPC.getProtocolInterfaces(protocolImpl.getClass());

      } else {
        if (!protocolClass.isAssignableFrom(protocolImpl.getClass())) {
          throw new IOException(
              "protocolClass "
                  + protocolClass
                  + " is not implemented by protocolImpl which is of class "
                  + protocolImpl.getClass());
        }
        // register protocol class and its super interfaces
        registerProtocolAndImpl(RPC.RpcKind.RPC_WRITABLE, protocolClass, protocolImpl);
        protocols = RPC.getProtocolInterfaces(protocolClass);
      }
      for (Class<?> p : protocols) {
        if (!p.equals(VersionedProtocol.class)) {
          registerProtocolAndImpl(RPC.RpcKind.RPC_WRITABLE, p, protocolImpl);
        }
      }
    }
Ejemplo n.º 9
0
  public NameNodeRpcServer(Configuration conf, NameNode nn) throws IOException {
    this.nn = nn;
    this.namesystem = nn.getNamesystem();
    this.metrics = NameNode.getNameNodeMetrics();

    int handlerCount =
        conf.getInt(DFS_NAMENODE_HANDLER_COUNT_KEY, DFS_NAMENODE_HANDLER_COUNT_DEFAULT);
    InetSocketAddress socAddr = nn.getRpcServerAddress(conf);

    InetSocketAddress dnSocketAddr = nn.getServiceRpcServerAddress(conf);
    if (dnSocketAddr != null) {
      int serviceHandlerCount =
          conf.getInt(
              DFS_NAMENODE_SERVICE_HANDLER_COUNT_KEY, DFS_NAMENODE_SERVICE_HANDLER_COUNT_DEFAULT);
      this.serviceRpcServer =
          RPC.getServer(
              NamenodeProtocols.class,
              this,
              dnSocketAddr.getHostName(),
              dnSocketAddr.getPort(),
              serviceHandlerCount,
              false,
              conf,
              namesystem.getDelegationTokenSecretManager());
      this.serviceRPCAddress = this.serviceRpcServer.getListenerAddress();
      nn.setRpcServiceServerAddress(conf, serviceRPCAddress);
    } else {
      serviceRpcServer = null;
      serviceRPCAddress = null;
    }
    this.server =
        RPC.getServer(
            NamenodeProtocols.class,
            this,
            socAddr.getHostName(),
            socAddr.getPort(),
            handlerCount,
            false,
            conf,
            namesystem.getDelegationTokenSecretManager());

    // set service-level authorization security policy
    if (serviceAuthEnabled =
        conf.getBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, false)) {
      this.server.refreshServiceAcl(conf, new HDFSPolicyProvider());
      if (this.serviceRpcServer != null) {
        this.serviceRpcServer.refreshServiceAcl(conf, new HDFSPolicyProvider());
      }
    }

    // The rpc-server port can be ephemeral... ensure we have the correct info
    this.rpcAddress = this.server.getListenerAddress();
    nn.setRpcServerAddress(conf, rpcAddress);
  }
Ejemplo n.º 10
0
  @Test
  public void testSlowRpc() throws IOException {
    System.out.println("Testing Slow RPC");
    // create a server with two handlers
    Server server =
        new RPC.Builder(conf)
            .setProtocol(TestProtocol.class)
            .setInstance(new TestImpl())
            .setBindAddress(ADDRESS)
            .setPort(0)
            .setNumHandlers(2)
            .setVerbose(false)
            .build();

    TestProtocol proxy = null;

    try {
      server.start();

      InetSocketAddress addr = NetUtils.getConnectAddress(server);

      // create a client
      proxy = RPC.getProxy(TestProtocol.class, TestProtocol.versionID, addr, conf);

      SlowRPC slowrpc = new SlowRPC(proxy);
      Thread thread = new Thread(slowrpc, "SlowRPC");
      thread.start(); // send a slow RPC, which won't return until two fast pings
      assertTrue("Slow RPC should not have finished1.", !slowrpc.isDone());

      proxy.slowPing(false); // first fast ping

      // verify that the first RPC is still stuck
      assertTrue("Slow RPC should not have finished2.", !slowrpc.isDone());

      proxy.slowPing(false); // second fast ping

      // Now the slow ping should be able to be executed
      while (!slowrpc.isDone()) {
        System.out.println("Waiting for slow RPC to get done.");
        try {
          Thread.sleep(1000);
        } catch (InterruptedException e) {
        }
      }
    } finally {
      server.stop();
      if (proxy != null) {
        RPC.stopProxy(proxy);
      }
      System.out.println("Down slow rpc testing");
    }
  }
Ejemplo n.º 11
0
 private static Object createNameNodeProxy(
     InetSocketAddress address, Configuration conf, UserGroupInformation ugi, Class<?> xface)
     throws IOException {
   RPC.setProtocolEngine(conf, xface, ProtobufRpcEngine.class);
   Object proxy =
       RPC.getProxy(
           xface,
           RPC.getProtocolVersion(xface),
           address,
           ugi,
           conf,
           NetUtils.getDefaultSocketFactory(conf));
   return proxy;
 }
Ejemplo n.º 12
0
  private void doRPCs(Configuration conf, boolean expectFailure) throws IOException {
    Server server =
        new RPC.Builder(conf)
            .setProtocol(TestProtocol.class)
            .setInstance(new TestImpl())
            .setBindAddress(ADDRESS)
            .setPort(0)
            .setNumHandlers(5)
            .setVerbose(true)
            .build();

    server.refreshServiceAcl(conf, new TestPolicyProvider());

    TestProtocol proxy = null;

    server.start();

    InetSocketAddress addr = NetUtils.getConnectAddress(server);

    try {
      proxy = RPC.getProxy(TestProtocol.class, TestProtocol.versionID, addr, conf);
      proxy.ping();

      if (expectFailure) {
        fail("Expect RPC.getProxy to fail with AuthorizationException!");
      }
    } catch (RemoteException e) {
      if (expectFailure) {
        assertTrue(e.unwrapRemoteException() instanceof AuthorizationException);
      } else {
        throw e;
      }
    } finally {
      server.stop();
      if (proxy != null) {
        RPC.stopProxy(proxy);
      }
      MetricsRecordBuilder rb = getMetrics(server.rpcMetrics.name());
      if (expectFailure) {
        assertCounter("RpcAuthorizationFailures", 1, rb);
      } else {
        assertCounter("RpcAuthorizationSuccesses", 1, rb);
      }
      // since we don't have authentication turned ON, we should see
      // 0 for the authentication successes and 0 for failure
      assertCounter("RpcAuthenticationFailures", 0, rb);
      assertCounter("RpcAuthenticationSuccesses", 0, rb);
    }
  }
Ejemplo n.º 13
0
  @Test
  public void testWrappedStopProxy() throws IOException {
    StoppedProtocol wrappedProxy =
        RPC.getProxy(StoppedProtocol.class, StoppedProtocol.versionID, null, conf);
    StoppedInvocationHandler invocationHandler =
        (StoppedInvocationHandler) Proxy.getInvocationHandler(wrappedProxy);

    StoppedProtocol proxy =
        (StoppedProtocol)
            RetryProxy.create(StoppedProtocol.class, wrappedProxy, RetryPolicies.RETRY_FOREVER);

    assertEquals(invocationHandler.getCloseCalled(), 0);
    RPC.stopProxy(proxy);
    assertEquals(invocationHandler.getCloseCalled(), 1);
  }
  protected void initialize(Configuration conf) throws IOException {
    InetSocketAddress socAddr = UtilizationCollector.getAddress(conf);
    int handlerCount = conf.getInt("mapred.resourceutilization.handler.count", 10);

    // create rpc server
    this.server =
        RPC.getServer(this, socAddr.getHostName(), socAddr.getPort(), handlerCount, false, conf);

    // The rpc-server port can be ephemeral... ensure we have the correct info
    this.serverAddress = this.server.getListenerAddress();
    LOG.info("Collector up at: " + this.serverAddress);

    // start RPC server
    this.server.start();

    // How long does the TaskTracker reports expire
    timeLimit = conf.getLong("mapred.resourceutilization.timelimit", DEFAULT_TIME_LIMIT);

    // How long do we consider a job is finished after it stops
    stopTimeLimit =
        conf.getLong("mapred.resourceutilization.stoptimelimit", DEFAULT_STOP_TIME_LIMIT);

    // How often do we aggregate the reports
    aggregatePeriod =
        conf.getLong("mapred.resourceutilization.aggregateperiod", DEFAULT_AGGREGATE_SLEEP_TIME);

    // Start the daemon thread to aggregate the TaskTracker reports
    this.aggregateDaemon = new Daemon(new AggregateRun());
    this.aggregateDaemon.start();
  }
 @Override
 protected void serviceStop() throws Exception {
   if (this.rmClient != null) {
     RPC.stopProxy(this.rmClient);
   }
   super.serviceStop();
 }
Ejemplo n.º 16
0
  @Test(timeout = 90000)
  public void testRPCInterruptedSimple() throws IOException {
    final Configuration conf = new Configuration();
    Server server =
        new RPC.Builder(conf)
            .setProtocol(TestProtocol.class)
            .setInstance(new TestImpl())
            .setBindAddress(ADDRESS)
            .setPort(0)
            .setNumHandlers(5)
            .setVerbose(true)
            .setSecretManager(null)
            .build();

    server.start();
    InetSocketAddress addr = NetUtils.getConnectAddress(server);

    final TestProtocol proxy = RPC.getProxy(TestProtocol.class, TestProtocol.versionID, addr, conf);
    // Connect to the server
    proxy.ping();
    // Interrupt self, try another call
    Thread.currentThread().interrupt();
    try {
      proxy.ping();
      fail("Interruption did not cause IPC to fail");
    } catch (IOException ioe) {
      if (!ioe.toString().contains("InterruptedException")) {
        throw ioe;
      }
      // clear interrupt status for future tests
      Thread.interrupted();
    }
  }
Ejemplo n.º 17
0
  /** Initialize SecondaryNameNode. */
  private void initialize(Configuration conf) throws IOException {
    // initiate Java VM metrics
    JvmMetrics.init("SecondaryNameNode", conf.get("session.id"));

    // Create connection to the namenode.
    shouldRun = true;
    nameNodeAddr = NameNode.getAddress(conf);

    this.conf = conf;
    this.namenode =
        (NamenodeProtocol)
            RPC.waitForProxy(
                NamenodeProtocol.class, NamenodeProtocol.versionID, nameNodeAddr, conf);

    // initialize checkpoint directories
    fsName = getInfoServer();
    checkpointDirs = FSImage.getCheckpointDirs(conf, "/tmp/hadoop/dfs/namesecondary");
    checkpointEditsDirs = FSImage.getCheckpointEditsDirs(conf, "/tmp/hadoop/dfs/namesecondary");
    checkpointImage = new CheckpointStorage(conf);
    checkpointImage.recoverCreate(checkpointDirs, checkpointEditsDirs);

    // Initialize other scheduling parameters from the configuration
    checkpointPeriod = conf.getLong("fs.checkpoint.period", 3600);
    checkpointSize = conf.getLong("fs.checkpoint.size", 4194304);

    // initialize the webserver for uploading files.
    String infoAddr =
        NetUtils.getServerAddress(
            conf,
            "dfs.secondary.info.bindAddress",
            "dfs.secondary.info.port",
            "dfs.secondary.http.address");
    InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr);
    infoBindAddress = infoSocAddr.getHostName();
    int tmpInfoPort = infoSocAddr.getPort();
    infoServer = new HttpServer("secondary", infoBindAddress, tmpInfoPort, tmpInfoPort == 0, conf);
    infoServer.setAttribute("name.system.image", checkpointImage);
    this.infoServer.setAttribute("name.conf", conf);
    infoServer.addInternalServlet("getimage", "/getimage", GetImageServlet.class);
    infoServer.start();

    // The web-server port can be ephemeral... ensure we have the correct info
    infoPort = infoServer.getPort();
    conf.set("dfs.secondary.http.address", infoBindAddress + ":" + infoPort);
    LOG.info("Secondary Web-server up at: " + infoBindAddress + ":" + infoPort);
    LOG.warn(
        "Checkpoint Period   :"
            + checkpointPeriod
            + " secs "
            + "("
            + checkpointPeriod / 60
            + " min)");
    LOG.warn(
        "Log Size Trigger    :"
            + checkpointSize
            + " bytes "
            + "("
            + checkpointSize / 1024
            + " KB)");
  }
Ejemplo n.º 18
0
 public Invocation(Method method, Object[] parameters) {
   this.methodName = method.getName();
   this.parameterClasses = method.getParameterTypes();
   this.parameters = parameters;
   rpcVersion = writableRpcVersion;
   if (method.getDeclaringClass().equals(VersionedProtocol.class)) {
     // VersionedProtocol is exempted from version check.
     clientVersion = 0;
     clientMethodsHash = 0;
   } else {
     this.clientVersion = RPC.getProtocolVersion(method.getDeclaringClass());
     this.clientMethodsHash =
         ProtocolSignature.getFingerprint(method.getDeclaringClass().getMethods());
   }
   this.declaringClassProtocolName = RPC.getProtocolName(method.getDeclaringClass());
 }
Ejemplo n.º 19
0
  private void doHealthChecks() throws InterruptedException {
    while (shouldRun) {
      HAServiceStatus status = null;
      boolean healthy = false;
      try {
        status = proxy.getServiceStatus();
        proxy.monitorHealth();
        healthy = true;
      } catch (HealthCheckFailedException e) {
        LOG.warn("Service health check failed for " + targetToMonitor + ": " + e.getMessage());
        enterState(State.SERVICE_UNHEALTHY);
      } catch (Throwable t) {
        LOG.warn(
            "Transport-level exception trying to monitor health of "
                + targetToMonitor
                + ": "
                + t.getLocalizedMessage());
        RPC.stopProxy(proxy);
        proxy = null;
        enterState(State.SERVICE_NOT_RESPONDING);
        Thread.sleep(sleepAfterDisconnectMillis);
        return;
      }

      if (status != null) {
        setLastServiceStatus(status);
      }
      if (healthy) {
        enterState(State.SERVICE_HEALTHY);
      }

      Thread.sleep(checkIntervalMillis);
    }
  }
Ejemplo n.º 20
0
  private static ClientProtocol createNNProxyWithClientProtocol(
      InetSocketAddress address,
      Configuration conf,
      UserGroupInformation ugi,
      boolean withRetries,
      AtomicBoolean fallbackToSimpleAuth)
      throws IOException {
    RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class, ProtobufRpcEngine.class);

    final RetryPolicy defaultPolicy =
        RetryUtils.getDefaultRetryPolicy(
            conf,
            HdfsClientConfigKeys.Retry.POLICY_ENABLED_KEY,
            HdfsClientConfigKeys.Retry.POLICY_ENABLED_DEFAULT,
            HdfsClientConfigKeys.Retry.POLICY_SPEC_KEY,
            HdfsClientConfigKeys.Retry.POLICY_SPEC_DEFAULT,
            SafeModeException.class.getName());

    final long version = RPC.getProtocolVersion(ClientNamenodeProtocolPB.class);
    ClientNamenodeProtocolPB proxy =
        RPC.getProtocolProxy(
                ClientNamenodeProtocolPB.class,
                version,
                address,
                ugi,
                conf,
                NetUtils.getDefaultSocketFactory(conf),
                org.apache.hadoop.ipc.Client.getTimeout(conf),
                defaultPolicy,
                fallbackToSimpleAuth)
            .getProxy();

    if (withRetries) { // create the proxy with retries

      Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>();
      ClientProtocol translatorProxy = new ClientNamenodeProtocolTranslatorPB(proxy);
      return (ClientProtocol)
          RetryProxy.create(
              ClientProtocol.class,
              new DefaultFailoverProxyProvider<ClientProtocol>(
                  ClientProtocol.class, translatorProxy),
              methodNameToPolicyMap,
              defaultPolicy);
    } else {
      return new ClientNamenodeProtocolTranslatorPB(proxy);
    }
  }
 static ClientDatanodeProtocolPB createClientDatanodeProtocolProxy(
     InetSocketAddress addr,
     UserGroupInformation ticket,
     Configuration conf,
     SocketFactory factory,
     int socketTimeout)
     throws IOException {
   RPC.setProtocolEngine(conf, ClientDatanodeProtocolPB.class, ProtobufRpcEngine.class);
   return RPC.getProxy(
       ClientDatanodeProtocolPB.class,
       RPC.getProtocolVersion(ClientDatanodeProtocolPB.class),
       addr,
       ticket,
       conf,
       factory,
       socketTimeout);
 }
Ejemplo n.º 22
0
 private void initProtocolMetaInfo(Configuration conf) {
   RPC.setProtocolEngine(conf, ProtocolMetaInfoPB.class, ProtobufRpcEngine.class);
   ProtocolMetaInfoServerSideTranslatorPB xlator =
       new ProtocolMetaInfoServerSideTranslatorPB(this);
   BlockingService protocolInfoBlockingService =
       ProtocolInfoService.newReflectiveBlockingService(xlator);
   addProtocol(
       RpcKind.RPC_PROTOCOL_BUFFER, ProtocolMetaInfoPB.class, protocolInfoBlockingService);
 }
 @Override
 public boolean isMethodSupported(String methodName) throws IOException {
   return RpcClientUtil.isMethodSupported(
       rpcProxy,
       ClientDatanodeProtocolPB.class,
       RPC.RpcKind.RPC_PROTOCOL_BUFFER,
       RPC.getProtocolVersion(ClientDatanodeProtocolPB.class),
       methodName);
 }
Ejemplo n.º 24
0
 public void shutdown() {
   if (executor != null) {
     executor.shutdownNow();
   }
   if (taskReporter != null) {
     taskReporter.shutdown();
   }
   if (umbilical != null) {
     RPC.stopProxy(umbilical);
   }
 }
Ejemplo n.º 25
0
 @Test
 public void testRpcMetrics() throws Exception {
   Configuration configuration = new Configuration();
   final int interval = 1;
   configuration.setBoolean(CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE, true);
   configuration.set(CommonConfigurationKeys.RPC_METRICS_PERCENTILES_INTERVALS_KEY, "" + interval);
   final Server server =
       new RPC.Builder(configuration)
           .setProtocol(TestProtocol.class)
           .setInstance(new TestImpl())
           .setBindAddress(ADDRESS)
           .setPort(0)
           .setNumHandlers(5)
           .setVerbose(true)
           .build();
   server.start();
   final TestProtocol proxy =
       RPC.getProxy(
           TestProtocol.class, TestProtocol.versionID, server.getListenerAddress(), configuration);
   try {
     for (int i = 0; i < 1000; i++) {
       proxy.ping();
       proxy.echo("" + i);
     }
     MetricsRecordBuilder rpcMetrics = getMetrics(server.getRpcMetrics().name());
     assertTrue(
         "Expected non-zero rpc queue time", getLongCounter("RpcQueueTimeNumOps", rpcMetrics) > 0);
     assertTrue(
         "Expected non-zero rpc processing time",
         getLongCounter("RpcProcessingTimeNumOps", rpcMetrics) > 0);
     MetricsAsserts.assertQuantileGauges("RpcQueueTime" + interval + "s", rpcMetrics);
     MetricsAsserts.assertQuantileGauges("RpcProcessingTime" + interval + "s", rpcMetrics);
   } finally {
     if (proxy != null) {
       RPC.stopProxy(proxy);
     }
     server.stop();
   }
 }
Ejemplo n.º 26
0
 public void testStandaloneClient() throws IOException {
   try {
     RPC.waitForProxy(
         TestProtocol.class,
         TestProtocol.versionID,
         new InetSocketAddress(ADDRESS, 20),
         conf,
         15000L);
     fail("We should not have reached here");
   } catch (ConnectException ioe) {
     // this is what we expected
   }
 }
Ejemplo n.º 27
0
 @Override
 protected void serviceStop() throws Exception {
   if (this.rmClient != null) {
     RPC.stopProxy(this.rmClient);
   }
   if (historyServiceEnabled) {
     historyClient.stop();
   }
   if (timelineServiceEnabled) {
     timelineClient.stop();
   }
   super.serviceStop();
 }
  /** Read the block length from one of the datanodes. */
  private long readBlockLength(LocatedBlock locatedblock) throws IOException {
    assert locatedblock != null : "LocatedBlock cannot be null";
    int replicaNotFoundCount = locatedblock.getLocations().length;

    for (DatanodeInfo datanode : locatedblock.getLocations()) {
      ClientDatanodeProtocol cdp = null;

      try {
        cdp =
            DFSUtil.createClientDatanodeProtocolProxy(
                datanode, dfsClient.conf, dfsClient.getConf().socketTimeout, locatedblock);

        final long n = cdp.getReplicaVisibleLength(locatedblock.getBlock());

        if (n >= 0) {
          return n;
        }
      } catch (IOException ioe) {
        if (ioe instanceof RemoteException
            && (((RemoteException) ioe).unwrapRemoteException()
                instanceof ReplicaNotFoundException)) {
          // special case : replica might not be on the DN, treat as 0 length
          replicaNotFoundCount--;
        }

        if (DFSClient.LOG.isDebugEnabled()) {
          DFSClient.LOG.debug(
              "Failed to getReplicaVisibleLength from datanode "
                  + datanode
                  + " for block "
                  + locatedblock.getBlock(),
              ioe);
        }
      } finally {
        if (cdp != null) {
          RPC.stopProxy(cdp);
        }
      }
    }

    // Namenode told us about these locations, but none know about the replica
    // means that we hit the race between pipeline creation start and end.
    // we require all 3 because some other exception could have happened
    // on a DN that has it.  we want to report that error
    if (replicaNotFoundCount == 0) {
      return 0;
    }

    throw new IOException("Cannot obtain block length for " + locatedblock);
  }
  public void initializeServer() throws IOException {

    String serverAddr = conf.get(CLUSTER_BALANCER_ADDR, "localhost:9143");
    InetSocketAddress addr = NetUtils.createSocketAddr(serverAddr);
    clusterDaemonServer = RPC.getServer(this, addr.getHostName(), addr.getPort(), conf);
    clusterDaemonServer.start();

    // Http server
    String infoServerAddr = conf.get(CLUSTER_HTTP_BALANCER_ADDR, "localhost:50143");
    InetSocketAddress infoAddr = NetUtils.createSocketAddr(infoServerAddr);
    infoServer =
        new HttpServer(
            "cb", infoAddr.getHostName(), infoAddr.getPort(), infoAddr.getPort() == 0, conf);
    infoServer.setAttribute("cluster.balancer", this);
    infoServer.start();
  }
Ejemplo n.º 30
0
  /** Start the JobTracker process, listen on the indicated port */
  JobTracker(Configuration conf) throws IOException {
    //
    // Grab some static constants
    //
    maxCurrentTasks = conf.getInt("mapred.tasktracker.tasks.maximum", 2);
    RETIRE_JOB_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.interval", 24 * 60 * 60 * 1000);
    RETIRE_JOB_CHECK_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.check", 60 * 1000);
    TASK_ALLOC_EPSILON = conf.getFloat("mapred.jobtracker.taskalloc.loadbalance.epsilon", 0.2f);
    PAD_FRACTION = conf.getFloat("mapred.jobtracker.taskalloc.capacitypad", 0.1f);
    MIN_SLOTS_FOR_PADDING = 3 * maxCurrentTasks;

    // This is a directory of temporary submission files.  We delete it
    // on startup, and can delete any files that we're done with
    this.conf = conf;
    JobConf jobConf = new JobConf(conf);
    this.systemDir = jobConf.getSystemDir();
    this.fs = FileSystem.get(conf);
    FileUtil.fullyDelete(fs, systemDir);
    fs.mkdirs(systemDir);

    // Same with 'localDir' except it's always on the local disk.
    jobConf.deleteLocalFiles(SUBDIR);

    // Set ports, start RPC servers, etc.
    InetSocketAddress addr = getAddress(conf);
    this.localMachine = addr.getHostName();
    this.port = addr.getPort();
    this.interTrackerServer = RPC.getServer(this, addr.getPort(), 10, false, conf);
    this.interTrackerServer.start();
    Properties p = System.getProperties();
    for (Iterator it = p.keySet().iterator(); it.hasNext(); ) {
      String key = (String) it.next();
      String val = (String) p.getProperty(key);
      LOG.info("Property '" + key + "' is " + val);
    }

    this.infoPort = conf.getInt("mapred.job.tracker.info.port", 50030);
    this.infoServer = new JobTrackerInfoServer(this, infoPort);
    this.infoServer.start();

    this.startTime = System.currentTimeMillis();

    new Thread(this.expireTrackers).start();
    new Thread(this.retireJobs).start();
    new Thread(this.initJobs).start();
  }