@Test(timeout = 60000)
  @SetupRpcConnection(setupRpcClient = false)
  public void testStubRecovery() throws Exception {
    RpcConnectionKey rpcConnectionKey =
        new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false);
    Properties connParams = new Properties();
    connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(1));
    BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams);

    EchoMessage echoMessage = EchoMessage.newBuilder().setMessage(MESSAGE).build();
    int repeat = 5;

    assertTrue(client.isConnected());
    BlockingInterface stub = client.getStub();

    client.close(); // close connection
    assertFalse(client.isConnected());

    for (int i = 0; i < repeat; i++) {
      try {
        EchoMessage response = stub.echo(null, echoMessage);
        assertEquals(MESSAGE, response.getMessage());
        assertTrue(client.isConnected());
      } finally {
        client.close(); // close connection
        assertFalse(client.isConnected());
      }
    }
  }
  @Test(timeout = 60000)
  @SetupRpcConnection(setupRpcClient = false)
  public void testRequestTimeoutOnBusy() throws Exception {
    RpcConnectionKey rpcConnectionKey =
        new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false);

    // 500 millis socket timeout
    Properties connParams = new Properties();
    connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + "");
    connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500));
    connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true");

    BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams);
    assertTrue(client.isConnected());

    BlockingInterface stub = client.getStub();
    EchoMessage echoMessage = EchoMessage.newBuilder().setMessage(MESSAGE).build();

    boolean expected = false;
    try {
      EchoMessage message = stub.busy(null, echoMessage); // 30 sec delay
      fail();
    } catch (TajoServiceException e) {
      expected = true;
    } finally {
      client.close();
    }
    assertTrue(expected);
  }
  @Test(timeout = 120000)
  @SetupRpcConnection(setupRpcClient = false)
  public void testUnresolvedAddress2() throws Exception {

    String hostAndPort = RpcUtils.normalizeInetSocketAddress(server.getListenAddress());
    RpcConnectionKey rpcConnectionKey =
        new RpcConnectionKey(RpcUtils.createUnresolved(hostAndPort), DummyProtocol.class, false);

    Properties connParams = new Properties();
    connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + "");

    BlockingRpcClient client =
        new BlockingRpcClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connParams);
    client.connect();
    assertTrue(client.isConnected());

    try {
      BlockingInterface stub = client.getStub();
      EchoMessage message = EchoMessage.newBuilder().setMessage(MESSAGE).build();
      EchoMessage response2 = stub.echo(null, message);
      assertEquals(MESSAGE, response2.getMessage());
    } finally {
      client.close();
    }
  }
  @Test(timeout = 120000)
  @SetupRpcConnection(setupRpcServer = false, setupRpcClient = false)
  public void testUnresolvedAddress() throws Exception {
    InetSocketAddress address = new InetSocketAddress("test", 0);
    boolean expected = false;
    BlockingRpcClient client = null;
    try {
      RpcConnectionKey rpcConnectionKey = new RpcConnectionKey(address, DummyProtocol.class, true);

      Properties connParams = new Properties();
      connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + "");

      client =
          new BlockingRpcClient(
              NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connParams);
      client.connect();
      fail();
    } catch (ConnectException e) {
      expected = true;
    } catch (Throwable throwable) {
      fail(throwable.getMessage());
    } finally {
      client.close();
    }
    assertTrue(expected);
  }
  @Test(timeout = 60000)
  @SetupRpcConnection(setupRpcServer = false, setupRpcClient = false)
  public void testClientRetryOnStartup() throws Exception {
    retries = 10;
    ServerSocket serverSocket = new ServerSocket(0);
    final InetSocketAddress address =
        new InetSocketAddress("127.0.0.1", serverSocket.getLocalPort());
    serverSocket.close();

    EchoMessage message = EchoMessage.newBuilder().setMessage(MESSAGE).build();

    // lazy startup
    Thread serverThread =
        new Thread(
            new Runnable() {
              @Override
              public void run() {
                try {
                  Thread.sleep(1000);
                  server =
                      new BlockingRpcServer(
                          DummyProtocol.class, new DummyProtocolBlockingImpl(), address, 2);
                } catch (Exception e) {
                  fail(e.getMessage());
                }
                server.start();
              }
            });
    serverThread.start();

    RpcConnectionKey rpcConnectionKey = new RpcConnectionKey(address, DummyProtocol.class, false);

    Properties connParams = new Properties();
    connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + "");

    BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams);
    assertTrue(client.isConnected());

    BlockingInterface stub = client.getStub();
    EchoMessage response = stub.echo(null, message);
    assertEquals(MESSAGE, response.getMessage());
    client.close();
    server.shutdown();
  }
  @Test(timeout = 60000)
  @SetupRpcConnection(setupRpcClient = false)
  public void testPingOnIdle() throws Exception {
    RpcConnectionKey rpcConnectionKey =
        new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false);

    // 500 millis socket timeout
    Properties connParams = new Properties();
    connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + "");
    connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500));
    connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true");

    BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams);
    assertTrue(client.isConnected());

    Thread.sleep(600);
    assertTrue(client.isConnected());

    Thread.sleep(600);
    assertTrue(client.isConnected());
    client.close();
  }
 public void tearDownRpcClient() throws Exception {
   if (client != null) {
     client.close();
     client = null;
   }
 }