@Test(timeout = 120000)
  @SetupRpcConnection(setupRpcServer = false, setupRpcClient = false)
  public void testUnresolvedAddress() throws Exception {
    InetSocketAddress address = new InetSocketAddress("test", 0);
    boolean expected = false;
    BlockingRpcClient client = null;
    try {
      RpcConnectionKey rpcConnectionKey = new RpcConnectionKey(address, DummyProtocol.class, true);

      Properties connParams = new Properties();
      connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + "");

      client =
          new BlockingRpcClient(
              NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connParams);
      client.connect();
      fail();
    } catch (ConnectException e) {
      expected = true;
    } catch (Throwable throwable) {
      fail(throwable.getMessage());
    } finally {
      client.close();
    }
    assertTrue(expected);
  }
  @Test(timeout = 60000)
  @SetupRpcConnection(setupRpcClient = false)
  public void testRequestTimeoutOnBusy() throws Exception {
    RpcConnectionKey rpcConnectionKey =
        new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false);

    // 500 millis socket timeout
    Properties connParams = new Properties();
    connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + "");
    connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500));
    connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true");

    BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams);
    assertTrue(client.isConnected());

    BlockingInterface stub = client.getStub();
    EchoMessage echoMessage = EchoMessage.newBuilder().setMessage(MESSAGE).build();

    boolean expected = false;
    try {
      EchoMessage message = stub.busy(null, echoMessage); // 30 sec delay
      fail();
    } catch (TajoServiceException e) {
      expected = true;
    } finally {
      client.close();
    }
    assertTrue(expected);
  }
  @Test(timeout = 120000)
  @SetupRpcConnection(setupRpcClient = false)
  public void testUnresolvedAddress2() throws Exception {

    String hostAndPort = RpcUtils.normalizeInetSocketAddress(server.getListenAddress());
    RpcConnectionKey rpcConnectionKey =
        new RpcConnectionKey(RpcUtils.createUnresolved(hostAndPort), DummyProtocol.class, false);

    Properties connParams = new Properties();
    connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + "");

    BlockingRpcClient client =
        new BlockingRpcClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connParams);
    client.connect();
    assertTrue(client.isConnected());

    try {
      BlockingInterface stub = client.getStub();
      EchoMessage message = EchoMessage.newBuilder().setMessage(MESSAGE).build();
      EchoMessage response2 = stub.echo(null, message);
      assertEquals(MESSAGE, response2.getMessage());
    } finally {
      client.close();
    }
  }
  @Test(timeout = 60000)
  @SetupRpcConnection(setupRpcServer = false, setupRpcClient = false)
  public void testClientRetryFailureOnStartup() throws Exception {
    retries = 2;
    ServerSocket serverSocket = new ServerSocket(0);
    final InetSocketAddress address =
        new InetSocketAddress("127.0.0.1", serverSocket.getLocalPort());
    serverSocket.close();

    EchoMessage message = EchoMessage.newBuilder().setMessage(MESSAGE).build();

    RpcConnectionKey rpcConnectionKey = new RpcConnectionKey(address, DummyProtocol.class, false);

    Properties connParams = new Properties();
    connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + "");

    BlockingRpcClient client =
        new BlockingRpcClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connParams);

    try {
      client.connect();
      fail();
    } catch (ConnectTimeoutException e) {
      assertFalse(e.getMessage(), client.isConnected());
    }

    BlockingInterface stub = client.getStub();
    try {
      EchoMessage response = stub.echo(null, message);
      fail();
    } catch (TajoServiceException e) {
      assertFalse(e.getMessage(), client.isConnected());
    }
    RpcClientManager.cleanup(client);
  }
  @Test(timeout = 60000)
  @SetupRpcConnection(setupRpcClient = false)
  public void testStubRecovery() throws Exception {
    RpcConnectionKey rpcConnectionKey =
        new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false);
    Properties connParams = new Properties();
    connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(1));
    BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams);

    EchoMessage echoMessage = EchoMessage.newBuilder().setMessage(MESSAGE).build();
    int repeat = 5;

    assertTrue(client.isConnected());
    BlockingInterface stub = client.getStub();

    client.close(); // close connection
    assertFalse(client.isConnected());

    for (int i = 0; i < repeat; i++) {
      try {
        EchoMessage response = stub.echo(null, echoMessage);
        assertEquals(MESSAGE, response.getMessage());
        assertTrue(client.isConnected());
      } finally {
        client.close(); // close connection
        assertFalse(client.isConnected());
      }
    }
  }
  public void setUpRpcClient() throws Exception {
    retries = 1;

    Properties connParams = new Properties();
    connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, "1");
    connParams.setProperty(
        RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(TimeUnit.SECONDS.toMillis(10)));
    connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true");

    RpcConnectionKey rpcConnectionKey =
        new RpcConnectionKey(
            RpcUtils.getConnectAddress(server.getListenAddress()), DummyProtocol.class, false);
    client = manager.newClient(rpcConnectionKey, connParams);
    assertTrue(client.isConnected());
    stub = client.getStub();
  }
  @Test(timeout = 60000)
  @SetupRpcConnection(setupRpcServer = false, setupRpcClient = false)
  public void testClientRetryOnStartup() throws Exception {
    retries = 10;
    ServerSocket serverSocket = new ServerSocket(0);
    final InetSocketAddress address =
        new InetSocketAddress("127.0.0.1", serverSocket.getLocalPort());
    serverSocket.close();

    EchoMessage message = EchoMessage.newBuilder().setMessage(MESSAGE).build();

    // lazy startup
    Thread serverThread =
        new Thread(
            new Runnable() {
              @Override
              public void run() {
                try {
                  Thread.sleep(1000);
                  server =
                      new BlockingRpcServer(
                          DummyProtocol.class, new DummyProtocolBlockingImpl(), address, 2);
                } catch (Exception e) {
                  fail(e.getMessage());
                }
                server.start();
              }
            });
    serverThread.start();

    RpcConnectionKey rpcConnectionKey = new RpcConnectionKey(address, DummyProtocol.class, false);

    Properties connParams = new Properties();
    connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + "");

    BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams);
    assertTrue(client.isConnected());

    BlockingInterface stub = client.getStub();
    EchoMessage response = stub.echo(null, message);
    assertEquals(MESSAGE, response.getMessage());
    client.close();
    server.shutdown();
  }
  @Test(timeout = 60000)
  @SetupRpcConnection(setupRpcClient = false)
  public void testIdleTimeoutWithActiveRequest() throws Exception {
    RpcConnectionKey rpcConnectionKey =
        new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false);

    // 500 millis socket timeout
    Properties connParams = new Properties();
    connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + "");
    connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500));

    BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams);
    assertTrue(client.isConnected());

    BlockingInterface stub = client.getStub();
    EchoMessage echoMessage = EchoMessage.newBuilder().setMessage(MESSAGE).build();

    EchoMessage message = stub.delay(null, echoMessage); // 3 sec delay
    assertEquals(message, echoMessage);
    assertTrue(client.isConnected());

    assertTrue(client.getActiveRequests() == 0);
    Thread.sleep(600); // timeout
    assertFalse(client.isConnected());
  }
  @Test(timeout = 60000)
  @SetupRpcConnection(setupRpcClient = false)
  public void testPingOnIdle() throws Exception {
    RpcConnectionKey rpcConnectionKey =
        new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false);

    // 500 millis socket timeout
    Properties connParams = new Properties();
    connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + "");
    connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500));
    connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true");

    BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams);
    assertTrue(client.isConnected());

    Thread.sleep(600);
    assertTrue(client.isConnected());

    Thread.sleep(600);
    assertTrue(client.isConnected());
    client.close();
  }
  @Test(timeout = 60000)
  public void testServerShutdown2() throws Exception {
    EchoMessage message = EchoMessage.newBuilder().setMessage(MESSAGE).build();

    tearDownRpcServer();
    boolean expect = false;
    try {
      BlockingInterface stub = client.getStub();
      EchoMessage response = stub.echo(null, message);
      fail(response.getMessage());
    } catch (TajoServiceException e) {
      expect = true;
    }
    assertTrue(expect);
  }
 public void tearDownRpcClient() throws Exception {
   if (client != null) {
     client.close();
     client = null;
   }
 }