Esempio n. 1
0
 @Test
 // disabled due to antlr incompatibility between Pig (which requires antlr 3.4) and Hive (3.0.x)
 public void testHiveServer() throws Exception {
   TServer server = ctx.getBean("hive-server", TServer.class);
   assertNotNull(server);
   assertTrue(server.isServing());
 }
 public synchronized void stop() {
   final TServer thriftServer = this.thriftServer;
   if (thriftServer != null) {
     this.service.stop();
     thriftServer.stop();
     final ThreadPoolExecutor connExecutor = this.thriftThreadPerConnExecutor;
     if (connExecutor != null) {
       connExecutor.shutdown();
     }
     this.thriftExecutor.shutdown();
     try {
       this.thriftMainThread.join(5000L);
       // force stop the executor if required
       if (this.thriftMainThread.isAlive()) {
         if (connExecutor != null) {
           connExecutor.shutdownNow();
         }
         this.thriftExecutor.shutdownNow();
         this.thriftMainThread.join();
       }
     } catch (InterruptedException ie) {
       Thread.currentThread().interrupt();
     }
   }
 }
Esempio n. 3
0
  public static void runServer(TProcessor processor, int port, int threads) {

    TNonblockingServerTransport serverTransport;
    TServer server;
    try {
      serverTransport = new TNonblockingServerSocket(port);

      if (threads == 1) {
        server = new TNonblockingServer(processor, serverTransport);
      } else {
        THsHaServer.Options serverOptions = new THsHaServer.Options();
        serverOptions.workerThreads = threads;
        server =
            new THsHaServer(
                new TProcessorFactory(processor),
                serverTransport,
                new TFramedTransport.Factory(),
                new TBinaryProtocol.Factory(),
                serverOptions);
      }
      Runtime.getRuntime()
          .addShutdownHook(
              new Thread(
                  new ShutdownListener(server, serverTransport), "Server Shutdown Listener"));
      logger.info("Starting the server on port {} with {} threads", port, threads);
      server.serve();
    } catch (TTransportException e) {
      logger.error("Thrift Transport error");
      logger.error(e.toString());
      System.exit(1);
    }
  }
Esempio n. 4
0
 public static void simple(SimpleIrcService.Processor processor) {
   try {
     TServerTransport serverTransport = new TServerSocket(9090);
     TServer server =
         new TThreadPoolServer(new TThreadPoolServer.Args(serverTransport).processor(processor));
     System.out.println("Starting the simple server...");
     server.serve();
   } catch (Exception e) {
     e.printStackTrace();
   }
 }
Esempio n. 5
0
 // Existing connections will keep our thread running: reach in with reflection and insist that
 // they shutdown.
 public static void stopTServer(TServer s) {
   if (s == null) return;
   s.stop();
   try {
     Field f = s.getClass().getDeclaredField("executorService_");
     f.setAccessible(true);
     ExecutorService es = (ExecutorService) f.get(s);
     es.shutdownNow();
   } catch (Exception e) {
     TServerUtils.log.error("Unable to call shutdownNow", e);
   }
 }
Esempio n. 6
0
 public static void StartsimpleServer(
     AdditionService.Processor<AdditionServiceHandler> processor) {
   try {
     TServerTransport serverTransport = new TServerSocket(9090);
     TServer server = new TSimpleServer(new TServer.Args(serverTransport).processor(processor));
     //            log.info("Starting the simple server.......");
     System.out.println("Starting the simple server......");
     server.serve();
   } catch (TTransportException e) {
     e.printStackTrace();
   }
 }
 public void start() {
   try {
     TServerSocket serverTransport = new TServerSocket(7911);
     BGService.Processor<BGService.Iface> processor =
         new BGService.Processor<BGService.Iface>(new BGVerticalServiceImpl(options));
     TServer server =
         new TThreadPoolServer(new TThreadPoolServer.Args(serverTransport).processor(processor));
     System.out.println("Starting Server on port 7911..");
     server.serve();
   } catch (TTransportException e) {
     e.printStackTrace();
   }
 }
Esempio n. 8
0
 public static void main(String args[]) {
   try {
     TServerSocket serverTransport = new TServerSocket(7777);
     Factory proFactory = new TBinaryProtocol.Factory();
     TProcessor processor = new RFIDService.Processor(new RFIDServiceImpl());
     TServer server =
         new TThreadPoolServer(new TThreadPoolServer.Args(serverTransport).processor(processor));
     System.out.println("Start server on port 7777...");
     server.serve();
   } catch (TTransportException e) {
     e.printStackTrace();
   }
 }
Esempio n. 9
0
 /**
  * Stop this TachyonWorker. Stop all the threads belong to this TachyonWorker.
  *
  * @throws IOException
  * @throws InterruptedException
  */
 public void stop() throws IOException, InterruptedException {
   mStop = true;
   mWorkerStorage.stop();
   mDataServer.close();
   mServer.stop();
   mServerTNonblockingServerSocket.close();
   mExecutorService.shutdown();
   while (!mDataServer.isClosed() || mServer.isServing() || mHeartbeatThread.isAlive()) {
     // TODO The reason to stop and close again is due to some issues in Thrift.
     mServer.stop();
     mServerTNonblockingServerSocket.close();
     CommonUtils.sleepMs(null, 100);
   }
   mHeartbeatThread.join();
 }
Esempio n. 10
0
 public void stop() {
   if (server != null && server.isServing()) {
     server.stop();
   }
   servingExecutor.shutdown();
   try {
     if (!servingExecutor.awaitTermination(5, TimeUnit.SECONDS)) {
       servingExecutor.shutdownNow();
     }
   } catch (InterruptedException e) {
     throw new FlumeException("Interrupted while waiting for server to be " + "shutdown.");
   }
   sourceCounter.stop();
   // Thrift will shutdown the executor passed to it.
   super.stop();
 }
Esempio n. 11
0
  public void run() {
    try {
      int port = 9090;
      TNonblockingServerTransport serverTransport = new TNonblockingServerSocket(port);
      VicDataService.Processor<VicDataServiceImpl> processor =
          new VicDataService.Processor<VicDataServiceImpl>(new VicDataServiceImpl());

      TServer server =
          new TNonblockingServer(new TNonblockingServer.Args(serverTransport).processor(processor));
      System.out.println("Starting server on port " + port + " ...");

      server.serve();
    } catch (TTransportException e) {
      e.printStackTrace();
    }
  }
Esempio n. 12
0
  /** Start the data server thread and heartbeat thread of this TachyonWorker. */
  public void start() throws IOException {
    login();

    mHeartbeatThread.start();

    LOG.info("The worker server started @ " + mWorkerAddress);
    mServer.serve();
    LOG.info("The worker server ends @ " + mWorkerAddress);
  }
Esempio n. 13
0
 public void shutdown() {
   this.interrupt();
   if (mMasterServiceServer != null) {
     mMasterServiceServer.stop();
   }
   if (mServerTNonblockingServerSocket != null) {
     mServerTNonblockingServerSocket.close();
   }
 }
Esempio n. 14
0
 public void run() {
   if (server != null) {
     server.stop();
   }
   if (transport != null) {
     transport.interrupt();
     transport.close();
   }
 }
Esempio n. 15
0
 @Override
 public void process(WatchedEvent event) {
   log.debug("event " + event.getPath() + " " + event.getType() + " " + event.getState());
   if (event.getState() == KeeperState.Expired) {
     log.warn("Trace server lost zookeeper registration at " + event.getPath());
     server.stop();
   } else if (event.getType() == EventType.NodeDeleted) {
     log.warn("Trace server zookeeper entry lost " + event.getPath());
     server.stop();
   }
   if (event.getPath() != null) {
     try {
       if (ZooReaderWriter.getInstance().exists(event.getPath(), this)) return;
     } catch (Exception ex) {
       log.error(ex, ex);
     }
     log.warn("Trace server unable to reset watch on zookeeper registration");
     server.stop();
   }
 }
  /**
   * 启动 Thrift 服务器
   *
   * @param args
   */
  public static void main(String[] args) {
    try {
      // hello 协议的处理类
      TProcessor hello = new Hello.Processor(new HelloServiceImpl());
      TProcessor world = new World.Processor(new WorldServiceImpl());

      TMultiplexedProcessor processor = new TMultiplexedProcessor();
      processor.registerProcessor("HelloService", hello);
      processor.registerProcessor("WorldService", world);

      // 服务传输层
      TServerTransport serverTransport = new TServerSocket(9090);
      TServer server =
          new TSimpleServer(new TThreadPoolServer.Args(serverTransport).processor(processor));
      System.out.println("Start server on port 9090...");
      server.serve();
    } catch (TTransportException e) {
      e.printStackTrace();
    }
  }
Esempio n. 17
0
 public void downServer() {
   server.stop();
   try {
     serverThread.join();
   } catch (InterruptedException e) {
     // we're probably shutting down
     LOG.debug("Interrupted waiting for server thread to exit.", e);
   }
   server = null;
   serverThread = null;
 }
Esempio n. 18
0
  //    @Test
  public void testCompactServerStart() {
    try {
      TServerTransport serverTransport = new TServerSocket(port);

      //            TThreadPoolServer.Args processor = new TThreadPoolServer.Args(serverTransport)
      //                    .inputTransportFactory(new TFramedTransport.Factory())
      //                    .outputTransportFactory(new TFramedTransport.Factory())
      //                    .protocolFactory(new TCompactProtocol.Factory())
      //                    .processor(new PingPongService.Processor<>(pingPongService));

      //            TThreadPoolServer server = new TThreadPoolServer(processor);

      System.out.println("Starting the server...");
      //            server.serve();

      TServer server = new TSimpleServer(new TSimpleServer.Args(serverTransport));
      server.serve();
    } catch (Exception e) {
      e.printStackTrace();
    }
  }
Esempio n. 19
0
  @Override
  public void start() {
    logger.info("Starting thrift source");
    ExecutorService sourceService;
    ThreadFactory threadFactory =
        new ThreadFactoryBuilder().setNameFormat("Flume Thrift IPC Thread %d").build();
    if (maxThreads == 0) {
      sourceService = Executors.newCachedThreadPool(threadFactory);
    } else {
      sourceService = Executors.newFixedThreadPool(maxThreads, threadFactory);
    }
    try {
      serverTransport = new TNonblockingServerSocket(new InetSocketAddress(bindAddress, port));
    } catch (TTransportException e) {
      throw new FlumeException("Failed to start Thrift Source.", e);
    }

    THsHaServer.Args thhsArgs = new THsHaServer.Args(serverTransport);
    thhsArgs.processor(new ThriftSourceProtocol.Processor(new ThriftSourceHandler()));
    //	thhsArgs.transportFactory(new TFramedTransport.Factory());
    thhsArgs.protocolFactory(new TBinaryProtocol.Factory());
    thhsArgs.executorService(sourceService);
    server = new THsHaServer(thhsArgs); // 半同步半异步的服务模型

    servingExecutor =
        Executors.newSingleThreadExecutor(
            new ThreadFactoryBuilder().setNameFormat("Flume Thrift Source I/O Boss").build());
    /** Start serving. */
    servingExecutor.submit(
        new Runnable() {
          @Override
          public void run() {
            server.serve();
          }
        });

    long timeAfterStart = System.currentTimeMillis();
    while (!server.isServing()) {
      try {
        if (System.currentTimeMillis() - timeAfterStart >= 10000) {
          throw new FlumeException("Thrift server failed to start!");
        }
        TimeUnit.MILLISECONDS.sleep(1000);
      } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new FlumeException("Interrupted while waiting for Thrift server" + " to start.", e);
      }
    }
    sourceCounter.start();
    logger.info("Started Thrift source.");
    super.start();
  }
Esempio n. 20
0
 public void run() throws Exception {
   SimpleTimer.getInstance()
       .schedule(
           new Runnable() {
             @Override
             public void run() {
               flush();
             }
           },
           1000,
           1000);
   server.serve();
 }
Esempio n. 21
0
  //    @Test
  public void start() {
    log.info("start thrift test server.......");
    try {

      TMultiplexedProcessor processor = new TMultiplexedProcessor();

      processor.registerProcessor(
          "pingPongService", new PingPongService.Processor<>(pingPongService));

      TCompactProtocol.Factory protocolFactory = new TCompactProtocol.Factory();

      TServerSocket socket = new TServerSocket(port);
      TServer server =
          new TThreadPoolServer(
              new TThreadPoolServer.Args(socket)
                  .processor(processor)
                  .protocolFactory(protocolFactory));

      server.serve();
    } catch (TException e) {
      e.printStackTrace();
    }
  }
Esempio n. 22
0
 public static void main(String[] args) {
   try {
     int port = 10000;
     if (args.length >= 1) {
       port = Integer.parseInt(args[0]);
     }
     TServerTransport serverTransport = new TServerSocket(port);
     ThriftHiveProcessorFactory hfactory = new ThriftHiveProcessorFactory(null);
     TThreadPoolServer.Options options = new TThreadPoolServer.Options();
     TServer server =
         new TThreadPoolServer(
             hfactory,
             serverTransport,
             new TTransportFactory(),
             new TTransportFactory(),
             new TBinaryProtocol.Factory(),
             new TBinaryProtocol.Factory(),
             options);
     HiveServerHandler.LOG.info("Starting hive server on port " + port);
     server.serve();
   } catch (Exception x) {
     x.printStackTrace();
   }
 }
Esempio n. 23
0
  /**
   * start serving the thrift server. doesn't return.
   *
   * @throws IOException
   * @throws TException
   */
  private void serve() throws IOException, TException {
    // set up the service handler
    HankSmartClient handler;
    try {
      handler = new HankSmartClient(coord, ringGroupName);
    } catch (DataNotFoundException e) {
      throw new RuntimeException(e);
    }

    // launch the thrift server
    TNonblockingServerSocket serverSocket =
        new TNonblockingServerSocket(configurator.getPortNumber());
    Args options = new THsHaServer.Args(serverSocket);
    options.processor(new SmartClient.Processor(handler));
    options.workerThreads(configurator.getNumThreads());
    options.protocolFactory(new TCompactProtocol.Factory());
    server = new THsHaServer(options);
    server.serve();
  }
Esempio n. 24
0
 /** Start serving the Thrift server. Returns when the server is up. */
 public void startServer() {
   Runnable r =
       new Runnable() {
         @Override
         public void run() {
           try {
             serve();
           } catch (Exception e) {
             LOG.fatal("Unexpected error in server main loop!", e);
           }
         }
       };
   serverThread = new Thread(r, "Client Thrift Server thread");
   serverThread.start();
   try {
     while (server == null || !server.isServing()) {
       LOG.debug("waiting for smart client daemon server to come online...");
       Thread.sleep(100);
     }
     LOG.debug("smart client server is online.");
   } catch (InterruptedException e) {
     throw new RuntimeException("Interrupted waiting for server thread to start", e);
   }
 }
Esempio n. 25
0
  public static TServer factory(ProcessorFactory processorFactory) {

    loadProperties();

    try {
      validate();
    } catch (Exception e) {
      e.printStackTrace();
    }

    TProtocolFactory tProtocolFactory = getTProtocolFactory();

    TTransportFactory tTransportFactory = getTTransportFactory();

    TServer serverEngine = null;

    TMultiplexedProcessor tProcessor = processorFactory.getProcessor();

    if (server_type.equals("nonblocking") || server_type.equals("threaded-selector")) {
      // Nonblocking servers
      TNonblockingServerSocket tNonblockingServerSocket = null;
      try {
        tNonblockingServerSocket =
            new TNonblockingServerSocket(
                new TNonblockingServerSocket.NonblockingAbstractServerSocketArgs().port(port));
      } catch (TTransportException e) {
        e.printStackTrace();
      }

      if (server_type.equals("nonblocking")) {
        // Nonblocking Server
        TNonblockingServer.Args tNonblockingServerArgs =
            new TNonblockingServer.Args(tNonblockingServerSocket);
        tNonblockingServerArgs.processor(tProcessor);
        tNonblockingServerArgs.protocolFactory(tProtocolFactory);
        tNonblockingServerArgs.transportFactory(tTransportFactory);

        serverEngine = new TNonblockingServer(tNonblockingServerArgs);
      } else { // server_type.equals("threaded-selector")
        // ThreadedSelector Server
        TThreadedSelectorServer.Args tThreadedSelectorServerArgs =
            new TThreadedSelectorServer.Args(tNonblockingServerSocket);
        tThreadedSelectorServerArgs.processor(tProcessor);
        tThreadedSelectorServerArgs.protocolFactory(tProtocolFactory);
        tThreadedSelectorServerArgs.transportFactory(tTransportFactory);

        serverEngine = new TThreadedSelectorServer(tThreadedSelectorServerArgs);
      }
    } else {
      // Blocking servers

      // SSL socket
      TServerSocket tServerSocket = null;
      if (ssl) {
        try {
          tServerSocket = TSSLTransportFactory.getServerSocket(port, 0);
        } catch (TTransportException e) {
          e.printStackTrace();
        }
      } else {
        try {
          tServerSocket =
              new TServerSocket(new TServerSocket.ServerSocketTransportArgs().port(port));
        } catch (TTransportException e) {
          e.printStackTrace();
        }
      }

      if (server_type.equals("simple")) {
        // Simple Server
        TServer.Args tServerArgs = new TServer.Args(tServerSocket);
        tServerArgs.processor(tProcessor);
        tServerArgs.protocolFactory(tProtocolFactory);
        tServerArgs.transportFactory(tTransportFactory);

        serverEngine = new TSimpleServer(tServerArgs);
      } else { // server_type.equals("threadpool")
        // ThreadPool Server
        TThreadPoolServer.Args tThreadPoolServerArgs = new TThreadPoolServer.Args(tServerSocket);
        tThreadPoolServerArgs.processor(tProcessor);
        tThreadPoolServerArgs.protocolFactory(tProtocolFactory);
        tThreadPoolServerArgs.transportFactory(tTransportFactory);

        serverEngine = new TThreadPoolServer(tThreadPoolServerArgs);
      }
    }
    // Set server event handler
    serverEngine.setServerEventHandler(new TServerEventHandlerImpl());

    return serverEngine;
  }
Esempio n. 26
0
 @Override
 public void run() {
   mMasterServiceServer.serve();
 }
 public final boolean isServing() {
   final TServer thriftServer = this.thriftServer;
   return thriftServer != null && thriftServer.isServing();
 }
 public void close() {
   server.stop();
 }
 @Override
 public void run() {
   server.serve();
 }
  @Test
  public void testIt() throws Exception {
    int server1Port = 12345;
    int server2Port = 12346;
    int server3Port = 12347;

    // launch server 1
    final MockPartitionServerHandler iface1 = new MockPartitionServerHandler(VALUE_1);
    TNonblockingServerTransport transport1 = createPartitionServerTransport(server1Port);
    final TServer server1 = createPartitionServer(transport1, iface1);
    Thread thread1 = new Thread(new ServerRunnable(server1), "mock partition server thread 1");
    thread1.start();

    // launch server 2;
    final MockPartitionServerHandler iface2 = new MockPartitionServerHandler(VALUE_2);
    TNonblockingServerTransport transport2 = createPartitionServerTransport(server2Port);
    final TServer server2 = createPartitionServer(transport2, iface2);
    Thread thread2 = new Thread(new ServerRunnable(server2), "mock partition server thread 2");
    thread2.start();

    // launch server 3;
    final MockPartitionServerHandler iface3 = new MockPartitionServerHandler(VALUE_3);
    TNonblockingServerTransport transport3 = createPartitionServerTransport(server3Port);
    final TServer server3 = createPartitionServer(transport3, iface3);
    Thread thread3 = new Thread(new ServerRunnable(server3), "mock partition server thread 3");
    thread3.start();

    final MockDomain existentDomain =
        new MockDomain(
            "existent_domain",
            0,
            2,
            new MapPartitioner(KEY_1, 0, KEY_2, 1, KEY_NOT_FOUND, 0),
            null,
            null,
            null);
    final MockDomain newDomain =
        new MockDomain("new_domain", 1, 1, new MapPartitioner(KEY_3, 0), null, null, null);

    final Host host1 =
        getHost(existentDomain, new PartitionServerAddress("localhost", server1Port), 0);
    final Host host2 =
        getHost(existentDomain, new PartitionServerAddress("localhost", server2Port), 1);
    final Host host3 = getHost(newDomain, new PartitionServerAddress("localhost", server3Port), 0);

    final Set<Host> mockRingHosts = new HashSet<Host>();
    mockRingHosts.add(host1);
    mockRingHosts.add(host2);

    final MockRing mockRing =
        new MockRing(null, null, 1) {
          @Override
          public Set<Host> getHosts() {
            return mockRingHosts;
          }
        };

    MockDomainGroup mockDomainGroup =
        new MockDomainGroup("myDomainGroup") {
          @Override
          public Set<DomainAndVersion> getDomainVersions() {
            return new HashSet<DomainAndVersion>(
                Arrays.asList(new DomainAndVersion(existentDomain, 1)));
          }
        };

    final MockRingGroup mockRingGroup =
        new MockRingGroup(mockDomainGroup, "myRingGroup", null) {
          @Override
          public Set<Ring> getRings() {
            return Collections.singleton((Ring) mockRing);
          }
        };

    Coordinator mockCoord =
        new MockCoordinator() {
          @Override
          public RingGroup getRingGroup(String ringGroupName) {
            return mockRingGroup;
          }

          @Override
          public Domain getDomain(String domainName) {
            if (domainName.equals("existent_domain")) {
              return existentDomain;
            } else if (domainName.equals("new_domain")) {
              return newDomain;
            } else {
              return null;
            }
          }
        };

    WaitUntil.orDie(
        new Condition() {
          @Override
          public boolean test() {
            return server1.isServing() && server2.isServing() && server3.isServing();
          }
        });

    try {
      final HankSmartClient client =
          new HankSmartClient(
              mockCoord, "myRingGroup", new HankSmartClientOptions().setQueryTimeoutMs(1000));
      final HankSmartClient cachingClient =
          new HankSmartClient(
              mockCoord,
              "myRingGroup",
              new HankSmartClientOptions()
                  .setResponseCacheEnabled(true)
                  .setResponseCacheNumItemsCapacity(1)
                  .setResponseCacheNumBytesCapacity(-1)
                  .setResponseCacheExpirationSeconds(1));

      // Test invalid get
      assertEquals(
          HankResponse.xception(HankException.no_such_domain(true)),
          client.get("nonexistent_domain", null));

      // Test get
      assertEquals(HankResponse.value(VALUE_1), client.get("existent_domain", KEY_1));
      assertEquals(HankResponse.value(VALUE_2), client.get("existent_domain", KEY_2));

      // Test invalid getBulk
      assertEquals(
          HankBulkResponse.xception(HankException.no_such_domain(true)),
          client.getBulk("nonexistent_domain", null));

      // Test getBulk
      HankBulkResponse bulkResponse1 = HankBulkResponse.responses(new ArrayList<HankResponse>());
      bulkResponse1.get_responses().add(HankResponse.value(VALUE_1));
      bulkResponse1.get_responses().add(HankResponse.value(VALUE_2));
      List<ByteBuffer> bulkRequest1 = new ArrayList<ByteBuffer>();
      bulkRequest1.add(KEY_1);
      bulkRequest1.add(KEY_2);
      assertEquals(bulkResponse1, client.getBulk("existent_domain", bulkRequest1));

      // Test get with null key
      try {
        client.get("existent_domain", null);
        fail("Should throw an exception.");
      } catch (NullKeyException e) {
        // Good
      }

      // Test get with empty key
      try {
        client.get("existent_domain", ByteBuffer.wrap(new byte[0]));
        fail("Should throw an exception.");
      } catch (EmptyKeyException e) {
        // Good
      }

      // Host is not available
      host1.setState(HostState.UPDATING);
      assertEquals(
          HankResponse.xception(HankException.no_connection_available(true)),
          client.get("existent_domain", KEY_1));

      // Host is offline but it's the only one, use it opportunistically
      host2.setState(HostState.OFFLINE);
      assertEquals(HankResponse.value(VALUE_2), client.get("existent_domain", KEY_2));

      host1.setState(HostState.SERVING);
      host2.setState(HostState.SERVING);

      // Test location changes

      // Add new host that has new domain
      mockRingHosts.add(host3);

      // Should not be able to query new domain
      assertTrue(client.get("new_domain", KEY_3).get_xception().is_set_no_replica());

      // Notify client of data location change
      client.onDataLocationChange(mockCoord.getRingGroup("myRingGroup"));

      // Should be able to query new domain when the client has done updating its cache
      WaitUntil.orDie(
          new Condition() {
            @Override
            public boolean test() {
              return HankResponse.value(VALUE_3).equals(client.get("new_domain", KEY_3));
            }
          });
      assertEquals(HankResponse.value(VALUE_3), client.get("new_domain", KEY_3));

      // TODO: Test not querying deletable partitions

      // Simulate servers that fail to perform gets
      iface1.setMode(MockPartitionServerHandler.Mode.FAILING);
      iface2.setMode(MockPartitionServerHandler.Mode.FAILING);

      assertTrue(client.get("existent_domain", KEY_1).get_xception().get_failed_retries() > 0);
      assertTrue(client.get("existent_domain", KEY_2).get_xception().get_failed_retries() > 0);

      // Simulate servers that throws an error
      iface1.setMode(MockPartitionServerHandler.Mode.THROWING_ERROR);
      iface2.setMode(MockPartitionServerHandler.Mode.THROWING_ERROR);

      assertTrue(client.get("existent_domain", KEY_1).get_xception().get_failed_retries() > 0);
      assertTrue(client.get("existent_domain", KEY_2).get_xception().get_failed_retries() > 0);

      // Simulate servers that hangs
      iface1.setMode(MockPartitionServerHandler.Mode.HANGING);
      iface2.setMode(MockPartitionServerHandler.Mode.HANGING);

      assertTrue(client.get("existent_domain", KEY_1).get_xception().get_failed_retries() > 0);
      assertTrue(client.get("existent_domain", KEY_2).get_xception().get_failed_retries() > 0);

      // Test caching
      iface1.setMode(MockPartitionServerHandler.Mode.NORMAL);
      iface2.setMode(MockPartitionServerHandler.Mode.NORMAL);
      iface3.setMode(MockPartitionServerHandler.Mode.NORMAL);

      iface1.clearNumRequests();

      // One request
      assertEquals(HankResponse.value(VALUE_1), cachingClient.get("existent_domain", KEY_1));
      assertEquals(1, iface1.getNumRequests());

      // One cached request
      assertEquals(HankResponse.value(VALUE_1), cachingClient.get("existent_domain", KEY_1));
      assertEquals(1, iface1.getNumRequests());

      iface1.clearNumRequests();

      // One not found request
      assertEquals(
          HankResponse.not_found(true), cachingClient.get("existent_domain", KEY_NOT_FOUND));
      assertEquals(1, iface1.getNumRequests());

      // One not found cached request
      assertEquals(
          HankResponse.not_found(true), cachingClient.get("existent_domain", KEY_NOT_FOUND));
      assertEquals(1, iface1.getNumRequests());

      // Wait for cache to expire
      Thread.sleep(2000);

      // Should not be in cache anymore
      assertEquals(
          HankResponse.not_found(true), cachingClient.get("existent_domain", KEY_NOT_FOUND));
      assertEquals(2, iface1.getNumRequests());

    } finally {
      server1.stop();
      server2.stop();
      thread1.join();
      thread2.join();
      transport1.close();
      transport2.close();
    }
  }