void printZkListeners(ZkClient client) throws Exception {
    Map<String, Set<IZkDataListener>> datalisteners = ZkTestHelper.getZkDataListener(client);
    Map<String, Set<IZkChildListener>> childListeners = ZkTestHelper.getZkChildListener(client);

    System.out.println("dataListeners {");
    for (String path : datalisteners.keySet()) {
      System.out.println("\t" + path + ": ");
      Set<IZkDataListener> set = datalisteners.get(path);
      for (IZkDataListener listener : set) {
        CallbackHandler handler = (CallbackHandler) listener;
        System.out.println("\t\t" + handler.getListener());
      }
    }
    System.out.println("}");

    System.out.println("childListeners {");
    for (String path : childListeners.keySet()) {
      System.out.println("\t" + path + ": ");
      Set<IZkChildListener> set = childListeners.get(path);
      for (IZkChildListener listener : set) {
        CallbackHandler handler = (CallbackHandler) listener;
        System.out.println("\t\t" + handler.getListener());
      }
    }
    System.out.println("}");
  }
  // @Test
  public void testDisconnectFlappingWindowController() throws Exception {
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    final String clusterName = className + "_" + methodName;

    TestHelper.setupCluster(
        clusterName,
        _zkaddr,
        12918, // participant port
        "localhost", // participant name prefix
        "TestDB", // resource name prefix
        1, // resources
        10, // partitions per resource
        5, // number of nodes
        3, // replicas
        "MasterSlave",
        true); // do rebalance

    // flapping time window to 5 sec
    System.setProperty("helixmanager.flappingTimeWindow", "5000");
    System.setProperty("helixmanager.maxDisconnectThreshold", "3");
    MockController manager2 = new MockController(_zkaddr, clusterName, null);
    manager2.connect();
    Thread.sleep(100);
    ZkClient zkClient = manager2.getZkClient();
    for (int i = 0; i < 2; i++) {
      ZkTestHelper.expireSession(zkClient);
      Thread.sleep(500);
      Assert.assertTrue(manager2.isConnected());
    }
    Thread.sleep(5000);
    // Old entries should be cleaned up
    for (int i = 0; i < 3; i++) {
      ZkTestHelper.expireSession(zkClient);
      Thread.sleep(500);
      Assert.assertTrue(manager2.isConnected());
    }
    ZkTestHelper.disconnectSession(zkClient);
    for (int i = 0; i < 20; i++) {
      Thread.sleep(500);
      if (!manager2.isConnected()) break;
    }
    Assert.assertFalse(manager2.isConnected());
  }
  @Test
  public void testDisconnectHistory() throws Exception {
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    final String clusterName = className + "_" + methodName;

    TestHelper.setupCluster(
        clusterName,
        _zkaddr,
        12918, // participant port
        "localhost", // participant name prefix
        "TestDB", // resource name prefix
        1, // resources
        10, // partitions per resource
        5, // number of nodes
        3, // replicas
        "MasterSlave",
        true); // do rebalance

    String instanceName = "localhost_" + (12918 + 0);
    MockParticipant manager = new MockParticipant(_zkaddr, clusterName, instanceName);

    manager.connect();
    ZkClient zkClient = manager.getZkClient();
    ZkTestHelper.expireSession(zkClient);
    for (int i = 0; i < 4; i++) {
      ZkTestHelper.expireSession(zkClient);
      Thread.sleep(1000);
      if (i < 5) {
        // System.out.println(i);
        Assert.assertTrue(manager.isConnected());
      }
    }
    ZkTestHelper.disconnectSession(zkClient);
    for (int i = 0; i < 20; i++) {
      Thread.sleep(500);
      System.out.println(i);
      if (!manager.isConnected()) break;
    }
    Assert.assertFalse(manager.isConnected());
  }
  @Test
  public void testCbHandlerLeakOnParticipantSessionExpiry() throws Exception {
    // Logger.getRootLogger().setLevel(Level.INFO);
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    final int n = 2;

    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));

    TestHelper.setupCluster(
        clusterName,
        ZK_ADDR,
        12918, // participant port
        "localhost", // participant name prefix
        "TestDB", // resource name prefix
        1, // resources
        32, // partitions per resource
        n, // number of nodes
        2, // replicas
        "MasterSlave",
        true); // do rebalance

    final ClusterControllerManager controller =
        new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();

    // start participants
    MockParticipantManager[] participants = new MockParticipantManager[n];
    for (int i = 0; i < n; i++) {
      String instanceName = "localhost_" + (12918 + i);

      participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
      participants[i].syncStart();
    }

    boolean result =
        ClusterStateVerifier.verifyByZkCallback(
            new ClusterStateVerifier.BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(result);
    final MockParticipantManager participantManagerToExpire = participants[1];

    // check controller zk-watchers
    result =
        TestHelper.verify(
            new TestHelper.Verifier() {

              @Override
              public boolean verify() throws Exception {
                Map<String, Set<String>> watchers = ZkTestHelper.getListenersBySession(ZK_ADDR);
                // Set<String> watchPaths = watchers.get("0x" + controllerManager.getSessionId());
                Set<String> watchPaths = watchers.get("0x" + controller.getSessionId());
                // System.out.println("controller watch paths: " + watchPaths);

                // controller should have 5 + 2n + m + (m+2)n zk-watchers
                // where n is number of nodes and m is number of resources
                return watchPaths.size() == (6 + 5 * n);
              }
            },
            500);
    Assert.assertTrue(result, "Controller should have 6 + 5*n zk-watchers.");

    // check participant zk-watchers
    result =
        TestHelper.verify(
            new TestHelper.Verifier() {

              @Override
              public boolean verify() throws Exception {
                Map<String, Set<String>> watchers = ZkTestHelper.getListenersBySession(ZK_ADDR);
                Set<String> watchPaths =
                    watchers.get("0x" + participantManagerToExpire.getSessionId());
                // System.out.println("participant watch paths: " + watchPaths);

                // participant should have 2 zk-watchers: 1 for MESSAGE and 1 for CONTROLLER
                return watchPaths.size() == 2;
              }
            },
            500);
    Assert.assertTrue(result, "Participant should have 2 zk-watchers.");

    // check HelixManager#_handlers
    // printHandlers(controllerManager);
    // printHandlers(participantManagerToExpire);
    int controllerHandlerNb = controller.getHandlers().size();
    int particHandlerNb = participantManagerToExpire.getHandlers().size();
    Assert.assertEquals(
        controllerHandlerNb,
        9,
        "HelixController should have 9 (5+2n) callback handlers for 2 (n) participant");
    Assert.assertEquals(
        particHandlerNb, 2, "HelixParticipant should have 2 (msg+cur-state) callback handlers");

    // expire the session of participant
    System.out.println("Expiring participant session...");
    String oldSessionId = participantManagerToExpire.getSessionId();

    ZkTestHelper.expireSession(participantManagerToExpire.getZkClient());
    String newSessionId = participantManagerToExpire.getSessionId();
    System.out.println(
        "Expried participant session. oldSessionId: "
            + oldSessionId
            + ", newSessionId: "
            + newSessionId);

    result =
        ClusterStateVerifier.verifyByPolling(
            new ClusterStateVerifier.BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(result);

    // check controller zk-watchers
    result =
        TestHelper.verify(
            new TestHelper.Verifier() {

              @Override
              public boolean verify() throws Exception {
                Map<String, Set<String>> watchers = ZkTestHelper.getListenersBySession(ZK_ADDR);
                Set<String> watchPaths = watchers.get("0x" + controller.getSessionId());
                // System.out.println("controller watch paths after session expiry: " + watchPaths);

                // controller should have 5 + 2n + m + (m+2)n zk-watchers
                // where n is number of nodes and m is number of resources
                return watchPaths.size() == (6 + 5 * n);
              }
            },
            500);
    Assert.assertTrue(result, "Controller should have 6 + 5*n zk-watchers after session expiry.");

    // check participant zk-watchers
    result =
        TestHelper.verify(
            new TestHelper.Verifier() {

              @Override
              public boolean verify() throws Exception {
                Map<String, Set<String>> watchers = ZkTestHelper.getListenersBySession(ZK_ADDR);
                Set<String> watchPaths =
                    watchers.get("0x" + participantManagerToExpire.getSessionId());
                // System.out.println("participant watch paths after session expiry: " +
                // watchPaths);

                // participant should have 2 zk-watchers: 1 for MESSAGE and 1 for CONTROLLER
                return watchPaths.size() == 2;
              }
            },
            500);
    Assert.assertTrue(result, "Participant should have 2 zk-watchers after session expiry.");

    // check handlers
    // printHandlers(controllerManager);
    // printHandlers(participantManagerToExpire);
    int handlerNb = controller.getHandlers().size();
    Assert.assertEquals(
        handlerNb,
        controllerHandlerNb,
        "controller callback handlers should not increase after participant session expiry");
    handlerNb = participantManagerToExpire.getHandlers().size();
    Assert.assertEquals(
        handlerNb,
        particHandlerNb,
        "participant callback handlers should not increase after participant session expiry");

    // clean up
    controller.syncStop();
    for (int i = 0; i < n; i++) {
      participants[i].syncStop();
    }

    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
  }
  @Test
  public void testRemoveUserCbHandlerOnPathRemoval() throws Exception {
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    final int n = 3;
    final String zkAddr = ZK_ADDR;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));

    TestHelper.setupCluster(
        clusterName,
        zkAddr,
        12918,
        "localhost",
        "TestDB",
        1, // resource
        32, // partitions
        n, // nodes
        2, // replicas
        "MasterSlave",
        true);

    final ClusterControllerManager controller =
        new ClusterControllerManager(zkAddr, clusterName, "controller_0");
    controller.syncStart();

    MockParticipantManager[] participants = new MockParticipantManager[n];
    for (int i = 0; i < n; i++) {
      String instanceName = "localhost_" + (12918 + i);
      participants[i] = new MockParticipantManager(zkAddr, clusterName, instanceName);
      participants[i].syncStart();

      // register a controller listener on participant_0
      if (i == 0) {
        // ZkHelixTestManager manager = participants[0].getManager();
        MockParticipantManager manager = participants[0];
        manager.addCurrentStateChangeListener(
            new CurrentStateChangeListener() {
              @Override
              public void onStateChange(
                  String instanceName,
                  List<CurrentState> statesInfo,
                  NotificationContext changeContext) {
                // To change body of implemented methods use File | Settings | File Templates.
                // System.out.println(instanceName + " on current-state change, type: " +
                // changeContext.getType());
              }
            },
            manager.getInstanceName(),
            manager.getSessionId());
      }
    }

    Boolean result =
        ClusterStateVerifier.verifyByZkCallback(
            new ClusterStateVerifier.BestPossAndExtViewZkVerifier(zkAddr, clusterName));
    Assert.assertTrue(result);

    MockParticipantManager participantToExpire = participants[0];
    String oldSessionId = participantToExpire.getSessionId();
    PropertyKeyBuilder keyBuilder = new PropertyKeyBuilder(clusterName);

    // check manager#hanlders
    Assert.assertEquals(
        participantToExpire.getHandlers().size(),
        3,
        "Should have 3 handlers: CURRENTSTATE/{sessionId}, CONTROLLER, and MESSAGES");

    // check zkclient#listeners
    Map<String, Set<IZkDataListener>> dataListeners =
        ZkTestHelper.getZkDataListener(participantToExpire.getZkClient());
    Map<String, Set<IZkChildListener>> childListeners =
        ZkTestHelper.getZkChildListener(participantToExpire.getZkClient());
    // printZkListeners(participantToExpire.getZkClient());
    Assert.assertEquals(
        dataListeners.size(),
        1,
        "Should have 1 path (CURRENTSTATE/{sessionId}/TestDB0) which has 1 data-listeners");
    String path =
        keyBuilder
            .currentState(participantToExpire.getInstanceName(), oldSessionId, "TestDB0")
            .getPath();
    Assert.assertEquals(
        dataListeners.get(path).size(), 1, "Should have 1 data-listeners on path: " + path);
    Assert.assertEquals(
        childListeners.size(),
        3,
        "Should have 3 paths (CURRENTSTATE/{sessionId}, CONTROLLER, and MESSAGES) each of which has 1 child-listener");
    path = keyBuilder.currentStates(participantToExpire.getInstanceName(), oldSessionId).getPath();
    Assert.assertEquals(
        childListeners.get(path).size(), 1, "Should have 1 child-listener on path: " + path);
    path = keyBuilder.messages(participantToExpire.getInstanceName()).getPath();
    Assert.assertEquals(
        childListeners.get(path).size(), 1, "Should have 1 child-listener on path: " + path);
    path = keyBuilder.controller().getPath();
    Assert.assertEquals(
        childListeners.get(path).size(), 1, "Should have 1 child-listener on path: " + path);

    // check zookeeper#watches on client side
    Map<String, List<String>> watchPaths =
        ZkTestHelper.getZkWatch(participantToExpire.getZkClient());
    // System.out.println("localhost_12918 zk-client side watchPaths: " + watchPaths + "\n");
    Assert.assertEquals(
        watchPaths.get("dataWatches").size(),
        4,
        "Should have 4 data-watches: CURRENTSTATE/{sessionId}, CURRENTSTATE/{sessionId}/TestDB, CONTROLLER, MESSAGES");
    Assert.assertEquals(
        watchPaths.get("childWatches").size(),
        3,
        "Should have 3 child-watches: CONTROLLER, MESSAGES, and CURRENTSTATE/{sessionId}");

    // expire localhost_12918
    System.out.println(
        "Expire participant: "
            + participantToExpire.getInstanceName()
            + ", session: "
            + participantToExpire.getSessionId());
    ZkTestHelper.expireSession(participantToExpire.getZkClient());
    String newSessionId = participantToExpire.getSessionId();
    System.out.println(
        participantToExpire.getInstanceName()
            + " oldSessionId: "
            + oldSessionId
            + ", newSessionId: "
            + newSessionId);
    result =
        ClusterStateVerifier.verifyByZkCallback(
            new ClusterStateVerifier.BestPossAndExtViewZkVerifier(zkAddr, clusterName));
    Assert.assertTrue(result);

    // check manager#hanlders
    Assert.assertEquals(
        participantToExpire.getHandlers().size(),
        2,
        "Should have 2 handlers: CONTROLLER and MESSAGES. CURRENTSTATE/{sessionId} handler should be removed by CallbackHandler#handleChildChange()");

    // check zkclient#listeners
    dataListeners = ZkTestHelper.getZkDataListener(participantToExpire.getZkClient());
    childListeners = ZkTestHelper.getZkChildListener(participantToExpire.getZkClient());
    // printZkListeners(participantToExpire.getZkClient());
    Assert.assertTrue(dataListeners.isEmpty(), "Should have no data-listeners");
    Assert.assertEquals(
        childListeners.size(),
        3,
        "Should have 3 paths (CURRENTSTATE/{oldSessionId}, CONTROLLER, and MESSAGES). "
            + "CONTROLLER and MESSAGE has 1 child-listener each. CURRENTSTATE/{oldSessionId} doesn't have listener (ZkClient doesn't remove empty childListener set. probably a ZkClient bug. see ZkClient#unsubscribeChildChange())");
    path = keyBuilder.currentStates(participantToExpire.getInstanceName(), oldSessionId).getPath();
    Assert.assertEquals(
        childListeners.get(path).size(), 0, "Should have no child-listener on path: " + path);
    path = keyBuilder.messages(participantToExpire.getInstanceName()).getPath();
    Assert.assertEquals(
        childListeners.get(path).size(), 1, "Should have 1 child-listener on path: " + path);
    path = keyBuilder.controller().getPath();
    Assert.assertEquals(
        childListeners.get(path).size(), 1, "Should have 1 child-listener on path: " + path);

    // check zookeeper#watches on client side
    watchPaths = ZkTestHelper.getZkWatch(participantToExpire.getZkClient());
    // System.out.println("localhost_12918 zk-client side watchPaths: " + watchPaths + "\n");
    Assert.assertEquals(
        watchPaths.get("dataWatches").size(),
        2,
        "Should have 2 data-watches: CONTROLLER and MESSAGES");
    Assert.assertEquals(
        watchPaths.get("childWatches").size(),
        2,
        "Should have 2 child-watches: CONTROLLER and MESSAGES");
    Assert.assertEquals(
        watchPaths.get("existWatches").size(),
        2,
        "Should have 2 exist-watches: CURRENTSTATE/{oldSessionId} and CURRENTSTATE/{oldSessionId}/TestDB0");

    // another session expiry on localhost_12918 should clear the two exist-watches on
    // CURRENTSTATE/{oldSessionId}
    System.out.println(
        "Expire participant: "
            + participantToExpire.getInstanceName()
            + ", session: "
            + participantToExpire.getSessionId());
    ZkTestHelper.expireSession(participantToExpire.getZkClient());
    result =
        ClusterStateVerifier.verifyByZkCallback(
            new ClusterStateVerifier.BestPossAndExtViewZkVerifier(zkAddr, clusterName));
    Assert.assertTrue(result);

    // check zookeeper#watches on client side
    watchPaths = ZkTestHelper.getZkWatch(participantToExpire.getZkClient());
    // System.out.println("localhost_12918 zk-client side watchPaths: " + watchPaths + "\n");
    Assert.assertEquals(
        watchPaths.get("dataWatches").size(),
        2,
        "Should have 2 data-watches: CONTROLLER and MESSAGES");
    Assert.assertEquals(
        watchPaths.get("childWatches").size(),
        2,
        "Should have 2 child-watches: CONTROLLER and MESSAGES");
    Assert.assertEquals(
        watchPaths.get("existWatches").size(),
        0,
        "Should have no exist-watches. exist-watches on CURRENTSTATE/{oldSessionId} and CURRENTSTATE/{oldSessionId}/TestDB0 should be cleared during handleNewSession");

    // Thread.sleep(1000);

    // clean up
    controller.syncStop();
    for (int i = 0; i < n; i++) {
      participants[i].syncStop();
    }

    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
  }
  @Test
  public void testSessionExpiryInTransition() throws Exception {
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    final String clusterName = className + "_" + methodName;
    int n = 1;
    CountDownLatch startCountdown = new CountDownLatch(1);
    CountDownLatch endCountdown = new CountDownLatch(1);

    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));

    MockParticipant[] participants = new MockParticipant[n];

    TestHelper.setupCluster(
        clusterName,
        _zkaddr,
        12918, // participant port
        "localhost", // participant name prefix
        "TestDB", // resource name prefix
        1, // resources
        1, // partitions per resource
        n, // number of nodes
        1, // replicas
        "MasterSlave",
        true); // do rebalance

    // start controller
    MockController controller = new MockController(_zkaddr, clusterName, "controller_0");
    controller.syncStart();

    // start participants
    for (int i = 0; i < n; i++) {
      String instanceName = "localhost_" + (12918 + i);
      participants[i] = new MockParticipant(_zkaddr, clusterName, instanceName);
      participants[i].setTransition(new SessionExpiryTransition(startCountdown, endCountdown));
      participants[i].syncStart();
    }

    // wait transition happens to trigger session expiry
    startCountdown.await();
    String oldSessionId = participants[0].getSessionId();
    System.out.println("oldSessionId: " + oldSessionId);
    ZkTestHelper.expireSession(participants[0].getZkClient());

    boolean result =
        ClusterStateVerifier.verifyByZkCallback(
            new BestPossAndExtViewZkVerifier(_zkaddr, clusterName));
    Assert.assertTrue(result);

    String newSessionId = participants[0].getSessionId();
    Assert.assertNotSame(newSessionId, oldSessionId);

    // assert interrupt exception error in old session
    String errPath =
        PropertyPathConfig.getPath(
            PropertyType.ERRORS,
            clusterName,
            "localhost_12918",
            oldSessionId,
            "TestDB0",
            "TestDB0_0");
    ZNRecord error = _zkclient.readData(errPath);
    Assert.assertNotNull(
        error,
        "InterruptedException should happen in old session since task is being cancelled during handleNewSession");
    String errString = new String(new ZNRecordSerializer().serialize(error));
    Assert.assertTrue(errString.indexOf("InterruptedException") != -1);

    // cleanup
    controller.syncStop();
    for (int i = 0; i < n; i++) {
      participants[i].syncStop();
    }

    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
  }
  @Test
  public void simpleSessionExpiryTest() throws Exception {
    // Logger.getRootLogger().setLevel(Level.WARN);

    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    final String clusterName = className + "_" + methodName;
    int n = 1;

    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));

    MockParticipant[] participants = new MockParticipant[n];

    TestHelper.setupCluster(
        clusterName,
        _zkaddr,
        12918, // participant port
        "localhost", // participant name prefix
        "TestDB", // resource name prefix
        1, // resources
        1, // partitions per resource
        n, // number of nodes
        1, // replicas
        "MasterSlave",
        true); // do rebalance

    // start controller
    MockController controller = new MockController(_zkaddr, clusterName, "controller_0");
    controller.syncStart();

    // start participants
    for (int i = 0; i < n; i++) {
      String instanceName = "localhost_" + (12918 + i);
      participants[i] = new MockParticipant(_zkaddr, clusterName, instanceName);
      participants[i].syncStart();
    }

    boolean result =
        ClusterStateVerifier.verifyByZkCallback(
            new BestPossAndExtViewZkVerifier(_zkaddr, clusterName));
    Assert.assertTrue(result);
    String oldSessionId = participants[0].getSessionId();

    // expire zk-connection on localhost_12918
    ZkTestHelper.expireSession(participants[0].getZkClient());

    // wait until session expiry callback happens
    TimeUnit.MILLISECONDS.sleep(100);

    result =
        ClusterStateVerifier.verifyByZkCallback(
            new BestPossAndExtViewZkVerifier(_zkaddr, clusterName));
    Assert.assertTrue(result);
    String newSessionId = participants[0].getSessionId();
    Assert.assertNotSame(newSessionId, oldSessionId);

    // cleanup
    controller.syncStop();
    for (int i = 0; i < n; i++) {
      participants[i].syncStop();
    }

    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
  }
  @Test
  public void testMultiCluster() throws Exception {
    String testName = TestUtil.getTestName();
    String clusterName = testName;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));

    // setup grand cluster
    final String grandClusterName = "GRAND_" + clusterName;
    TestHelper.setupCluster(
        grandClusterName, _zkaddr, 0, "controller", null, 0, 0, 1, 0, null, true);

    MockMultiClusterController multiClusterController =
        new MockMultiClusterController(_zkaddr, grandClusterName, "controller_0");
    multiClusterController.syncStart();

    // setup cluster
    _setupTool.addCluster(clusterName, true);
    _setupTool.activateCluster(clusterName, "GRAND_" + clusterName, true); // addCluster2

    boolean result;
    result =
        ClusterStateVerifier.verifyByPolling(
            new ClusterStateVerifier.BestPossAndExtViewZkVerifier(_zkaddr, "GRAND_" + clusterName));
    Assert.assertTrue(result);

    // add node/resource, and do rebalance
    final int nodeNr = 2;
    for (int i = 0; i < nodeNr - 1; i++) {
      int port = 12918 + i;
      _setupTool.addInstanceToCluster(clusterName, "localhost_" + port);
    }

    _setupTool.addResourceToCluster(clusterName, "TestDB0", 1, "LeaderStandby");
    _setupTool.rebalanceStorageCluster(clusterName, "TestDB0", 1);

    MockParticipant[] participants = new MockParticipant[nodeNr];
    for (int i = 0; i < nodeNr - 1; i++) {
      String instanceName = "localhost_" + (12918 + i);
      participants[i] = new MockParticipant(_zkaddr, clusterName, instanceName);
      participants[i].syncStart();
    }

    result =
        ClusterStateVerifier.verifyByPolling(
            new ClusterStateVerifier.BestPossAndExtViewZkVerifier(_zkaddr, clusterName));
    Assert.assertTrue(result);

    // check if controller_0 has message listener for localhost_12918
    String msgPath =
        PropertyPathConfig.getPath(PropertyType.MESSAGES, clusterName, "localhost_12918");
    int numberOfListeners = ZkTestHelper.numberOfListeners(_zkaddr, msgPath);
    // System.out.println("numberOfListeners(" + msgPath + "): " + numberOfListeners);
    Assert.assertEquals(numberOfListeners, 2); // 1 of participant, and 1 of controller

    _setupTool.addInstanceToCluster(clusterName, "localhost_12919");
    _setupTool.rebalanceStorageCluster(clusterName, "TestDB0", 2);

    participants[nodeNr - 1] = new MockParticipant(_zkaddr, clusterName, "localhost_12919");
    participants[nodeNr - 1].syncStart();
    result =
        ClusterStateVerifier.verifyByPolling(
            new ClusterStateVerifier.BestPossAndExtViewZkVerifier(_zkaddr, clusterName));
    Assert.assertTrue(result);
    // check if controller_0 has message listener for localhost_12919
    msgPath = PropertyPathConfig.getPath(PropertyType.MESSAGES, clusterName, "localhost_12919");
    numberOfListeners = ZkTestHelper.numberOfListeners(_zkaddr, msgPath);
    // System.out.println("numberOfListeners(" + msgPath + "): " + numberOfListeners);
    Assert.assertEquals(numberOfListeners, 2); // 1 of participant, and 1 of controller

    // clean up
    multiClusterController.syncStop();
    for (int i = 0; i < nodeNr; i++) {
      participants[i].syncStop();
    }

    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
  }
  @Test
  public void testLiveInstanceInfoProvider() throws Exception {
    System.out.println(
        "START "
            + className
            + ".testLiveInstanceInfoProvider() at "
            + new Date(System.currentTimeMillis()));
    final String clusterName = CLUSTER_PREFIX + "_" + className + "_liveInstanceInfoProvider";
    class provider implements LiveInstanceInfoProvider {
      boolean _flag = false;

      public provider(boolean genSessionId) {
        _flag = genSessionId;
      }

      @Override
      public ZNRecord getAdditionalLiveInstanceInfo() {
        ZNRecord record = new ZNRecord("info");
        record.setSimpleField("simple", "value");
        List<String> listFieldVal = new ArrayList<String>();
        listFieldVal.add("val1");
        listFieldVal.add("val2");
        listFieldVal.add("val3");
        record.setListField("list", listFieldVal);
        Map<String, String> mapFieldVal = new HashMap<String, String>();
        mapFieldVal.put("k1", "val1");
        mapFieldVal.put("k2", "val2");
        mapFieldVal.put("k3", "val3");
        record.setMapField("map", mapFieldVal);
        if (_flag) {
          record.setSimpleField("SESSION_ID", "value");
          record.setSimpleField("LIVE_INSTANCE", "value");
          record.setSimpleField("Others", "value");
        }
        return record;
      }
    }

    TestHelper.setupEmptyCluster(_gZkClient, clusterName);
    int[] ids = {0, 1, 2, 3, 4, 5};
    setupInstances(clusterName, ids);

    // ///////////////////
    ZKHelixManager manager =
        new ZKHelixManager(clusterName, "localhost_0", InstanceType.PARTICIPANT, ZK_ADDR);
    manager.connect();
    HelixDataAccessor accessor = manager.getHelixDataAccessor();

    LiveInstance liveInstance =
        accessor.getProperty(accessor.keyBuilder().liveInstance("localhost_0"));
    Assert.assertTrue(liveInstance.getRecord().getListFields().size() == 0);
    Assert.assertTrue(liveInstance.getRecord().getMapFields().size() == 0);
    Assert.assertTrue(liveInstance.getRecord().getSimpleFields().size() == 3);

    manager = new ZKHelixManager(clusterName, "localhost_1", InstanceType.PARTICIPANT, ZK_ADDR);
    manager.setLiveInstanceInfoProvider(new provider(false));

    manager.connect();
    accessor = manager.getHelixDataAccessor();

    liveInstance = accessor.getProperty(accessor.keyBuilder().liveInstance("localhost_1"));
    Assert.assertTrue(liveInstance.getRecord().getListFields().size() == 1);
    Assert.assertTrue(liveInstance.getRecord().getMapFields().size() == 1);
    Assert.assertTrue(liveInstance.getRecord().getSimpleFields().size() == 4);

    manager = new ZKHelixManager(clusterName, "localhost_2", InstanceType.PARTICIPANT, ZK_ADDR);
    manager.setLiveInstanceInfoProvider(new provider(true));

    manager.connect();
    accessor = manager.getHelixDataAccessor();

    liveInstance = accessor.getProperty(accessor.keyBuilder().liveInstance("localhost_2"));
    Assert.assertTrue(liveInstance.getRecord().getListFields().size() == 1);
    Assert.assertTrue(liveInstance.getRecord().getMapFields().size() == 1);
    Assert.assertTrue(liveInstance.getRecord().getSimpleFields().size() == 5);
    Assert.assertFalse(liveInstance.getSessionId().equals("value"));
    Assert.assertFalse(liveInstance.getLiveInstance().equals("value"));

    // //////////////////////////////////

    ZkHelixTestManager manager2 =
        new ZkHelixTestManager(clusterName, "localhost_3", InstanceType.PARTICIPANT, ZK_ADDR);
    manager2.setLiveInstanceInfoProvider(new provider(true));

    manager2.connect();
    accessor = manager2.getHelixDataAccessor();

    liveInstance = accessor.getProperty(accessor.keyBuilder().liveInstance("localhost_3"));
    Assert.assertTrue(liveInstance.getRecord().getListFields().size() == 1);
    Assert.assertTrue(liveInstance.getRecord().getMapFields().size() == 1);
    Assert.assertTrue(liveInstance.getRecord().getSimpleFields().size() == 5);
    Assert.assertFalse(liveInstance.getSessionId().equals("value"));
    Assert.assertFalse(liveInstance.getLiveInstance().equals("value"));
    String sessionId = liveInstance.getSessionId();

    ZkTestHelper.expireSession(manager2.getZkClient());
    Thread.sleep(1000);

    liveInstance = accessor.getProperty(accessor.keyBuilder().liveInstance("localhost_3"));
    Assert.assertTrue(liveInstance.getRecord().getListFields().size() == 1);
    Assert.assertTrue(liveInstance.getRecord().getMapFields().size() == 1);
    Assert.assertTrue(liveInstance.getRecord().getSimpleFields().size() == 5);
    Assert.assertFalse(liveInstance.getSessionId().equals("value"));
    Assert.assertFalse(liveInstance.getLiveInstance().equals("value"));
    Assert.assertFalse(sessionId.equals(liveInstance.getSessionId()));

    System.out.println(
        "END "
            + className
            + ".testLiveInstanceInfoProvider() at "
            + new Date(System.currentTimeMillis()));
  }