@Test
  public void testStandalone() throws Exception {
    String testName = TestUtil.getTestName();
    String clusterName = testName; // className + "_standalone";
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));

    final int nodeNr = 5;

    TestHelper.setupCluster(
        clusterName,
        _zkaddr,
        12918,
        "localhost",
        "TestDB",
        1,
        20,
        nodeNr - 1,
        3,
        "MasterSlave",
        true);

    MockParticipant[] participants = new MockParticipant[nodeNr];
    for (int i = 0; i < nodeNr - 1; i++) {
      String instanceName = "localhost_" + (12918 + i);
      participants[i] = new MockParticipant(_zkaddr, clusterName, instanceName);
      participants[i].syncStart();
    }

    MockController controller = new MockController(_zkaddr, clusterName, "controller_0");
    controller.syncStart();

    boolean result;
    result =
        ClusterStateVerifier.verifyByPolling(
            new ClusterStateVerifier.BestPossAndExtViewZkVerifier(_zkaddr, clusterName));
    Assert.assertTrue(result);
    String msgPath =
        PropertyPathConfig.getPath(PropertyType.MESSAGES, clusterName, "localhost_12918");
    result = checkHandlers(controller.getHandlers(), msgPath);
    Assert.assertTrue(result);

    _setupTool.addInstanceToCluster(clusterName, "localhost_12922");
    _setupTool.rebalanceStorageCluster(clusterName, "TestDB0", 3);

    participants[nodeNr - 1] = new MockParticipant(_zkaddr, clusterName, "localhost_12922");
    new Thread(participants[nodeNr - 1]).start();
    result =
        ClusterStateVerifier.verifyByPolling(
            new ClusterStateVerifier.BestPossAndExtViewZkVerifier(_zkaddr, clusterName));
    Assert.assertTrue(result);
    msgPath = PropertyPathConfig.getPath(PropertyType.MESSAGES, clusterName, "localhost_12922");
    result = checkHandlers(controller.getHandlers(), msgPath);
    Assert.assertTrue(result);

    // clean up
    controller.syncStop();
    for (int i = 0; i < nodeNr; i++) {
      participants[i].syncStop();
    }

    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
  }
  @Test
  public void testMultiCluster() throws Exception {
    String testName = TestUtil.getTestName();
    String clusterName = testName;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));

    // setup grand cluster
    final String grandClusterName = "GRAND_" + clusterName;
    TestHelper.setupCluster(
        grandClusterName, _zkaddr, 0, "controller", null, 0, 0, 1, 0, null, true);

    MockMultiClusterController multiClusterController =
        new MockMultiClusterController(_zkaddr, grandClusterName, "controller_0");
    multiClusterController.syncStart();

    // setup cluster
    _setupTool.addCluster(clusterName, true);
    _setupTool.activateCluster(clusterName, "GRAND_" + clusterName, true); // addCluster2

    boolean result;
    result =
        ClusterStateVerifier.verifyByPolling(
            new ClusterStateVerifier.BestPossAndExtViewZkVerifier(_zkaddr, "GRAND_" + clusterName));
    Assert.assertTrue(result);

    // add node/resource, and do rebalance
    final int nodeNr = 2;
    for (int i = 0; i < nodeNr - 1; i++) {
      int port = 12918 + i;
      _setupTool.addInstanceToCluster(clusterName, "localhost_" + port);
    }

    _setupTool.addResourceToCluster(clusterName, "TestDB0", 1, "LeaderStandby");
    _setupTool.rebalanceStorageCluster(clusterName, "TestDB0", 1);

    MockParticipant[] participants = new MockParticipant[nodeNr];
    for (int i = 0; i < nodeNr - 1; i++) {
      String instanceName = "localhost_" + (12918 + i);
      participants[i] = new MockParticipant(_zkaddr, clusterName, instanceName);
      participants[i].syncStart();
    }

    result =
        ClusterStateVerifier.verifyByPolling(
            new ClusterStateVerifier.BestPossAndExtViewZkVerifier(_zkaddr, clusterName));
    Assert.assertTrue(result);

    // check if controller_0 has message listener for localhost_12918
    String msgPath =
        PropertyPathConfig.getPath(PropertyType.MESSAGES, clusterName, "localhost_12918");
    int numberOfListeners = ZkTestHelper.numberOfListeners(_zkaddr, msgPath);
    // System.out.println("numberOfListeners(" + msgPath + "): " + numberOfListeners);
    Assert.assertEquals(numberOfListeners, 2); // 1 of participant, and 1 of controller

    _setupTool.addInstanceToCluster(clusterName, "localhost_12919");
    _setupTool.rebalanceStorageCluster(clusterName, "TestDB0", 2);

    participants[nodeNr - 1] = new MockParticipant(_zkaddr, clusterName, "localhost_12919");
    participants[nodeNr - 1].syncStart();
    result =
        ClusterStateVerifier.verifyByPolling(
            new ClusterStateVerifier.BestPossAndExtViewZkVerifier(_zkaddr, clusterName));
    Assert.assertTrue(result);
    // check if controller_0 has message listener for localhost_12919
    msgPath = PropertyPathConfig.getPath(PropertyType.MESSAGES, clusterName, "localhost_12919");
    numberOfListeners = ZkTestHelper.numberOfListeners(_zkaddr, msgPath);
    // System.out.println("numberOfListeners(" + msgPath + "): " + numberOfListeners);
    Assert.assertEquals(numberOfListeners, 2); // 1 of participant, and 1 of controller

    // clean up
    multiClusterController.syncStop();
    for (int i = 0; i < nodeNr; i++) {
      participants[i].syncStop();
    }

    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
  }