@Override
  public void onLiveInstanceChange(
      List<LiveInstance> liveInstances, NotificationContext changeContext) {
    if (connectionPool == null) {
      LOGGER.info("init hasn't been called yet on the live instances listener...");
      return;
    }

    LOGGER.info("Connection pool found, moving on...");

    for (LiveInstance instance : liveInstances) {

      String instanceId = instance.getInstanceName();
      String sessionId = instance.getSessionId();

      if (instanceId.startsWith("Broker_")) {
        LOGGER.info("skipping broker instances {}", instanceId);
        continue;
      }

      String namePortStr = instanceId.split("Server_")[1];
      String hostName = namePortStr.split("_")[0];
      int port;
      try {
        port = Integer.parseInt(namePortStr.split("_")[1]);
      } catch (Exception e) {
        LOGGER.warn("Port for server instance " + instanceId + " does not appear to be numeric", e);
        port = CommonConstants.Helix.DEFAULT_SERVER_NETTY_PORT;
      }
      ServerInstance ins = new ServerInstance(hostName, port);

      if (liveInstanceToSessionIdMap.containsKey(instanceId)) {
        // sessionId has changed
        LOGGER.info(
            "found instance Id : {} with new session Id : {} old session Id {}",
            instanceId,
            sessionId,
            liveInstanceToSessionIdMap.get(instanceId));
        if (!sessionId.equals(liveInstanceToSessionIdMap.get(instanceId))) {
          try {
            connectionPool.validatePool(ins, DO_NOT_RECREATE);
            liveInstanceToSessionIdMap.put(instanceId, sessionId);
          } catch (Exception e) {
            LOGGER.error(
                "Error trying to validate & destroy dead connections for {}", instanceId, e);
          }
        }
      } else {
        LOGGER.info("found instance Id : {} with new session Id : {}", instanceId, sessionId);
        // we don't have this instanceId
        // lets first check if the connection is valid or not
        try {
          connectionPool.validatePool(ins, DO_NOT_RECREATE);
          liveInstanceToSessionIdMap.put(instanceId, sessionId);
        } catch (Exception e) {
          LOGGER.error("Error trying to destroy dead connections for {}", instanceId, e);
        }
      }
    }
  }
  @Override
  public boolean isLeader() {
    if (_instanceType != InstanceType.CONTROLLER
        && _instanceType != InstanceType.CONTROLLER_PARTICIPANT) {
      return false;
    }

    if (!isConnected()) {
      return false;
    }

    try {
      LiveInstance leader = _dataAccessor.getProperty(_keyBuilder.controllerLeader());
      if (leader != null) {
        String leaderName = leader.getInstanceName();
        String sessionId = leader.getSessionId();
        if (leaderName != null
            && leaderName.equals(_instanceName)
            && sessionId != null
            && sessionId.equals(_sessionId)) {
          return true;
        }
      }
    } catch (Exception e) {
      // log
    }
    return false;
  }
  protected void setupLiveInstances(int numLiveInstances) {
    // setup liveInstances
    for (int i = 0; i < numLiveInstances; i++) {
      LiveInstance liveInstance = new LiveInstance("localhost_" + i);
      liveInstance.setSessionId("session_" + i);

      Builder keyBuilder = accessor.keyBuilder();
      accessor.setProperty(keyBuilder.liveInstance("localhost_" + i), liveInstance);
    }
  }
  @Test
  public void testOnConnectedAndDisconnecting() throws Exception {
    // Logger.getRootLogger().setLevel(Level.INFO);
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    int n = 2;

    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));

    TestHelper.setupCluster(
        clusterName,
        _zkaddr,
        12918, // participant port
        "localhost", // participant name prefix
        "TestDB", // resource name prefix
        1, // resources
        32, // partitions per resource
        n, // number of nodes
        2, // replicas
        "MasterSlave",
        true); // do rebalance

    // create connection
    HelixConnection connection = new ZkHelixConnection(_zkaddr);
    connection.connect();

    // start controller
    ClusterId clusterId = ClusterId.from(clusterName);
    ControllerId controllerId = ControllerId.from("controller");
    HelixController controller = connection.createController(clusterId, controllerId);
    controller.start();

    // check leader znode exists
    HelixDataAccessor accessor = connection.createDataAccessor(clusterId);
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    LiveInstance leader = accessor.getProperty(keyBuilder.controllerLeader());
    Assert.assertNotNull(leader);
    Assert.assertEquals(leader.getInstanceName(), controllerId.stringify());

    // stop participant
    controller.stop();

    // check leader znode is gone
    Assert.assertNull(accessor.getProperty(keyBuilder.controllerLeader()));

    // clean up
    connection.disconnect();

    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
  }
  private void prepare(
      String controllerVersion, String participantVersion, String minSupportedParticipantVersion) {
    List<String> instances =
        Arrays.asList("localhost_0", "localhost_1", "localhost_2", "localhost_3", "localhost_4");
    int partitions = 10;
    int replicas = 1;

    // set ideal state
    String resourceName = "testResource";
    ZNRecord record =
        DefaultTwoStateStrategy.calculateIdealState(
            instances, partitions, replicas, resourceName, "MASTER", "SLAVE");
    IdealState idealState = new IdealState(record);
    idealState.setStateModelDefId(StateModelDefinitionId.from("MasterSlave"));

    PropertyKeyBuilder keyBuilder = accessor.keyBuilder();
    accessor.setProperty(keyBuilder.idealStates(resourceName), idealState);

    // set live instances
    record = new ZNRecord("localhost_0");
    if (participantVersion != null) {
      record.setSimpleField(LiveInstanceProperty.HELIX_VERSION.toString(), participantVersion);
    }
    LiveInstance liveInstance = new LiveInstance(record);
    liveInstance.setSessionId("session_0");
    accessor.setProperty(keyBuilder.liveInstance("localhost_0"), liveInstance);
    InstanceConfig config = new InstanceConfig(liveInstance.getInstanceName());
    accessor.setProperty(keyBuilder.instanceConfig(config.getInstanceName()), config);

    if (controllerVersion != null) {
      ((Mocks.MockManager) manager).setVersion(controllerVersion);
    }

    if (minSupportedParticipantVersion != null) {
      manager
          .getProperties()
          .getProperties()
          .put("minimum_supported_version.participant", minSupportedParticipantVersion);
    }
    event.addAttribute("helixmanager", manager);
    runStage(event, new ReadClusterDataStage());
  }
  /**
   * Remove leader znode externally should invoke another round of leader-election this simulates
   * the race condition in {@link
   * ZkHelixLeaderElection#onControllerChange(org.apache.helix.NotificationContext)}
   *
   * @throws Exception
   */
  @Test
  public void testRemoveLeaderZnode() throws Exception {

    // Logger.getRootLogger().setLevel(Level.INFO);
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    int n = 2;

    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));

    TestHelper.setupCluster(
        clusterName,
        _zkaddr,
        12918, // participant port
        "localhost", // participant name prefix
        "TestDB", // resource name prefix
        1, // resources
        32, // partitions per resource
        n, // number of nodes
        2, // replicas
        "MasterSlave",
        true); // do rebalance

    // create connection
    HelixConnection connection = new ZkHelixConnection(_zkaddr);
    connection.connect();

    // start controller
    ClusterId clusterId = ClusterId.from(clusterName);
    final ControllerId controllerId = ControllerId.from("controller");

    // start controller
    HelixController controller = connection.createController(clusterId, controllerId);
    controller.start();

    // check live-instance znode for localhost_12918 exists
    final HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, _baseAccessor);
    final PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    LiveInstance leader = accessor.getProperty(keyBuilder.controllerLeader());
    Assert.assertNotNull(leader);
    Assert.assertEquals(leader.getInstanceName(), controllerId.stringify());

    // remove leader znode externally
    accessor.removeProperty(keyBuilder.controllerLeader());

    // verify leader is re-elected
    boolean result =
        TestHelper.verify(
            new TestHelper.Verifier() {

              @Override
              public boolean verify() throws Exception {
                LiveInstance leader = accessor.getProperty(keyBuilder.controllerLeader());
                if (leader == null) {
                  return false;
                }

                return leader.getInstanceName().equals(controllerId.stringify());
              }
            },
            3 * 1000);

    Assert.assertTrue(result, "Fail to re-elect a new leader");

    // clean up
    connection.disconnect();

    // check leader znode is gone
    Assert.assertNull(accessor.getProperty(keyBuilder.controllerLeader()));

    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
  }
  @Test
  public void testLiveInstanceInfoProvider() throws Exception {
    System.out.println(
        "START "
            + className
            + ".testLiveInstanceInfoProvider() at "
            + new Date(System.currentTimeMillis()));
    final String clusterName = CLUSTER_PREFIX + "_" + className + "_liveInstanceInfoProvider";
    class provider implements LiveInstanceInfoProvider {
      boolean _flag = false;

      public provider(boolean genSessionId) {
        _flag = genSessionId;
      }

      @Override
      public ZNRecord getAdditionalLiveInstanceInfo() {
        ZNRecord record = new ZNRecord("info");
        record.setSimpleField("simple", "value");
        List<String> listFieldVal = new ArrayList<String>();
        listFieldVal.add("val1");
        listFieldVal.add("val2");
        listFieldVal.add("val3");
        record.setListField("list", listFieldVal);
        Map<String, String> mapFieldVal = new HashMap<String, String>();
        mapFieldVal.put("k1", "val1");
        mapFieldVal.put("k2", "val2");
        mapFieldVal.put("k3", "val3");
        record.setMapField("map", mapFieldVal);
        if (_flag) {
          record.setSimpleField("SESSION_ID", "value");
          record.setSimpleField("LIVE_INSTANCE", "value");
          record.setSimpleField("Others", "value");
        }
        return record;
      }
    }

    TestHelper.setupEmptyCluster(_gZkClient, clusterName);
    int[] ids = {0, 1, 2, 3, 4, 5};
    setupInstances(clusterName, ids);

    // ///////////////////
    ZKHelixManager manager =
        new ZKHelixManager(clusterName, "localhost_0", InstanceType.PARTICIPANT, ZK_ADDR);
    manager.connect();
    HelixDataAccessor accessor = manager.getHelixDataAccessor();

    LiveInstance liveInstance =
        accessor.getProperty(accessor.keyBuilder().liveInstance("localhost_0"));
    Assert.assertTrue(liveInstance.getRecord().getListFields().size() == 0);
    Assert.assertTrue(liveInstance.getRecord().getMapFields().size() == 0);
    Assert.assertTrue(liveInstance.getRecord().getSimpleFields().size() == 3);

    manager = new ZKHelixManager(clusterName, "localhost_1", InstanceType.PARTICIPANT, ZK_ADDR);
    manager.setLiveInstanceInfoProvider(new provider(false));

    manager.connect();
    accessor = manager.getHelixDataAccessor();

    liveInstance = accessor.getProperty(accessor.keyBuilder().liveInstance("localhost_1"));
    Assert.assertTrue(liveInstance.getRecord().getListFields().size() == 1);
    Assert.assertTrue(liveInstance.getRecord().getMapFields().size() == 1);
    Assert.assertTrue(liveInstance.getRecord().getSimpleFields().size() == 4);

    manager = new ZKHelixManager(clusterName, "localhost_2", InstanceType.PARTICIPANT, ZK_ADDR);
    manager.setLiveInstanceInfoProvider(new provider(true));

    manager.connect();
    accessor = manager.getHelixDataAccessor();

    liveInstance = accessor.getProperty(accessor.keyBuilder().liveInstance("localhost_2"));
    Assert.assertTrue(liveInstance.getRecord().getListFields().size() == 1);
    Assert.assertTrue(liveInstance.getRecord().getMapFields().size() == 1);
    Assert.assertTrue(liveInstance.getRecord().getSimpleFields().size() == 5);
    Assert.assertFalse(liveInstance.getSessionId().equals("value"));
    Assert.assertFalse(liveInstance.getLiveInstance().equals("value"));

    // //////////////////////////////////

    ZkHelixTestManager manager2 =
        new ZkHelixTestManager(clusterName, "localhost_3", InstanceType.PARTICIPANT, ZK_ADDR);
    manager2.setLiveInstanceInfoProvider(new provider(true));

    manager2.connect();
    accessor = manager2.getHelixDataAccessor();

    liveInstance = accessor.getProperty(accessor.keyBuilder().liveInstance("localhost_3"));
    Assert.assertTrue(liveInstance.getRecord().getListFields().size() == 1);
    Assert.assertTrue(liveInstance.getRecord().getMapFields().size() == 1);
    Assert.assertTrue(liveInstance.getRecord().getSimpleFields().size() == 5);
    Assert.assertFalse(liveInstance.getSessionId().equals("value"));
    Assert.assertFalse(liveInstance.getLiveInstance().equals("value"));
    String sessionId = liveInstance.getSessionId();

    ZkTestHelper.expireSession(manager2.getZkClient());
    Thread.sleep(1000);

    liveInstance = accessor.getProperty(accessor.keyBuilder().liveInstance("localhost_3"));
    Assert.assertTrue(liveInstance.getRecord().getListFields().size() == 1);
    Assert.assertTrue(liveInstance.getRecord().getMapFields().size() == 1);
    Assert.assertTrue(liveInstance.getRecord().getSimpleFields().size() == 5);
    Assert.assertFalse(liveInstance.getSessionId().equals("value"));
    Assert.assertFalse(liveInstance.getLiveInstance().equals("value"));
    Assert.assertFalse(sessionId.equals(liveInstance.getSessionId()));

    System.out.println(
        "END "
            + className
            + ".testLiveInstanceInfoProvider() at "
            + new Date(System.currentTimeMillis()));
  }