@Override
public void onLiveInstanceChange(
List<LiveInstance> liveInstances, NotificationContext changeContext) {
if (connectionPool == null) {
LOGGER.info("init hasn't been called yet on the live instances listener...");
return;
}
LOGGER.info("Connection pool found, moving on...");
for (LiveInstance instance : liveInstances) {
String instanceId = instance.getInstanceName();
String sessionId = instance.getSessionId();
if (instanceId.startsWith("Broker_")) {
LOGGER.info("skipping broker instances {}", instanceId);
continue;
}
String namePortStr = instanceId.split("Server_")[1];
String hostName = namePortStr.split("_")[0];
int port;
try {
port = Integer.parseInt(namePortStr.split("_")[1]);
} catch (Exception e) {
LOGGER.warn("Port for server instance " + instanceId + " does not appear to be numeric", e);
port = CommonConstants.Helix.DEFAULT_SERVER_NETTY_PORT;
}
ServerInstance ins = new ServerInstance(hostName, port);
if (liveInstanceToSessionIdMap.containsKey(instanceId)) {
// sessionId has changed
LOGGER.info(
"found instance Id : {} with new session Id : {} old session Id {}",
instanceId,
sessionId,
liveInstanceToSessionIdMap.get(instanceId));
if (!sessionId.equals(liveInstanceToSessionIdMap.get(instanceId))) {
try {
connectionPool.validatePool(ins, DO_NOT_RECREATE);
liveInstanceToSessionIdMap.put(instanceId, sessionId);
} catch (Exception e) {
LOGGER.error(
"Error trying to validate & destroy dead connections for {}", instanceId, e);
}
}
} else {
LOGGER.info("found instance Id : {} with new session Id : {}", instanceId, sessionId);
// we don't have this instanceId
// lets first check if the connection is valid or not
try {
connectionPool.validatePool(ins, DO_NOT_RECREATE);
liveInstanceToSessionIdMap.put(instanceId, sessionId);
} catch (Exception e) {
LOGGER.error("Error trying to destroy dead connections for {}", instanceId, e);
}
}
}
}
@Override
public boolean isLeader() {
if (_instanceType != InstanceType.CONTROLLER
&& _instanceType != InstanceType.CONTROLLER_PARTICIPANT) {
return false;
}
if (!isConnected()) {
return false;
}
try {
LiveInstance leader = _dataAccessor.getProperty(_keyBuilder.controllerLeader());
if (leader != null) {
String leaderName = leader.getInstanceName();
String sessionId = leader.getSessionId();
if (leaderName != null
&& leaderName.equals(_instanceName)
&& sessionId != null
&& sessionId.equals(_sessionId)) {
return true;
}
}
} catch (Exception e) {
// log
}
return false;
}
protected void setupLiveInstances(int numLiveInstances) {
// setup liveInstances
for (int i = 0; i < numLiveInstances; i++) {
LiveInstance liveInstance = new LiveInstance("localhost_" + i);
liveInstance.setSessionId("session_" + i);
Builder keyBuilder = accessor.keyBuilder();
accessor.setProperty(keyBuilder.liveInstance("localhost_" + i), liveInstance);
}
}
@Test
public void testOnConnectedAndDisconnecting() throws Exception {
// Logger.getRootLogger().setLevel(Level.INFO);
String className = TestHelper.getTestClassName();
String methodName = TestHelper.getTestMethodName();
String clusterName = className + "_" + methodName;
int n = 2;
System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
TestHelper.setupCluster(
clusterName,
_zkaddr,
12918, // participant port
"localhost", // participant name prefix
"TestDB", // resource name prefix
1, // resources
32, // partitions per resource
n, // number of nodes
2, // replicas
"MasterSlave",
true); // do rebalance
// create connection
HelixConnection connection = new ZkHelixConnection(_zkaddr);
connection.connect();
// start controller
ClusterId clusterId = ClusterId.from(clusterName);
ControllerId controllerId = ControllerId.from("controller");
HelixController controller = connection.createController(clusterId, controllerId);
controller.start();
// check leader znode exists
HelixDataAccessor accessor = connection.createDataAccessor(clusterId);
PropertyKey.Builder keyBuilder = accessor.keyBuilder();
LiveInstance leader = accessor.getProperty(keyBuilder.controllerLeader());
Assert.assertNotNull(leader);
Assert.assertEquals(leader.getInstanceName(), controllerId.stringify());
// stop participant
controller.stop();
// check leader znode is gone
Assert.assertNull(accessor.getProperty(keyBuilder.controllerLeader()));
// clean up
connection.disconnect();
System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
private void prepare(
String controllerVersion, String participantVersion, String minSupportedParticipantVersion) {
List<String> instances =
Arrays.asList("localhost_0", "localhost_1", "localhost_2", "localhost_3", "localhost_4");
int partitions = 10;
int replicas = 1;
// set ideal state
String resourceName = "testResource";
ZNRecord record =
DefaultTwoStateStrategy.calculateIdealState(
instances, partitions, replicas, resourceName, "MASTER", "SLAVE");
IdealState idealState = new IdealState(record);
idealState.setStateModelDefId(StateModelDefinitionId.from("MasterSlave"));
PropertyKeyBuilder keyBuilder = accessor.keyBuilder();
accessor.setProperty(keyBuilder.idealStates(resourceName), idealState);
// set live instances
record = new ZNRecord("localhost_0");
if (participantVersion != null) {
record.setSimpleField(LiveInstanceProperty.HELIX_VERSION.toString(), participantVersion);
}
LiveInstance liveInstance = new LiveInstance(record);
liveInstance.setSessionId("session_0");
accessor.setProperty(keyBuilder.liveInstance("localhost_0"), liveInstance);
InstanceConfig config = new InstanceConfig(liveInstance.getInstanceName());
accessor.setProperty(keyBuilder.instanceConfig(config.getInstanceName()), config);
if (controllerVersion != null) {
((Mocks.MockManager) manager).setVersion(controllerVersion);
}
if (minSupportedParticipantVersion != null) {
manager
.getProperties()
.getProperties()
.put("minimum_supported_version.participant", minSupportedParticipantVersion);
}
event.addAttribute("helixmanager", manager);
runStage(event, new ReadClusterDataStage());
}
/**
* Remove leader znode externally should invoke another round of leader-election this simulates
* the race condition in {@link
* ZkHelixLeaderElection#onControllerChange(org.apache.helix.NotificationContext)}
*
* @throws Exception
*/
@Test
public void testRemoveLeaderZnode() throws Exception {
// Logger.getRootLogger().setLevel(Level.INFO);
String className = TestHelper.getTestClassName();
String methodName = TestHelper.getTestMethodName();
String clusterName = className + "_" + methodName;
int n = 2;
System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
TestHelper.setupCluster(
clusterName,
_zkaddr,
12918, // participant port
"localhost", // participant name prefix
"TestDB", // resource name prefix
1, // resources
32, // partitions per resource
n, // number of nodes
2, // replicas
"MasterSlave",
true); // do rebalance
// create connection
HelixConnection connection = new ZkHelixConnection(_zkaddr);
connection.connect();
// start controller
ClusterId clusterId = ClusterId.from(clusterName);
final ControllerId controllerId = ControllerId.from("controller");
// start controller
HelixController controller = connection.createController(clusterId, controllerId);
controller.start();
// check live-instance znode for localhost_12918 exists
final HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, _baseAccessor);
final PropertyKey.Builder keyBuilder = accessor.keyBuilder();
LiveInstance leader = accessor.getProperty(keyBuilder.controllerLeader());
Assert.assertNotNull(leader);
Assert.assertEquals(leader.getInstanceName(), controllerId.stringify());
// remove leader znode externally
accessor.removeProperty(keyBuilder.controllerLeader());
// verify leader is re-elected
boolean result =
TestHelper.verify(
new TestHelper.Verifier() {
@Override
public boolean verify() throws Exception {
LiveInstance leader = accessor.getProperty(keyBuilder.controllerLeader());
if (leader == null) {
return false;
}
return leader.getInstanceName().equals(controllerId.stringify());
}
},
3 * 1000);
Assert.assertTrue(result, "Fail to re-elect a new leader");
// clean up
connection.disconnect();
// check leader znode is gone
Assert.assertNull(accessor.getProperty(keyBuilder.controllerLeader()));
System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
@Test
public void testLiveInstanceInfoProvider() throws Exception {
System.out.println(
"START "
+ className
+ ".testLiveInstanceInfoProvider() at "
+ new Date(System.currentTimeMillis()));
final String clusterName = CLUSTER_PREFIX + "_" + className + "_liveInstanceInfoProvider";
class provider implements LiveInstanceInfoProvider {
boolean _flag = false;
public provider(boolean genSessionId) {
_flag = genSessionId;
}
@Override
public ZNRecord getAdditionalLiveInstanceInfo() {
ZNRecord record = new ZNRecord("info");
record.setSimpleField("simple", "value");
List<String> listFieldVal = new ArrayList<String>();
listFieldVal.add("val1");
listFieldVal.add("val2");
listFieldVal.add("val3");
record.setListField("list", listFieldVal);
Map<String, String> mapFieldVal = new HashMap<String, String>();
mapFieldVal.put("k1", "val1");
mapFieldVal.put("k2", "val2");
mapFieldVal.put("k3", "val3");
record.setMapField("map", mapFieldVal);
if (_flag) {
record.setSimpleField("SESSION_ID", "value");
record.setSimpleField("LIVE_INSTANCE", "value");
record.setSimpleField("Others", "value");
}
return record;
}
}
TestHelper.setupEmptyCluster(_gZkClient, clusterName);
int[] ids = {0, 1, 2, 3, 4, 5};
setupInstances(clusterName, ids);
// ///////////////////
ZKHelixManager manager =
new ZKHelixManager(clusterName, "localhost_0", InstanceType.PARTICIPANT, ZK_ADDR);
manager.connect();
HelixDataAccessor accessor = manager.getHelixDataAccessor();
LiveInstance liveInstance =
accessor.getProperty(accessor.keyBuilder().liveInstance("localhost_0"));
Assert.assertTrue(liveInstance.getRecord().getListFields().size() == 0);
Assert.assertTrue(liveInstance.getRecord().getMapFields().size() == 0);
Assert.assertTrue(liveInstance.getRecord().getSimpleFields().size() == 3);
manager = new ZKHelixManager(clusterName, "localhost_1", InstanceType.PARTICIPANT, ZK_ADDR);
manager.setLiveInstanceInfoProvider(new provider(false));
manager.connect();
accessor = manager.getHelixDataAccessor();
liveInstance = accessor.getProperty(accessor.keyBuilder().liveInstance("localhost_1"));
Assert.assertTrue(liveInstance.getRecord().getListFields().size() == 1);
Assert.assertTrue(liveInstance.getRecord().getMapFields().size() == 1);
Assert.assertTrue(liveInstance.getRecord().getSimpleFields().size() == 4);
manager = new ZKHelixManager(clusterName, "localhost_2", InstanceType.PARTICIPANT, ZK_ADDR);
manager.setLiveInstanceInfoProvider(new provider(true));
manager.connect();
accessor = manager.getHelixDataAccessor();
liveInstance = accessor.getProperty(accessor.keyBuilder().liveInstance("localhost_2"));
Assert.assertTrue(liveInstance.getRecord().getListFields().size() == 1);
Assert.assertTrue(liveInstance.getRecord().getMapFields().size() == 1);
Assert.assertTrue(liveInstance.getRecord().getSimpleFields().size() == 5);
Assert.assertFalse(liveInstance.getSessionId().equals("value"));
Assert.assertFalse(liveInstance.getLiveInstance().equals("value"));
// //////////////////////////////////
ZkHelixTestManager manager2 =
new ZkHelixTestManager(clusterName, "localhost_3", InstanceType.PARTICIPANT, ZK_ADDR);
manager2.setLiveInstanceInfoProvider(new provider(true));
manager2.connect();
accessor = manager2.getHelixDataAccessor();
liveInstance = accessor.getProperty(accessor.keyBuilder().liveInstance("localhost_3"));
Assert.assertTrue(liveInstance.getRecord().getListFields().size() == 1);
Assert.assertTrue(liveInstance.getRecord().getMapFields().size() == 1);
Assert.assertTrue(liveInstance.getRecord().getSimpleFields().size() == 5);
Assert.assertFalse(liveInstance.getSessionId().equals("value"));
Assert.assertFalse(liveInstance.getLiveInstance().equals("value"));
String sessionId = liveInstance.getSessionId();
ZkTestHelper.expireSession(manager2.getZkClient());
Thread.sleep(1000);
liveInstance = accessor.getProperty(accessor.keyBuilder().liveInstance("localhost_3"));
Assert.assertTrue(liveInstance.getRecord().getListFields().size() == 1);
Assert.assertTrue(liveInstance.getRecord().getMapFields().size() == 1);
Assert.assertTrue(liveInstance.getRecord().getSimpleFields().size() == 5);
Assert.assertFalse(liveInstance.getSessionId().equals("value"));
Assert.assertFalse(liveInstance.getLiveInstance().equals("value"));
Assert.assertFalse(sessionId.equals(liveInstance.getSessionId()));
System.out.println(
"END "
+ className
+ ".testLiveInstanceInfoProvider() at "
+ new Date(System.currentTimeMillis()));
}
