@Override
public void onCallback(NotificationContext context) {
LOG.info(
"START: MasterSlaveRebalancer.onCallback running at "
+ _context.getHelixManager().getInstanceName());
if (context.getType().equals(NotificationContext.Type.FINALIZE)) {
LOG.info(
"END: MasterSlaveRebalancer.onCallback FINALIZE callback invoked. Likely lost connection to Helix");
return;
}
HelixManager manager = context.getManager();
String clusterName = manager.getClusterName();
HelixAdmin helixAdmin = manager.getClusterManagmentTool();
IdealState idealState =
helixAdmin.getResourceIdealState(clusterName, MySQLConstants.MASTER_SLAVE_RESOURCE_NAME);
if (idealState == null) {
LOG.info(
"END: MasterSlaveRebalancer.onCallback. "
+ MySQLConstants.MASTER_SLAVE_RESOURCE_NAME
+ " is not yet created");
}
PropertyKey.Builder builder = new PropertyKey.Builder(clusterName);
Map<String, LiveInstance> liveInstancesMap =
manager.getHelixDataAccessor().getChildValuesMap(builder.liveInstances());
Map<String, InstanceConfig> instanceConfigs =
manager.getHelixDataAccessor().getChildValuesMap(builder.instanceConfigs());
IdealState newIdealState = new IdealState(idealState.getId());
newIdealState.getRecord().setSimpleFields(idealState.getRecord().getSimpleFields());
newIdealState.getRecord().setListFields(idealState.getRecord().getListFields());
for (String partition : idealState.getPartitionSet()) {
Map<String, String> instanceStateMap = idealState.getInstanceStateMap(partition);
String currMaster = null;
Set<String> slaveSet = new TreeSet<String>();
for (String instance : instanceStateMap.keySet()) {
if ("MASTER".equalsIgnoreCase(instanceStateMap.get(instance))) {
currMaster = instance;
}
if ("SLAVE".equalsIgnoreCase(instanceStateMap.get(instance))) {
slaveSet.add(instance);
}
}
String newMaster = currMaster;
if (!liveInstancesMap.containsKey(currMaster)
|| !instanceConfigs.get(currMaster).getInstanceEnabled()) {
// need to find a new master.
newMaster = findNewMaster(liveInstancesMap, instanceConfigs, currMaster, slaveSet);
}
for (String instance : instanceStateMap.keySet()) {
if (instance.equalsIgnoreCase(newMaster)) {
newIdealState.setPartitionState(partition, instance, "MASTER");
} else {
newIdealState.setPartitionState(partition, instance, "SLAVE");
}
}
}
if (!idealState.equals(newIdealState)) {
LOG.info("New idealstate computed.");
LOG.info(newIdealState.toString());
manager
.getClusterManagmentTool()
.setResourceIdealState(
clusterName, MySQLConstants.MASTER_SLAVE_RESOURCE_NAME, newIdealState);
} else {
LOG.info("No change in IdealState");
}
LOG.info("END: MasterSlaveRebalancer.onCallback");
}
@Test
public void testAlertActionDisableNode() throws InterruptedException {
ConfigScope scope = new ConfigScopeBuilder().forCluster(CLUSTER_NAME).build();
Map<String, String> properties = new HashMap<String, String>();
properties.put("healthChange.enabled", "true");
_setupTool.getClusterManagementTool().setConfig(scope, properties);
String alertStr1 =
"EXP(decay(1.0)(localhost_*.TestStat@DB=db1.TestMetric1))CMP(GREATER)CON(20)ACTION(DISABLE_INSTANCE)";
String alertStr2 =
"EXP(decay(1.0)(localhost_*.TestStat@DB=db1.TestMetric2))CMP(GREATER)CON(120)ACTION(DISABLE_INSTANCE)";
String alertStr3 =
"EXP(decay(1.0)(localhost_*.TestStat@DB=TestDB;Partition=*.TestMetric2))CMP(GREATER)CON(160)ACTION(DISABLE_PARTITION)";
_setupTool.getClusterManagementTool().addAlert(CLUSTER_NAME, alertStr1);
_setupTool.getClusterManagementTool().addAlert(CLUSTER_NAME, alertStr2);
_setupTool.getClusterManagementTool().addAlert(CLUSTER_NAME, alertStr3);
int[] metrics1 = {10, 15, 22, 12, 16};
int[] metrics2 = {22, 115, 22, 163, 16};
int[] metrics3 = {0, 0, 0, 0, 0};
setHealthData(metrics1, metrics2);
String controllerName = CONTROLLER_PREFIX + "_0";
HelixManager manager = _startCMResultMap.get(controllerName)._manager;
HealthStatsAggregationTask task =
new HealthStatsAggregationTask(_startCMResultMap.get(controllerName)._manager);
task.run();
Thread.sleep(4000);
HelixDataAccessor helixDataAccessor = manager.getHelixDataAccessor();
Builder keyBuilder = helixDataAccessor.keyBuilder();
boolean result =
ClusterStateVerifier.verifyByZkCallback(
new BestPossAndExtViewZkVerifier(ZK_ADDR, CLUSTER_NAME));
Assert.assertTrue(result);
Builder kb = manager.getHelixDataAccessor().keyBuilder();
ExternalView externalView =
manager.getHelixDataAccessor().getProperty(kb.externalView("TestDB"));
// Test the DISABLE_INSTANCE alerts
String participant1 = "localhost_" + (START_PORT + 3);
String participant2 = "localhost_" + (START_PORT + 2);
ConfigAccessor configAccessor = manager.getConfigAccessor();
scope =
new ConfigScopeBuilder()
.forCluster(manager.getClusterName())
.forParticipant(participant1)
.build();
String isEnabled = configAccessor.get(scope, "HELIX_ENABLED");
Assert.assertFalse(Boolean.parseBoolean(isEnabled));
scope =
new ConfigScopeBuilder()
.forCluster(manager.getClusterName())
.forParticipant(participant2)
.build();
isEnabled = configAccessor.get(scope, "HELIX_ENABLED");
Assert.assertFalse(Boolean.parseBoolean(isEnabled));
for (String partitionName : externalView.getRecord().getMapFields().keySet()) {
for (String hostName : externalView.getRecord().getMapField(partitionName).keySet()) {
if (hostName.equals(participant1) || hostName.equals(participant2)) {
Assert.assertEquals(
externalView.getRecord().getMapField(partitionName).get(hostName), "OFFLINE");
}
}
}
// enable the disabled instances
setHealthData(metrics3, metrics3);
task.run();
Thread.sleep(1000);
manager.getClusterManagmentTool().enableInstance(manager.getClusterName(), participant2, true);
manager.getClusterManagmentTool().enableInstance(manager.getClusterName(), participant1, true);
result =
ClusterStateVerifier.verifyByZkCallback(
new BestPossAndExtViewZkVerifier(ZK_ADDR, CLUSTER_NAME));
Assert.assertTrue(result);
// Test the DISABLE_PARTITION case
int[] metrics4 = {22, 115, 22, 16, 163};
setHealthData2(metrics4);
task.run();
scope =
new ConfigScopeBuilder()
.forCluster(manager.getClusterName())
.forParticipant(participant1)
.build();
isEnabled = configAccessor.get(scope, "HELIX_ENABLED");
Assert.assertTrue(Boolean.parseBoolean(isEnabled));
scope =
new ConfigScopeBuilder()
.forCluster(manager.getClusterName())
.forParticipant(participant2)
.build();
isEnabled = configAccessor.get(scope, "HELIX_ENABLED");
Assert.assertTrue(Boolean.parseBoolean(isEnabled));
result =
ClusterStateVerifier.verifyByZkCallback(
new BestPossAndExtViewZkVerifier(ZK_ADDR, CLUSTER_NAME));
Assert.assertTrue(result);
String participant3 = "localhost_" + (START_PORT + 4);
externalView = manager.getHelixDataAccessor().getProperty(kb.externalView("TestDB"));
Assert.assertTrue(
externalView
.getRecord()
.getMapField("TestDB_3")
.get(participant3)
.equalsIgnoreCase("OFFLINE"));
InstanceConfig nodeConfig =
helixDataAccessor.getProperty(keyBuilder.instanceConfig(participant3));
Assert.assertTrue(
nodeConfig
.getRecord()
.getListField(InstanceConfigProperty.HELIX_DISABLED_PARTITION.toString())
.contains("TestDB_3"));
}