@Override public void onCallback(NotificationContext context) { LOG.info( "START: MasterSlaveRebalancer.onCallback running at " + _context.getHelixManager().getInstanceName()); if (context.getType().equals(NotificationContext.Type.FINALIZE)) { LOG.info( "END: MasterSlaveRebalancer.onCallback FINALIZE callback invoked. Likely lost connection to Helix"); return; } HelixManager manager = context.getManager(); String clusterName = manager.getClusterName(); HelixAdmin helixAdmin = manager.getClusterManagmentTool(); IdealState idealState = helixAdmin.getResourceIdealState(clusterName, MySQLConstants.MASTER_SLAVE_RESOURCE_NAME); if (idealState == null) { LOG.info( "END: MasterSlaveRebalancer.onCallback. " + MySQLConstants.MASTER_SLAVE_RESOURCE_NAME + " is not yet created"); } PropertyKey.Builder builder = new PropertyKey.Builder(clusterName); Map<String, LiveInstance> liveInstancesMap = manager.getHelixDataAccessor().getChildValuesMap(builder.liveInstances()); Map<String, InstanceConfig> instanceConfigs = manager.getHelixDataAccessor().getChildValuesMap(builder.instanceConfigs()); IdealState newIdealState = new IdealState(idealState.getId()); newIdealState.getRecord().setSimpleFields(idealState.getRecord().getSimpleFields()); newIdealState.getRecord().setListFields(idealState.getRecord().getListFields()); for (String partition : idealState.getPartitionSet()) { Map<String, String> instanceStateMap = idealState.getInstanceStateMap(partition); String currMaster = null; Set<String> slaveSet = new TreeSet<String>(); for (String instance : instanceStateMap.keySet()) { if ("MASTER".equalsIgnoreCase(instanceStateMap.get(instance))) { currMaster = instance; } if ("SLAVE".equalsIgnoreCase(instanceStateMap.get(instance))) { slaveSet.add(instance); } } String newMaster = currMaster; if (!liveInstancesMap.containsKey(currMaster) || !instanceConfigs.get(currMaster).getInstanceEnabled()) { // need to find a new master. newMaster = findNewMaster(liveInstancesMap, instanceConfigs, currMaster, slaveSet); } for (String instance : instanceStateMap.keySet()) { if (instance.equalsIgnoreCase(newMaster)) { newIdealState.setPartitionState(partition, instance, "MASTER"); } else { newIdealState.setPartitionState(partition, instance, "SLAVE"); } } } if (!idealState.equals(newIdealState)) { LOG.info("New idealstate computed."); LOG.info(newIdealState.toString()); manager .getClusterManagmentTool() .setResourceIdealState( clusterName, MySQLConstants.MASTER_SLAVE_RESOURCE_NAME, newIdealState); } else { LOG.info("No change in IdealState"); } LOG.info("END: MasterSlaveRebalancer.onCallback"); }
@Test public void testAlertActionDisableNode() throws InterruptedException { ConfigScope scope = new ConfigScopeBuilder().forCluster(CLUSTER_NAME).build(); Map<String, String> properties = new HashMap<String, String>(); properties.put("healthChange.enabled", "true"); _setupTool.getClusterManagementTool().setConfig(scope, properties); String alertStr1 = "EXP(decay(1.0)(localhost_*.TestStat@DB=db1.TestMetric1))CMP(GREATER)CON(20)ACTION(DISABLE_INSTANCE)"; String alertStr2 = "EXP(decay(1.0)(localhost_*.TestStat@DB=db1.TestMetric2))CMP(GREATER)CON(120)ACTION(DISABLE_INSTANCE)"; String alertStr3 = "EXP(decay(1.0)(localhost_*.TestStat@DB=TestDB;Partition=*.TestMetric2))CMP(GREATER)CON(160)ACTION(DISABLE_PARTITION)"; _setupTool.getClusterManagementTool().addAlert(CLUSTER_NAME, alertStr1); _setupTool.getClusterManagementTool().addAlert(CLUSTER_NAME, alertStr2); _setupTool.getClusterManagementTool().addAlert(CLUSTER_NAME, alertStr3); int[] metrics1 = {10, 15, 22, 12, 16}; int[] metrics2 = {22, 115, 22, 163, 16}; int[] metrics3 = {0, 0, 0, 0, 0}; setHealthData(metrics1, metrics2); String controllerName = CONTROLLER_PREFIX + "_0"; HelixManager manager = _startCMResultMap.get(controllerName)._manager; HealthStatsAggregationTask task = new HealthStatsAggregationTask(_startCMResultMap.get(controllerName)._manager); task.run(); Thread.sleep(4000); HelixDataAccessor helixDataAccessor = manager.getHelixDataAccessor(); Builder keyBuilder = helixDataAccessor.keyBuilder(); boolean result = ClusterStateVerifier.verifyByZkCallback( new BestPossAndExtViewZkVerifier(ZK_ADDR, CLUSTER_NAME)); Assert.assertTrue(result); Builder kb = manager.getHelixDataAccessor().keyBuilder(); ExternalView externalView = manager.getHelixDataAccessor().getProperty(kb.externalView("TestDB")); // Test the DISABLE_INSTANCE alerts String participant1 = "localhost_" + (START_PORT + 3); String participant2 = "localhost_" + (START_PORT + 2); ConfigAccessor configAccessor = manager.getConfigAccessor(); scope = new ConfigScopeBuilder() .forCluster(manager.getClusterName()) .forParticipant(participant1) .build(); String isEnabled = configAccessor.get(scope, "HELIX_ENABLED"); Assert.assertFalse(Boolean.parseBoolean(isEnabled)); scope = new ConfigScopeBuilder() .forCluster(manager.getClusterName()) .forParticipant(participant2) .build(); isEnabled = configAccessor.get(scope, "HELIX_ENABLED"); Assert.assertFalse(Boolean.parseBoolean(isEnabled)); for (String partitionName : externalView.getRecord().getMapFields().keySet()) { for (String hostName : externalView.getRecord().getMapField(partitionName).keySet()) { if (hostName.equals(participant1) || hostName.equals(participant2)) { Assert.assertEquals( externalView.getRecord().getMapField(partitionName).get(hostName), "OFFLINE"); } } } // enable the disabled instances setHealthData(metrics3, metrics3); task.run(); Thread.sleep(1000); manager.getClusterManagmentTool().enableInstance(manager.getClusterName(), participant2, true); manager.getClusterManagmentTool().enableInstance(manager.getClusterName(), participant1, true); result = ClusterStateVerifier.verifyByZkCallback( new BestPossAndExtViewZkVerifier(ZK_ADDR, CLUSTER_NAME)); Assert.assertTrue(result); // Test the DISABLE_PARTITION case int[] metrics4 = {22, 115, 22, 16, 163}; setHealthData2(metrics4); task.run(); scope = new ConfigScopeBuilder() .forCluster(manager.getClusterName()) .forParticipant(participant1) .build(); isEnabled = configAccessor.get(scope, "HELIX_ENABLED"); Assert.assertTrue(Boolean.parseBoolean(isEnabled)); scope = new ConfigScopeBuilder() .forCluster(manager.getClusterName()) .forParticipant(participant2) .build(); isEnabled = configAccessor.get(scope, "HELIX_ENABLED"); Assert.assertTrue(Boolean.parseBoolean(isEnabled)); result = ClusterStateVerifier.verifyByZkCallback( new BestPossAndExtViewZkVerifier(ZK_ADDR, CLUSTER_NAME)); Assert.assertTrue(result); String participant3 = "localhost_" + (START_PORT + 4); externalView = manager.getHelixDataAccessor().getProperty(kb.externalView("TestDB")); Assert.assertTrue( externalView .getRecord() .getMapField("TestDB_3") .get(participant3) .equalsIgnoreCase("OFFLINE")); InstanceConfig nodeConfig = helixDataAccessor.getProperty(keyBuilder.instanceConfig(participant3)); Assert.assertTrue( nodeConfig .getRecord() .getListField(InstanceConfigProperty.HELIX_DISABLED_PARTITION.toString()) .contains("TestDB_3")); }