protected List<IdealState> setupIdealState( int nodes, String[] resources, int partitions, int replicas) { List<IdealState> idealStates = new ArrayList<IdealState>(); List<String> instances = new ArrayList<String>(); for (int i = 0; i < nodes; i++) { instances.add("localhost_" + i); } for (int i = 0; i < resources.length; i++) { String resourceName = resources[i]; ZNRecord record = new ZNRecord(resourceName); for (int p = 0; p < partitions; p++) { List<String> value = new ArrayList<String>(); for (int r = 0; r < replicas; r++) { value.add("localhost_" + (p + r + 1) % nodes); } record.setListField(resourceName + "_" + p, value); } IdealState idealState = new IdealState(record); idealState.setStateModelDefRef("MasterSlave"); idealState.setIdealStateMode(IdealStateModeProperty.AUTO.toString()); idealState.setNumPartitions(partitions); idealStates.add(idealState); // System.out.println(idealState); Builder keyBuilder = accessor.keyBuilder(); accessor.setProperty(keyBuilder.idealStates(resourceName), idealState); } return idealStates; }
@Override public void addConfigChangeListener( ScopedConfigChangeListener listener, ConfigScopeProperty scope) throws Exception { Builder keyBuilder = new Builder(_clusterName); PropertyKey propertyKey = null; switch (scope) { case CLUSTER: propertyKey = keyBuilder.clusterConfigs(); break; case PARTICIPANT: propertyKey = keyBuilder.instanceConfigs(); break; case RESOURCE: propertyKey = keyBuilder.resourceConfigs(); break; default: break; } if (propertyKey != null) { addListener( listener, propertyKey, ChangeType.CONFIG, new EventType[] {EventType.NodeChildrenChanged}); } else { LOG.error("Can't add listener to config scope: " + scope); } }
/** * Get the {@link PropertyKey} for this message * * @param keyBuilder PropertyKey Builder * @param instanceName target instance * @return message PropertyKey */ public PropertyKey getKey(Builder keyBuilder, String instanceName) { if (isControlerMsg()) { return keyBuilder.controllerMessage(getId()); } else { return keyBuilder.message(instanceName, getId()); } }
protected void setupLiveInstances(int numLiveInstances) { // setup liveInstances for (int i = 0; i < numLiveInstances; i++) { LiveInstance liveInstance = new LiveInstance("localhost_" + i); liveInstance.setSessionId("session_" + i); Builder keyBuilder = accessor.keyBuilder(); accessor.setProperty(keyBuilder.liveInstance("localhost_" + i), liveInstance); } }
/* * stateMap: partition->instance->state */ public static boolean verifyState( ZkClient zkclient, String clusterName, String resourceName, Map<String, Map<String, String>> expectStateMap, String op) { boolean result = true; ZkBaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(zkclient); ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor); Builder keyBuilder = accessor.keyBuilder(); ExternalView extView = accessor.getProperty(keyBuilder.externalView(resourceName)); Map<String, Map<String, String>> actualStateMap = extView.getRecord().getMapFields(); for (String partition : actualStateMap.keySet()) { for (String expectPartiton : expectStateMap.keySet()) { if (!partition.matches(expectPartiton)) { continue; } Map<String, String> actualInstanceStateMap = actualStateMap.get(partition); Map<String, String> expectInstanceStateMap = expectStateMap.get(expectPartiton); for (String instance : actualInstanceStateMap.keySet()) { for (String expectInstance : expectStateMap.get(expectPartiton).keySet()) { if (!instance.matches(expectInstance)) { continue; } String actualState = actualInstanceStateMap.get(instance); String expectState = expectInstanceStateMap.get(expectInstance); boolean equals = expectState.equals(actualState); if (op.equals("==") && !equals || op.equals("!=") && equals) { System.out.println( partition + "/" + instance + " state mismatch. actual state: " + actualState + ", but expect: " + expectState + ", op: " + op); result = false; } } } } } return result; }
protected void setupStateModel() { ZNRecord masterSlave = new StateModelConfigGenerator().generateConfigForMasterSlave(); Builder keyBuilder = accessor.keyBuilder(); accessor.setProperty( keyBuilder.stateModelDef(masterSlave.getId()), new StateModelDefinition(masterSlave)); ZNRecord leaderStandby = new StateModelConfigGenerator().generateConfigForLeaderStandby(); accessor.setProperty( keyBuilder.stateModelDef(leaderStandby.getId()), new StateModelDefinition(leaderStandby)); ZNRecord onlineOffline = new StateModelConfigGenerator().generateConfigForOnlineOffline(); accessor.setProperty( keyBuilder.stateModelDef(onlineOffline.getId()), new StateModelDefinition(onlineOffline)); }
@Override public boolean isLeader() { if (_instanceType != InstanceType.CONTROLLER && _instanceType != InstanceType.CONTROLLER_PARTICIPANT) { return false; } if (!isConnected()) { return false; } try { LiveInstance leader = _dataAccessor.getProperty(_keyBuilder.controllerLeader()); if (leader != null) { String leaderName = leader.getInstanceName(); String sessionId = leader.getSessionId(); if (leaderName != null && leaderName.equals(_instanceName) && sessionId != null && sessionId.equals(_sessionId)) { return true; } } } catch (Exception e) { // log } return false; }
@Override public boolean verify() { try { HelixDataAccessor accessor = new ZKHelixDataAccessor(_clusterName, new ZkBaseDataAccessor<ZNRecord>(_client)); Builder keyBuilder = accessor.keyBuilder(); int numberOfPartitions = accessor .getProperty(keyBuilder.idealStates(_resourceName)) .getRecord() .getListFields() .size(); ClusterDataCache cache = new ClusterDataCache(); cache.refresh(accessor); String masterValue = cache .getStateModelDef(cache.getIdealState(_resourceName).getStateModelDefRef()) .getStatesPriorityList() .get(0); int replicas = Integer.parseInt(cache.getIdealState(_resourceName).getReplicas()); String instanceGroupTag = cache.getIdealState(_resourceName).getInstanceGroupTag(); int instances = 0; for (String liveInstanceName : cache.getLiveInstances().keySet()) { if (cache.getInstanceConfigMap().get(liveInstanceName).containsTag(instanceGroupTag)) { instances++; } } if (instances == 0) { instances = cache.getLiveInstances().size(); } return verifyBalanceExternalView( accessor.getProperty(keyBuilder.externalView(_resourceName)).getRecord(), numberOfPartitions, masterValue, replicas, instances); } catch (Exception e) { return false; } }
void setHealthData2(int[] val1) { for (int i = 0; i < NODE_NR; i++) { String instanceName = PARTICIPANT_PREFIX + "_" + (START_PORT + i); HelixManager manager = _startCMResultMap.get(instanceName)._manager; ZNRecord record = new ZNRecord(_stat); Map<String, String> valMap = new HashMap<String, String>(); valMap.put(metricName2, val1[i] + ""); record.setSimpleField("TimeStamp", new Date().getTime() + ""); record.setMapField("TestStat@DB=TestDB;Partition=TestDB_3", valMap); HelixDataAccessor helixDataAccessor = manager.getHelixDataAccessor(); Builder keyBuilder = helixDataAccessor.keyBuilder(); helixDataAccessor.setProperty( keyBuilder.healthReport(manager.getInstanceName(), record.getId()), new HealthStat(record)); } try { Thread.sleep(1000); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } }
@Test public void testCustomizedIdealStateRebalancer() throws InterruptedException { _setupTool.addResourceToCluster(CLUSTER_NAME, db2, 60, "MasterSlave"); _setupTool.addResourceProperty( CLUSTER_NAME, db2, IdealStateProperty.REBALANCER_CLASS_NAME.toString(), TestCustomizedIdealStateRebalancer.TestRebalancer.class.getName()); _setupTool.addResourceProperty( CLUSTER_NAME, db2, IdealStateProperty.REBALANCE_MODE.toString(), RebalanceMode.USER_DEFINED.toString()); _setupTool.rebalanceStorageCluster(CLUSTER_NAME, db2, 3); boolean result = ClusterStateVerifier.verifyByZkCallback( new ExternalViewBalancedVerifier(_gZkClient, CLUSTER_NAME, db2)); Assert.assertTrue(result); Thread.sleep(1000); HelixDataAccessor accessor = new ZKHelixDataAccessor(CLUSTER_NAME, new ZkBaseDataAccessor<ZNRecord>(_gZkClient)); Builder keyBuilder = accessor.keyBuilder(); ExternalView ev = accessor.getProperty(keyBuilder.externalView(db2)); Assert.assertEquals(ev.getPartitionSet().size(), 60); for (String partition : ev.getPartitionSet()) { Assert.assertEquals(ev.getStateMap(partition).size(), 1); } IdealState is = accessor.getProperty(keyBuilder.idealStates(db2)); for (String partition : is.getPartitionSet()) { Assert.assertEquals(is.getPreferenceList(partition).size(), 3); Assert.assertEquals(is.getInstanceStateMap(partition).size(), 3); } Assert.assertTrue(testRebalancerCreated); Assert.assertTrue(testRebalancerInvoked); }
void handleNewSessionAsController() { if (_leaderElectionHandler != null) { _leaderElectionHandler.init(); } else { _leaderElectionHandler = new CallbackHandler( this, _zkclient, _keyBuilder.controller(), new DistributedLeaderElection(this, _controller, _controllerTimerTasks), new EventType[] { EventType.NodeChildrenChanged, EventType.NodeDeleted, EventType.NodeCreated }, ChangeType.CONTROLLER); } }
@Test public void testBasic() throws Exception { // Logger.getRootLogger().setLevel(Level.INFO); String className = TestHelper.getTestClassName(); String methodName = TestHelper.getTestMethodName(); String clusterName = className + "_" + methodName; final int n = 5; System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis())); MockParticipantManager[] participants = new MockParticipantManager[n]; TestHelper.setupCluster( clusterName, ZK_ADDR, 12918, // participant port "localhost", // participant name prefix "TestDB", // resource name prefix 1, // resources 10, // partitions per resource n, // number of nodes 3, // replicas "MasterSlave", true); // do rebalance ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0"); controller.syncStart(); // start participants for (int i = 0; i < n; i++) { String instanceName = "localhost_" + (12918 + i); participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName); participants[i].syncStart(); } boolean result = ClusterStateVerifier.verifyByZkCallback( new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName)); Assert.assertTrue(result); // add a new idealState without registering message handling factory ClusterSetup setupTool = new ClusterSetup(ZK_ADDR); setupTool.addResourceToCluster(clusterName, "TestDB1", 16, "MasterSlave"); ZkBaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient); ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor); Builder keyBuilder = accessor.keyBuilder(); IdealState idealState = accessor.getProperty(keyBuilder.idealStates("TestDB1")); idealState.setStateModelFactoryName("TestDB1_Factory"); accessor.setProperty(keyBuilder.idealStates("TestDB1"), idealState); setupTool.rebalanceStorageCluster(clusterName, "TestDB1", 3); // assert that we have received OFFLINE->SLAVE messages for all partitions int totalMsgs = 0; for (int retry = 0; retry < 5; retry++) { Thread.sleep(100); totalMsgs = 0; for (int i = 0; i < n; i++) { List<Message> msgs = accessor.getChildValues(keyBuilder.messages(participants[i].getInstanceName())); totalMsgs += msgs.size(); } if (totalMsgs == 48) // partition# x replicas break; } Assert.assertEquals( totalMsgs, 48, "Should accumulated 48 unprocessed messages (1 O->S per partition per replica) because TestDB1 is added without state-model-factory but was " + totalMsgs); // register "TestDB1_Factory" state model factory // Logger.getRootLogger().setLevel(Level.INFO); for (int i = 0; i < n; i++) { participants[i] .getStateMachineEngine() .registerStateModelFactory("MasterSlave", new MockMSModelFactory(), "TestDB1_Factory"); } result = ClusterStateVerifier.verifyByZkCallback( new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName)); Assert.assertTrue(result); // clean up // wait for all zk callbacks done controller.syncStop(); for (int i = 0; i < 5; i++) { participants[i].syncStop(); } System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis())); }
@Test public void testAlertActionDisableNode() throws InterruptedException { ConfigScope scope = new ConfigScopeBuilder().forCluster(CLUSTER_NAME).build(); Map<String, String> properties = new HashMap<String, String>(); properties.put("healthChange.enabled", "true"); _setupTool.getClusterManagementTool().setConfig(scope, properties); String alertStr1 = "EXP(decay(1.0)(localhost_*.TestStat@DB=db1.TestMetric1))CMP(GREATER)CON(20)ACTION(DISABLE_INSTANCE)"; String alertStr2 = "EXP(decay(1.0)(localhost_*.TestStat@DB=db1.TestMetric2))CMP(GREATER)CON(120)ACTION(DISABLE_INSTANCE)"; String alertStr3 = "EXP(decay(1.0)(localhost_*.TestStat@DB=TestDB;Partition=*.TestMetric2))CMP(GREATER)CON(160)ACTION(DISABLE_PARTITION)"; _setupTool.getClusterManagementTool().addAlert(CLUSTER_NAME, alertStr1); _setupTool.getClusterManagementTool().addAlert(CLUSTER_NAME, alertStr2); _setupTool.getClusterManagementTool().addAlert(CLUSTER_NAME, alertStr3); int[] metrics1 = {10, 15, 22, 12, 16}; int[] metrics2 = {22, 115, 22, 163, 16}; int[] metrics3 = {0, 0, 0, 0, 0}; setHealthData(metrics1, metrics2); String controllerName = CONTROLLER_PREFIX + "_0"; HelixManager manager = _startCMResultMap.get(controllerName)._manager; HealthStatsAggregationTask task = new HealthStatsAggregationTask(_startCMResultMap.get(controllerName)._manager); task.run(); Thread.sleep(4000); HelixDataAccessor helixDataAccessor = manager.getHelixDataAccessor(); Builder keyBuilder = helixDataAccessor.keyBuilder(); boolean result = ClusterStateVerifier.verifyByZkCallback( new BestPossAndExtViewZkVerifier(ZK_ADDR, CLUSTER_NAME)); Assert.assertTrue(result); Builder kb = manager.getHelixDataAccessor().keyBuilder(); ExternalView externalView = manager.getHelixDataAccessor().getProperty(kb.externalView("TestDB")); // Test the DISABLE_INSTANCE alerts String participant1 = "localhost_" + (START_PORT + 3); String participant2 = "localhost_" + (START_PORT + 2); ConfigAccessor configAccessor = manager.getConfigAccessor(); scope = new ConfigScopeBuilder() .forCluster(manager.getClusterName()) .forParticipant(participant1) .build(); String isEnabled = configAccessor.get(scope, "HELIX_ENABLED"); Assert.assertFalse(Boolean.parseBoolean(isEnabled)); scope = new ConfigScopeBuilder() .forCluster(manager.getClusterName()) .forParticipant(participant2) .build(); isEnabled = configAccessor.get(scope, "HELIX_ENABLED"); Assert.assertFalse(Boolean.parseBoolean(isEnabled)); for (String partitionName : externalView.getRecord().getMapFields().keySet()) { for (String hostName : externalView.getRecord().getMapField(partitionName).keySet()) { if (hostName.equals(participant1) || hostName.equals(participant2)) { Assert.assertEquals( externalView.getRecord().getMapField(partitionName).get(hostName), "OFFLINE"); } } } // enable the disabled instances setHealthData(metrics3, metrics3); task.run(); Thread.sleep(1000); manager.getClusterManagmentTool().enableInstance(manager.getClusterName(), participant2, true); manager.getClusterManagmentTool().enableInstance(manager.getClusterName(), participant1, true); result = ClusterStateVerifier.verifyByZkCallback( new BestPossAndExtViewZkVerifier(ZK_ADDR, CLUSTER_NAME)); Assert.assertTrue(result); // Test the DISABLE_PARTITION case int[] metrics4 = {22, 115, 22, 16, 163}; setHealthData2(metrics4); task.run(); scope = new ConfigScopeBuilder() .forCluster(manager.getClusterName()) .forParticipant(participant1) .build(); isEnabled = configAccessor.get(scope, "HELIX_ENABLED"); Assert.assertTrue(Boolean.parseBoolean(isEnabled)); scope = new ConfigScopeBuilder() .forCluster(manager.getClusterName()) .forParticipant(participant2) .build(); isEnabled = configAccessor.get(scope, "HELIX_ENABLED"); Assert.assertTrue(Boolean.parseBoolean(isEnabled)); result = ClusterStateVerifier.verifyByZkCallback( new BestPossAndExtViewZkVerifier(ZK_ADDR, CLUSTER_NAME)); Assert.assertTrue(result); String participant3 = "localhost_" + (START_PORT + 4); externalView = manager.getHelixDataAccessor().getProperty(kb.externalView("TestDB")); Assert.assertTrue( externalView .getRecord() .getMapField("TestDB_3") .get(participant3) .equalsIgnoreCase("OFFLINE")); InstanceConfig nodeConfig = helixDataAccessor.getProperty(keyBuilder.instanceConfig(participant3)); Assert.assertTrue( nodeConfig .getRecord() .getListField(InstanceConfigProperty.HELIX_DISABLED_PARTITION.toString()) .contains("TestDB_3")); }
public static IdealState getTableIdealState(HelixManager manager, String resourceName) { final HelixDataAccessor accessor = manager.getHelixDataAccessor(); final Builder builder = accessor.keyBuilder(); return accessor.getProperty(builder.idealStates(resourceName)); }
@Test() public void testController() throws Exception { System.out.println( "START " + className + ".testController() at " + new Date(System.currentTimeMillis())); final String clusterName = CLUSTER_PREFIX + "_" + className + "_controller"; // basic test if (_gZkClient.exists("/" + clusterName)) { _gZkClient.deleteRecursive("/" + clusterName); } ZKHelixManager controller = new ZKHelixManager(clusterName, null, InstanceType.CONTROLLER, ZK_ADDR); try { controller.connect(); Assert.fail("Should throw HelixException if initial cluster structure is not setup"); } catch (HelixException e) { // OK } TestHelper.setupEmptyCluster(_gZkClient, clusterName); controller.connect(); AssertJUnit.assertTrue(controller.isConnected()); controller.connect(); AssertJUnit.assertTrue(controller.isConnected()); MockListener listener = new MockListener(); listener.reset(); try { controller.addControllerListener(null); Assert.fail("Should throw HelixException"); } catch (HelixException e) { // OK } Builder keyBuilder = new Builder(controller.getClusterName()); controller.addControllerListener(listener); AssertJUnit.assertTrue(listener.isControllerChangeListenerInvoked); controller.removeListener(keyBuilder.controller(), listener); ZkHelixPropertyStore<ZNRecord> store = controller.getHelixPropertyStore(); ZNRecord record = new ZNRecord("node_1"); int options = 0; store.set("/node_1", record, AccessOption.PERSISTENT); Stat stat = new Stat(); record = store.get("/node_1", stat, options); AssertJUnit.assertEquals("node_1", record.getId()); controller.getMessagingService(); controller.getHealthReportCollector(); controller.getClusterManagmentTool(); controller.handleNewSession(); controller.disconnect(); AssertJUnit.assertFalse(controller.isConnected()); System.out.println( "END " + className + ".testController() at " + new Date(System.currentTimeMillis())); }