protected List<IdealState> setupIdealState(
int nodes, String[] resources, int partitions, int replicas) {
List<IdealState> idealStates = new ArrayList<IdealState>();
List<String> instances = new ArrayList<String>();
for (int i = 0; i < nodes; i++) {
instances.add("localhost_" + i);
}
for (int i = 0; i < resources.length; i++) {
String resourceName = resources[i];
ZNRecord record = new ZNRecord(resourceName);
for (int p = 0; p < partitions; p++) {
List<String> value = new ArrayList<String>();
for (int r = 0; r < replicas; r++) {
value.add("localhost_" + (p + r + 1) % nodes);
}
record.setListField(resourceName + "_" + p, value);
}
IdealState idealState = new IdealState(record);
idealState.setStateModelDefRef("MasterSlave");
idealState.setIdealStateMode(IdealStateModeProperty.AUTO.toString());
idealState.setNumPartitions(partitions);
idealStates.add(idealState);
// System.out.println(idealState);
Builder keyBuilder = accessor.keyBuilder();
accessor.setProperty(keyBuilder.idealStates(resourceName), idealState);
}
return idealStates;
}
@Override
public void addConfigChangeListener(
ScopedConfigChangeListener listener, ConfigScopeProperty scope) throws Exception {
Builder keyBuilder = new Builder(_clusterName);
PropertyKey propertyKey = null;
switch (scope) {
case CLUSTER:
propertyKey = keyBuilder.clusterConfigs();
break;
case PARTICIPANT:
propertyKey = keyBuilder.instanceConfigs();
break;
case RESOURCE:
propertyKey = keyBuilder.resourceConfigs();
break;
default:
break;
}
if (propertyKey != null) {
addListener(
listener,
propertyKey,
ChangeType.CONFIG,
new EventType[] {EventType.NodeChildrenChanged});
} else {
LOG.error("Can't add listener to config scope: " + scope);
}
}
/**
* Get the {@link PropertyKey} for this message
*
* @param keyBuilder PropertyKey Builder
* @param instanceName target instance
* @return message PropertyKey
*/
public PropertyKey getKey(Builder keyBuilder, String instanceName) {
if (isControlerMsg()) {
return keyBuilder.controllerMessage(getId());
} else {
return keyBuilder.message(instanceName, getId());
}
}
protected void setupLiveInstances(int numLiveInstances) {
// setup liveInstances
for (int i = 0; i < numLiveInstances; i++) {
LiveInstance liveInstance = new LiveInstance("localhost_" + i);
liveInstance.setSessionId("session_" + i);
Builder keyBuilder = accessor.keyBuilder();
accessor.setProperty(keyBuilder.liveInstance("localhost_" + i), liveInstance);
}
}
/*
* stateMap: partition->instance->state
*/
public static boolean verifyState(
ZkClient zkclient,
String clusterName,
String resourceName,
Map<String, Map<String, String>> expectStateMap,
String op) {
boolean result = true;
ZkBaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(zkclient);
ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor);
Builder keyBuilder = accessor.keyBuilder();
ExternalView extView = accessor.getProperty(keyBuilder.externalView(resourceName));
Map<String, Map<String, String>> actualStateMap = extView.getRecord().getMapFields();
for (String partition : actualStateMap.keySet()) {
for (String expectPartiton : expectStateMap.keySet()) {
if (!partition.matches(expectPartiton)) {
continue;
}
Map<String, String> actualInstanceStateMap = actualStateMap.get(partition);
Map<String, String> expectInstanceStateMap = expectStateMap.get(expectPartiton);
for (String instance : actualInstanceStateMap.keySet()) {
for (String expectInstance : expectStateMap.get(expectPartiton).keySet()) {
if (!instance.matches(expectInstance)) {
continue;
}
String actualState = actualInstanceStateMap.get(instance);
String expectState = expectInstanceStateMap.get(expectInstance);
boolean equals = expectState.equals(actualState);
if (op.equals("==") && !equals || op.equals("!=") && equals) {
System.out.println(
partition
+ "/"
+ instance
+ " state mismatch. actual state: "
+ actualState
+ ", but expect: "
+ expectState
+ ", op: "
+ op);
result = false;
}
}
}
}
}
return result;
}
protected void setupStateModel() {
ZNRecord masterSlave = new StateModelConfigGenerator().generateConfigForMasterSlave();
Builder keyBuilder = accessor.keyBuilder();
accessor.setProperty(
keyBuilder.stateModelDef(masterSlave.getId()), new StateModelDefinition(masterSlave));
ZNRecord leaderStandby = new StateModelConfigGenerator().generateConfigForLeaderStandby();
accessor.setProperty(
keyBuilder.stateModelDef(leaderStandby.getId()), new StateModelDefinition(leaderStandby));
ZNRecord onlineOffline = new StateModelConfigGenerator().generateConfigForOnlineOffline();
accessor.setProperty(
keyBuilder.stateModelDef(onlineOffline.getId()), new StateModelDefinition(onlineOffline));
}
@Override
public boolean isLeader() {
if (_instanceType != InstanceType.CONTROLLER
&& _instanceType != InstanceType.CONTROLLER_PARTICIPANT) {
return false;
}
if (!isConnected()) {
return false;
}
try {
LiveInstance leader = _dataAccessor.getProperty(_keyBuilder.controllerLeader());
if (leader != null) {
String leaderName = leader.getInstanceName();
String sessionId = leader.getSessionId();
if (leaderName != null
&& leaderName.equals(_instanceName)
&& sessionId != null
&& sessionId.equals(_sessionId)) {
return true;
}
}
} catch (Exception e) {
// log
}
return false;
}
@Override
public boolean verify() {
try {
HelixDataAccessor accessor =
new ZKHelixDataAccessor(_clusterName, new ZkBaseDataAccessor<ZNRecord>(_client));
Builder keyBuilder = accessor.keyBuilder();
int numberOfPartitions =
accessor
.getProperty(keyBuilder.idealStates(_resourceName))
.getRecord()
.getListFields()
.size();
ClusterDataCache cache = new ClusterDataCache();
cache.refresh(accessor);
String masterValue =
cache
.getStateModelDef(cache.getIdealState(_resourceName).getStateModelDefRef())
.getStatesPriorityList()
.get(0);
int replicas = Integer.parseInt(cache.getIdealState(_resourceName).getReplicas());
String instanceGroupTag = cache.getIdealState(_resourceName).getInstanceGroupTag();
int instances = 0;
for (String liveInstanceName : cache.getLiveInstances().keySet()) {
if (cache.getInstanceConfigMap().get(liveInstanceName).containsTag(instanceGroupTag)) {
instances++;
}
}
if (instances == 0) {
instances = cache.getLiveInstances().size();
}
return verifyBalanceExternalView(
accessor.getProperty(keyBuilder.externalView(_resourceName)).getRecord(),
numberOfPartitions,
masterValue,
replicas,
instances);
} catch (Exception e) {
return false;
}
}
void setHealthData2(int[] val1) {
for (int i = 0; i < NODE_NR; i++) {
String instanceName = PARTICIPANT_PREFIX + "_" + (START_PORT + i);
HelixManager manager = _startCMResultMap.get(instanceName)._manager;
ZNRecord record = new ZNRecord(_stat);
Map<String, String> valMap = new HashMap<String, String>();
valMap.put(metricName2, val1[i] + "");
record.setSimpleField("TimeStamp", new Date().getTime() + "");
record.setMapField("TestStat@DB=TestDB;Partition=TestDB_3", valMap);
HelixDataAccessor helixDataAccessor = manager.getHelixDataAccessor();
Builder keyBuilder = helixDataAccessor.keyBuilder();
helixDataAccessor.setProperty(
keyBuilder.healthReport(manager.getInstanceName(), record.getId()),
new HealthStat(record));
}
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
@Test
public void testCustomizedIdealStateRebalancer() throws InterruptedException {
_setupTool.addResourceToCluster(CLUSTER_NAME, db2, 60, "MasterSlave");
_setupTool.addResourceProperty(
CLUSTER_NAME,
db2,
IdealStateProperty.REBALANCER_CLASS_NAME.toString(),
TestCustomizedIdealStateRebalancer.TestRebalancer.class.getName());
_setupTool.addResourceProperty(
CLUSTER_NAME,
db2,
IdealStateProperty.REBALANCE_MODE.toString(),
RebalanceMode.USER_DEFINED.toString());
_setupTool.rebalanceStorageCluster(CLUSTER_NAME, db2, 3);
boolean result =
ClusterStateVerifier.verifyByZkCallback(
new ExternalViewBalancedVerifier(_gZkClient, CLUSTER_NAME, db2));
Assert.assertTrue(result);
Thread.sleep(1000);
HelixDataAccessor accessor =
new ZKHelixDataAccessor(CLUSTER_NAME, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
Builder keyBuilder = accessor.keyBuilder();
ExternalView ev = accessor.getProperty(keyBuilder.externalView(db2));
Assert.assertEquals(ev.getPartitionSet().size(), 60);
for (String partition : ev.getPartitionSet()) {
Assert.assertEquals(ev.getStateMap(partition).size(), 1);
}
IdealState is = accessor.getProperty(keyBuilder.idealStates(db2));
for (String partition : is.getPartitionSet()) {
Assert.assertEquals(is.getPreferenceList(partition).size(), 3);
Assert.assertEquals(is.getInstanceStateMap(partition).size(), 3);
}
Assert.assertTrue(testRebalancerCreated);
Assert.assertTrue(testRebalancerInvoked);
}
void handleNewSessionAsController() {
if (_leaderElectionHandler != null) {
_leaderElectionHandler.init();
} else {
_leaderElectionHandler =
new CallbackHandler(
this,
_zkclient,
_keyBuilder.controller(),
new DistributedLeaderElection(this, _controller, _controllerTimerTasks),
new EventType[] {
EventType.NodeChildrenChanged, EventType.NodeDeleted, EventType.NodeCreated
},
ChangeType.CONTROLLER);
}
}
@Test
public void testBasic() throws Exception {
// Logger.getRootLogger().setLevel(Level.INFO);
String className = TestHelper.getTestClassName();
String methodName = TestHelper.getTestMethodName();
String clusterName = className + "_" + methodName;
final int n = 5;
System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
MockParticipantManager[] participants = new MockParticipantManager[n];
TestHelper.setupCluster(
clusterName,
ZK_ADDR,
12918, // participant port
"localhost", // participant name prefix
"TestDB", // resource name prefix
1, // resources
10, // partitions per resource
n, // number of nodes
3, // replicas
"MasterSlave",
true); // do rebalance
ClusterControllerManager controller =
new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
controller.syncStart();
// start participants
for (int i = 0; i < n; i++) {
String instanceName = "localhost_" + (12918 + i);
participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
participants[i].syncStart();
}
boolean result =
ClusterStateVerifier.verifyByZkCallback(
new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
Assert.assertTrue(result);
// add a new idealState without registering message handling factory
ClusterSetup setupTool = new ClusterSetup(ZK_ADDR);
setupTool.addResourceToCluster(clusterName, "TestDB1", 16, "MasterSlave");
ZkBaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient);
ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor);
Builder keyBuilder = accessor.keyBuilder();
IdealState idealState = accessor.getProperty(keyBuilder.idealStates("TestDB1"));
idealState.setStateModelFactoryName("TestDB1_Factory");
accessor.setProperty(keyBuilder.idealStates("TestDB1"), idealState);
setupTool.rebalanceStorageCluster(clusterName, "TestDB1", 3);
// assert that we have received OFFLINE->SLAVE messages for all partitions
int totalMsgs = 0;
for (int retry = 0; retry < 5; retry++) {
Thread.sleep(100);
totalMsgs = 0;
for (int i = 0; i < n; i++) {
List<Message> msgs =
accessor.getChildValues(keyBuilder.messages(participants[i].getInstanceName()));
totalMsgs += msgs.size();
}
if (totalMsgs == 48) // partition# x replicas
break;
}
Assert.assertEquals(
totalMsgs,
48,
"Should accumulated 48 unprocessed messages (1 O->S per partition per replica) because TestDB1 is added without state-model-factory but was "
+ totalMsgs);
// register "TestDB1_Factory" state model factory
// Logger.getRootLogger().setLevel(Level.INFO);
for (int i = 0; i < n; i++) {
participants[i]
.getStateMachineEngine()
.registerStateModelFactory("MasterSlave", new MockMSModelFactory(), "TestDB1_Factory");
}
result =
ClusterStateVerifier.verifyByZkCallback(
new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
Assert.assertTrue(result);
// clean up
// wait for all zk callbacks done
controller.syncStop();
for (int i = 0; i < 5; i++) {
participants[i].syncStop();
}
System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
@Test
public void testAlertActionDisableNode() throws InterruptedException {
ConfigScope scope = new ConfigScopeBuilder().forCluster(CLUSTER_NAME).build();
Map<String, String> properties = new HashMap<String, String>();
properties.put("healthChange.enabled", "true");
_setupTool.getClusterManagementTool().setConfig(scope, properties);
String alertStr1 =
"EXP(decay(1.0)(localhost_*.TestStat@DB=db1.TestMetric1))CMP(GREATER)CON(20)ACTION(DISABLE_INSTANCE)";
String alertStr2 =
"EXP(decay(1.0)(localhost_*.TestStat@DB=db1.TestMetric2))CMP(GREATER)CON(120)ACTION(DISABLE_INSTANCE)";
String alertStr3 =
"EXP(decay(1.0)(localhost_*.TestStat@DB=TestDB;Partition=*.TestMetric2))CMP(GREATER)CON(160)ACTION(DISABLE_PARTITION)";
_setupTool.getClusterManagementTool().addAlert(CLUSTER_NAME, alertStr1);
_setupTool.getClusterManagementTool().addAlert(CLUSTER_NAME, alertStr2);
_setupTool.getClusterManagementTool().addAlert(CLUSTER_NAME, alertStr3);
int[] metrics1 = {10, 15, 22, 12, 16};
int[] metrics2 = {22, 115, 22, 163, 16};
int[] metrics3 = {0, 0, 0, 0, 0};
setHealthData(metrics1, metrics2);
String controllerName = CONTROLLER_PREFIX + "_0";
HelixManager manager = _startCMResultMap.get(controllerName)._manager;
HealthStatsAggregationTask task =
new HealthStatsAggregationTask(_startCMResultMap.get(controllerName)._manager);
task.run();
Thread.sleep(4000);
HelixDataAccessor helixDataAccessor = manager.getHelixDataAccessor();
Builder keyBuilder = helixDataAccessor.keyBuilder();
boolean result =
ClusterStateVerifier.verifyByZkCallback(
new BestPossAndExtViewZkVerifier(ZK_ADDR, CLUSTER_NAME));
Assert.assertTrue(result);
Builder kb = manager.getHelixDataAccessor().keyBuilder();
ExternalView externalView =
manager.getHelixDataAccessor().getProperty(kb.externalView("TestDB"));
// Test the DISABLE_INSTANCE alerts
String participant1 = "localhost_" + (START_PORT + 3);
String participant2 = "localhost_" + (START_PORT + 2);
ConfigAccessor configAccessor = manager.getConfigAccessor();
scope =
new ConfigScopeBuilder()
.forCluster(manager.getClusterName())
.forParticipant(participant1)
.build();
String isEnabled = configAccessor.get(scope, "HELIX_ENABLED");
Assert.assertFalse(Boolean.parseBoolean(isEnabled));
scope =
new ConfigScopeBuilder()
.forCluster(manager.getClusterName())
.forParticipant(participant2)
.build();
isEnabled = configAccessor.get(scope, "HELIX_ENABLED");
Assert.assertFalse(Boolean.parseBoolean(isEnabled));
for (String partitionName : externalView.getRecord().getMapFields().keySet()) {
for (String hostName : externalView.getRecord().getMapField(partitionName).keySet()) {
if (hostName.equals(participant1) || hostName.equals(participant2)) {
Assert.assertEquals(
externalView.getRecord().getMapField(partitionName).get(hostName), "OFFLINE");
}
}
}
// enable the disabled instances
setHealthData(metrics3, metrics3);
task.run();
Thread.sleep(1000);
manager.getClusterManagmentTool().enableInstance(manager.getClusterName(), participant2, true);
manager.getClusterManagmentTool().enableInstance(manager.getClusterName(), participant1, true);
result =
ClusterStateVerifier.verifyByZkCallback(
new BestPossAndExtViewZkVerifier(ZK_ADDR, CLUSTER_NAME));
Assert.assertTrue(result);
// Test the DISABLE_PARTITION case
int[] metrics4 = {22, 115, 22, 16, 163};
setHealthData2(metrics4);
task.run();
scope =
new ConfigScopeBuilder()
.forCluster(manager.getClusterName())
.forParticipant(participant1)
.build();
isEnabled = configAccessor.get(scope, "HELIX_ENABLED");
Assert.assertTrue(Boolean.parseBoolean(isEnabled));
scope =
new ConfigScopeBuilder()
.forCluster(manager.getClusterName())
.forParticipant(participant2)
.build();
isEnabled = configAccessor.get(scope, "HELIX_ENABLED");
Assert.assertTrue(Boolean.parseBoolean(isEnabled));
result =
ClusterStateVerifier.verifyByZkCallback(
new BestPossAndExtViewZkVerifier(ZK_ADDR, CLUSTER_NAME));
Assert.assertTrue(result);
String participant3 = "localhost_" + (START_PORT + 4);
externalView = manager.getHelixDataAccessor().getProperty(kb.externalView("TestDB"));
Assert.assertTrue(
externalView
.getRecord()
.getMapField("TestDB_3")
.get(participant3)
.equalsIgnoreCase("OFFLINE"));
InstanceConfig nodeConfig =
helixDataAccessor.getProperty(keyBuilder.instanceConfig(participant3));
Assert.assertTrue(
nodeConfig
.getRecord()
.getListField(InstanceConfigProperty.HELIX_DISABLED_PARTITION.toString())
.contains("TestDB_3"));
}
public static IdealState getTableIdealState(HelixManager manager, String resourceName) {
final HelixDataAccessor accessor = manager.getHelixDataAccessor();
final Builder builder = accessor.keyBuilder();
return accessor.getProperty(builder.idealStates(resourceName));
}
@Test()
public void testController() throws Exception {
System.out.println(
"START " + className + ".testController() at " + new Date(System.currentTimeMillis()));
final String clusterName = CLUSTER_PREFIX + "_" + className + "_controller";
// basic test
if (_gZkClient.exists("/" + clusterName)) {
_gZkClient.deleteRecursive("/" + clusterName);
}
ZKHelixManager controller =
new ZKHelixManager(clusterName, null, InstanceType.CONTROLLER, ZK_ADDR);
try {
controller.connect();
Assert.fail("Should throw HelixException if initial cluster structure is not setup");
} catch (HelixException e) {
// OK
}
TestHelper.setupEmptyCluster(_gZkClient, clusterName);
controller.connect();
AssertJUnit.assertTrue(controller.isConnected());
controller.connect();
AssertJUnit.assertTrue(controller.isConnected());
MockListener listener = new MockListener();
listener.reset();
try {
controller.addControllerListener(null);
Assert.fail("Should throw HelixException");
} catch (HelixException e) {
// OK
}
Builder keyBuilder = new Builder(controller.getClusterName());
controller.addControllerListener(listener);
AssertJUnit.assertTrue(listener.isControllerChangeListenerInvoked);
controller.removeListener(keyBuilder.controller(), listener);
ZkHelixPropertyStore<ZNRecord> store = controller.getHelixPropertyStore();
ZNRecord record = new ZNRecord("node_1");
int options = 0;
store.set("/node_1", record, AccessOption.PERSISTENT);
Stat stat = new Stat();
record = store.get("/node_1", stat, options);
AssertJUnit.assertEquals("node_1", record.getId());
controller.getMessagingService();
controller.getHealthReportCollector();
controller.getClusterManagmentTool();
controller.handleNewSession();
controller.disconnect();
AssertJUnit.assertFalse(controller.isConnected());
System.out.println(
"END " + className + ".testController() at " + new Date(System.currentTimeMillis()));
}