/**
* Check if the ideal state adheres to a rule
*
* @param idealState the ideal state to check
* @param rule the rules of a valid ideal state
* @return true if the ideal state is a superset of the entries of the rule, false otherwise
*/
private boolean idealStateMatchesRule(IdealState idealState, Map<String, String> rule) {
Map<String, String> simpleFields = idealState.getRecord().getSimpleFields();
for (String key : rule.keySet()) {
String value = rule.get(key);
if (!simpleFields.containsKey(key) || !value.equals(simpleFields.get(key))) {
return false;
}
}
return true;
}
public void testEspressoStorageClusterIdealState(int partitions, int nodes, int replica)
throws Exception {
List<String> storageNodes = new ArrayList<String>();
for (int i = 0; i < partitions; i++) {
storageNodes.add("localhost:123" + i);
}
List<String> relays = new ArrayList<String>();
for (int i = 0; i < nodes; i++) {
relays.add("relay:123" + i);
}
IdealState idealstate =
IdealStateCalculatorForEspressoRelay.calculateRelayIdealState(
storageNodes, relays, "TEST", replica, "Leader", "Standby", "LeaderStandby");
Assert.assertEquals(
idealstate.getRecord().getListFields().size(),
idealstate.getRecord().getMapFields().size());
Map<String, Integer> countMap = new TreeMap<String, Integer>();
for (String key : idealstate.getRecord().getListFields().keySet()) {
Assert.assertEquals(
idealstate.getRecord().getListFields().get(key).size(),
idealstate.getRecord().getMapFields().get(key).size());
List<String> list = idealstate.getRecord().getListFields().get(key);
Map<String, String> map = idealstate.getRecord().getMapFields().get(key);
Assert.assertEquals(list.size(), replica);
for (String val : list) {
if (!countMap.containsKey(val)) {
countMap.put(val, 1);
} else {
countMap.put(val, countMap.get(val) + 1);
}
Assert.assertTrue(map.containsKey(val));
}
}
for (String nodeName : countMap.keySet()) {
Assert.assertTrue(countMap.get(nodeName) <= partitions * replica / nodes + 1);
// System.out.println(nodeName + " " + countMap.get(nodeName));
}
System.out.println();
}
@Test
public void testSchemataSM() throws Exception {
String className = TestHelper.getTestClassName();
String methodName = TestHelper.getTestMethodName();
String clusterName = className + "_" + methodName;
int n = 5;
MockParticipant[] participants = new MockParticipant[n];
System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
TestHelper.setupCluster(
clusterName,
_zkaddr,
12918, // participant start port
"localhost", // participant name prefix
"TestSchemata", // resource name prefix
1, // resources
1, // partitions per resource
n, // number of nodes
0, // replicas
"STORAGE_DEFAULT_SM_SCHEMATA",
false); // don't rebalance
// rebalance ideal-state to use ANY_LIVEINSTANCE for preference list
ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, _baseAccessor);
PropertyKey.Builder keyBuilder = accessor.keyBuilder();
PropertyKey key = keyBuilder.idealStates("TestSchemata0");
IdealState idealState = accessor.getProperty(key);
idealState.setReplicas(HelixConstants.StateModelToken.ANY_LIVEINSTANCE.toString());
idealState
.getRecord()
.setListField(
"TestSchemata0_0",
Arrays.asList(HelixConstants.StateModelToken.ANY_LIVEINSTANCE.toString()));
accessor.setProperty(key, idealState);
MockController controller = new MockController(_zkaddr, clusterName, "controller");
controller.syncStart();
// start n-1 participants
for (int i = 1; i < n; i++) {
String instanceName = "localhost_" + (12918 + i);
participants[i] = new MockParticipant(_zkaddr, clusterName, instanceName);
participants[i].syncStart();
}
boolean result =
ClusterStateVerifier.verifyByZkCallback(
new BestPossAndExtViewZkVerifier(_zkaddr, clusterName));
Assert.assertTrue(result);
// start the remaining 1 participant
participants[0] = new MockParticipant(_zkaddr, clusterName, "localhost_12918");
participants[0].syncStart();
// make sure we have all participants in MASTER state
result =
ClusterStateVerifier.verifyByZkCallback(
new BestPossAndExtViewZkVerifier(_zkaddr, clusterName));
Assert.assertTrue(result);
key = keyBuilder.externalView("TestSchemata0");
ExternalView externalView = accessor.getProperty(key);
Map<String, String> stateMap = externalView.getStateMap("TestSchemata0_0");
Assert.assertNotNull(stateMap);
Assert.assertEquals(stateMap.size(), n, "all " + n + " participants should be in Master state");
for (int i = 0; i < n; i++) {
String instanceName = "localhost_" + (12918 + i);
Assert.assertNotNull(stateMap.get(instanceName));
Assert.assertEquals(stateMap.get(instanceName), "MASTER");
}
// clean up
controller.syncStop();
for (int i = 0; i < n; i++) {
participants[i].syncStop();
}
System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
@Override
public void onCallback(NotificationContext context) {
LOG.info(
"START: MasterSlaveRebalancer.onCallback running at "
+ _context.getHelixManager().getInstanceName());
if (context.getType().equals(NotificationContext.Type.FINALIZE)) {
LOG.info(
"END: MasterSlaveRebalancer.onCallback FINALIZE callback invoked. Likely lost connection to Helix");
return;
}
HelixManager manager = context.getManager();
String clusterName = manager.getClusterName();
HelixAdmin helixAdmin = manager.getClusterManagmentTool();
IdealState idealState =
helixAdmin.getResourceIdealState(clusterName, MySQLConstants.MASTER_SLAVE_RESOURCE_NAME);
if (idealState == null) {
LOG.info(
"END: MasterSlaveRebalancer.onCallback. "
+ MySQLConstants.MASTER_SLAVE_RESOURCE_NAME
+ " is not yet created");
}
PropertyKey.Builder builder = new PropertyKey.Builder(clusterName);
Map<String, LiveInstance> liveInstancesMap =
manager.getHelixDataAccessor().getChildValuesMap(builder.liveInstances());
Map<String, InstanceConfig> instanceConfigs =
manager.getHelixDataAccessor().getChildValuesMap(builder.instanceConfigs());
IdealState newIdealState = new IdealState(idealState.getId());
newIdealState.getRecord().setSimpleFields(idealState.getRecord().getSimpleFields());
newIdealState.getRecord().setListFields(idealState.getRecord().getListFields());
for (String partition : idealState.getPartitionSet()) {
Map<String, String> instanceStateMap = idealState.getInstanceStateMap(partition);
String currMaster = null;
Set<String> slaveSet = new TreeSet<String>();
for (String instance : instanceStateMap.keySet()) {
if ("MASTER".equalsIgnoreCase(instanceStateMap.get(instance))) {
currMaster = instance;
}
if ("SLAVE".equalsIgnoreCase(instanceStateMap.get(instance))) {
slaveSet.add(instance);
}
}
String newMaster = currMaster;
if (!liveInstancesMap.containsKey(currMaster)
|| !instanceConfigs.get(currMaster).getInstanceEnabled()) {
// need to find a new master.
newMaster = findNewMaster(liveInstancesMap, instanceConfigs, currMaster, slaveSet);
}
for (String instance : instanceStateMap.keySet()) {
if (instance.equalsIgnoreCase(newMaster)) {
newIdealState.setPartitionState(partition, instance, "MASTER");
} else {
newIdealState.setPartitionState(partition, instance, "SLAVE");
}
}
}
if (!idealState.equals(newIdealState)) {
LOG.info("New idealstate computed.");
LOG.info(newIdealState.toString());
manager
.getClusterManagmentTool()
.setResourceIdealState(
clusterName, MySQLConstants.MASTER_SLAVE_RESOURCE_NAME, newIdealState);
} else {
LOG.info("No change in IdealState");
}
LOG.info("END: MasterSlaveRebalancer.onCallback");
}
@Test
public void testExpandCluster() throws Exception {
String DB2 = "TestDB2";
int partitions = 100;
int replica = 3;
_setupTool.addResourceToCluster(CLUSTER_NAME, DB2, partitions, STATE_MODEL);
_setupTool.rebalanceStorageCluster(CLUSTER_NAME, DB2, replica, "keyX");
String DB3 = "TestDB3";
_setupTool.addResourceToCluster(CLUSTER_NAME, DB3, partitions, STATE_MODEL);
IdealState testDB0 =
_setupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, TEST_DB);
IdealState testDB2 =
_setupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, DB2);
IdealState testDB3 =
_setupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, DB3);
for (int i = 0; i < 5; i++) {
String storageNodeName = "localhost_" + (27960 + i);
_setupTool.addInstanceToCluster(CLUSTER_NAME, storageNodeName);
}
String command = "-zkSvr localhost:2183 -expandCluster " + CLUSTER_NAME;
ClusterSetup.processCommandLineArgs(command.split(" "));
IdealState testDB0_1 =
_setupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, TEST_DB);
IdealState testDB2_1 =
_setupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, DB2);
IdealState testDB3_1 =
_setupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, DB3);
Map<String, Object> resultOld2 = RebalanceUtil.buildInternalIdealState(testDB2);
Map<String, Object> result2 = RebalanceUtil.buildInternalIdealState(testDB2_1);
TestEspressoStorageClusterIdealState.Verify(result2, partitions, replica - 1);
Double masterKeepRatio = 0.0, slaveKeepRatio = 0.0;
double[] result = TestEspressoStorageClusterIdealState.compareResult(resultOld2, result2);
masterKeepRatio = result[0];
slaveKeepRatio = result[1];
Assert.assertTrue(masterKeepRatio > 0.49 && masterKeepRatio < 0.51);
Assert.assertTrue(testDB3_1.getRecord().getListFields().size() == 0);
// partitions should stay as same
Assert.assertTrue(
testDB0_1
.getRecord()
.getListFields()
.keySet()
.containsAll(testDB0.getRecord().getListFields().keySet()));
Assert.assertTrue(
testDB0_1.getRecord().getListFields().size() == testDB0.getRecord().getListFields().size());
Assert.assertTrue(
testDB2_1
.getRecord()
.getMapFields()
.keySet()
.containsAll(testDB2.getRecord().getMapFields().keySet()));
Assert.assertTrue(
testDB2_1.getRecord().getMapFields().size() == testDB2.getRecord().getMapFields().size());
Assert.assertTrue(
testDB3_1
.getRecord()
.getMapFields()
.keySet()
.containsAll(testDB3.getRecord().getMapFields().keySet()));
Assert.assertTrue(
testDB3_1.getRecord().getMapFields().size() == testDB3.getRecord().getMapFields().size());
Map<String, Object> resultOld = RebalanceUtil.buildInternalIdealState(testDB0);
Map<String, Object> resultNew = RebalanceUtil.buildInternalIdealState(testDB0_1);
result = TestEspressoStorageClusterIdealState.compareResult(resultOld, resultNew);
masterKeepRatio = result[0];
slaveKeepRatio = result[1];
Assert.assertTrue(masterKeepRatio > 0.49 && masterKeepRatio < 0.51);
}
public static IdealState calculateRelayIdealState(
List<String> partitions,
List<String> instances,
String resultRecordName,
int replica,
String firstValue,
String restValue,
String stateModelName) {
Collections.sort(partitions);
Collections.sort(instances);
if (instances.size() % replica != 0) {
throw new HelixException("Instances must be divided by replica");
}
IdealState result = new IdealState(resultRecordName);
result.setNumPartitions(partitions.size());
result.setReplicas("" + replica);
result.setStateModelDefId(StateModelDefinitionId.from(stateModelName));
int groups = instances.size() / replica;
int remainder = instances.size() % replica;
int remainder2 = partitions.size() % groups;
int storageNodeGroupSize = partitions.size() / groups;
for (int i = 0; i < groups; i++) {
int relayStart = 0, relayEnd = 0, storageNodeStart = 0, storageNodeEnd = 0;
if (i < remainder) {
relayStart = (replica + 1) * i;
relayEnd = (replica + 1) * (i + 1);
} else {
relayStart = (replica + 1) * remainder + replica * (i - remainder);
relayEnd = relayStart + replica;
}
// System.out.println("relay start :" + relayStart + " relayEnd:" + relayEnd);
if (i < remainder2) {
storageNodeStart = (storageNodeGroupSize + 1) * i;
storageNodeEnd = (storageNodeGroupSize + 1) * (i + 1);
} else {
storageNodeStart =
(storageNodeGroupSize + 1) * remainder2 + storageNodeGroupSize * (i - remainder2);
storageNodeEnd = storageNodeStart + storageNodeGroupSize;
}
// System.out.println("storageNodeStart :" + storageNodeStart + " storageNodeEnd:" +
// storageNodeEnd);
List<String> snBatch = partitions.subList(storageNodeStart, storageNodeEnd);
List<String> relayBatch = instances.subList(relayStart, relayEnd);
Map<String, List<String>> sublistFields =
calculateSubIdealState(snBatch, relayBatch, replica);
result.getRecord().getListFields().putAll(sublistFields);
}
for (String snName : result.getRecord().getListFields().keySet()) {
Map<String, String> mapField = new TreeMap<String, String>();
List<String> relayCandidates = result.getRecord().getListField(snName);
mapField.put(relayCandidates.get(0), firstValue);
for (int i = 1; i < relayCandidates.size(); i++) {
mapField.put(relayCandidates.get(i), restValue);
}
result.getRecord().getMapFields().put(snName, mapField);
}
System.out.println();
return result;
}