@Test
public void simpleIntegrationTest() throws Exception {
// Logger.getRootLogger().setLevel(Level.INFO);
String className = TestHelper.getTestClassName();
String methodName = TestHelper.getTestMethodName();
String clusterName = className + "_" + methodName;
int n = 1;
System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
TestHelper.setupCluster(
clusterName,
_zkaddr,
12918, // participant port
"localhost", // participant name prefix
"TestDB", // resource name prefix
1, // resources
4, // partitions per resource
n, // number of nodes
1, // replicas
"MasterSlave",
true); // do rebalance
HelixManager participant =
new ZKHelixManager(clusterName, "localhost_12918", InstanceType.PARTICIPANT, _zkaddr);
participant
.getStateMachineEngine()
.registerStateModelFactory(StateModelDefId.MasterSlave, new MockMSModelFactory());
participant.connect();
HelixManager controller =
new ZKHelixManager(clusterName, "controller_0", InstanceType.CONTROLLER, _zkaddr);
controller.connect();
boolean result =
ClusterStateVerifier.verifyByZkCallback(
new BestPossAndExtViewZkVerifier(_zkaddr, clusterName));
Assert.assertTrue(result);
// cleanup
controller.disconnect();
participant.disconnect();
// verify all live-instances and leader nodes are gone
ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, _baseAccessor);
PropertyKey.Builder keyBuilder = accessor.keyBuilder();
Assert.assertNull(accessor.getProperty(keyBuilder.liveInstance("localhost_12918")));
Assert.assertNull(accessor.getProperty(keyBuilder.controllerLeader()));
System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
/*
* stateMap: partition->instance->state
*/
public static boolean verifyState(
ZkClient zkclient,
String clusterName,
String resourceName,
Map<String, Map<String, String>> expectStateMap,
String op) {
boolean result = true;
ZkBaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(zkclient);
ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor);
Builder keyBuilder = accessor.keyBuilder();
ExternalView extView = accessor.getProperty(keyBuilder.externalView(resourceName));
Map<String, Map<String, String>> actualStateMap = extView.getRecord().getMapFields();
for (String partition : actualStateMap.keySet()) {
for (String expectPartiton : expectStateMap.keySet()) {
if (!partition.matches(expectPartiton)) {
continue;
}
Map<String, String> actualInstanceStateMap = actualStateMap.get(partition);
Map<String, String> expectInstanceStateMap = expectStateMap.get(expectPartiton);
for (String instance : actualInstanceStateMap.keySet()) {
for (String expectInstance : expectStateMap.get(expectPartiton).keySet()) {
if (!instance.matches(expectInstance)) {
continue;
}
String actualState = actualInstanceStateMap.get(instance);
String expectState = expectInstanceStateMap.get(expectInstance);
boolean equals = expectState.equals(actualState);
if (op.equals("==") && !equals || op.equals("!=") && equals) {
System.out.println(
partition
+ "/"
+ instance
+ " state mismatch. actual state: "
+ actualState
+ ", but expect: "
+ expectState
+ ", op: "
+ op);
result = false;
}
}
}
}
}
return result;
}
@Test
public void testSchemataSM() throws Exception {
String className = TestHelper.getTestClassName();
String methodName = TestHelper.getTestMethodName();
String clusterName = className + "_" + methodName;
int n = 5;
MockParticipant[] participants = new MockParticipant[n];
System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
TestHelper.setupCluster(
clusterName,
_zkaddr,
12918, // participant start port
"localhost", // participant name prefix
"TestSchemata", // resource name prefix
1, // resources
1, // partitions per resource
n, // number of nodes
0, // replicas
"STORAGE_DEFAULT_SM_SCHEMATA",
false); // don't rebalance
// rebalance ideal-state to use ANY_LIVEINSTANCE for preference list
ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, _baseAccessor);
PropertyKey.Builder keyBuilder = accessor.keyBuilder();
PropertyKey key = keyBuilder.idealStates("TestSchemata0");
IdealState idealState = accessor.getProperty(key);
idealState.setReplicas(HelixConstants.StateModelToken.ANY_LIVEINSTANCE.toString());
idealState
.getRecord()
.setListField(
"TestSchemata0_0",
Arrays.asList(HelixConstants.StateModelToken.ANY_LIVEINSTANCE.toString()));
accessor.setProperty(key, idealState);
MockController controller = new MockController(_zkaddr, clusterName, "controller");
controller.syncStart();
// start n-1 participants
for (int i = 1; i < n; i++) {
String instanceName = "localhost_" + (12918 + i);
participants[i] = new MockParticipant(_zkaddr, clusterName, instanceName);
participants[i].syncStart();
}
boolean result =
ClusterStateVerifier.verifyByZkCallback(
new BestPossAndExtViewZkVerifier(_zkaddr, clusterName));
Assert.assertTrue(result);
// start the remaining 1 participant
participants[0] = new MockParticipant(_zkaddr, clusterName, "localhost_12918");
participants[0].syncStart();
// make sure we have all participants in MASTER state
result =
ClusterStateVerifier.verifyByZkCallback(
new BestPossAndExtViewZkVerifier(_zkaddr, clusterName));
Assert.assertTrue(result);
key = keyBuilder.externalView("TestSchemata0");
ExternalView externalView = accessor.getProperty(key);
Map<String, String> stateMap = externalView.getStateMap("TestSchemata0_0");
Assert.assertNotNull(stateMap);
Assert.assertEquals(stateMap.size(), n, "all " + n + " participants should be in Master state");
for (int i = 0; i < n; i++) {
String instanceName = "localhost_" + (12918 + i);
Assert.assertNotNull(stateMap.get(instanceName));
Assert.assertEquals(stateMap.get(instanceName), "MASTER");
}
// clean up
controller.syncStop();
for (int i = 0; i < n; i++) {
participants[i].syncStop();
}
System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
@Test
public void testBasic() throws Exception {
// Logger.getRootLogger().setLevel(Level.INFO);
String className = TestHelper.getTestClassName();
String methodName = TestHelper.getTestMethodName();
String clusterName = className + "_" + methodName;
final int n = 5;
System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
MockParticipantManager[] participants = new MockParticipantManager[n];
TestHelper.setupCluster(
clusterName,
ZK_ADDR,
12918, // participant port
"localhost", // participant name prefix
"TestDB", // resource name prefix
1, // resources
10, // partitions per resource
n, // number of nodes
3, // replicas
"MasterSlave",
true); // do rebalance
ClusterControllerManager controller =
new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
controller.syncStart();
// start participants
for (int i = 0; i < n; i++) {
String instanceName = "localhost_" + (12918 + i);
participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
participants[i].syncStart();
}
boolean result =
ClusterStateVerifier.verifyByZkCallback(
new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
Assert.assertTrue(result);
// add a new idealState without registering message handling factory
ClusterSetup setupTool = new ClusterSetup(ZK_ADDR);
setupTool.addResourceToCluster(clusterName, "TestDB1", 16, "MasterSlave");
ZkBaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient);
ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor);
Builder keyBuilder = accessor.keyBuilder();
IdealState idealState = accessor.getProperty(keyBuilder.idealStates("TestDB1"));
idealState.setStateModelFactoryName("TestDB1_Factory");
accessor.setProperty(keyBuilder.idealStates("TestDB1"), idealState);
setupTool.rebalanceStorageCluster(clusterName, "TestDB1", 3);
// assert that we have received OFFLINE->SLAVE messages for all partitions
int totalMsgs = 0;
for (int retry = 0; retry < 5; retry++) {
Thread.sleep(100);
totalMsgs = 0;
for (int i = 0; i < n; i++) {
List<Message> msgs =
accessor.getChildValues(keyBuilder.messages(participants[i].getInstanceName()));
totalMsgs += msgs.size();
}
if (totalMsgs == 48) // partition# x replicas
break;
}
Assert.assertEquals(
totalMsgs,
48,
"Should accumulated 48 unprocessed messages (1 O->S per partition per replica) because TestDB1 is added without state-model-factory but was "
+ totalMsgs);
// register "TestDB1_Factory" state model factory
// Logger.getRootLogger().setLevel(Level.INFO);
for (int i = 0; i < n; i++) {
participants[i]
.getStateMachineEngine()
.registerStateModelFactory("MasterSlave", new MockMSModelFactory(), "TestDB1_Factory");
}
result =
ClusterStateVerifier.verifyByZkCallback(
new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
Assert.assertTrue(result);
// clean up
// wait for all zk callbacks done
controller.syncStop();
for (int i = 0; i < 5; i++) {
participants[i].syncStop();
}
System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}