private void waitForCleanStore(MetaDataStateStore store) {
while (true) {
try {
FunctionalTestUtils.count(store);
} catch (Exception ex) {
System.out.println(ex);
UtilWaitThread.sleep(250);
continue;
}
break;
}
}
@Test(timeout = 5 * 60 * 1000)
public void test() throws Exception {
// make some tablets, spread 'em around
Connector c = getConnector();
Credentials creds = new Credentials("root", new PasswordToken(ROOT_PASSWORD));
String table = this.getUniqueNames(1)[0];
c.securityOperations().grantTablePermission("root", MetadataTable.NAME, TablePermission.WRITE);
c.securityOperations().grantTablePermission("root", RootTable.NAME, TablePermission.WRITE);
c.tableOperations().create(table);
SortedSet<Text> partitions = new TreeSet<Text>();
for (String part : "a b c d e f g h i j k l m n o p q r s t u v w x y z".split(" ")) {
partitions.add(new Text(part));
}
c.tableOperations().addSplits(table, partitions);
// scan the metadata table and get the two table location states
Set<TServerInstance> states = new HashSet<TServerInstance>();
Set<TabletLocationState> oldLocations = new HashSet<TabletLocationState>();
MetaDataStateStore store = new MetaDataStateStore(c.getInstance(), creds, null);
while (states.size() < 2) {
UtilWaitThread.sleep(250);
oldLocations.clear();
for (TabletLocationState tls : store) {
if (tls.current != null) {
states.add(tls.current);
oldLocations.add(tls);
}
}
}
assertEquals(2, states.size());
// Kill a tablet server... we don't care which one... wait for everything to be reassigned
cluster.killProcess(
ServerType.TABLET_SERVER,
cluster.getProcesses().get(ServerType.TABLET_SERVER).iterator().next());
// Find out which tablet server remains
while (true) {
UtilWaitThread.sleep(1000);
states.clear();
boolean allAssigned = true;
for (TabletLocationState tls : store) {
if (tls != null && tls.current != null) {
states.add(tls.current);
} else {
allAssigned = false;
}
}
System.out.println(states + " size " + states.size() + " allAssigned " + allAssigned);
if (states.size() != 2 && allAssigned == true) break;
}
assertEquals(1, states.size());
// pick an assigned tablet and assign it to the old tablet
TabletLocationState moved = null;
for (TabletLocationState old : oldLocations) {
if (!states.contains(old.current)) {
moved = old;
}
}
assertNotEquals(null, moved);
// throw a mutation in as if we were the dying tablet
BatchWriter bw = c.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
Mutation assignment = new Mutation(moved.extent.getMetadataEntry());
moved.current.putLocation(assignment);
bw.addMutation(assignment);
bw.close();
// wait for the master to fix the problem
waitForCleanStore(store);
// now jam up the metadata table
bw = c.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
assignment =
new Mutation(new KeyExtent(new Text(MetadataTable.ID), null, null).getMetadataEntry());
moved.current.putLocation(assignment);
bw.addMutation(assignment);
bw.close();
waitForCleanStore(new RootTabletStateStore(c.getInstance(), creds, null));
}