@Test(timeout = 60000)
public void testLedgerCloseWithConsistentLength() throws Exception {
ClientConfiguration conf = new ClientConfiguration();
conf.setZkServers(zkUtil.getZooKeeperConnectString()).setReadTimeout(1);
BookKeeper bkc = new BookKeeper(conf);
LedgerHandle lh = bkc.createLedger(6, 3, DigestType.CRC32, new byte[] {});
final CountDownLatch latch = new CountDownLatch(1);
stopBKCluster();
final AtomicInteger i = new AtomicInteger(0xdeadbeef);
AsyncCallback.AddCallback cb =
new AsyncCallback.AddCallback() {
@Override
public void addComplete(int rc, LedgerHandle lh, long entryId, Object ctx) {
i.set(rc);
latch.countDown();
}
};
lh.asyncAddEntry("Test Entry".getBytes(), cb, null);
latch.await();
assertEquals(i.get(), BKException.Code.NotEnoughBookiesException);
assertEquals(0, lh.getLength());
assertEquals(LedgerHandle.INVALID_ENTRY_ID, lh.getLastAddConfirmed());
startBKCluster();
LedgerHandle newLh = bkc.openLedger(lh.getId(), DigestType.CRC32, new byte[] {});
assertEquals(0, newLh.getLength());
assertEquals(LedgerHandle.INVALID_ENTRY_ID, newLh.getLastAddConfirmed());
}
private void verifyMetadataConsistency(int numEntries, LedgerHandle lh) throws Exception {
final CountDownLatch addDoneLatch = new CountDownLatch(1);
final CountDownLatch deadIOLatch = new CountDownLatch(1);
final CountDownLatch recoverDoneLatch = new CountDownLatch(1);
final CountDownLatch failedLatch = new CountDownLatch(1);
// kill first bookie to replace with a unauthorize bookie
BookieSocketAddress bookie = lh.getLedgerMetadata().currentEnsemble.get(0);
ServerConfiguration conf = killBookie(bookie);
// replace a unauthorize bookie
startUnauthorizedBookie(conf, addDoneLatch);
// kill second bookie to replace with a dead bookie
bookie = lh.getLedgerMetadata().currentEnsemble.get(1);
conf = killBookie(bookie);
// replace a slow dead bookie
startDeadBookie(conf, deadIOLatch);
// tried to add entries
for (int i = 0; i < numEntries; i++) {
lh.asyncAddEntry(
"data".getBytes(),
new AddCallback() {
@Override
public void addComplete(int rc, LedgerHandle lh, long entryId, Object ctx) {
if (BKException.Code.OK != rc) {
failedLatch.countDown();
deadIOLatch.countDown();
}
if (0 == entryId) {
try {
recoverDoneLatch.await();
} catch (InterruptedException ie) {
}
}
}
},
null);
}
// add finished
addDoneLatch.countDown();
// wait until entries failed due to UnauthorizedAccessException
failedLatch.await();
// simulate the ownership of this ledger is transfer to another host
LOG.info("Recover ledger {}.", lh.getId());
ClientConfiguration newConf = new ClientConfiguration();
newConf.addConfiguration(baseClientConf);
BookKeeper newBkc = new BookKeeperTestClient(newConf.setReadTimeout(1));
LedgerHandle recoveredLh = newBkc.openLedger(lh.getId(), digestType, "".getBytes());
LOG.info("Recover ledger {} done.", lh.getId());
recoverDoneLatch.countDown();
// wait a bit until add operations failed from second bookie due to IOException
TimeUnit.SECONDS.sleep(5);
// open the ledger again to make sure we ge the right last confirmed.
LedgerHandle newLh = newBkc.openLedger(lh.getId(), digestType, "".getBytes());
assertEquals(
"Metadata should be consistent across different opened ledgers",
recoveredLh.getLastAddConfirmed(),
newLh.getLastAddConfirmed());
}